def __init__(self, bootstrapHost: str = 'bootstrap'): self.server = Server() self.server.listen(8469) self.has_list = [] # Hack to get the "deafult" IP s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.connect(('1.1.1.1', 80)) self.local_ip = s.getsockname()[0] s.close() #self.local_ip = socket.gethostbyname(socket.gethostname()) bootstrap_ip = socket.gethostbyname(bootstrapHost) self.bootstrap_node = (bootstrap_ip, 8469) self.loop = asyncio.get_event_loop() self.loop.set_debug(True) self.loop.run_until_complete( self.server.bootstrap([self.bootstrap_node])) neighbors = self.server.bootstrappableNeighbors() for node in neighbors: print("DHT Peer found! {0}:{1}".format(node[0], node[1])) print("Starting TCP transfer server") self.tcpThread = threading.Thread(target=self.startTCPServer) self.tcpThread.start()
def test_get(self): expected = "rohan" key = "A" # getkey("127.0.0.1",8468,key) # self.assertEqual(expected,result) def done(result): # print "Key result:" # print result self.assertEqual(expected, result) reactor.stop() def bootstrapDone(found, server, key): if len(found) == 0: print "Could not connect to the bootstrap server." reactor.stop() server.get(key).addCallback(done) server = Server() temp_port = random.randint(9000, 9999) server.listen( temp_port ) # Any node wishing to query the network essentially registers itself as a server/node with every node it communicates with server.bootstrap([("127.0.0.1", 8468)]).addCallback(bootstrapDone, server, key) reactor.run()
def test_custom_protocol(self): # pylint: disable=no-self-use """ A subclass of Server which overrides the protocol_class attribute will have an instance of that class as its protocol after its listen() method is called. """ # Make a custom Protocol and Server to go with hit. class CoconutProtocol(KademliaProtocol): pass class HuskServer(Server): protocol_class = CoconutProtocol # An ordinary server does NOT have a CoconutProtocol as its protocol... loop = asyncio.get_event_loop() server = Server() loop.run_until_complete(server.listen(8469)) assert not isinstance(server.protocol, CoconutProtocol) server.stop() # ...but our custom server does. husk_server = HuskServer() loop.run_until_complete(husk_server.listen(8469)) assert isinstance(husk_server.protocol, CoconutProtocol) husk_server.stop()
def refresh_mirror( working_dir, first_block, last_block ): """ Refresh the mirror: * process all new invalidations * grab and mirror any new profiles from the DHT This gets called by Twisted every time there ought to be a new block. """ from twisted.python import log from kademlia.network import Server # make soure our bitcoind cached block index is up-to-speed nameop_sequence = sync_blockchain( working_dir, first_block, last_block ) if nameop_sequence is None: log.error("sync_blockchain(%s-%s) failed" % (first_block, last_block)) return None # synchronize name registrations... server = Server() server.listen( dht.plugin.DHT_SERVER_PORT ) server.bootstrap( dht.plugin.DEFAULT_DHT_SERVERS ).addCallback( connect_done, server ) pass
async def run(): server = Server() await server.listen(8469) bootstrap_node = (sys.argv[2], int(sys.argv[3])) await server.bootstrap([bootstrap_node]) await server.set(sys.argv[4], sys.argv[5]) server.stop()
def initialize_dht( bootstrap_ip_port_pair_list: List[Tuple[str, int]], listen_port: int, ) -> None: """ connects to the distributed hash table if no bootstrap ip port pair list is given, it starts a new dht :param bootstrap_ip_port_pair_list: list of ip port tuples to connect to \ the dht :param listen_port: port to listen on :return: instance of server """ global _node_instance get_logger("kademlia") logger.debug("set up DHT: %s", str(bootstrap_ip_port_pair_list)) node = Server(storage=DropPeerDHTStorage()) node.listen(listen_port) loop = asyncio.get_event_loop() if len(bootstrap_ip_port_pair_list) > 0: loop.run_until_complete(node.bootstrap(bootstrap_ip_port_pair_list)) _node_instance = node
def __init__(self, config, bootstrapNeighbors): if os.path.isfile(config['dht.state.cache']): self.kserver = Server.loadState(config['dht.state.cache']) else: self.kserver = Server() self.kserver.bootstrap(bootstrapNeighbors) self.kserver.saveStateRegularly(config['dht.state.cache'], 60)
def start_node(port, BTIp="", BTPort=0): handler = logging.StreamHandler() formatter = logging.Formatter('%(asctime)s - %(name)s\ - %(levelname)s - %(message)s') handler.setFormatter(formatter) # DEBUG if DEBUG: log = logging.getLogger('kademlia') log.addHandler(handler) log.setLevel(logging.DEBUG) loop = asyncio.get_event_loop() if DEBUG: loop.set_debug(True) server = Server() loop.run_until_complete(server.listen(port)) # the first peer don't do that if BTPort != 0: bootstrap_node = (BTIp, int(BTPort)) loop.run_until_complete(server.bootstrap([bootstrap_node])) return (server, loop)
class P2PConnection(object): ''' Class for comunication with the p2p Kademlia network ''' def __init__(self): self.server = Server() self.server.listen(8468)
def send(port, key, message): handler = logging.StreamHandler() formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') handler.setFormatter(formatter) log = logging.getLogger('kademlia') log.addHandler(handler) log.setLevel(logging.DEBUG) loop = asyncio.get_event_loop() loop.set_debug(True) server = Server() server.listen(8469) bootstrap_node = ("0.0.0.0", int(port)) loop.run_until_complete(server.bootstrap([bootstrap_node])) result = loop.run_until_complete(server.get(key)) loop.run_until_complete( server.set(key, str(result) + str(datetime.datetime.now()) + " " + message)) result = loop.run_until_complete(server.get(key)) server.stop() #loop.close() print("************************************************") print(key, "\n", result + "\n") print("************************************************")
def test_custom_protocol(self): """ A subclass of Server which overrides the protocol_class attribute will have an instance of that class as its protocol after its listen() method is called. """ # Make a custom Protocol and Server to go with hit. class CoconutProtocol(KademliaProtocol): pass class HuskServer(Server): protocol_class = CoconutProtocol # An ordinary server does NOT have a CoconutProtocol as its protocol... loop = asyncio.get_event_loop() server = Server() loop.run_until_complete(server.listen(8469)) self.assertNotIsInstance(server.protocol, CoconutProtocol) server.stop() # ...but our custom server does. husk_server = HuskServer() loop.run_until_complete(husk_server.listen(8469)) self.assertIsInstance(husk_server.protocol, CoconutProtocol) husk_server.stop()
def __init__(self, port=DEFAULT_PORT, evt_loop=None): self.set_logging() if not evt_loop: evt_loop = asyncio.get_event_loop() self.loop = evt_loop self.node = Server() self.port = port self.start_server()
def bootstrap_node(event_loop): server = Server() event_loop.run_until_complete(server.listen(8468)) try: yield ('127.0.0.1', 8468) finally: server.stop()
def __init__(self, name="KademliaUser%d" % random.randint(1, 1000), port=emu_config.kademlia_default_port, peerlist=None): super(KademliaUser, self).__init__(name=name) self.kademliaServer = Server() self.port = port self.peerlist = peerlist if peerlist is not None else []
def peer_kademlia_join(): temp = Server() loop.run_until_complete(temp.listen(5678)) loop.run_until_complete(temp.bootstrap([(master_ip, 5678)])) msg = stamp("{} has joined the P2P network.".format(peer_name)) print(msg) global node node = temp
def __init__(self, ksize=20, alpha=3, id=None, storage=None): storage = storage or ForgetfulStorageFix() Server.__init__(self, ksize, alpha, id, storage=storage) self.set_keys=set([]) self.protocol = KademliaProtocolAppend(self.node, self.storage, ksize, set_keys=self.set_keys) if kademlia_version != '0.5': _log.error("#################################################") _log.error("### EXPECTING VERSION 0.5 of kademlia package ###") _log.error("#################################################")
def __init__(self, ip_address=BOOSTRAP_IP, port=BOOSTRAP_PORT): self.server = Server() self.server.listen(UDHT_PORT) self.loop = asyncio.get_event_loop() self.loop.set_debug(True) bootstrap_node = (ip_address, int(port)) self.loop.run_until_complete(self.server.bootstrap([bootstrap_node]))
async def run(): server = Server() await server.listen(8469) bootstrap_node = (sys.argv[1], int(sys.argv[2])) await server.bootstrap([bootstrap_node]) result = await server.get(sys.argv[3]) print("Get result:", result) server.stop()
async def test_storing(bootstrap_node): server = Server() await server.listen(bootstrap_node[1] + 1) await server.bootstrap([bootstrap_node]) await server.set('key', 'value') result = await server.get('key') assert result == 'value' server.stop()
def __init__(self, *initial_peers: Tuple[str, int], port=8081, start=False): super().__init__() self.port, self.initial_peers = port, initial_peers self._pipe, self.pipe = mp.Pipe(duplex=False) self.server = Server() if start: self.start()
def test_default_protocol(self): """ An ordinary Server object will initially not have a protocol, but will have a KademliaProtocol object as its protocol after its listen() method is called. """ server = Server() self.assertIsNone(server.protocol) server.listen(8469) self.assertIsInstance(server.protocol, KademliaProtocol) server.stop()
def test_default_protocol(self): # pylint: disable=no-self-use """ An ordinary Server object will initially not have a protocol, but will have a KademliaProtocol object as its protocol after its listen() method is called. """ loop = asyncio.get_event_loop() server = Server() assert server.protocol is None loop.run_until_complete(server.listen(8469)) assert isinstance(server.protocol, KademliaProtocol) server.stop()
def test_default_protocol(self): """ An ordinary Server object will initially not have a protocol, but will have a KademliaProtocol object as its protocol after its listen() method is called. """ loop = asyncio.get_event_loop() server = Server() self.assertIsNone(server.protocol) loop.run_until_complete(server.listen(8469)) self.assertIsInstance(server.protocol, KademliaProtocol) server.stop()
async def run(): server = Server() await server.listen(8469) bootstrap_node = (sys.argv[1], int(sys.argv[2])) await server.bootstrap([bootstrap_node]) await server.set(sys.argv[3], sys.argv[4]) print("sleeping for 5 seconds") time.sleep(5) print("waking up to fetch stored value") result = await server.get(sys.argv[3]) print("Get result:", result) server.stop()
def __init__(self, *initial_peers: Tuple[str, int], port=8081, start=False, daemon=True): super().__init__() self.port, self.initial_peers = port, initial_peers self._pipe, self.pipe = mp.Pipe(duplex=False) self.ready = mp.Event() self.server = Server() self.daemon = daemon if start: self.run_in_background(await_ready=True)
def initiate(self): if self.server is None: self.loop = asyncio.get_event_loop() self.server = Server(ksize=100, alpha=10) self.loop.run_until_complete(self.server.listen(self.port)) handler = logging.StreamHandler() formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') handler.setFormatter(formatter) log = logging.getLogger('kademlia') log.addHandler(handler) log.setLevel(logging.DEBUG) self.loop.set_debug(True)
class DHTResolver(object): def __init__(self, config, bootstrapNeighbors): self.config = config self.log = Logger(system=self) if os.path.isfile(config['dht.state.cache']): self.kserver = Server.loadState(config['dht.state.cache']) else: self.kserver = Server() self.kserver.bootstrap(bootstrapNeighbors) self.kserver.saveStateRegularly(config['dht.state.cache'], 60) def getProtocol(self): return self.kserver.protocol def getPublicKey(self, keyId): """ Get the public key from the network, and return only if the key is the one that matches the keyId based on hash. """ def verify(key): if key is not None and PublicKey(key).getKeyId() == keyId: return key return None self.log.debug("Getting key text for key id %s" % keyId) return self.kserver.get(keyId).addCallback(verify) def resolve(self, keyId): def parse(locations): self.log.debug("Locations for %s: %s" % (keyId, locations)) results = [] if locations is None or locations == "": return results for location in locations.split(','): host, port = location.split(':') results.append((host, int(port))) return results d = self.kserver.get("%s-location" % keyId) return d.addCallback(parse) def announceLocation(self, myKeyId, myPublicKey): def announce(ips): ips = self.localAddresses() + ips ipports = map(lambda ip: "%s:%i" % (ip, self.config['s2s.port']), ips) return self.kserver.set("%s-location" % myKeyId, ",".join(ipports)) d = self.kserver.set(myKeyId, str(myPublicKey)) d.addCallback(lambda _: self.kserver.inetVisibleIP()) return d.addCallback(announce) def localAddresses(self): result = [] for iface in netifaces.interfaces(): addys = netifaces.ifaddresses(iface).get(netifaces.AF_INET) result += [ addy['addr'] for addy in (addys or []) if addy['addr'] != '127.0.0.1' ] return result
def master_kademlia_join(loop): node = Server() loop.run_until_complete(node.listen(5678)) try: msg = stamp("{} has joined the P2P network.".format(master_name)) send_to_hub(msg) print(msg) loop.run_forever() except KeyboardInterrupt: pass finally: node.stop() loop.close() return node
class DHT: """docstring for Server""" def __init__(self, ip_address=BOOSTRAP_IP, port=BOOSTRAP_PORT): self.server = Server() self.server.listen(UDHT_PORT) self.loop = asyncio.get_event_loop() self.loop.set_debug(True) bootstrap_node = (ip_address, int(port)) self.loop.run_until_complete(self.server.bootstrap([bootstrap_node])) def stop(self): self.server.stop() self.loop.close() def __getitem__(self, key): result = Empty() try: result = loads(self.loop.run_until_complete(self.server.get(key))) except TypeError: pass if isinstance(result, Empty): raise KeyError return result def __setitem__(self, key, item): self.loop.run_until_complete(self.server.set(key, dumps(item))) def __delitem__(self, key): self.loop.run_until_complete(self.server.set(key, dumps(Empty())))
def __init__(self, listenPort): handler = logging.StreamHandler() formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') handler.setFormatter(formatter) log = logging.getLogger('kademlia') log.addHandler(handler) log.setLevel(logging.DEBUG) self.loop = asyncio.get_event_loop() self.loop.set_debug(True) asyncioreactor.install(eventloop=self.loop) self.server = Server() self.server.listen(listenPort)
def __init__(self, name="app_default",port=4020): self._name = name self.number_of_nodes = 0 self.list_of_nodes = [] self._queues = [DeferredQueue()] * 3 # web process pointer self._web = "" # here is the logic to join the DHT, currently using an implementation of Kademlia from entangled.kademlia import node knownNodes = [("127.0.0.1", 4021)] self._ip_address = "127.0.0.1" #self._node = node.Node(4020) #self._node.joinNetwork(knownNodes) from kademlia.network import Server self._serverNode = Server() # listen on port 5556 self._serverNode.listen(5556) # bootstrap with known nodes self._serverNode.bootstrap([("127.0.0.1",5557)]).addCallback(self.bootStrapDone)
def dht_init(blockchain_index): """ Establish our connection to the DHT, and give it the requisite state it needs (i.e. the index over the blockchain). """ global dht_server global dht_blockchain_index dht_blockchain_index = blockchain_index dht_server = Server(storage=BlockStorage()) dht_server.listen(DHT_SERVER_PORT) bootstrap_servers = hostname_to_ip(DEFAULT_DHT_SERVERS) dht_server.bootstrap(bootstrap_servers)
async def test_storing(bootstrap_node): with server_context(Server()) as server: await server.listen(bootstrap_node[1] + 1) await server.bootstrap([bootstrap_node]) await server.set('key', 'value') result = await server.get('key') assert result == 'value'
def uat(wc, option_flags, option_format, myReason): from twisted.internet import reactor from twisted.python import log from kademlia.network import Server import sys log.startLogging(sys.stdout) def query(wc, query_string): wc = wc.canonicalize() query_string = "@prefix : <http://test#>.\n" + query_string query_file = "query.tmp.n3" o = open(query_file, "w") o.write(query_string) o.close() filterContext = _store.load(query_file, flags=option_flags[option_format], referer="", why=myReason, topLevel=True) wc.reopen() print "applying...", filterContext, "to", wc applyRules(wc, filterContext); return wc def done(result): print "Key result:", result #reactor.stop() def setDone(result, server): server.get("a key").addCallback(done) def kademliaBootstrapDone(found, server): server.set("a key", "a value").addCallback(setDone, server) server = Server() server.listen(8468) server.bootstrap([("1.2.3.4", 8468)]).addCallback(kademliaBootstrapDone, server) wc = query(wc, "[ a :banana ].") #reactor.run() return wc
class KadNode: @staticmethod def set_logging(): handler = logging.StreamHandler() formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') handler.setFormatter(formatter) log = logging.getLogger('kademlia') log.addHandler(handler) log.setLevel(logging.DEBUG) def __init__(self, port=DEFAULT_PORT, evt_loop=None): self.set_logging() if not evt_loop: evt_loop = asyncio.get_event_loop() self.loop = evt_loop self.node = Server() self.port = port self.start_server() def start_server(self): def start_loop(node, port): server_loop = asyncio.new_event_loop() server_loop.run_until_complete(node.listen(port)) try: server_loop.run_forever() except KeyboardInterrupt: pass finally: node.stop() server_loop.close() # t = Thread(target=start_loop, args=(self.node, self.port)) t.start() def boot_to(self, host, port): self.loop.run_until_complete(self.node.bootstrap([ (host, port), ])) def set(self, key, value): self.loop.run_until_complete(self.node.set(key, value)) def get(self, key): result = self.loop.run_until_complete(self.node.get(key)) return result
def chat(port, chatname): node = Server() node.listen(port) loop = asyncio.get_event_loop() loop.run_until_complete(node.bootstrap([("0.0.0.0", 8468)])) chatlist = loop.run_until_complete(node.get("chatlist")) loop.run_until_complete(node.set("chatlist", str(chatlist) + "\n" + chatname + "\n")) node.stop()
def user(port, username): node = Server() node.listen(port) loop = asyncio.get_event_loop() loop.run_until_complete(node.bootstrap([("0.0.0.0", 8468)])) userlist = loop.run_until_complete(node.get("userlist")) loop.run_until_complete(node.set("userlist", str(userlist) + "\n" + username + "\n")) node.stop()
def start_node(self): """Each blackbox instance is a node which other nodes can connect to""" print("Node operating at " + str(self.opening_port)) handler = logging.StreamHandler() formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') handler.setFormatter(formatter) log = logging.getLogger('kademlia') log.addHandler(handler) log.setLevel(logging.DEBUG) self.loop = asyncio.get_event_loop() self.loop.set_debug(True) self.node = Server() self.loop.run_until_complete(self.node.listen(self.opening_port)) bootstrap_node = (self.ip, int(self.connecting_port)) self.loop.run_until_complete(self.node.bootstrap([bootstrap_node])) self.loop.run_until_complete(self.node.set("nodes", True))
class P2p(object): def __init__(self): self.server = Server() self.loop = None def run(self): loop = asyncio.get_event_loop() self.loop = loop loop.run_until_complete(self.server.listen(listen_port)) self.loop.run_until_complete(self.server.bootstrap([(bootstrap_host, bootstrap_port)])) loop.run_forever() def get_nodes(self): nodes = [] for bucket in self.server.protocol.router.buckets: nodes.extend(bucket.get_nodes()) return nodes
def kad_server_join(network_port, profile_port, neighbor_ip, neighbor_port, username): handler = logging.StreamHandler() formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') handler.setFormatter(formatter) log = logging.getLogger('kademlia') log.addHandler(handler) log.setLevel(logging.DEBUG) aio = asyncio.new_event_loop() kad = Server() aio.run_until_complete(kad.listen(network_port)) aio.run_until_complete(kad.bootstrap([(neighbor_ip, neighbor_port)])) aio.run_until_complete(asyncio.sleep(1)) # set a value for the key "my-key" on the network aio.run_until_complete( kad.set(username, 'http://' + socket.gethostname() + ':' + str(profile_port))) aio.run_until_complete(asyncio.sleep(2)) # run forever since we are the first node try: main_loop(aio, kad) except KeyboardInterrupt: pass finally: kad.stop() aio.close()
def __init__(self, name="KademliaBot%d" % random.randint(1, 1000), port=emu_config.kademlia_default_port, peerlist=None): super(KademliaBot, self).__init__(name=name) self.kademliaServer = Server() handlers = [("/current_command", KademliaCommandHandler, {"kserver": self.kademliaServer})] app = tornado.web.Application(handlers, autoreload=False) self.httpserver = tornado.httpserver.HTTPServer(app) self.port = port self.peerlist = peerlist if peerlist is not None else []
def getKey(ip,port,key): def done(result): print "Key result:" print result reactor.stop() def bootstrapDone(found, server, key): if len(found) == 0: print "Could not connect to the bootstrap server." reactor.stop() server.get(key).addCallback(done) server = Server() temp_port = random.randint(9000,9999) server.listen(temp_port) # Any node wishing to query the network essentially registers itself as a server/node with every node it communicates with server.bootstrap([(ip, port)]).addCallback(bootstrapDone, server, key) reactor.run()
def helper(file, ip, bootstrapPort, localPort, send, debug): # No loss yet ! config.totalLoss = 0 config.debug = debug # Start Server if debug == 'loads': log.startLogging(sys.stdout) server = Server() # Start own server on port 5678 server.listen(localPort) # Bootstrap with a known ip address of an existing kad server server.bootstrap([(ip, bootstrapPort)]).addCallback(started, server, send, file, debug) reactor.run()
def dht_run(config_file=DEFAULT_CONFIG_FILE): """ Run the blockstackd RPC server, optionally in the foreground. """ # start DHT server observer = log.FileLogObserver(sys.stdout, log.INFO) observer.start() dht_opts = get_dht_opts(config_file) dht_servers_str = dht_opts['servers'] dht_port = dht_opts['port'] dht_servers = parse_dht_servers( dht_servers_str ) dht_server = Server(storage=BlockStorage()) bootstrap_servers = hostname_to_ip(dht_servers) dht_server.bootstrap(bootstrap_servers) reactor.listenUDP( dht_port, dht_server.protocol ) reactor.run()
def __init__(self, appDeployer=False): self._connected = False self._app_deployer = appDeployer # optional... self._number_of_nodes = 0 self._list_of_nodes =[] # HERE--> Implementation specific node instanciation from kademlia.network import Server() self._node = Server()
def connect(self, port): if(self._app_deployer): from twisted.application import service, internet from twisted.python.log import ILogObserver from twisted.internet import reactor, task import sys, os sys.path.append(os.path.dirname(__file__)) from kademlia.network import Server from kademlia import log application = service.Application("kademlia") application.setComponent(ILogObserver, log.FileLogObserver(sys.stdout, log.INFO).emit) if os.path.isfile('cache.pickle'): kserver = Server.loadState('cache.pickle') else: kserver = Server() kserver.bootstrap([("127.0.0.1", port)]) kserver.saveStateRegularly('cache.pickle', 10) server = internet.UDPServer(8468, kserver.protocol) server.setServiceParent(application) else: self._node.listen(port) self._node.bootstrap([("127.0.0.1",port)]).addCallback(self.bootStrapDone)
def bootstrap(self, addrs): """ Bootstrap the server by connecting to other known nodes in the network. Args: addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses are acceptable - hostnames will cause an error. """ # if the transport hasn't been initialized yet, wait a second if self.protocol.transport is None: return task.deferLater(reactor, .2, self.bootstrap, addrs) else: return Server.bootstrap(self, addrs)
def __init__(self, appDeployer, uuid): self._connected = False self._app_deployer = appDeployer # optional... self._number_of_nodes = 0 self._list_of_nodes =[] # logging capabilities self._log = Logger(system=self) # HERE--> Implementation specific node instanciation from kademlia.network import Server self._node = Server() self._node.log.level = 4
class KademliaUser(Runnable): """Allows the KademliaUser to be run in its own thread.""" def __init__(self, name="KademliaUser%d"%random.randint(1, 1000), port=emu_config.kademlia_default_port, peerlist=None): super(KademliaUser, self).__init__(name=name) self.kademliaServer = Server() self.port = port self.peerlist = peerlist if peerlist is not None else [] def start(self): """Implements start() from the superclass.""" self._startKademlia() try: IOLoop.current().start() except socket.error as ex: logging.warning("Could not start the KademliaUser: %s"%ex) def _startKademlia(self): possible_interfaces = [iface for iface in netifaces.interfaces() if iface_searchterm in iface and netifaces.ifaddresses(iface).has_key(netifaces.AF_INET)] if len(possible_interfaces) == 0: logging.error("No suitable interfaces found, tried the following: %s"%netifaces.interfaces()) logging.debug("Interfaces: %s"%netifaces.ifaddresses(possible_interfaces[0])) ipAddr = netifaces.ifaddresses(possible_interfaces[0])[netifaces.AF_INET][0]["addr"] logging.debug("Node %s starts with %s on %s"%(self.name, self.peerlist, ipAddr)) self.kademliaServer.listen(self.port, interface=ipAddr) serverDeferred = self.kademliaServer.bootstrap([(peer, emu_config.kademlia_default_port) for peer in self.peerlist]) serverDeferred.addCallback(self.executeBot) serverDeferred.addErrback(self.errback) def executeBot(self, peersfound=None): """Method that is called regularly and checks for new commands""" self.kademliaServer.get(generateRandomString(length=2)).addCallbacks(self.ignoreInput, self.errback) self.kademliaServer.set(generateRandomString(length=2), generateRandomString()).addCallbacks(self.ignoreInput, self.errback) reactor.callLater(emu_config.botcommand_timeout, self.executeBot) def ignoreInput(self, *args, **kwargs): """Gets whatever is the result of the Kademlia GET request and ignores it""" pass def errback(self, failure, *args, **kwargs): """Given to defereds to report errors. Ignores those errors.""" logging.debug( "Kademlia Error (for the legitimate user, so non-existing keys are not a problem) in %s: %s"%(self.name, failure)) pass
class BitHaikuDHT: log = logging.getLogger(__name__) def __init__(self, configuration): self.configuration = configuration self.server = Server() self.server.listen(configuration.ports.dht) self.server.bootstrap([(configuration.local_address, configuration.ports.dht)]) def add(self, ip): self.log.error("Bootstrapping " + ip) self.server.bootstrap([(ip, self.configuration.ports.dht)]) def find_owner(self, data): key = value = hashlib.sha256(data).hexdigest() self.server.set(key, value) digest = kademlia.utils.digest(hash) node = Node(digest) nearest = self.server.protocol.router.findNeighbors(node) return nearest[0].ip
def __init__(self, appDeployer, uuid): self._connected = False self._app_deployer = appDeployer # optional... self._number_of_nodes = 0 self._list_of_nodes =[] # logging capabilities self._log = Logger(system=self) # HERE--> Implementation specific node instanciation from kademlia.network import Server import kademlia import os path = os.path.dirname(kademlia.__file__) print "PATH TO KADEMLIA: --> ", path self._node = Server() self._node.log.level = 4
class DHTResolver(object): def __init__(self, config, bootstrapNeighbors): if os.path.isfile(config['dht.state.cache']): self.kserver = Server.loadState(config['dht.state.cache']) else: self.kserver = Server() self.kserver.bootstrap(bootstrapNeighbors) self.kserver.saveStateRegularly(config['dht.state.cache'], 60) def getProtocol(self): return self.kserver.protocol def resolve(self, keyId): return self.kserver.get(keyId) def announceLocation(self, myKeyId): def announce(ips): if len(ips) == 0: return defer.succeed(False) return self.kserver.set(myKeyId, ips[0]) d = self.kserver.inetVisibleIP() return d.addCallback(announce)
#announce to master that we exist, then check for ack server.set(key.hexdigest(),str(bot.id)).addCallback(check_ack,server,bot) def setup(ip_list, server): #check that it got a result back #print str(server.node.long_id) if not len(ip_list): print "Could not determine my ip, retrying" server.inetVisibleIP().addCallback(setup,server) myip = most_common(ip_list) idhash = get_hash(str(server.node.long_id)) bot = botnode(myip,port,str(server.node.long_id),idhash) callhome(server,bot) def bootstrapDone(found, server): if len(found) == 0: print "Could not connect to the bootstrap server." reactor.stop() exit(0) server.inetVisibleIP().addCallback(setup,server) server = Server() server.listen(myport) server.bootstrap([(bootstrap_ip, port)]).addCallback(bootstrapDone, server) reactor.run() #bootstrap is at 192.168.56.101:8468
class Application (object): """ This object represent an application in the Cloud, and as indicated in the paper relating to the P2P cloud architecture proposal, these can be seen as being slices of cloud. The concept of application within this project reflect that of a container and/or a seperation layer from the overlay network. Consisting of process pool (Web/Worker/Data), and corresponding queues. cherry_py_instance: is the Instanciated object that contains the logic for the web_server """ def bootStrapDone(self, server): print self print server contacts = self.inetVisibleIP() print contacts def __init__(self, name="app_default",port=4020): self._name = name self.number_of_nodes = 0 self.list_of_nodes = [] self._queues = [DeferredQueue()] * 3 # web process pointer self._web = "" # here is the logic to join the DHT, currently using an implementation of Kademlia from entangled.kademlia import node knownNodes = [("127.0.0.1", 4021)] self._ip_address = "127.0.0.1" #self._node = node.Node(4020) #self._node.joinNetwork(knownNodes) from kademlia.network import Server self._serverNode = Server() # listen on port 5556 self._serverNode.listen(5556) # bootstrap with known nodes self._serverNode.bootstrap([("127.0.0.1",5557)]).addCallback(self.bootStrapDone) def __str__(self): return "Application: [name] = " + self._name + " [number_of_nodes] = " + str(self.number_of_nodes) #" [list_of_nodes] = " + str(self.list_of_nodes) def print_contacts(self): print "app_node --> printcontacts" self._node.printContacts() def addNode (self, node): self.list_of_nodes.append(node) self.number_of_nodes += 1 # next we need to make it a process... def addTask (self, task): # simply add the task to the corresponding queue self._queues[task.type].put(task) def getTask (self, task_type): # simply return the top elt of the queue # and add a callback to fire it's deferred return self._queues[task_type].get().addCallback(task_func) def launchApplication(self): print "--> Launch Application : ", self._name result = self._node.iterativeStore(self._name, self._ip_address) print "called iterative store" return result def retrieveKnownNodes(self, results): d = Deferred() d.addCallback(self._node.knownNodes) d.callback(results) print results print d return d @inlineCallbacks def analyzeResources(self): #retrieve list of known nodes from DHT (overlay network) nodes = yield self._node.knownNodes() print "analyzeResources() ..." print "Nodes: ", nodes #print self import resAdvProtocol results = resAdvProtocol.seek_resources(nodes) print "inside:", results defer.returnValue(results) #return candidates @inlineCallbacks def run(self): # attempt to read from the web queue print "using the run method" task_type = 0 # temp variable to hold a snapshot of the number of nodes in the app. temp_nodes = self.number_of_nodes # ctr ctr = 0 #thread = task.deferLater(reactor,6, self.launchApplication) # then publish the name of the application that you've deployed #thread.addCallback(self.launchApplication) # Analyze available resources... # first extract the list of known nodes, to subscribe to their resource # specification adverstizement # call the resAdvProtocol and print available resources and stats #candidates = seek_resources(known_nodes) from twisted.internet.protocol import Protocol, Factory from twisted.internet import reactor from twisted.internet.defer import Deferred #import cloudProtocol from twisted.internet import reactor, task result = [] #thread = task.deferLater(reactor,1, self.analyzeResources, result) results = yield self.analyzeResources() # d = Deferred() # factory = cloudProtocol.CloudClientFactory(d) # factory.protocol = cloudProtocol.CloudComms # factory.clients = [] #reactor.listenTCP(64000, factory) #print "Cloud Comms server started" #factory.protocol.message(factory.protocol,"Server's MEssage!") #reactor.run() # simply loop forever over each queues #while(True): """
from twisted.internet import reactor from twisted.python import log from kademlia.network import Server import sys # log to std out log.startLogging(sys.stdout) def quit(result): print "Key result:", result reactor.stop() def get(result, server): return server.get("a key").addCallback(quit) def done(found, server): log.msg("Found nodes: %s" % found) return server.set("a key", "a value").addCallback(get, server) server = Server() # next line, or use reactor.listenUDP(5678, server.protocol) server.listen(5678) server.bootstrap([('127.0.0.1', 1234)]).addCallback(done, server) reactor.run()
import logging import asyncio from kademlia.network import Server handler = logging.StreamHandler() formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') handler.setFormatter(formatter) log = logging.getLogger('kademlia') log.addHandler(handler) log.setLevel(logging.DEBUG) loop = asyncio.get_event_loop() loop.set_debug(True) server = Server() loop.run_until_complete(server.listen(8468)) try: loop.run_forever() except KeyboardInterrupt: pass finally: server.stop() loop.close()
def twisted_main(args): log_observer = log.FileLogObserver(sys.stdout, log.INFO) log_observer.start() # Load state root = appdirs.user_cache_dir(args.instancename, 'zarbosoft') mkdirs(root) state = {} try: with open(os.path.join(root, 'state.json'), 'r') as prestate: state = json.loads(prestate.read()) except Exception as e: if args.verbose: log_info('Failed to load state: {}'.format(e)) republish = state.get('republish', {}) # Set up kademlia kserver = Server( ksize=state.get('ksize', 20), alpha=state.get('alpha', 3), seed=binascii.unhexlify(state['seed']) if 'seed' in state else None, storage=Storage(args)) bootstraps = map(tuple, state.get('bootstrap', [])) for bootstrap in args.bootstrap: bhost, bport = bootstrap.split(':', 2) bport = int(bport) bhost_ip = yield reactor.resolve(bhost) bootstraps.append((bhost_ip, bport)) if args.verbose: log_info('Bootstrapping hosts: {}'.format(bootstraps)) kserver.bootstrap(bootstraps) udpserver = internet.UDPServer(args.dhtport, kserver.protocol) udpserver.startService() # Set up state saver def save_state(): if args.verbose: log_info('Saving state') state['ksize'] = kserver.ksize state['alpha'] = kserver.alpha state['seed'] = binascii.hexlify(kserver.node.seed) state['republish'] = republish state['bootstrap'] = kserver.bootstrappableNeighbors() with open(os.path.join(root, 'state.json.1'), 'w') as prestate: prestate.write(json.dumps(state)) os.rename( os.path.join(root, 'state.json.1'), os.path.join(root, 'state.json')) save_state_loop = LoopingCall(save_state) save_state_loop.start(60) # Set up value republisher def start_republish(): @defer.inlineCallbacks def republish_call(): for key, val in republish.items(): if args.verbose: log_info('Republishing {}'.format(key)) yield kserver.set(key, val) republish_loop = LoopingCall(republish_call) republish_loop.start(1 * 60 * 60 * 24) deferLater(reactor, 60, start_republish) # Set up webserver with open(res('redirect_template.html'), 'r') as template: redirect_template = template.read() class Resource(resource.Resource): def getChild(self, child, request): return self def render_GET(self, request): key = urllib.unquote(request.path[1:]) if key == 'setup': with open(res('browser_setup.html'), 'r') as static: return static.read() if key == 'icon-bizast-off.png': with open(res('icon-bizast-off.png'), 'r') as static: return static.read() if key.startswith(webprotocol): key = key[len(webprotocol):] if key.startswith(webprotocol2): key = key[len(webprotocol2):] if key.count(':') != 1: raise ValueError('Invalid resource id') try: key, path = key.split('/', 1) path = '/' + path except ValueError: path = '' def respond(value): if not value: request.write(NoResource().render(request)) else: valid, ign, ign = validate(args, None, value, None) if not valid: request.write(NoResource('Received invalid resource: {}'.format(value)).render(request)) else: value = json.loads(value) if any('text/html' in val for val in request.requestHeaders.getRawHeaders('Accept', [])): message = value['message'] + path if urlmatch.match(message) and '\'' not in message and '"' not in message: request.write(redirect_template.format( resource=message).encode('utf-8')) else: request.write(message.encode('utf-8')) else: request.write(json.dumps(value)) request.finish() log.msg('GET: key [{}]'.format(key)) d = kserver.get(key) d.addCallback(respond) return server.NOT_DONE_YET def render_POST(self, request): value = request.content.getvalue() valid, rec_key, fingerprint = validate(args, None, value, None) if not valid: raise ValueError('Failed verification') log.msg('SET: key [{}] = val [{}]'.format(rec_key, value)) republish[rec_key] = value def respond(result): request.write('Success') request.finish() d = kserver.set(rec_key, value) d.addCallback(respond) return server.NOT_DONE_YET def render_DELETE(self, request): key = urllib.unquote(request.path[1:]) if key.startswith(webprotocol): key = key[len(webprotocol):] if key.count(':') != 1: raise ValueError('Invalid resource id') if key not in republish: raise ValueError('Not republishing key {}'.format(key)) del republish[key] return 'Success' webserver = internet.TCPServer(args.webport, server.Site(Resource())) webserver.startService()