def __init__(self, bootstrapHost: str = 'bootstrap'):
self.server = Server()
self.server.listen(8469)
self.has_list = []
# Hack to get the "deafult" IP
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('1.1.1.1', 80))
self.local_ip = s.getsockname()[0]
s.close()
#self.local_ip = socket.gethostbyname(socket.gethostname())
bootstrap_ip = socket.gethostbyname(bootstrapHost)
self.bootstrap_node = (bootstrap_ip, 8469)
self.loop = asyncio.get_event_loop()
self.loop.set_debug(True)
self.loop.run_until_complete(
self.server.bootstrap([self.bootstrap_node]))
neighbors = self.server.bootstrappableNeighbors()
for node in neighbors:
print("DHT Peer found! {0}:{1}".format(node[0], node[1]))
print("Starting TCP transfer server")
self.tcpThread = threading.Thread(target=self.startTCPServer)
self.tcpThread.start()
def test_get(self):
expected = "rohan"
key = "A"
# getkey("127.0.0.1",8468,key)
# self.assertEqual(expected,result)
def done(result):
# print "Key result:"
# print result
self.assertEqual(expected, result)
reactor.stop()
def bootstrapDone(found, server, key):
if len(found) == 0:
print "Could not connect to the bootstrap server."
reactor.stop()
server.get(key).addCallback(done)
server = Server()
temp_port = random.randint(9000, 9999)
server.listen(
temp_port
) # Any node wishing to query the network essentially registers itself as a server/node with every node it communicates with
server.bootstrap([("127.0.0.1", 8468)]).addCallback(bootstrapDone, server, key)
reactor.run()
def test_custom_protocol(self): # pylint: disable=no-self-use
"""
A subclass of Server which overrides the protocol_class attribute will
have an instance of that class as its protocol after its listen()
method is called.
"""
# Make a custom Protocol and Server to go with hit.
class CoconutProtocol(KademliaProtocol):
pass
class HuskServer(Server):
protocol_class = CoconutProtocol
# An ordinary server does NOT have a CoconutProtocol as its protocol...
loop = asyncio.get_event_loop()
server = Server()
loop.run_until_complete(server.listen(8469))
assert not isinstance(server.protocol, CoconutProtocol)
server.stop()
# ...but our custom server does.
husk_server = HuskServer()
loop.run_until_complete(husk_server.listen(8469))
assert isinstance(husk_server.protocol, CoconutProtocol)
husk_server.stop()
def refresh_mirror( working_dir, first_block, last_block ):
"""
Refresh the mirror:
* process all new invalidations
* grab and mirror any new profiles from the DHT
This gets called by Twisted every time there ought to be a new block.
"""
from twisted.python import log
from kademlia.network import Server
# make soure our bitcoind cached block index is up-to-speed
nameop_sequence = sync_blockchain( working_dir, first_block, last_block )
if nameop_sequence is None:
log.error("sync_blockchain(%s-%s) failed" % (first_block, last_block))
return None
# synchronize name registrations...
server = Server()
server.listen( dht.plugin.DHT_SERVER_PORT )
server.bootstrap( dht.plugin.DEFAULT_DHT_SERVERS ).addCallback( connect_done, server )
pass
async def run():
server = Server()
await server.listen(8469)
bootstrap_node = (sys.argv[2], int(sys.argv[3]))
await server.bootstrap([bootstrap_node])
await server.set(sys.argv[4], sys.argv[5])
server.stop()
def initialize_dht(
bootstrap_ip_port_pair_list: List[Tuple[str, int]],
listen_port: int,
) -> None:
"""
connects to the distributed hash table
if no bootstrap ip port pair list is given, it starts a new dht
:param bootstrap_ip_port_pair_list: list of ip port tuples to connect to \
the dht
:param listen_port: port to listen on
:return: instance of server
"""
global _node_instance
get_logger("kademlia")
logger.debug("set up DHT: %s", str(bootstrap_ip_port_pair_list))
node = Server(storage=DropPeerDHTStorage())
node.listen(listen_port)
loop = asyncio.get_event_loop()
if len(bootstrap_ip_port_pair_list) > 0:
loop.run_until_complete(node.bootstrap(bootstrap_ip_port_pair_list))
_node_instance = node
def __init__(self, config, bootstrapNeighbors):
if os.path.isfile(config['dht.state.cache']):
self.kserver = Server.loadState(config['dht.state.cache'])
else:
self.kserver = Server()
self.kserver.bootstrap(bootstrapNeighbors)
self.kserver.saveStateRegularly(config['dht.state.cache'], 60)
def start_node(port, BTIp="", BTPort=0):
handler = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s - %(name)s\
- %(levelname)s - %(message)s')
handler.setFormatter(formatter)
# DEBUG
if DEBUG:
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
loop = asyncio.get_event_loop()
if DEBUG:
loop.set_debug(True)
server = Server()
loop.run_until_complete(server.listen(port))
# the first peer don't do that
if BTPort != 0:
bootstrap_node = (BTIp, int(BTPort))
loop.run_until_complete(server.bootstrap([bootstrap_node]))
return (server, loop)
class P2PConnection(object):
'''
Class for comunication with the p2p Kademlia network
'''
def __init__(self):
self.server = Server()
self.server.listen(8468)
def send(port, key, message):
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
loop = asyncio.get_event_loop()
loop.set_debug(True)
server = Server()
server.listen(8469)
bootstrap_node = ("0.0.0.0", int(port))
loop.run_until_complete(server.bootstrap([bootstrap_node]))
result = loop.run_until_complete(server.get(key))
loop.run_until_complete(
server.set(key,
str(result) + str(datetime.datetime.now()) + " " +
message))
result = loop.run_until_complete(server.get(key))
server.stop()
#loop.close()
print("************************************************")
print(key, "\n", result + "\n")
print("************************************************")
def test_custom_protocol(self):
"""
A subclass of Server which overrides the protocol_class attribute will
have an instance of that class as its protocol after its listen()
method is called.
"""
# Make a custom Protocol and Server to go with hit.
class CoconutProtocol(KademliaProtocol):
pass
class HuskServer(Server):
protocol_class = CoconutProtocol
# An ordinary server does NOT have a CoconutProtocol as its protocol...
loop = asyncio.get_event_loop()
server = Server()
loop.run_until_complete(server.listen(8469))
self.assertNotIsInstance(server.protocol, CoconutProtocol)
server.stop()
# ...but our custom server does.
husk_server = HuskServer()
loop.run_until_complete(husk_server.listen(8469))
self.assertIsInstance(husk_server.protocol, CoconutProtocol)
husk_server.stop()
def __init__(self, port=DEFAULT_PORT, evt_loop=None):
self.set_logging()
if not evt_loop:
evt_loop = asyncio.get_event_loop()
self.loop = evt_loop
self.node = Server()
self.port = port
self.start_server()
def bootstrap_node(event_loop):
server = Server()
event_loop.run_until_complete(server.listen(8468))
try:
yield ('127.0.0.1', 8468)
finally:
server.stop()
def bootstrap_node(event_loop):
server = Server()
event_loop.run_until_complete(server.listen(8468))
try:
yield ('127.0.0.1', 8468)
finally:
server.stop()
def __init__(self,
name="KademliaUser%d" % random.randint(1, 1000),
port=emu_config.kademlia_default_port,
peerlist=None):
super(KademliaUser, self).__init__(name=name)
self.kademliaServer = Server()
self.port = port
self.peerlist = peerlist if peerlist is not None else []
def peer_kademlia_join():
temp = Server()
loop.run_until_complete(temp.listen(5678))
loop.run_until_complete(temp.bootstrap([(master_ip, 5678)]))
msg = stamp("{} has joined the P2P network.".format(peer_name))
print(msg)
global node
node = temp
def __init__(self, ksize=20, alpha=3, id=None, storage=None):
storage = storage or ForgetfulStorageFix()
Server.__init__(self, ksize, alpha, id, storage=storage)
self.set_keys=set([])
self.protocol = KademliaProtocolAppend(self.node, self.storage, ksize, set_keys=self.set_keys)
if kademlia_version != '0.5':
_log.error("#################################################")
_log.error("### EXPECTING VERSION 0.5 of kademlia package ###")
_log.error("#################################################")
def __init__(self, ip_address=BOOSTRAP_IP, port=BOOSTRAP_PORT):
self.server = Server()
self.server.listen(UDHT_PORT)
self.loop = asyncio.get_event_loop()
self.loop.set_debug(True)
bootstrap_node = (ip_address, int(port))
self.loop.run_until_complete(self.server.bootstrap([bootstrap_node]))
async def run():
server = Server()
await server.listen(8469)
bootstrap_node = (sys.argv[1], int(sys.argv[2]))
await server.bootstrap([bootstrap_node])
result = await server.get(sys.argv[3])
print("Get result:", result)
server.stop()
async def test_storing(bootstrap_node):
server = Server()
await server.listen(bootstrap_node[1] + 1)
await server.bootstrap([bootstrap_node])
await server.set('key', 'value')
result = await server.get('key')
assert result == 'value'
server.stop()
async def test_storing(bootstrap_node):
server = Server()
await server.listen(bootstrap_node[1] + 1)
await server.bootstrap([bootstrap_node])
await server.set('key', 'value')
result = await server.get('key')
assert result == 'value'
server.stop()
def __init__(self,
*initial_peers: Tuple[str, int],
port=8081,
start=False):
super().__init__()
self.port, self.initial_peers = port, initial_peers
self._pipe, self.pipe = mp.Pipe(duplex=False)
self.server = Server()
if start:
self.start()
def test_default_protocol(self):
"""
An ordinary Server object will initially not have a protocol, but will
have a KademliaProtocol object as its protocol after its listen()
method is called.
"""
server = Server()
self.assertIsNone(server.protocol)
server.listen(8469)
self.assertIsInstance(server.protocol, KademliaProtocol)
server.stop()
def test_default_protocol(self): # pylint: disable=no-self-use
"""
An ordinary Server object will initially not have a protocol, but will
have a KademliaProtocol object as its protocol after its listen()
method is called.
"""
loop = asyncio.get_event_loop()
server = Server()
assert server.protocol is None
loop.run_until_complete(server.listen(8469))
assert isinstance(server.protocol, KademliaProtocol)
server.stop()
def test_default_protocol(self):
"""
An ordinary Server object will initially not have a protocol, but will
have a KademliaProtocol object as its protocol after its listen()
method is called.
"""
loop = asyncio.get_event_loop()
server = Server()
self.assertIsNone(server.protocol)
loop.run_until_complete(server.listen(8469))
self.assertIsInstance(server.protocol, KademliaProtocol)
server.stop()
async def run():
server = Server()
await server.listen(8469)
bootstrap_node = (sys.argv[1], int(sys.argv[2]))
await server.bootstrap([bootstrap_node])
await server.set(sys.argv[3], sys.argv[4])
print("sleeping for 5 seconds")
time.sleep(5)
print("waking up to fetch stored value")
result = await server.get(sys.argv[3])
print("Get result:", result)
server.stop()
def __init__(self,
*initial_peers: Tuple[str, int],
port=8081,
start=False,
daemon=True):
super().__init__()
self.port, self.initial_peers = port, initial_peers
self._pipe, self.pipe = mp.Pipe(duplex=False)
self.ready = mp.Event()
self.server = Server()
self.daemon = daemon
if start:
self.run_in_background(await_ready=True)
def initiate(self):
if self.server is None:
self.loop = asyncio.get_event_loop()
self.server = Server(ksize=100, alpha=10)
self.loop.run_until_complete(self.server.listen(self.port))
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
self.loop.set_debug(True)
class DHTResolver(object):
def __init__(self, config, bootstrapNeighbors):
self.config = config
self.log = Logger(system=self)
if os.path.isfile(config['dht.state.cache']):
self.kserver = Server.loadState(config['dht.state.cache'])
else:
self.kserver = Server()
self.kserver.bootstrap(bootstrapNeighbors)
self.kserver.saveStateRegularly(config['dht.state.cache'], 60)
def getProtocol(self):
return self.kserver.protocol
def getPublicKey(self, keyId):
"""
Get the public key from the network, and return only if the key is
the one that matches the keyId based on hash.
"""
def verify(key):
if key is not None and PublicKey(key).getKeyId() == keyId:
return key
return None
self.log.debug("Getting key text for key id %s" % keyId)
return self.kserver.get(keyId).addCallback(verify)
def resolve(self, keyId):
def parse(locations):
self.log.debug("Locations for %s: %s" % (keyId, locations))
results = []
if locations is None or locations == "":
return results
for location in locations.split(','):
host, port = location.split(':')
results.append((host, int(port)))
return results
d = self.kserver.get("%s-location" % keyId)
return d.addCallback(parse)
def announceLocation(self, myKeyId, myPublicKey):
def announce(ips):
ips = self.localAddresses() + ips
ipports = map(lambda ip: "%s:%i" % (ip, self.config['s2s.port']), ips)
return self.kserver.set("%s-location" % myKeyId, ",".join(ipports))
d = self.kserver.set(myKeyId, str(myPublicKey))
d.addCallback(lambda _: self.kserver.inetVisibleIP())
return d.addCallback(announce)
def localAddresses(self):
result = []
for iface in netifaces.interfaces():
addys = netifaces.ifaddresses(iface).get(netifaces.AF_INET)
result += [ addy['addr'] for addy in (addys or []) if addy['addr'] != '127.0.0.1' ]
return result
def master_kademlia_join(loop):
node = Server()
loop.run_until_complete(node.listen(5678))
try:
msg = stamp("{} has joined the P2P network.".format(master_name))
send_to_hub(msg)
print(msg)
loop.run_forever()
except KeyboardInterrupt:
pass
finally:
node.stop()
loop.close()
return node
class DHT:
"""docstring for Server"""
def __init__(self, ip_address=BOOSTRAP_IP, port=BOOSTRAP_PORT):
self.server = Server()
self.server.listen(UDHT_PORT)
self.loop = asyncio.get_event_loop()
self.loop.set_debug(True)
bootstrap_node = (ip_address, int(port))
self.loop.run_until_complete(self.server.bootstrap([bootstrap_node]))
def stop(self):
self.server.stop()
self.loop.close()
def __getitem__(self, key):
result = Empty()
try:
result = loads(self.loop.run_until_complete(self.server.get(key)))
except TypeError:
pass
if isinstance(result, Empty):
raise KeyError
return result
def __setitem__(self, key, item):
self.loop.run_until_complete(self.server.set(key, dumps(item)))
def __delitem__(self, key):
self.loop.run_until_complete(self.server.set(key, dumps(Empty())))
def __init__(self, listenPort):
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
self.loop = asyncio.get_event_loop()
self.loop.set_debug(True)
asyncioreactor.install(eventloop=self.loop)
self.server = Server()
self.server.listen(listenPort)
def __init__(self, name="app_default",port=4020):
self._name = name
self.number_of_nodes = 0
self.list_of_nodes = []
self._queues = [DeferredQueue()] * 3
# web process pointer
self._web = ""
# here is the logic to join the DHT, currently using an implementation of Kademlia
from entangled.kademlia import node
knownNodes = [("127.0.0.1", 4021)]
self._ip_address = "127.0.0.1"
#self._node = node.Node(4020)
#self._node.joinNetwork(knownNodes)
from kademlia.network import Server
self._serverNode = Server()
# listen on port 5556
self._serverNode.listen(5556)
# bootstrap with known nodes
self._serverNode.bootstrap([("127.0.0.1",5557)]).addCallback(self.bootStrapDone)
def dht_init(blockchain_index):
"""
Establish our connection to the DHT, and give
it the requisite state it needs (i.e. the index
over the blockchain).
"""
global dht_server
global dht_blockchain_index
dht_blockchain_index = blockchain_index
dht_server = Server(storage=BlockStorage())
dht_server.listen(DHT_SERVER_PORT)
bootstrap_servers = hostname_to_ip(DEFAULT_DHT_SERVERS)
dht_server.bootstrap(bootstrap_servers)
async def test_storing(bootstrap_node):
with server_context(Server()) as server:
await server.listen(bootstrap_node[1] + 1)
await server.bootstrap([bootstrap_node])
await server.set('key', 'value')
result = await server.get('key')
assert result == 'value'
def uat(wc, option_flags, option_format, myReason):
from twisted.internet import reactor
from twisted.python import log
from kademlia.network import Server
import sys
log.startLogging(sys.stdout)
def query(wc, query_string):
wc = wc.canonicalize()
query_string = "@prefix : <http://test#>.\n" + query_string
query_file = "query.tmp.n3"
o = open(query_file, "w")
o.write(query_string)
o.close()
filterContext = _store.load(query_file,
flags=option_flags[option_format],
referer="",
why=myReason, topLevel=True)
wc.reopen()
print "applying...", filterContext, "to", wc
applyRules(wc, filterContext);
return wc
def done(result):
print "Key result:", result
#reactor.stop()
def setDone(result, server):
server.get("a key").addCallback(done)
def kademliaBootstrapDone(found, server):
server.set("a key", "a value").addCallback(setDone, server)
server = Server()
server.listen(8468)
server.bootstrap([("1.2.3.4", 8468)]).addCallback(kademliaBootstrapDone, server)
wc = query(wc, "[ a :banana ].")
#reactor.run()
return wc
class KadNode:
@staticmethod
def set_logging():
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
def __init__(self, port=DEFAULT_PORT, evt_loop=None):
self.set_logging()
if not evt_loop:
evt_loop = asyncio.get_event_loop()
self.loop = evt_loop
self.node = Server()
self.port = port
self.start_server()
def start_server(self):
def start_loop(node, port):
server_loop = asyncio.new_event_loop()
server_loop.run_until_complete(node.listen(port))
try:
server_loop.run_forever()
except KeyboardInterrupt:
pass
finally:
node.stop()
server_loop.close()
#
t = Thread(target=start_loop, args=(self.node, self.port))
t.start()
def boot_to(self, host, port):
self.loop.run_until_complete(self.node.bootstrap([
(host, port),
]))
def set(self, key, value):
self.loop.run_until_complete(self.node.set(key, value))
def get(self, key):
result = self.loop.run_until_complete(self.node.get(key))
return result
def chat(port, chatname):
node = Server()
node.listen(port)
loop = asyncio.get_event_loop()
loop.run_until_complete(node.bootstrap([("0.0.0.0", 8468)]))
chatlist = loop.run_until_complete(node.get("chatlist"))
loop.run_until_complete(node.set("chatlist", str(chatlist) + "\n" + chatname + "\n"))
node.stop()
def user(port, username):
node = Server()
node.listen(port)
loop = asyncio.get_event_loop()
loop.run_until_complete(node.bootstrap([("0.0.0.0", 8468)]))
userlist = loop.run_until_complete(node.get("userlist"))
loop.run_until_complete(node.set("userlist", str(userlist) + "\n" + username + "\n"))
node.stop()
def start_node(self):
"""Each blackbox instance is a node which other nodes can connect to"""
print("Node operating at " + str(self.opening_port))
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
self.loop = asyncio.get_event_loop()
self.loop.set_debug(True)
self.node = Server()
self.loop.run_until_complete(self.node.listen(self.opening_port))
bootstrap_node = (self.ip, int(self.connecting_port))
self.loop.run_until_complete(self.node.bootstrap([bootstrap_node]))
self.loop.run_until_complete(self.node.set("nodes", True))
class P2p(object):
def __init__(self):
self.server = Server()
self.loop = None
def run(self):
loop = asyncio.get_event_loop()
self.loop = loop
loop.run_until_complete(self.server.listen(listen_port))
self.loop.run_until_complete(self.server.bootstrap([(bootstrap_host, bootstrap_port)]))
loop.run_forever()
def get_nodes(self):
nodes = []
for bucket in self.server.protocol.router.buckets:
nodes.extend(bucket.get_nodes())
return nodes
def kad_server_join(network_port, profile_port, neighbor_ip, neighbor_port,
username):
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
aio = asyncio.new_event_loop()
kad = Server()
aio.run_until_complete(kad.listen(network_port))
aio.run_until_complete(kad.bootstrap([(neighbor_ip, neighbor_port)]))
aio.run_until_complete(asyncio.sleep(1))
# set a value for the key "my-key" on the network
aio.run_until_complete(
kad.set(username,
'http://' + socket.gethostname() + ':' + str(profile_port)))
aio.run_until_complete(asyncio.sleep(2))
# run forever since we are the first node
try:
main_loop(aio, kad)
except KeyboardInterrupt:
pass
finally:
kad.stop()
aio.close()
def __init__(self, name="KademliaBot%d" % random.randint(1, 1000),
port=emu_config.kademlia_default_port, peerlist=None):
super(KademliaBot, self).__init__(name=name)
self.kademliaServer = Server()
handlers = [("/current_command", KademliaCommandHandler, {"kserver": self.kademliaServer})]
app = tornado.web.Application(handlers, autoreload=False)
self.httpserver = tornado.httpserver.HTTPServer(app)
self.port = port
self.peerlist = peerlist if peerlist is not None else []
def getKey(ip,port,key): def done(result): print "Key result:" print result reactor.stop() def bootstrapDone(found, server, key): if len(found) == 0: print "Could not connect to the bootstrap server." reactor.stop() server.get(key).addCallback(done) server = Server() temp_port = random.randint(9000,9999) server.listen(temp_port) # Any node wishing to query the network essentially registers itself as a server/node with every node it communicates with server.bootstrap([(ip, port)]).addCallback(bootstrapDone, server, key) reactor.run()
def helper(file, ip, bootstrapPort, localPort, send, debug):
# No loss yet !
config.totalLoss = 0
config.debug = debug
# Start Server
if debug == 'loads': log.startLogging(sys.stdout)
server = Server()
# Start own server on port 5678
server.listen(localPort)
# Bootstrap with a known ip address of an existing kad server
server.bootstrap([(ip, bootstrapPort)]).addCallback(started, server, send, file, debug)
reactor.run()
def dht_run(config_file=DEFAULT_CONFIG_FILE):
"""
Run the blockstackd RPC server, optionally in the foreground.
"""
# start DHT server
observer = log.FileLogObserver(sys.stdout, log.INFO)
observer.start()
dht_opts = get_dht_opts(config_file)
dht_servers_str = dht_opts['servers']
dht_port = dht_opts['port']
dht_servers = parse_dht_servers( dht_servers_str )
dht_server = Server(storage=BlockStorage())
bootstrap_servers = hostname_to_ip(dht_servers)
dht_server.bootstrap(bootstrap_servers)
reactor.listenUDP( dht_port, dht_server.protocol )
reactor.run()
def __init__(self, appDeployer=False):
self._connected = False
self._app_deployer = appDeployer
# optional...
self._number_of_nodes = 0
self._list_of_nodes =[]
# HERE--> Implementation specific node instanciation
from kademlia.network import Server()
self._node = Server()
def connect(self, port):
if(self._app_deployer):
from twisted.application import service, internet
from twisted.python.log import ILogObserver
from twisted.internet import reactor, task
import sys, os
sys.path.append(os.path.dirname(__file__))
from kademlia.network import Server
from kademlia import log
application = service.Application("kademlia")
application.setComponent(ILogObserver, log.FileLogObserver(sys.stdout, log.INFO).emit)
if os.path.isfile('cache.pickle'):
kserver = Server.loadState('cache.pickle')
else:
kserver = Server()
kserver.bootstrap([("127.0.0.1", port)])
kserver.saveStateRegularly('cache.pickle', 10)
server = internet.UDPServer(8468, kserver.protocol)
server.setServiceParent(application)
else:
self._node.listen(port)
self._node.bootstrap([("127.0.0.1",port)]).addCallback(self.bootStrapDone)
def bootstrap(self, addrs):
"""
Bootstrap the server by connecting to other known nodes in the network.
Args:
addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses
are acceptable - hostnames will cause an error.
"""
# if the transport hasn't been initialized yet, wait a second
if self.protocol.transport is None:
return task.deferLater(reactor, .2, self.bootstrap, addrs)
else:
return Server.bootstrap(self, addrs)
def __init__(self, appDeployer, uuid):
self._connected = False
self._app_deployer = appDeployer
# optional...
self._number_of_nodes = 0
self._list_of_nodes =[]
# logging capabilities
self._log = Logger(system=self)
# HERE--> Implementation specific node instanciation
from kademlia.network import Server
self._node = Server()
self._node.log.level = 4
class KademliaUser(Runnable):
"""Allows the KademliaUser to be run in its own thread."""
def __init__(self, name="KademliaUser%d"%random.randint(1, 1000),
port=emu_config.kademlia_default_port, peerlist=None):
super(KademliaUser, self).__init__(name=name)
self.kademliaServer = Server()
self.port = port
self.peerlist = peerlist if peerlist is not None else []
def start(self):
"""Implements start() from the superclass."""
self._startKademlia()
try:
IOLoop.current().start()
except socket.error as ex:
logging.warning("Could not start the KademliaUser: %s"%ex)
def _startKademlia(self):
possible_interfaces = [iface for iface in netifaces.interfaces() if iface_searchterm in iface
and netifaces.ifaddresses(iface).has_key(netifaces.AF_INET)]
if len(possible_interfaces) == 0:
logging.error("No suitable interfaces found, tried the following: %s"%netifaces.interfaces())
logging.debug("Interfaces: %s"%netifaces.ifaddresses(possible_interfaces[0]))
ipAddr = netifaces.ifaddresses(possible_interfaces[0])[netifaces.AF_INET][0]["addr"]
logging.debug("Node %s starts with %s on %s"%(self.name, self.peerlist, ipAddr))
self.kademliaServer.listen(self.port, interface=ipAddr)
serverDeferred = self.kademliaServer.bootstrap([(peer, emu_config.kademlia_default_port) for peer in self.peerlist])
serverDeferred.addCallback(self.executeBot)
serverDeferred.addErrback(self.errback)
def executeBot(self, peersfound=None):
"""Method that is called regularly and checks for new commands"""
self.kademliaServer.get(generateRandomString(length=2)).addCallbacks(self.ignoreInput, self.errback)
self.kademliaServer.set(generateRandomString(length=2), generateRandomString()).addCallbacks(self.ignoreInput,
self.errback)
reactor.callLater(emu_config.botcommand_timeout, self.executeBot)
def ignoreInput(self, *args, **kwargs):
"""Gets whatever is the result of the Kademlia GET request and ignores it"""
pass
def errback(self, failure, *args, **kwargs):
"""Given to defereds to report errors. Ignores those errors."""
logging.debug(
"Kademlia Error (for the legitimate user, so non-existing keys are not a problem) in %s: %s"%(self.name, failure))
pass
class BitHaikuDHT:
log = logging.getLogger(__name__)
def __init__(self, configuration):
self.configuration = configuration
self.server = Server()
self.server.listen(configuration.ports.dht)
self.server.bootstrap([(configuration.local_address, configuration.ports.dht)])
def add(self, ip):
self.log.error("Bootstrapping " + ip)
self.server.bootstrap([(ip, self.configuration.ports.dht)])
def find_owner(self, data):
key = value = hashlib.sha256(data).hexdigest()
self.server.set(key, value)
digest = kademlia.utils.digest(hash)
node = Node(digest)
nearest = self.server.protocol.router.findNeighbors(node)
return nearest[0].ip
def __init__(self, appDeployer, uuid):
self._connected = False
self._app_deployer = appDeployer
# optional...
self._number_of_nodes = 0
self._list_of_nodes =[]
# logging capabilities
self._log = Logger(system=self)
# HERE--> Implementation specific node instanciation
from kademlia.network import Server
import kademlia
import os
path = os.path.dirname(kademlia.__file__)
print "PATH TO KADEMLIA: --> ", path
self._node = Server()
self._node.log.level = 4
class DHTResolver(object):
def __init__(self, config, bootstrapNeighbors):
if os.path.isfile(config['dht.state.cache']):
self.kserver = Server.loadState(config['dht.state.cache'])
else:
self.kserver = Server()
self.kserver.bootstrap(bootstrapNeighbors)
self.kserver.saveStateRegularly(config['dht.state.cache'], 60)
def getProtocol(self):
return self.kserver.protocol
def resolve(self, keyId):
return self.kserver.get(keyId)
def announceLocation(self, myKeyId):
def announce(ips):
if len(ips) == 0:
return defer.succeed(False)
return self.kserver.set(myKeyId, ips[0])
d = self.kserver.inetVisibleIP()
return d.addCallback(announce)
#announce to master that we exist, then check for ack
server.set(key.hexdigest(),str(bot.id)).addCallback(check_ack,server,bot)
def setup(ip_list, server):
#check that it got a result back
#print str(server.node.long_id)
if not len(ip_list):
print "Could not determine my ip, retrying"
server.inetVisibleIP().addCallback(setup,server)
myip = most_common(ip_list)
idhash = get_hash(str(server.node.long_id))
bot = botnode(myip,port,str(server.node.long_id),idhash)
callhome(server,bot)
def bootstrapDone(found, server):
if len(found) == 0:
print "Could not connect to the bootstrap server."
reactor.stop()
exit(0)
server.inetVisibleIP().addCallback(setup,server)
server = Server()
server.listen(myport)
server.bootstrap([(bootstrap_ip, port)]).addCallback(bootstrapDone, server)
reactor.run()
#bootstrap is at 192.168.56.101:8468
class Application (object):
"""
This object represent an application in the Cloud, and as indicated in the paper
relating to the P2P cloud architecture proposal, these can be seen as being slices
of cloud.
The concept of application within this project reflect that of a container and/or a seperation
layer from the overlay network.
Consisting of process pool (Web/Worker/Data), and corresponding queues.
cherry_py_instance: is the Instanciated object that contains the logic for the web_server
"""
def bootStrapDone(self, server):
print self
print server
contacts = self.inetVisibleIP()
print contacts
def __init__(self, name="app_default",port=4020):
self._name = name
self.number_of_nodes = 0
self.list_of_nodes = []
self._queues = [DeferredQueue()] * 3
# web process pointer
self._web = ""
# here is the logic to join the DHT, currently using an implementation of Kademlia
from entangled.kademlia import node
knownNodes = [("127.0.0.1", 4021)]
self._ip_address = "127.0.0.1"
#self._node = node.Node(4020)
#self._node.joinNetwork(knownNodes)
from kademlia.network import Server
self._serverNode = Server()
# listen on port 5556
self._serverNode.listen(5556)
# bootstrap with known nodes
self._serverNode.bootstrap([("127.0.0.1",5557)]).addCallback(self.bootStrapDone)
def __str__(self):
return "Application: [name] = " + self._name + " [number_of_nodes] = " + str(self.number_of_nodes)
#" [list_of_nodes] = " + str(self.list_of_nodes)
def print_contacts(self):
print "app_node --> printcontacts"
self._node.printContacts()
def addNode (self, node):
self.list_of_nodes.append(node)
self.number_of_nodes += 1
# next we need to make it a process...
def addTask (self, task):
# simply add the task to the corresponding queue
self._queues[task.type].put(task)
def getTask (self, task_type):
# simply return the top elt of the queue
# and add a callback to fire it's deferred
return self._queues[task_type].get().addCallback(task_func)
def launchApplication(self):
print "--> Launch Application : ", self._name
result = self._node.iterativeStore(self._name, self._ip_address)
print "called iterative store"
return result
def retrieveKnownNodes(self, results):
d = Deferred()
d.addCallback(self._node.knownNodes)
d.callback(results)
print results
print d
return d
@inlineCallbacks
def analyzeResources(self):
#retrieve list of known nodes from DHT (overlay network)
nodes = yield self._node.knownNodes()
print "analyzeResources() ..."
print "Nodes: ", nodes
#print self
import resAdvProtocol
results = resAdvProtocol.seek_resources(nodes)
print "inside:", results
defer.returnValue(results)
#return candidates
@inlineCallbacks
def run(self):
# attempt to read from the web queue
print "using the run method"
task_type = 0
# temp variable to hold a snapshot of the number of nodes in the app.
temp_nodes = self.number_of_nodes
# ctr
ctr = 0
#thread = task.deferLater(reactor,6, self.launchApplication)
# then publish the name of the application that you've deployed
#thread.addCallback(self.launchApplication)
# Analyze available resources...
# first extract the list of known nodes, to subscribe to their resource
# specification adverstizement
# call the resAdvProtocol and print available resources and stats
#candidates = seek_resources(known_nodes)
from twisted.internet.protocol import Protocol, Factory
from twisted.internet import reactor
from twisted.internet.defer import Deferred
#import cloudProtocol
from twisted.internet import reactor, task
result = []
#thread = task.deferLater(reactor,1, self.analyzeResources, result)
results = yield self.analyzeResources()
# d = Deferred()
# factory = cloudProtocol.CloudClientFactory(d)
# factory.protocol = cloudProtocol.CloudComms
# factory.clients = []
#reactor.listenTCP(64000, factory)
#print "Cloud Comms server started"
#factory.protocol.message(factory.protocol,"Server's MEssage!")
#reactor.run()
# simply loop forever over each queues
#while(True):
"""
from twisted.internet import reactor
from twisted.python import log
from kademlia.network import Server
import sys
# log to std out
log.startLogging(sys.stdout)
def quit(result):
print "Key result:", result
reactor.stop()
def get(result, server):
return server.get("a key").addCallback(quit)
def done(found, server):
log.msg("Found nodes: %s" % found)
return server.set("a key", "a value").addCallback(get, server)
server = Server()
# next line, or use reactor.listenUDP(5678, server.protocol)
server.listen(5678)
server.bootstrap([('127.0.0.1', 1234)]).addCallback(done, server)
reactor.run()
import logging
import asyncio
from kademlia.network import Server
handler = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
loop = asyncio.get_event_loop()
loop.set_debug(True)
server = Server()
loop.run_until_complete(server.listen(8468))
try:
loop.run_forever()
except KeyboardInterrupt:
pass
finally:
server.stop()
loop.close()
def twisted_main(args):
log_observer = log.FileLogObserver(sys.stdout, log.INFO)
log_observer.start()
# Load state
root = appdirs.user_cache_dir(args.instancename, 'zarbosoft')
mkdirs(root)
state = {}
try:
with open(os.path.join(root, 'state.json'), 'r') as prestate:
state = json.loads(prestate.read())
except Exception as e:
if args.verbose:
log_info('Failed to load state: {}'.format(e))
republish = state.get('republish', {})
# Set up kademlia
kserver = Server(
ksize=state.get('ksize', 20),
alpha=state.get('alpha', 3),
seed=binascii.unhexlify(state['seed']) if 'seed' in state else None,
storage=Storage(args))
bootstraps = map(tuple, state.get('bootstrap', []))
for bootstrap in args.bootstrap:
bhost, bport = bootstrap.split(':', 2)
bport = int(bport)
bhost_ip = yield reactor.resolve(bhost)
bootstraps.append((bhost_ip, bport))
if args.verbose:
log_info('Bootstrapping hosts: {}'.format(bootstraps))
kserver.bootstrap(bootstraps)
udpserver = internet.UDPServer(args.dhtport, kserver.protocol)
udpserver.startService()
# Set up state saver
def save_state():
if args.verbose:
log_info('Saving state')
state['ksize'] = kserver.ksize
state['alpha'] = kserver.alpha
state['seed'] = binascii.hexlify(kserver.node.seed)
state['republish'] = republish
state['bootstrap'] = kserver.bootstrappableNeighbors()
with open(os.path.join(root, 'state.json.1'), 'w') as prestate:
prestate.write(json.dumps(state))
os.rename(
os.path.join(root, 'state.json.1'),
os.path.join(root, 'state.json'))
save_state_loop = LoopingCall(save_state)
save_state_loop.start(60)
# Set up value republisher
def start_republish():
@defer.inlineCallbacks
def republish_call():
for key, val in republish.items():
if args.verbose:
log_info('Republishing {}'.format(key))
yield kserver.set(key, val)
republish_loop = LoopingCall(republish_call)
republish_loop.start(1 * 60 * 60 * 24)
deferLater(reactor, 60, start_republish)
# Set up webserver
with open(res('redirect_template.html'), 'r') as template:
redirect_template = template.read()
class Resource(resource.Resource):
def getChild(self, child, request):
return self
def render_GET(self, request):
key = urllib.unquote(request.path[1:])
if key == 'setup':
with open(res('browser_setup.html'), 'r') as static:
return static.read()
if key == 'icon-bizast-off.png':
with open(res('icon-bizast-off.png'), 'r') as static:
return static.read()
if key.startswith(webprotocol):
key = key[len(webprotocol):]
if key.startswith(webprotocol2):
key = key[len(webprotocol2):]
if key.count(':') != 1:
raise ValueError('Invalid resource id')
try:
key, path = key.split('/', 1)
path = '/' + path
except ValueError:
path = ''
def respond(value):
if not value:
request.write(NoResource().render(request))
else:
valid, ign, ign = validate(args, None, value, None)
if not valid:
request.write(NoResource('Received invalid resource: {}'.format(value)).render(request))
else:
value = json.loads(value)
if any('text/html' in val for val in request.requestHeaders.getRawHeaders('Accept', [])):
message = value['message'] + path
if urlmatch.match(message) and '\'' not in message and '"' not in message:
request.write(redirect_template.format(
resource=message).encode('utf-8'))
else:
request.write(message.encode('utf-8'))
else:
request.write(json.dumps(value))
request.finish()
log.msg('GET: key [{}]'.format(key))
d = kserver.get(key)
d.addCallback(respond)
return server.NOT_DONE_YET
def render_POST(self, request):
value = request.content.getvalue()
valid, rec_key, fingerprint = validate(args, None, value, None)
if not valid:
raise ValueError('Failed verification')
log.msg('SET: key [{}] = val [{}]'.format(rec_key, value))
republish[rec_key] = value
def respond(result):
request.write('Success')
request.finish()
d = kserver.set(rec_key, value)
d.addCallback(respond)
return server.NOT_DONE_YET
def render_DELETE(self, request):
key = urllib.unquote(request.path[1:])
if key.startswith(webprotocol):
key = key[len(webprotocol):]
if key.count(':') != 1:
raise ValueError('Invalid resource id')
if key not in republish:
raise ValueError('Not republishing key {}'.format(key))
del republish[key]
return 'Success'
webserver = internet.TCPServer(args.webport, server.Site(Resource()))
webserver.startService()
