class DHT:
"""docstring for Server"""
def __init__(self, ip_address=BOOSTRAP_IP, port=BOOSTRAP_PORT):
self.server = Server()
self.server.listen(UDHT_PORT)
self.loop = asyncio.get_event_loop()
self.loop.set_debug(True)
bootstrap_node = (ip_address, int(port))
self.loop.run_until_complete(self.server.bootstrap([bootstrap_node]))
def stop(self):
self.server.stop()
self.loop.close()
def __getitem__(self, key):
result = Empty()
try:
result = loads(self.loop.run_until_complete(self.server.get(key)))
except TypeError:
pass
if isinstance(result, Empty):
raise KeyError
return result
def __setitem__(self, key, item):
self.loop.run_until_complete(self.server.set(key, dumps(item)))
def __delitem__(self, key):
self.loop.run_until_complete(self.server.set(key, dumps(Empty())))
class DHTResolver(object):
def __init__(self, config, bootstrapNeighbors):
self.config = config
self.log = Logger(system=self)
if os.path.isfile(config['dht.state.cache']):
self.kserver = Server.loadState(config['dht.state.cache'])
else:
self.kserver = Server()
self.kserver.bootstrap(bootstrapNeighbors)
self.kserver.saveStateRegularly(config['dht.state.cache'], 60)
def getProtocol(self):
return self.kserver.protocol
def getPublicKey(self, keyId):
"""
Get the public key from the network, and return only if the key is
the one that matches the keyId based on hash.
"""
def verify(key):
if key is not None and PublicKey(key).getKeyId() == keyId:
return key
return None
self.log.debug("Getting key text for key id %s" % keyId)
return self.kserver.get(keyId).addCallback(verify)
def resolve(self, keyId):
def parse(locations):
self.log.debug("Locations for %s: %s" % (keyId, locations))
results = []
if locations is None or locations == "":
return results
for location in locations.split(','):
host, port = location.split(':')
results.append((host, int(port)))
return results
d = self.kserver.get("%s-location" % keyId)
return d.addCallback(parse)
def announceLocation(self, myKeyId, myPublicKey):
def announce(ips):
ips = self.localAddresses() + ips
ipports = map(lambda ip: "%s:%i" % (ip, self.config['s2s.port']), ips)
return self.kserver.set("%s-location" % myKeyId, ",".join(ipports))
d = self.kserver.set(myKeyId, str(myPublicKey))
d.addCallback(lambda _: self.kserver.inetVisibleIP())
return d.addCallback(announce)
def localAddresses(self):
result = []
for iface in netifaces.interfaces():
addys = netifaces.ifaddresses(iface).get(netifaces.AF_INET)
result += [ addy['addr'] for addy in (addys or []) if addy['addr'] != '127.0.0.1' ]
return result
class KademliaUser(Runnable):
"""Allows the KademliaUser to be run in its own thread."""
def __init__(self, name="KademliaUser%d"%random.randint(1, 1000),
port=emu_config.kademlia_default_port, peerlist=None):
super(KademliaUser, self).__init__(name=name)
self.kademliaServer = Server()
self.port = port
self.peerlist = peerlist if peerlist is not None else []
def start(self):
"""Implements start() from the superclass."""
self._startKademlia()
try:
IOLoop.current().start()
except socket.error as ex:
logging.warning("Could not start the KademliaUser: %s"%ex)
def _startKademlia(self):
possible_interfaces = [iface for iface in netifaces.interfaces() if iface_searchterm in iface
and netifaces.ifaddresses(iface).has_key(netifaces.AF_INET)]
if len(possible_interfaces) == 0:
logging.error("No suitable interfaces found, tried the following: %s"%netifaces.interfaces())
logging.debug("Interfaces: %s"%netifaces.ifaddresses(possible_interfaces[0]))
ipAddr = netifaces.ifaddresses(possible_interfaces[0])[netifaces.AF_INET][0]["addr"]
logging.debug("Node %s starts with %s on %s"%(self.name, self.peerlist, ipAddr))
self.kademliaServer.listen(self.port, interface=ipAddr)
serverDeferred = self.kademliaServer.bootstrap([(peer, emu_config.kademlia_default_port) for peer in self.peerlist])
serverDeferred.addCallback(self.executeBot)
serverDeferred.addErrback(self.errback)
def executeBot(self, peersfound=None):
"""Method that is called regularly and checks for new commands"""
self.kademliaServer.get(generateRandomString(length=2)).addCallbacks(self.ignoreInput, self.errback)
self.kademliaServer.set(generateRandomString(length=2), generateRandomString()).addCallbacks(self.ignoreInput,
self.errback)
reactor.callLater(emu_config.botcommand_timeout, self.executeBot)
def ignoreInput(self, *args, **kwargs):
"""Gets whatever is the result of the Kademlia GET request and ignores it"""
pass
def errback(self, failure, *args, **kwargs):
"""Given to defereds to report errors. Ignores those errors."""
logging.debug(
"Kademlia Error (for the legitimate user, so non-existing keys are not a problem) in %s: %s"%(self.name, failure))
pass
def kad_server_join(network_port, profile_port, neighbor_ip, neighbor_port,
username):
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
aio = asyncio.new_event_loop()
kad = Server()
aio.run_until_complete(kad.listen(network_port))
aio.run_until_complete(kad.bootstrap([(neighbor_ip, neighbor_port)]))
aio.run_until_complete(asyncio.sleep(1))
# set a value for the key "my-key" on the network
aio.run_until_complete(
kad.set(username,
'http://' + socket.gethostname() + ':' + str(profile_port)))
aio.run_until_complete(asyncio.sleep(2))
# run forever since we are the first node
try:
main_loop(aio, kad)
except KeyboardInterrupt:
pass
finally:
kad.stop()
aio.close()
def send(port, key, message):
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
loop = asyncio.get_event_loop()
loop.set_debug(True)
server = Server()
server.listen(8469)
bootstrap_node = ("0.0.0.0", int(port))
loop.run_until_complete(server.bootstrap([bootstrap_node]))
result = loop.run_until_complete(server.get(key))
loop.run_until_complete(
server.set(key,
str(result) + str(datetime.datetime.now()) + " " +
message))
result = loop.run_until_complete(server.get(key))
server.stop()
#loop.close()
print("************************************************")
print(key, "\n", result + "\n")
print("************************************************")
class BitHaikuDHT:
log = logging.getLogger(__name__)
def __init__(self, configuration):
self.configuration = configuration
self.server = Server()
self.server.listen(configuration.ports.dht)
self.server.bootstrap([(configuration.local_address, configuration.ports.dht)])
def add(self, ip):
self.log.error("Bootstrapping " + ip)
self.server.bootstrap([(ip, self.configuration.ports.dht)])
def find_owner(self, data):
key = value = hashlib.sha256(data).hexdigest()
self.server.set(key, value)
digest = kademlia.utils.digest(hash)
node = Node(digest)
nearest = self.server.protocol.router.findNeighbors(node)
return nearest[0].ip
def user(port, username):
node = Server()
node.listen(port)
loop = asyncio.get_event_loop()
loop.run_until_complete(node.bootstrap([("0.0.0.0", 8468)]))
userlist = loop.run_until_complete(node.get("userlist"))
loop.run_until_complete(node.set("userlist", str(userlist) + "\n" + username + "\n"))
node.stop()
def chat(port, chatname):
node = Server()
node.listen(port)
loop = asyncio.get_event_loop()
loop.run_until_complete(node.bootstrap([("0.0.0.0", 8468)]))
chatlist = loop.run_until_complete(node.get("chatlist"))
loop.run_until_complete(node.set("chatlist", str(chatlist) + "\n" + chatname + "\n"))
node.stop()
class KadNode:
@staticmethod
def set_logging():
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
def __init__(self, port=DEFAULT_PORT, evt_loop=None):
self.set_logging()
if not evt_loop:
evt_loop = asyncio.get_event_loop()
self.loop = evt_loop
self.node = Server()
self.port = port
self.start_server()
def start_server(self):
def start_loop(node, port):
server_loop = asyncio.new_event_loop()
server_loop.run_until_complete(node.listen(port))
try:
server_loop.run_forever()
except KeyboardInterrupt:
pass
finally:
node.stop()
server_loop.close()
#
t = Thread(target=start_loop, args=(self.node, self.port))
t.start()
def boot_to(self, host, port):
self.loop.run_until_complete(self.node.bootstrap([
(host, port),
]))
def set(self, key, value):
self.loop.run_until_complete(self.node.set(key, value))
def get(self, key):
result = self.loop.run_until_complete(self.node.get(key))
return result
def send(port, chatname, username, message):
node = Server()
node.listen(port)
loop = asyncio.get_event_loop()
loop.run_until_complete(node.bootstrap([("0.0.0.0", 8468)]))
chat = loop.run_until_complete(node.get(chatname))
loop.run_until_complete(
node.set(chatname,
str(chat) + "\n" + username + ": " + message + "\n"))
new_chat = loop.run_until_complete(node.get(chatname))
node.stop()
print("************************************************")
print(chatname)
print("************************************************")
print("\n" + new_chat + "\n")
print("************************************************")
def test_custom_event_loop(self):
custom_loop = asyncio.new_event_loop()
server = Server()
server.listen(8468)
custom_loop = asyncio.new_event_loop()
server2 = Server(custom_event_loop=custom_loop)
server_thread = threading.Thread(target=setup_extra_server,
args=[server2, custom_loop])
server_thread.start()
# testing using the custom event loop
loop = asyncio.get_event_loop()
loop.run_until_complete(server.bootstrap([("localhost", 8469)]))
loop.run_until_complete(server.set("test", "test1"))
rec_value = loop.run_until_complete(server.get("test"))
server.stop()
stop_extra_server(server2, custom_loop)
server_thread.join()
assert rec_value == "test1"
class DHTResolver(object):
def __init__(self, config, bootstrapNeighbors):
if os.path.isfile(config['dht.state.cache']):
self.kserver = Server.loadState(config['dht.state.cache'])
else:
self.kserver = Server()
self.kserver.bootstrap(bootstrapNeighbors)
self.kserver.saveStateRegularly(config['dht.state.cache'], 60)
def getProtocol(self):
return self.kserver.protocol
def resolve(self, keyId):
return self.kserver.get(keyId)
def announceLocation(self, myKeyId):
def announce(ips):
if len(ips) == 0:
return defer.succeed(False)
return self.kserver.set(myKeyId, ips[0])
d = self.kserver.inetVisibleIP()
return d.addCallback(announce)
class CDataBase(object):
def __init__(self):
self.server = None
self.loop = None
self.port = 10023
def initiate(self):
if self.server is None:
self.loop = asyncio.get_event_loop()
self.server = Server(ksize=100, alpha=10)
self.loop.run_until_complete(self.server.listen(self.port))
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
self.loop.set_debug(True)
def set(self, key, value):
self.loop = asyncio.get_event_loop()
self.loop.run_until_complete(self.server.set(key, value))
return self.get(key)
def get(self, key):
self.loop = asyncio.get_event_loop()
return self.loop.run_until_complete(self.server.get(key))
def bootstrap(self, nodes):
self.loop = asyncio.get_event_loop()
bootstrap_node = []
try:
for n in nodes:
bootstrap_node.append((n, self.port))
self.loop.run_until_complete(self.server.bootstrap(bootstrap_node))
except:
pass
class Node:
"""This is the object that should be created to start listening
as an active node on the network.
Attributes:
connecting_port: The port to connect to
ip: The ip to connect to
block_height: The height of the blockchain of the node
nodes: If other nodes are connected or not
node: The actual node
loop: The loop tied to the node
verified_block: Boolean that check if a block_hash is correct or not
"""
def __init__(self, connecting_port, ip):
"""
Create a node instance. This will start listening on the given port.
Args:
connecting_port: The port to connect to
ip: The ip to connect to
"""
self.connecting_port = connecting_port
self.opening_port = randint(8800, 9000)
self.TARGET_MAX = 0x0000FFFFFFFFFFFFFFFFFFFFFFF000000000000000000000000000000000000
self.target = 1
self.ip = ip
self.block_height = 0
self.nodes_connected = None
self.node = None
self.loop = None
self.verified_block = False
if self.connecting_port is None:
bootstrap_node = Thread(target=self.bootstrap)
bootstrap_node.start()
else:
self.start_node()
self.get_blockheight()
def bootstrap(self):
"""Bootstrap the server by connecting to other known nodes in the network."""
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
print("Node operating at " + str(self.opening_port))
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
self.loop.set_debug(True)
self.node = Server()
self.loop.run_until_complete(self.node.listen(self.opening_port))
try:
self.loop.run_forever()
except KeyboardInterrupt:
pass
finally:
self.node.stop()
self.loop.close()
def start_node(self):
"""Each blackbox instance is a node which other nodes can connect to"""
print("Node operating at " + str(self.opening_port))
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
self.loop = asyncio.get_event_loop()
self.loop.set_debug(True)
self.node = Server()
self.loop.run_until_complete(self.node.listen(self.opening_port))
bootstrap_node = (self.ip, int(self.connecting_port))
self.loop.run_until_complete(self.node.bootstrap([bootstrap_node]))
self.loop.run_until_complete(self.node.set("nodes", True))
def get_blockheight(self):
"""When other nodes join the network they need to start mining at the most
up-to-date block"""
block_height = self.loop.run_until_complete(
self.node.get("block_height"))
if self.block_height is None or block_height is None or 0:
self.get_blockheight()
return
if (block_height > self.block_height):
self.block_height = block_height
def set_hash(self, block_hash):
"""When a new block is found, sets in the DHT the associated block_hash and block_height
Args:
block_hash: The hash of the block found by the participating node
"""
key = "blk" + str(self.block_height)
if self.connecting_port is None and block_hash is not None:
asyncio.run_coroutine_threadsafe(
self.node.set("block_height", self.block_height + 1),
self.loop)
asyncio.run_coroutine_threadsafe(self.node.set(key, block_hash),
self.loop)
elif block_hash is not None and self.block_height is not 0:
self.loop.run_until_complete(
self.node.set("block_height", self.block_height + 1))
self.loop.run_until_complete(self.node.set(key, block_hash))
def get_last_block(self):
key = "blk" + str(self.block_height)
if self.connecting_port is None:
return asyncio.run_coroutine_threadsafe(self.node.get(key),
self.loop)
return self.loop.run_until_complete(self.node.get(key))
def broadcast(self, hash_broadcasted, difficulty):
"""
Args:
hash_broadcasted: The hash that a node broadcasted to the network
difficulty: The current difficulty on the network
"""
if self.connecting_port is None:
hash_broadcasted = hash_broadcasted.result()
if hash_broadcasted is not None:
self.target = self.TARGET_MAX / difficulty
if int(hash_broadcasted, 16) < int(self.target):
if self.connecting_port is None:
asyncio.run_coroutine_threadsafe(
self.node.set("verified_block", True), self.loop)
else:
self.loop.run_until_complete(
self.node.set("verified_block", True))
return True
return False
def verify(self, difficulty):
"""Verify if a block_hash found by a node is a correct one
Args:
difficulty: The current difficulty of the network
"""
key = "blk" + str(self.block_height)
if self.connecting_port is None:
self.nodes_connected = asyncio.run_coroutine_threadsafe(
self.node.get("nodes"), self.loop)
if self.nodes_connected.result() is not True:
asyncio.run_coroutine_threadsafe(
self.node.set("verified_block", True), self.loop)
return True
else:
hash_broadcasted = asyncio.run_coroutine_threadsafe(
self.node.get(key), self.loop)
return (self.broadcast(hash_broadcasted, difficulty))
else:
hash_broadcasted = self.loop.run_until_complete(self.node.get(key))
return (self.broadcast(hash_broadcasted, difficulty))
class IPFSNode():
def __init__(self, bootstrapHost: str = 'bootstrap'):
self.server = Server()
self.server.listen(8469)
self.has_list = []
# Hack to get the "deafult" IP
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('1.1.1.1', 80))
self.local_ip = s.getsockname()[0]
s.close()
#self.local_ip = socket.gethostbyname(socket.gethostname())
bootstrap_ip = socket.gethostbyname(bootstrapHost)
self.bootstrap_node = (bootstrap_ip, 8469)
self.loop = asyncio.get_event_loop()
self.loop.set_debug(True)
self.loop.run_until_complete(
self.server.bootstrap([self.bootstrap_node]))
neighbors = self.server.bootstrappableNeighbors()
for node in neighbors:
print("DHT Peer found! {0}:{1}".format(node[0], node[1]))
print("Starting TCP transfer server")
self.tcpThread = threading.Thread(target=self.startTCPServer)
self.tcpThread.start()
def startTCPServer(self):
host = '0.0.0.0'
port = 9528
self.running = True
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((host, port))
s.listen(10)
while self.running:
conn, addr = s.accept()
print("Connection from " + str(addr))
sys.stdout.flush()
while 1:
data = conn.recv(4096)
if not data:
break
# Check if data is correct format
data = data.decode('utf-8')
if (len(data) == 133
or len(data) == 134) and data[0:5] == 'hash=':
requestedHash = data[5:134]
print("looking for hash " + requestedHash)
sys.stdout.flush()
#Find requested hash in our data
found = False
for has_chunk in self.has_list:
if has_chunk[1] in requestedHash:
conn.send(has_chunk[0]) #Send the found data
found = True
break
if not found:
conn.send(b"notfound")
else:
conn.send(b"invalid_request")
print("Request: " + data)
sys.stdout.flush()
conn.close()
s.close()
def chunkFile(self, data: str):
chunks = []
for i in range(0, len(data), block_size):
chunk = data[i:i + block_size]
hsh = hashlib.sha512(chunk).hexdigest()
chunks.append({"data": chunk, "hash": hsh, "size": len(chunk)})
return chunks
def setDHTKey(self, key: str, val: str):
self.loop.run_until_complete(self.server.set(key, val))
def getDHTKey(self, key: str) -> str:
return self.loop.run_until_complete(self.server.get(key))
def addFile(self, filepath: str, compression: bool):
with open(filepath, 'rb') as f:
data = f.read()
fileHash = hashlib.sha512(data).hexdigest()
fileLen = len(data)
if compression: #Compress data if necessary
data = zlib.compress(data, level=7)
fileChunks = self.chunkFile(data)
fileName = os.path.basename(filepath)
ipfsList = List(fileHash, fileName, fileLen, compression)
ipfsBlobs = []
for chunk in fileChunks:
ipfsBlob = Blob()
ipfsBlob.setData(chunk['data'], chunk['hash'])
ipfsBlobs.append(ipfsBlob)
ipfsList.addLink(chunk['hash'], chunk['size'])
#Add data to our has list
self.has_list.append((chunk['data'], chunk['hash']))
print(ipfsList)
if debug:
print(ipfsList.getData())
print("\n\n\n")
for blob in ipfsBlobs:
print(blob.getData())
if self.server:
#Need to add data onto DHT network
self.setDHTKey(fileHash,
str(ipfsList)) #Add master file record to DHT
for blob in ipfsBlobs: #Add items onto DHT
block = blob.getData()
if len(
block['data']
) > 1024: #If the block is bigger than 1k, add to local TCP server
self.setDHTKey(block['hash'], 'ip=' + self.local_ip)
else: #Otherwise store directly on DHT
self.setDHTKey(blob.getData()['hash'],
blob.getData()['data'])
return fileHash
def TCPGet(self, host, hsh):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host, 9528))
if not isinstance(hsh, bytes):
hsh = hsh.encode()
s.send(b'hash=' + hsh + b'\n')
data = s.recv(8192)
s.close()
return data
def getFile(self, hsh: str) -> (bytes, dict):
masterFileRecord = self.getDHTKey(hsh) #Get metadata
metadata = None
#Convert from string dictionary to python dictionary
if masterFileRecord and len(masterFileRecord) > 1:
masterFileRecord = masterFileRecord.replace("'", "\"").replace(
'True', 'true').replace('False',
'false') #transform into valid JSON
metadata = json.loads(masterFileRecord)
else:
raise Exception("Unable to locate file record on network!")
fileContents = b''
for link in metadata['links']:
DHTData = self.getDHTKey(link['hash'])
data = None
if not DHTData:
raise Exception("Unable to get part of file with hash " +
link['hash'])
if DHTData[0:3] == 'ip=': #Need to get data via TCP not DHT
data = self.TCPGet(DHTData[3:], link['hash'])
else:
data = DHTData
if len(data) != link['size']:
raise Exception(
"Hash value ({}) has invalid or corrupted length".format(
link['hash']))
fileContents += data
if metadata['compression']:
fileContents = zlib.decompress(fileContents)
return (fileContents, metadata)
def __del__(self):
self.server.stop()
self.loop.close()
self.tcpThread.join()
class P2PConnection:
def __init__(self, listenPort):
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
self.loop = asyncio.get_event_loop()
self.loop.set_debug(True)
asyncioreactor.install(eventloop=self.loop)
self.server = Server()
self.server.listen(listenPort)
def connect_p2p(self, bootstrap_address):
future = ensure_future(
self.server.bootstrap([(bootstrap_address, 8468)]))
d = Deferred.fromFuture(future)
return d
def get_key(self, content):
return hashlib.sha256(str.encode(content)).hexdigest()
def set_chat_info(self, chatuuid, groupname):
chat_info = {"name": groupname}
chat_info_str = json.dumps(chat_info)
key = chatuuid
def done(res, key):
print("Stored key:'{}' in network".format(key))
d = self._send(key, chat_info_str)
d.addCallback(done, key)
d.addErrback(self.send_failed)
return d
def get_chat_info(self, chatuuid):
d = self.get(chatuuid)
return d
def _send(self, key, data):
print("Start storing key:'{}' in P2P-network".format(key))
fut = ensure_future(self.server.set(key, data))
d = Deferred.fromFuture(fut)
# from twisted.internet import reactor
# d.addTimeout(30, reactor, self.send_failed)
return d
def send(self, message):
key = self.get_key(message)
d = self._send(key, message)
return d, key
def send_failed(self, err):
#TODO: Auto resend to network or ask user to resend?
err.trap(TimeoutError)
print('P2P send timed out')
def get(self, key):
fut = ensure_future(self.server.get(key))
d = Deferred.fromFuture(fut)
return d
class DHTNode(mp.Process):
UID_DELIMETER = '.' # splits expert uids over this delimeter
HEARTBEAT_EXPIRATION = 120 # expert is inactive iff it fails to post timestamp for *this many seconds*
make_key = "{}::{}".format
def __init__(self,
*initial_peers: Tuple[str, int],
port=8081,
start=False,
daemon=True):
super().__init__()
self.port, self.initial_peers = port, initial_peers
self._pipe, self.pipe = mp.Pipe(duplex=False)
self.ready = mp.Event()
self.server = Server()
self.daemon = daemon
if start:
self.run_in_background(await_ready=True)
def run(self) -> None:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(self.server.listen(self.port))
loop.run_until_complete(self.server.bootstrap(self.initial_peers))
run_forever(loop.run_forever)
self.ready.set()
while True:
method, args, kwargs = self._pipe.recv()
getattr(self, method)(*args, **kwargs)
def run_in_background(self, await_ready=True, timeout=None):
"""
Starts DHTNode in a background process. if await_ready, this method will wait until background dht
is ready to process incoming requests or for :timeout: seconds max.
"""
self.start()
if await_ready and not self.ready.wait(timeout=timeout):
raise TimeoutError(
"Server didn't notify .ready in {timeout} seconds")
def shutdown(self) -> None:
""" Shuts down the dht process """
if self.is_alive():
self.kill()
else:
warnings.warn(
"DHT shutdown has no effect: dht process is already not alive")
def get_experts(self,
uids: List[str],
heartbeat_expiration=HEARTBEAT_EXPIRATION
) -> List[Optional[RemoteExpert]]:
""" Find experts across DHT using their ids; Return a list of [RemoteExpert if found else None]"""
future, _future = SharedFuture.make_pair()
self.pipe.send(('_get_experts', [],
dict(uids=uids,
heartbeat_expiration=heartbeat_expiration,
future=_future)))
return future.result()
def _get_experts(self, uids: List[str], heartbeat_expiration: float,
future: SharedFuture):
loop = asyncio.get_event_loop()
lookup_futures = [
asyncio.run_coroutine_threadsafe(
self.server.get(self.make_key('expert', uid)), loop)
for uid in uids
]
current_time = datetime.datetime.now()
experts = [None] * len(uids)
for i, (uid, lookup) in enumerate(zip(uids, lookup_futures)):
if lookup.result() is not None:
(host,
port), timestamp = PickleSerializer.loads(lookup.result())
if (current_time -
timestamp).total_seconds() <= heartbeat_expiration:
experts[i] = RemoteExpert(uid=uid, host=host, port=port)
future.set_result(experts)
def declare_experts(self, uids: List[str], addr, port, wait_timeout=0):
"""
Make experts available to DHT; update timestamps if already available
:param uids: a list of expert ids to update
:param addr: hostname that can be used to call this expert
:param port: port that can be used to call this expert
:param wait_timeout: if wait_timeout > 0, waits for the procedure to finish
"""
done_event = mp.Event() if wait_timeout else None
self.pipe.send(('_declare_experts', [],
dict(uids=list(uids),
addr=addr,
port=port,
done_event=done_event)))
if done_event is not None:
done_event.wait(wait_timeout)
def _declare_experts(self, uids: List[str], addr: str, port: int,
done_event: Optional[mp.Event]):
loop = asyncio.get_event_loop()
timestamp = datetime.datetime.now()
expert_metadata = PickleSerializer.dumps(((addr, port), timestamp))
prefix_metadata = PickleSerializer.dumps(timestamp)
unique_prefixes = set()
for uid in uids:
asyncio.run_coroutine_threadsafe(
self.server.set(self.make_key('expert', uid), expert_metadata),
loop)
uid_parts = uid.split(self.UID_DELIMETER)
unique_prefixes.update([
self.UID_DELIMETER.join(uid_parts[:i + 1])
for i in range(len(uid_parts))
])
for prefix in unique_prefixes:
asyncio.run_coroutine_threadsafe(
self.server.set(self.make_key('prefix', prefix),
prefix_metadata), loop)
if done_event is not None:
done_event.set()
def first_k_active(self,
prefixes: List[str],
k: int,
heartbeat_expiration=HEARTBEAT_EXPIRATION,
max_prefetch=None):
"""
Find k prefixes with active experts; may return less if there aren't enough; used for DMoE beam search
:param prefixes: a list of uid prefixes ordered from highest to lowest priority
:param k: return at most *this many* active prefixes
:param heartbeat_expiration: consider expert active if his last heartbeat was sent at most this many seconds ago
:param max_prefetch: pre-dispatch up to *this many* asynchronous expert requests, defaults to pre-dispatch = k
:returns: a list of at most :k: prefixes that have at least one active expert each;
"""
future, _future = SharedFuture.make_pair()
self.pipe.send(('_first_k_active', [],
dict(prefixes=prefixes,
k=k,
heartbeat_expiration=heartbeat_expiration,
max_prefetch=max_prefetch or k,
future=_future)))
return future.result()
def _first_k_active(self, prefixes: List[str], k, heartbeat_expiration,
max_prefetch, future: SharedFuture):
loop = asyncio.get_event_loop()
lookup_prefetch = [
asyncio.run_coroutine_threadsafe(
self.server.get(self.make_key('prefix', prefix)), loop)
for prefix in prefixes[:max_prefetch]
]
current_time = datetime.datetime.now()
active_prefixes = []
for i, prefix in enumerate(prefixes):
lookup = lookup_prefetch[i]
if lookup.result() is not None:
timestamp = PickleSerializer.loads(lookup.result())
if (current_time -
timestamp).total_seconds() <= heartbeat_expiration:
active_prefixes.append(prefix)
if len(active_prefixes) >= k:
future.set_result(active_prefixes)
return
# pre-dispatch the next request in line
if len(lookup_prefetch) < len(prefixes):
lookup_prefetch.append(
asyncio.run_coroutine_threadsafe(
self.server.get(
self.make_key('prefix',
prefixes[len(lookup_prefetch)])),
loop))
# could not find enough active prefixes; return what we can
future.set_result(active_prefixes)
# OTHER_PORTS = [5580, 6680]
OTHER_PORTS = [5580, 6680, 7780] # omit node 1 and 3
HOST_URL = '127.0.0.1'
loop = asyncio.get_event_loop()
node = Server(node_id=name)
node.listen(PORT)
# Bootstrap this node on others
boots = []
for port in OTHER_PORTS + [PORT]:
boots.append((HOST_URL, port))
log.critical("bootstrapping node..")
loop.run_until_complete(node.bootstrap(boots))
log.critical("bootstrap done.")
log.critical("getting my-key...")
result = loop.run_until_complete(node.get("my-key"))
log.critical("got result: {}".format(result))
log.critical("setting new key 'node-4-info' from node_4...")
loop.run_until_complete(node.set("node-4-info", "dank stuff"))
log.critical("done setting.")
log.critical("\n\nFetching node info for node_id = NODE_3")
result = loop.run_until_complete(node.lookup_ip('NODE_3'))
log.critical("got result from lookup: {}".format(result))
log.critical("running forever")
loop.run_forever()
# backup = random.randint(1,numNodes)
# print('Backup machine for machine',numNodes,'is machine',backup)
# Start listening on the given PORT
loop.run_until_complete(node.listen(args.host))
# Bootstrap the node by connecting to other known nodes, in this case
# replace 123.123.123.123 with the IP of another node and optionally
# give as many ip/port combos as you can for other nodes.
loop.run_until_complete(node.bootstrap([('0.0.0.0', args.port)]))
print('DHT node running on ', ('0.0.0.0', args.host))
# set a value for the key "my-key" on the network
for lv in myLogicalVolumes:
loop.run_until_complete(node.set(lv, args.host + 1))
# get the value associated with "my-key" from the network
#result = loop.run_until_complete(node.get("my-key"))
#print(result)
client_server = NodeServer("0.0.0.0", args.host + 1)
client = NodeClient("0.0.0.0", args.host + 2, node)
thread1 = threading.Thread(target=client_server.listen)
thread2 = threading.Thread(target=client.get)
print('Node client running on ', ('0.0.0.0', args.host + 2))
thread1.start()
thread2.start()
try:
import time
loop = asyncio.get_event_loop()
loop.set_debug(True)
value = 0
server = Server()
bootstrap_node = ("0.0.0.0", 8469 + int(sys.argv[1]))
loop.run_until_complete(server.listen(8470 + int(sys.argv[1])))
loop.run_until_complete(server.bootstrap([bootstrap_node]))
f = open("devicefile.txt", "a")
for device in range(int(sys.argv[1]) * 10, int(sys.argv[1]) * 10 + 10):
try:
print("inserting", device)
result = loop.run_until_complete(server.set(device, value))
f.write("1")
except:
pass
f.close()
try:
loop.run_forever()
except KeyboardInterrupt:
pass
finally:
server.stop()
loop.close()
import logging
import asyncio
import sys
from kademlia.network import Server
if len(sys.argv) != 5:
print(
"Usage: python set.py <bootstrap node> <bootstrap port> <key> <value>")
sys.exit(1)
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
loop = asyncio.get_event_loop()
loop.set_debug(True)
server = Server()
loop.run_until_complete(server.listen(8469))
bootstrap_node = (sys.argv[1], int(sys.argv[2]))
loop.run_until_complete(server.bootstrap([bootstrap_node]))
loop.run_until_complete(server.set(sys.argv[3], sys.argv[4]))
server.stop()
loop.close()
print(sys.argv[3], ": ", sys.argv[4])
import logging
import asyncio
import sys
from kademlia.network import Server
if len(sys.argv) != 5:
print("Usage: python set.py <bootstrap node> <bootstrap port> <key> <value>")
sys.exit(1)
handler = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
loop = asyncio.get_event_loop()
loop.set_debug(True)
server = Server()
loop.run_until_complete(server.listen(8469))
bootstrap_node = (sys.argv[1], int(sys.argv[2]))
loop.run_until_complete(server.bootstrap([bootstrap_node]))
loop.run_until_complete(server.set(sys.argv[3], sys.argv[4]))
server.stop()
loop.close()
from kademlia.network import Server
loop = asyncio.get_event_loop()
loop.set_debug(True)
server = Server()
loop.run_until_complete(server.listen(8468))
bootstrap_node = ('192.168.1.240', 8468)
loop.run_until_complete(server.bootstrap([bootstrap_node]))
avg_response_time = 0
key = 'performance_test'
value = 'testValue'
loop.run_until_complete(server.set(key, value))
for i in range(500):
option = random.randint(1, 3)
start = time.time()
if option == 1: # read
loop.run_until_complete(server.get(key))
elif option == 2: # write
loop.run_until_complete(server.set(key, value))
else:
loop.run_until_complete(server.delete(key))
end = time.time()
print('Operation time: %d' % (end-start))
avg_response_time += (end-start)
avg_response_time /= 500
print('Average respone time: %d' % avg_response_time)
from Crypto import Signature, PublicKey, Hash
from kademlia.network import Server
if sys.argc != 5:
print("Usage: python set.py <bootstrap node> <bootstrap port> <key_file> <job_desc_file>")
sys.exit(1)
loop = asyncio.get_event_loop()
server = Server()
server.listen(8469)
bootstrap_node = (sys.argv[1], int(sys.argv[2]))
loop.run_until_complete(server.bootstrap([bootstrap_node]))
with open(sys.argv[3], 'r') as f:
key = PublicKey.RSA.import_key(f.read())
verifier = Signature.pkcs1_15.new()
value = {}
with open(sys.argv[4], 'r') as f:
value['job_data'] = f.read()
hash = Hash.SHA256.new(data=value['job_data']).digest()
value['sig'] = verifier.sign(hash)
loop.run_until_complete(server.set('task', value))
server.stop()
loop.close()
class NetworkInterface (object):
# Create a NetworkInterface object to accomplish all network related tasks
def __init__(self, appDeployer, uuid):
self._connected = False
self._app_deployer = appDeployer
# optional...
self._number_of_nodes = 0
self._list_of_nodes =[]
# logging capabilities
self._log = Logger(system=self)
# HERE--> Implementation specific node instanciation
from kademlia.network import Server
self._node = Server()
self._node.log.level = 4
# END OF SECTION
def bootStrapDone(self, server):
#contacts = self._node.inetVisibleIP()
print "BOOOOTTTT STAPPP IT"
def retrieveContacts(self):
"""
NEED TO FIND A WAY TO RETRIEVE THE LIST OF NEIGHBORS !!!
"""
# !!! DOES EXACTLY THE SAME AS bootstrappableNeighbors !!!
for bucket in self._node.protocol.router.buckets:
print bucket.getNodes()
# !!! bootstrappableNeighbors returns only the list of neighbors that you provided as !!!
# !!! a bootstrap list, that are also online !!!
neighbors = self._node.bootstrappableNeighbors()
print neighbors
return neighbors
def connect(self,fromPort,toPort,ip='127.0.0.1'):
self._log.debug('Connecting...')
#print "in connect ... "
#print "now listening on port: ",fromPort
self._node.listen(fromPort)
return self._node.bootstrap([(ip,toPort)]).addCallback(self.bootStrapDone)
# This function is used to set a value in the DHT
def setDone(self,result):
print result
print "set is done"
deferred = Deferred()
return deferred
def set(self, key, value):
def _processKey(result, key, values):
print result, key, values
deferred = Deferred()
# upon recovering the value of the key
if result == None:
deferred = self._node.set(key, values)
return deferred
#.addCallback(self.setDone)
else:
for value in values:
if value not in result:
# append + publish
result.append(value)
else:
self._log.info("Value is already in the corresponding key.")
deferred = self._node.set(key, result)
return deferred
# Only application deployers are allowed to write to the DHT.
if self._app_deployer != False:
deferred = Deferred()
# Two possible keys are allowed to be written to, the template key and their respective application key
if ('template' == key or self._uuid == key) and key != None:
# HERE --> Implementation Specific Code
print " ::: ", self, " ::: ", key, " ::: ", value, " <----------------------------"
# if writing to the template, retrieve the value first then append to it if necessary
if key == 'template':
deferred = self._node.get(key)
deferred.addCallback(_processKey, key, value)
return deferred
#self._node.set(key, value).addCallback(self.setDone)
# END OF SECTION
# Not Allowed to write to the DHT.
else:
self._log.info("Only application deployers are allowed to write values into the DHT!")
def done(self,result):
print "self: ", self
print "Key result:", result
def get(self,result, key):
# HERE --> Implementation Specific Code
print result, " ::: ", self, " ::: ", key, " <----------------------------"
deferred = self._node.get(key)
deferred.addCallback(self.done)
return deferred
import logging
import asyncio
import sys
from kademlia.network import Server
if len(sys.argv) != 5:
print(
"Usage: python set.py <bootstrap node> <bootstrap port> <key> <value>")
sys.exit(1)
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
loop = asyncio.get_event_loop()
loop.set_debug(True)
server = Server()
server.listen(8469)
bootstrap_node = (sys.argv[1], int(sys.argv[2]))
loop.run_until_complete(server.bootstrap([bootstrap_node]))
cipherkey = cilok.urlEncode16(sys.argv[3])
loop.run_until_complete(server.set(cipherkey, sys.argv[4]))
server.stop()
loop.close()
import asyncio
from kademlia.network import Server
# Create a node and start listening on port 5678
node = Server()
node.listen(8468)
# Bootstrap the node by connecting to other known nodes, in this case
# replace 123.123.123.123 with the IP of another node and optionally
# give as many ip/port combos as you can for other nodes.
loop = asyncio.get_event_loop()
loop.run_until_complete(node.bootstrap([("192.168.31.124", 8468)]))
# set a value for the key "my-key" on the network
loop.run_until_complete(node.set("my-key", "my awesome value"))
# get the value associated with "my-key" from the network
result = loop.run_until_complete(node.get("my-key"))
print(result)
ATK_HOLDERS = None
result = loop.run_until_complete(node.get('ATK_FILE'))
print(result)
while True:
if not result is None:
ATK_HOLDERS = result.split(',')
# Get attack file
some_ip_port = choice(ATK_HOLDERS).split(':')
print('[+]Result {}'.format(some_ip_port))
local_msg = {'cmd':'GET_ATK', 'data' : None}
send_msg(json.dumps(local_msg), some_ip_port[0], some_ip_port[1])
if os.path.exists(ATK_FILE):
loop.run_until_complete(node.set("ATK_FILE", "{},{}:{}".format(result, MY_IP, UDP_PORT)))
break
# Wait for 2 seconds and then retry
time.sleep(2)
else:
loop.run_until_complete(node.set("ATK_FILE", "{}:{}".format(MY_IP, UDP_PORT)))
break
# Start the attack module??
os.system("python3 {} {}&".format(ATK_FILE, BOOTSTARP_IP))
if __name__ == '__main__':
try:
print('[+] Starting main.')
class KademliaUser(Runnable):
"""Allows the KademliaUser to be run in its own thread."""
def __init__(self,
name="KademliaUser%d" % random.randint(1, 1000),
port=emu_config.kademlia_default_port,
peerlist=None):
super(KademliaUser, self).__init__(name=name)
self.kademliaServer = Server()
self.port = port
self.peerlist = peerlist if peerlist is not None else []
def start(self):
"""Implements start() from the superclass."""
self._startKademlia()
try:
IOLoop.current().start()
except socket.error as ex:
logging.warning("Could not start the KademliaUser: %s" % ex)
def _startKademlia(self):
possible_interfaces = [
iface for iface in netifaces.interfaces()
if iface_searchterm in iface
and netifaces.ifaddresses(iface).has_key(netifaces.AF_INET)
]
if len(possible_interfaces) == 0:
logging.error(
"No suitable interfaces found, tried the following: %s" %
netifaces.interfaces())
logging.debug("Interfaces: %s" %
netifaces.ifaddresses(possible_interfaces[0]))
ipAddr = netifaces.ifaddresses(
possible_interfaces[0])[netifaces.AF_INET][0]["addr"]
logging.debug("Node %s starts with %s on %s" %
(self.name, self.peerlist, ipAddr))
self.kademliaServer.listen(self.port, interface=ipAddr)
serverDeferred = self.kademliaServer.bootstrap([
(peer, emu_config.kademlia_default_port) for peer in self.peerlist
])
serverDeferred.addCallback(self.executeBot)
serverDeferred.addErrback(self.errback)
def executeBot(self, peersfound=None):
"""Method that is called regularly and checks for new commands"""
self.kademliaServer.get(generateRandomString(length=2)).addCallbacks(
self.ignoreInput, self.errback)
self.kademliaServer.set(generateRandomString(length=2),
generateRandomString()).addCallbacks(
self.ignoreInput, self.errback)
reactor.callLater(emu_config.botcommand_timeout, self.executeBot)
def ignoreInput(self, *args, **kwargs):
"""Gets whatever is the result of the Kademlia GET request and ignores it"""
pass
def errback(self, failure, *args, **kwargs):
"""Given to defereds to report errors. Ignores those errors."""
logging.debug(
"Kademlia Error (for the legitimate user, so non-existing keys are not a problem) in %s: %s"
% (self.name, failure))
pass
class App(object):
def __init__(self):
self.parser = argparse.ArgumentParser(
description='A p2p key-value databse.')
def parse_commandline(self):
self.parser.add_argument(
'-f',
'--first',
action='store_true',
dest='first_node',
default=False,
help="Start the first node in the p2p network")
self.parser.add_argument(
'-b',
'--bootstrap',
nargs=2,
dest='bootstrap',
metavar=('<address>', '<port>'),
help=
"Start a node and connect to bootstrap node in existing network")
self.parser.add_argument('-p',
'--port',
nargs=1,
dest='port',
metavar=('<port>'),
default=[8468],
help='Server listen to this port.')
self.parser.add_argument(
'--flooding',
dest='flooding',
nargs=3,
metavar=('<iprange_l>', '<iprange_r>', '<ports>'),
default=None,
help=
"Flooding neighbors instead of explicit bootstrapping given node. "
"Format of <ports>:"
"I: 9468,8469,8890 (equals [9468, 8469, 8890])"
"II: 8469~8891 (equals list(range(8469, 8892)))"
"Example: 192.168.1.1 192.168.2.0 8468~8470")
self.parser.add_argument(
'-t',
'--timeout',
nargs=1,
dest='wait_timeout',
metavar='<timeout>',
default=(5, ),
help='Wait timeout for bootstrapping and flooding')
self.parser.add_argument(
'-r',
'--record',
action='store_true',
dest='record',
default=False,
help='If True, record active neighbors in the 0-th k-bucket')
options = self.parser.parse_args()
try:
return (options.flooding, options.bootstrap, options.port,
options.wait_timeout, options.record)
except:
print(
'To join existing network, you need a bootstrap node or flooding setting.'
)
# if options.first_node: return (options.flooding, options.bootstrap, options.port)
# else:
# try:
# return (options.flooding, options.bootstrap, options.port)
# except:
# print('To join existing network, you need a bootstrap node to connect the exist network.')
def print_help(self):
commands = {
'help': 'print this help',
'get': 'get the value for given key',
'put': 'update the value of the specified key',
'delete': 'delete the key in the storage system',
'quit': 'exit the system'
}
self.parser.print_help()
print('\nCLI commands:')
for command, description in commands.items():
print("%-10s %s" % (command, description))
def quit(self):
self.server.stop()
self.loop.close()
def start_loop(self):
self.loop.set_debug(True)
asyncio.set_event_loop(self.loop)
try:
self.loop.run_forever()
except KeyboardInterrupt:
pass
finally:
self.server.stop()
self.loop.close()
def run(self):
print(
"""Welcome to this p2p key-value system. To find out what other commands exist, type 'help'"""
)
# log
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
# command information
flooding_nodes, bootstrap_node, port, timeout, record = self.parse_commandline(
)
# run subthread
self.loop = asyncio.new_event_loop()
self.t = Thread(target=self.start_loop)
self.server = Server(timeout=int(timeout[0]), record=record)
self.t.setDaemon(True)
self.t.start()
# self.server.reset(timeout=int(timeout[0]), record=record)
asyncio.run_coroutine_threadsafe(self.server.listen(int(port[0])),
self.loop).result()
if bootstrap_node is not None:
bootstrap_node = (bootstrap_node[0], int(bootstrap_node[1]))
re = asyncio.run_coroutine_threadsafe(
self.server.bootstrap([bootstrap_node]), self.loop).result()
elif flooding_nodes is not None:
iprange_l, iprange_r, ports = flooding_nodes
if '~' in ports:
l, r = ports.split('~')
ports = list(range(int(l), int(r)))
else:
str_ports = ports.split(',')
ports = [int(i) for i in str_ports]
re = asyncio.run_coroutine_threadsafe(
self.server.bootstrap(flooding=True,
iprange_l=iprange_l,
iprange_r=iprange_r,
ports=ports), self.loop).result()
while True:
sleep(0.3)
try:
io = input('Command: ').lstrip().rstrip()
if io == 'help':
self.print_help()
elif io == 'get':
print("Usage: <key>")
args = input().split(' ')
if len(args) != 1:
print("Number of parameters does not match.")
else:
result = asyncio.run_coroutine_threadsafe(
self.server.get(args[0]), self.loop).result()
result = result[0] if result else None
print("Get result:", result)
elif io == 'put':
print("Usage: <key> <value>")
args = input().split(' ')
if len(args) != 2:
print('Number of parameters dose not match.')
else:
asyncio.run_coroutine_threadsafe(
self.server.set(args[0], args[1]),
self.loop).result()
elif io == 'delete':
print("Usage: <key>")
args = input().split(' ')
if len(args) != 1:
print('Number of parameters dose not match.')
else:
asyncio.run_coroutine_threadsafe(
self.server.delete(args[0]), self.loop).result()
elif io == 'quit':
print('Bye ~ Have a nice day.')
self.loop.call_soon_threadsafe(self.quit)
break
else:
print('Sorry! Invalid command.')
except EOFError:
self.loop.call_soon_threadsafe(self.quit)
break
def main(argv):
port_given = False
key_given = False
set_key = False
get_key = False
try:
opts, args = getopt.getopt(argv, "p:k:s:g:", ["set=", "get="])
except getopt.GetoptError:
print(
"python3 new_node.py -p <port> -k <key> -s <set_value> -g <get_key>"
)
sys.exit(2)
for opt, arg in opts:
if opt == "-p":
port_given = True
port = arg
elif opt == "-k":
key_given = True
key = arg
elif opt in ("-s", "set="):
set_key = True
key_value = arg
elif opt in ("-g", "get="):
get_key = True
gkey = arg
if port_given is False:
print(
"python3 new_node.py -p <port> -k <key> -s <set_value> -g <get_key>"
)
sys.exit(1)
node = Server()
node.listen(port)
# Bootstrap the node by connecting to other known nodes, in this case
# ("0.0.0.0", 1111) is the starting node.
loop = asyncio.get_event_loop()
loop.run_until_complete(node.bootstrap([STARTER_NODE]))
# set a user specified value for the user specified key on the network
if set_key is True:
if key_given is True:
loop.run_until_complete(node.set(key, key_value))
else:
print(
"python3 new_node.py -p <port> -k <key> -s <set_value> -g <get_key>"
)
sys.exit(1)
# get the value associated with the user specified key from the network
if key_given is True or get_key is True:
if get_key is True:
result = loop.run_until_complete(node.get(gkey))
print(result)
elif set_key is False:
result = loop.run_until_complete(node.get(key))
print(result)
try:
loop.run_forever()
except KeyboardInterrupt:
pass
finally:
node.stop()
loop.close()
sys.exit(0)
madt_client = MADT_Client()
status = '1'
log = ''
while True: # Main Testing Loop
wait_for(asyncio.sleep(1))
try:
key = uuid.uuid4().hex
value = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua"
# Saving data. Kademlia's node.set will return False if it can't save data,
# in this case we'll send status=1 to the madt monitoring system and node
# on the graph will turn red
log = 'setting key = ' + key
status = '1'
if not wait_for(node.set(key, value)):
madt_client.send(status, log, len(node.storage.data))
continue
# If we've managed to successfully save data, we need to load it and compare in
# order to completelly test functionality of Kademlia. node.get will return False
# on failure,in this case we'll send status=2 to the madt monitoring system and
# node on the graph will turn yellow
log += '\n' + 'getting key'
status = '2'
ret_val = wait_for(node.get(key))
if not ret_val:
madt_client.send(status, log, len(node.storage.data),
len(node.storage.data))
continue
# Comparing initial and retrieved value, sending status=3 if they're different
import logging
import asyncio
import sys
from kademlia.network import Server
if len(sys.argv) != 3:
print("Usage: python set.py <key> <value>")
sys.exit(1)
logging.basicConfig(level=logging.DEBUG)
loop = asyncio.get_event_loop()
loop.set_debug(True)
server = Server()
server.listen(8469)
loop.run_until_complete(server.bootstrap([("127.0.0.1", 8468)]))
loop.run_until_complete(server.set(sys.argv[1], sys.argv[2]))
server.stop()
loop.close()
class NetworkInterface (object):
# Create a NetworkInterface object to accomplish all network related tasks
def __init__(self, appDeployer=False):
self._connected = False
self._app_deployer = appDeployer
# optional...
self._number_of_nodes = 0
self._list_of_nodes =[]
# logging capabilities
self._log = Logger(system=self)
# HERE--> Implementation specific node instanciation
from kademlia.network import Server
self._node = Server()
self._node.log.level = 4
# END OF SECTION
def bootStrapDone(self, server):
#contacts = self._node.inetVisibleIP()
print "BOOOOTTTT STAPPP IT"
def retrieveContacts(self):
"""
NEED TO FIND A WAY TO RETRIEVE THE LIST OF NEIGHBORS !!!
"""
# !!! DOES EXACTLY THE SAME AS bootstrappableNeighbors !!!
for bucket in self._node.protocol.router.buckets:
print bucket.getNodes()
# !!! bootstrappableNeighbors returns only the list of neighbors that you provided as !!!
# !!! a bootstrap list, that are also online !!!
neighbors = self._node.bootstrappableNeighbors()
print neighbors
return neighbors
def connect(self,fromPort,toPort,ip='127.0.0.1'):
self._log.debug('Connecting...')
#print "in connect ... "
#print "now listening on port: ",fromPort
self._node.listen(fromPort)
return self._node.bootstrap([(ip,toPort)]).addCallback(self.bootStrapDone)
# This function is used to set a value in the DHT
def setDone(self,result):
print result
print "set is done"
def set(self,result, key, value):
# HERE --> Implementation Specific Code
print result, " ::: ", self, " ::: ", key, " ::: ", value, " <----------------------------"
self._node.set(key, value).addCallback(self.setDone)
# END OF SECTION
def done(self,result):
print "self: ", self
print "Key result:", result
def get(self,result, key):
# HERE --> Implementation Specific Code
print result, " ::: ", self, " ::: ", key, " <----------------------------"
self._node.get(key).addCallback(self.done)
