def setUp(self):
# create peers
self.swarm = []
for i in range(TEST_SWARM_SIZE):
peer = Server()
bootstrap_peers = [
("127.0.0.1", PORT + x) for x in range(i)
][-1:] # each peer only knows of the last peer
peer.bootstrap(bootstrap_peers)
peer.listen(PORT + i)
self.swarm.append(peer)
# stabalize network overlay
time.sleep(10)
for peer in self.swarm:
peer.bootstrap(peer.bootstrappableNeighbors())
time.sleep(10)
for peer in self.swarm:
peer.bootstrap(peer.bootstrappableNeighbors())
time.sleep(10)
import asyncio
import sys
from kademlia.network import Server
from kademlia.routing import RoutingTable
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
loop = asyncio.get_event_loop()
loop.set_debug(True)
server = Server()
server.listen(8469)
bootstrap_node = ('127.0.0.1', 8468)
loop.run_until_complete(server.bootstrap([bootstrap_node]))
#loop.run_until_complete(server.set("key", "a"*1024*4))
print(server.bootstrappableNeighbors())
result = loop.run_until_complete(server.get('key'))
server.stop()
loop.close()
print("Get result:", result)
class NetworkInterface (object):
# Create a NetworkInterface object to accomplish all network related tasks
def __init__(self, appDeployer, uuid):
self._connected = False
self._app_deployer = appDeployer
# optional...
self._number_of_nodes = 0
self._list_of_nodes =[]
# logging capabilities
self._log = Logger(system=self)
# HERE--> Implementation specific node instanciation
from kademlia.network import Server
self._node = Server()
self._node.log.level = 4
# END OF SECTION
def bootStrapDone(self, server):
#contacts = self._node.inetVisibleIP()
print "BOOOOTTTT STAPPP IT"
def retrieveContacts(self):
"""
NEED TO FIND A WAY TO RETRIEVE THE LIST OF NEIGHBORS !!!
"""
# !!! DOES EXACTLY THE SAME AS bootstrappableNeighbors !!!
for bucket in self._node.protocol.router.buckets:
print bucket.getNodes()
# !!! bootstrappableNeighbors returns only the list of neighbors that you provided as !!!
# !!! a bootstrap list, that are also online !!!
neighbors = self._node.bootstrappableNeighbors()
print neighbors
return neighbors
def connect(self,fromPort,toPort,ip='127.0.0.1'):
self._log.debug('Connecting...')
#print "in connect ... "
#print "now listening on port: ",fromPort
self._node.listen(fromPort)
return self._node.bootstrap([(ip,toPort)]).addCallback(self.bootStrapDone)
# This function is used to set a value in the DHT
def setDone(self,result):
print result
print "set is done"
deferred = Deferred()
return deferred
def set(self, key, value):
def _processKey(result, key, values):
print result, key, values
deferred = Deferred()
# upon recovering the value of the key
if result == None:
deferred = self._node.set(key, values)
return deferred
#.addCallback(self.setDone)
else:
for value in values:
if value not in result:
# append + publish
result.append(value)
else:
self._log.info("Value is already in the corresponding key.")
deferred = self._node.set(key, result)
return deferred
# Only application deployers are allowed to write to the DHT.
if self._app_deployer != False:
deferred = Deferred()
# Two possible keys are allowed to be written to, the template key and their respective application key
if ('template' == key or self._uuid == key) and key != None:
# HERE --> Implementation Specific Code
print " ::: ", self, " ::: ", key, " ::: ", value, " <----------------------------"
# if writing to the template, retrieve the value first then append to it if necessary
if key == 'template':
deferred = self._node.get(key)
deferred.addCallback(_processKey, key, value)
return deferred
#self._node.set(key, value).addCallback(self.setDone)
# END OF SECTION
# Not Allowed to write to the DHT.
else:
self._log.info("Only application deployers are allowed to write values into the DHT!")
def done(self,result):
print "self: ", self
print "Key result:", result
def get(self,result, key):
# HERE --> Implementation Specific Code
print result, " ::: ", self, " ::: ", key, " <----------------------------"
deferred = self._node.get(key)
deferred.addCallback(self.done)
return deferred
class IPFSNode():
def __init__(self, bootstrapHost: str = 'bootstrap'):
self.server = Server()
self.server.listen(8469)
self.has_list = []
# Hack to get the "deafult" IP
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('1.1.1.1', 80))
self.local_ip = s.getsockname()[0]
s.close()
#self.local_ip = socket.gethostbyname(socket.gethostname())
bootstrap_ip = socket.gethostbyname(bootstrapHost)
self.bootstrap_node = (bootstrap_ip, 8469)
self.loop = asyncio.get_event_loop()
self.loop.set_debug(True)
self.loop.run_until_complete(
self.server.bootstrap([self.bootstrap_node]))
neighbors = self.server.bootstrappableNeighbors()
for node in neighbors:
print("DHT Peer found! {0}:{1}".format(node[0], node[1]))
print("Starting TCP transfer server")
self.tcpThread = threading.Thread(target=self.startTCPServer)
self.tcpThread.start()
def startTCPServer(self):
host = '0.0.0.0'
port = 9528
self.running = True
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((host, port))
s.listen(10)
while self.running:
conn, addr = s.accept()
print("Connection from " + str(addr))
sys.stdout.flush()
while 1:
data = conn.recv(4096)
if not data:
break
# Check if data is correct format
data = data.decode('utf-8')
if (len(data) == 133
or len(data) == 134) and data[0:5] == 'hash=':
requestedHash = data[5:134]
print("looking for hash " + requestedHash)
sys.stdout.flush()
#Find requested hash in our data
found = False
for has_chunk in self.has_list:
if has_chunk[1] in requestedHash:
conn.send(has_chunk[0]) #Send the found data
found = True
break
if not found:
conn.send(b"notfound")
else:
conn.send(b"invalid_request")
print("Request: " + data)
sys.stdout.flush()
conn.close()
s.close()
def chunkFile(self, data: str):
chunks = []
for i in range(0, len(data), block_size):
chunk = data[i:i + block_size]
hsh = hashlib.sha512(chunk).hexdigest()
chunks.append({"data": chunk, "hash": hsh, "size": len(chunk)})
return chunks
def setDHTKey(self, key: str, val: str):
self.loop.run_until_complete(self.server.set(key, val))
def getDHTKey(self, key: str) -> str:
return self.loop.run_until_complete(self.server.get(key))
def addFile(self, filepath: str, compression: bool):
with open(filepath, 'rb') as f:
data = f.read()
fileHash = hashlib.sha512(data).hexdigest()
fileLen = len(data)
if compression: #Compress data if necessary
data = zlib.compress(data, level=7)
fileChunks = self.chunkFile(data)
fileName = os.path.basename(filepath)
ipfsList = List(fileHash, fileName, fileLen, compression)
ipfsBlobs = []
for chunk in fileChunks:
ipfsBlob = Blob()
ipfsBlob.setData(chunk['data'], chunk['hash'])
ipfsBlobs.append(ipfsBlob)
ipfsList.addLink(chunk['hash'], chunk['size'])
#Add data to our has list
self.has_list.append((chunk['data'], chunk['hash']))
print(ipfsList)
if debug:
print(ipfsList.getData())
print("\n\n\n")
for blob in ipfsBlobs:
print(blob.getData())
if self.server:
#Need to add data onto DHT network
self.setDHTKey(fileHash,
str(ipfsList)) #Add master file record to DHT
for blob in ipfsBlobs: #Add items onto DHT
block = blob.getData()
if len(
block['data']
) > 1024: #If the block is bigger than 1k, add to local TCP server
self.setDHTKey(block['hash'], 'ip=' + self.local_ip)
else: #Otherwise store directly on DHT
self.setDHTKey(blob.getData()['hash'],
blob.getData()['data'])
return fileHash
def TCPGet(self, host, hsh):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host, 9528))
if not isinstance(hsh, bytes):
hsh = hsh.encode()
s.send(b'hash=' + hsh + b'\n')
data = s.recv(8192)
s.close()
return data
def getFile(self, hsh: str) -> (bytes, dict):
masterFileRecord = self.getDHTKey(hsh) #Get metadata
metadata = None
#Convert from string dictionary to python dictionary
if masterFileRecord and len(masterFileRecord) > 1:
masterFileRecord = masterFileRecord.replace("'", "\"").replace(
'True', 'true').replace('False',
'false') #transform into valid JSON
metadata = json.loads(masterFileRecord)
else:
raise Exception("Unable to locate file record on network!")
fileContents = b''
for link in metadata['links']:
DHTData = self.getDHTKey(link['hash'])
data = None
if not DHTData:
raise Exception("Unable to get part of file with hash " +
link['hash'])
if DHTData[0:3] == 'ip=': #Need to get data via TCP not DHT
data = self.TCPGet(DHTData[3:], link['hash'])
else:
data = DHTData
if len(data) != link['size']:
raise Exception(
"Hash value ({}) has invalid or corrupted length".format(
link['hash']))
fileContents += data
if metadata['compression']:
fileContents = zlib.decompress(fileContents)
return (fileContents, metadata)
def __del__(self):
self.server.stop()
self.loop.close()
self.tcpThread.join()
class NetworkInterface (object):
# Create a NetworkInterface object to accomplish all network related tasks
def __init__(self, appDeployer=False):
self._connected = False
self._app_deployer = appDeployer
# optional...
self._number_of_nodes = 0
self._list_of_nodes =[]
# logging capabilities
self._log = Logger(system=self)
# HERE--> Implementation specific node instanciation
from kademlia.network import Server
self._node = Server()
self._node.log.level = 4
# END OF SECTION
def bootStrapDone(self, server):
#contacts = self._node.inetVisibleIP()
print "BOOOOTTTT STAPPP IT"
def retrieveContacts(self):
"""
NEED TO FIND A WAY TO RETRIEVE THE LIST OF NEIGHBORS !!!
"""
# !!! DOES EXACTLY THE SAME AS bootstrappableNeighbors !!!
for bucket in self._node.protocol.router.buckets:
print bucket.getNodes()
# !!! bootstrappableNeighbors returns only the list of neighbors that you provided as !!!
# !!! a bootstrap list, that are also online !!!
neighbors = self._node.bootstrappableNeighbors()
print neighbors
return neighbors
def connect(self,fromPort,toPort,ip='127.0.0.1'):
self._log.debug('Connecting...')
#print "in connect ... "
#print "now listening on port: ",fromPort
self._node.listen(fromPort)
return self._node.bootstrap([(ip,toPort)]).addCallback(self.bootStrapDone)
# This function is used to set a value in the DHT
def setDone(self,result):
print result
print "set is done"
def set(self,result, key, value):
# HERE --> Implementation Specific Code
print result, " ::: ", self, " ::: ", key, " ::: ", value, " <----------------------------"
self._node.set(key, value).addCallback(self.setDone)
# END OF SECTION
def done(self,result):
print "self: ", self
print "Key result:", result
def get(self,result, key):
# HERE --> Implementation Specific Code
print result, " ::: ", self, " ::: ", key, " <----------------------------"
self._node.get(key).addCallback(self.done)
