class DHT:
"""docstring for Server"""
def __init__(self, ip_address=BOOSTRAP_IP, port=BOOSTRAP_PORT):
self.server = Server()
self.server.listen(UDHT_PORT)
self.loop = asyncio.get_event_loop()
self.loop.set_debug(True)
bootstrap_node = (ip_address, int(port))
self.loop.run_until_complete(self.server.bootstrap([bootstrap_node]))
def stop(self):
self.server.stop()
self.loop.close()
def __getitem__(self, key):
result = Empty()
try:
result = loads(self.loop.run_until_complete(self.server.get(key)))
except TypeError:
pass
if isinstance(result, Empty):
raise KeyError
return result
def __setitem__(self, key, item):
self.loop.run_until_complete(self.server.set(key, dumps(item)))
def __delitem__(self, key):
self.loop.run_until_complete(self.server.set(key, dumps(Empty())))
async def run_test():
server = Server()
def async_return(result):
f = asyncio.Future()
f.set_result(result)
return f
get_signed_value = get_signed_value_with_keys(priv_key_path='kademlia/tests/resources/key.der',
pub_key_path='kademlia/tests/resources/public.der')
get_signed_message = get_signed_message_with_keys(priv_key_path='kademlia/tests/resources/key.der',
pub_key_path='kademlia/tests/resources/public.der')
key_test = 'test key'
dkey_test = digest(key_test)
data = json.dumps(get_signed_value(dkey_test, 'data', PersistMode.SECURED).to_json())
value = get_signed_value(dkey_test, data, PersistMode.SECURED)
server._call_remote_persist = Mock(return_value=async_return(True))
server.get = Mock(return_value=async_return(get_signed_message(dkey_test, data)))
server.set_digest = Mock(return_value=async_return(True))
Server._get_dtl_record = Mock(return_value=True)
await server.set('test key', value)
server.get.assert_called_with('test key')
server.stop()
def kad_server_join(network_port, profile_port, neighbor_ip, neighbor_port,
username):
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
aio = asyncio.new_event_loop()
kad = Server()
aio.run_until_complete(kad.listen(network_port))
aio.run_until_complete(kad.bootstrap([(neighbor_ip, neighbor_port)]))
aio.run_until_complete(asyncio.sleep(1))
# set a value for the key "my-key" on the network
aio.run_until_complete(
kad.set(username,
'http://' + socket.gethostname() + ':' + str(profile_port)))
aio.run_until_complete(asyncio.sleep(2))
# run forever since we are the first node
try:
main_loop(aio, kad)
except KeyboardInterrupt:
pass
finally:
kad.stop()
aio.close()
async def run():
server = Server()
await server.listen(8469)
bootstrap_node = (sys.argv[2], int(sys.argv[3]))
await server.bootstrap([bootstrap_node])
await server.set(sys.argv[4], sys.argv[5])
server.stop()
def send(port, key, message):
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
loop = asyncio.get_event_loop()
loop.set_debug(True)
server = Server()
server.listen(8469)
bootstrap_node = ("0.0.0.0", int(port))
loop.run_until_complete(server.bootstrap([bootstrap_node]))
result = loop.run_until_complete(server.get(key))
loop.run_until_complete(
server.set(key,
str(result) + str(datetime.datetime.now()) + " " +
message))
result = loop.run_until_complete(server.get(key))
server.stop()
#loop.close()
print("************************************************")
print(key, "\n", result + "\n")
print("************************************************")
def test_custom_protocol(self):
"""
A subclass of Server which overrides the protocol_class attribute will
have an instance of that class as its protocol after its listen()
method is called.
"""
# Make a custom Protocol and Server to go with hit.
class CoconutProtocol(KademliaProtocol):
pass
class HuskServer(Server):
protocol_class = CoconutProtocol
# An ordinary server does NOT have a CoconutProtocol as its protocol...
server = Server()
server.listen(8469)
self.assertNotIsInstance(server.protocol, CoconutProtocol)
server.stop()
# ...but our custom server does.
husk_server = HuskServer()
husk_server.listen(8469)
self.assertIsInstance(husk_server.protocol, CoconutProtocol)
husk_server.stop()
def test_custom_protocol(self):
"""
A subclass of Server which overrides the protocol_class attribute will
have an instance of that class as its protocol after its listen()
method is called.
"""
# Make a custom Protocol and Server to go with hit.
class CoconutProtocol(KademliaProtocol):
pass
class HuskServer(Server):
protocol_class = CoconutProtocol
# An ordinary server does NOT have a CoconutProtocol as its protocol...
loop = asyncio.get_event_loop()
server = Server()
loop.run_until_complete(server.listen(8469))
self.assertNotIsInstance(server.protocol, CoconutProtocol)
server.stop()
# ...but our custom server does.
husk_server = HuskServer()
loop.run_until_complete(husk_server.listen(8469))
self.assertIsInstance(husk_server.protocol, CoconutProtocol)
husk_server.stop()
def test_custom_protocol(self): # pylint: disable=no-self-use
"""
A subclass of Server which overrides the protocol_class attribute will
have an instance of that class as its protocol after its listen()
method is called.
"""
# Make a custom Protocol and Server to go with hit.
class CoconutProtocol(KademliaProtocol):
pass
class HuskServer(Server):
protocol_class = CoconutProtocol
# An ordinary server does NOT have a CoconutProtocol as its protocol...
loop = asyncio.get_event_loop()
server = Server()
loop.run_until_complete(server.listen(8469))
assert not isinstance(server.protocol, CoconutProtocol)
server.stop()
# ...but our custom server does.
husk_server = HuskServer()
loop.run_until_complete(husk_server.listen(8469))
assert isinstance(husk_server.protocol, CoconutProtocol)
husk_server.stop()
def bootstrap_node(event_loop):
server = Server()
event_loop.run_until_complete(server.listen(8468))
try:
yield ('127.0.0.1', 8468)
finally:
server.stop()
def bootstrap_node(event_loop):
server = Server()
event_loop.run_until_complete(server.listen(8468))
try:
yield ('127.0.0.1', 8468)
finally:
server.stop()
async def run():
server = Server()
await server.listen(8469)
bootstrap_node = (sys.argv[1], int(sys.argv[2]))
await server.bootstrap([bootstrap_node])
result = await server.get(sys.argv[3])
print("Get result:", result)
server.stop()
async def test_storing(bootstrap_node):
server = Server()
await server.listen(bootstrap_node[1] + 1)
await server.bootstrap([bootstrap_node])
await server.set('key', 'value')
result = await server.get('key')
assert result == 'value'
server.stop()
async def test_storing(bootstrap_node):
server = Server()
await server.listen(bootstrap_node[1] + 1)
await server.bootstrap([bootstrap_node])
await server.set('key', 'value')
result = await server.get('key')
assert result == 'value'
server.stop()
def test_default_protocol(self):
"""
An ordinary Server object will initially not have a protocol, but will
have a KademliaProtocol object as its protocol after its listen()
method is called.
"""
server = Server()
self.assertIsNone(server.protocol)
server.listen(8469)
self.assertIsInstance(server.protocol, KademliaProtocol)
server.stop()
async def run():
server = Server()
await server.listen(8469)
bootstrap_node = (sys.argv[1], int(sys.argv[2]))
await server.bootstrap([bootstrap_node])
await server.set(sys.argv[3], sys.argv[4])
print("sleeping for 5 seconds")
time.sleep(5)
print("waking up to fetch stored value")
result = await server.get(sys.argv[3])
print("Get result:", result)
server.stop()
def test_default_protocol(self):
"""
An ordinary Server object will initially not have a protocol, but will
have a KademliaProtocol object as its protocol after its listen()
method is called.
"""
loop = asyncio.get_event_loop()
server = Server()
self.assertIsNone(server.protocol)
loop.run_until_complete(server.listen(8469))
self.assertIsInstance(server.protocol, KademliaProtocol)
server.stop()
def test_default_protocol(self): # pylint: disable=no-self-use
"""
An ordinary Server object will initially not have a protocol, but will
have a KademliaProtocol object as its protocol after its listen()
method is called.
"""
loop = asyncio.get_event_loop()
server = Server()
assert server.protocol is None
loop.run_until_complete(server.listen(8469))
assert isinstance(server.protocol, KademliaProtocol)
server.stop()
def chat(port, chatname):
node = Server()
node.listen(port)
loop = asyncio.get_event_loop()
loop.run_until_complete(node.bootstrap([("0.0.0.0", 8468)]))
chatlist = loop.run_until_complete(node.get("chatlist"))
loop.run_until_complete(node.set("chatlist", str(chatlist) + "\n" + chatname + "\n"))
node.stop()
def user(port, username):
node = Server()
node.listen(port)
loop = asyncio.get_event_loop()
loop.run_until_complete(node.bootstrap([("0.0.0.0", 8468)]))
userlist = loop.run_until_complete(node.get("userlist"))
loop.run_until_complete(node.set("userlist", str(userlist) + "\n" + username + "\n"))
node.stop()
def master_kademlia_join(loop):
node = Server()
loop.run_until_complete(node.listen(5678))
try:
msg = stamp("{} has joined the P2P network.".format(master_name))
send_to_hub(msg)
print(msg)
loop.run_forever()
except KeyboardInterrupt:
pass
finally:
node.stop()
loop.close()
return node
def launch_bootstrap():
server = Server()
server.listen(8469)
loop = asyncio.get_event_loop()
loop.set_debug(True)
try:
loop.run_forever()
except KeyboardInterrupt:
pass
finally:
server.stop()
loop.close()
def run(self):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.set_debug(True)
server = Server()
loop.run_until_complete(server.listen(self.port))
if self.bootstrap_port is not None:
bootstrap_node = (self.bootstrap_address, self.bootstrap_port)
loop.run_until_complete(server.bootstrap([bootstrap_node]))
try:
loop.run_forever()
except KeyboardInterrupt:
pass
finally:
server.stop()
loop.close()
def send(port, chatname, username, message):
node = Server()
node.listen(port)
loop = asyncio.get_event_loop()
loop.run_until_complete(node.bootstrap([("0.0.0.0", 8468)]))
chat = loop.run_until_complete(node.get(chatname))
loop.run_until_complete(
node.set(chatname,
str(chat) + "\n" + username + ": " + message + "\n"))
new_chat = loop.run_until_complete(node.get(chatname))
node.stop()
print("************************************************")
print(chatname)
print("************************************************")
print("\n" + new_chat + "\n")
print("************************************************")
def kad_client(neighbor_ip, neighbor_port, username):
#handler = logging.StreamHandler()
#formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
#handler.setFormatter(formatter)
#log = logging.getLogger('kademlia')
#log.addHandler(handler)
#log.setLevel(logging.DEBUG)
aio = asyncio.get_event_loop()
kad = Server()
aio.run_until_complete(kad.listen(8889))
aio.run_until_complete(kad.bootstrap([(neighbor_ip, neighbor_port)]))
resp = aio.run_until_complete(get_user_profile(kad, username))
print("resp.json(): " + resp.json())
# print(resp.json())
kad.stop()
aio.close()
def test_custom_event_loop(self):
custom_loop = asyncio.new_event_loop()
server = Server()
server.listen(8468)
custom_loop = asyncio.new_event_loop()
server2 = Server(custom_event_loop=custom_loop)
server_thread = threading.Thread(target=setup_extra_server,
args=[server2, custom_loop])
server_thread.start()
# testing using the custom event loop
loop = asyncio.get_event_loop()
loop.run_until_complete(server.bootstrap([("localhost", 8469)]))
loop.run_until_complete(server.set("test", "test1"))
rec_value = loop.run_until_complete(server.get("test"))
server.stop()
stop_extra_server(server2, custom_loop)
server_thread.join()
assert rec_value == "test1"
def main():
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
server = Server()
server.listen(1111)
loop = asyncio.get_event_loop()
loop.set_debug(True)
try:
loop.run_forever()
except KeyboardInterrupt:
pass
finally:
server.stop()
loop.close()
sys.exit(0)
def main(argv):
port_given = False
key_given = False
set_key = False
get_key = False
try:
opts, args = getopt.getopt(argv, "p:k:s:g:", ["set=", "get="])
except getopt.GetoptError:
print(
"python3 new_node.py -p <port> -k <key> -s <set_value> -g <get_key>"
)
sys.exit(2)
for opt, arg in opts:
if opt == "-p":
port_given = True
port = arg
elif opt == "-k":
key_given = True
key = arg
elif opt in ("-s", "set="):
set_key = True
key_value = arg
elif opt in ("-g", "get="):
get_key = True
gkey = arg
if port_given is False:
print(
"python3 new_node.py -p <port> -k <key> -s <set_value> -g <get_key>"
)
sys.exit(1)
node = Server()
node.listen(port)
# Bootstrap the node by connecting to other known nodes, in this case
# ("0.0.0.0", 1111) is the starting node.
loop = asyncio.get_event_loop()
loop.run_until_complete(node.bootstrap([STARTER_NODE]))
# set a user specified value for the user specified key on the network
if set_key is True:
if key_given is True:
loop.run_until_complete(node.set(key, key_value))
else:
print(
"python3 new_node.py -p <port> -k <key> -s <set_value> -g <get_key>"
)
sys.exit(1)
# get the value associated with the user specified key from the network
if key_given is True or get_key is True:
if get_key is True:
result = loop.run_until_complete(node.get(gkey))
print(result)
elif set_key is False:
result = loop.run_until_complete(node.get(key))
print(result)
try:
loop.run_forever()
except KeyboardInterrupt:
pass
finally:
node.stop()
loop.close()
sys.exit(0)
import logging
from kademlia.network import Server
import os
rootNodeAddress = os.getenv('ROOTNODEADDRESS')
rootNodePort = int(os.getenv('ROOTNODEPORT'))
loop = asyncio.get_event_loop()
handler = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
loop.set_debug(True)
# Create a node and start listening on port 8468
node = Server()
loop.run_until_complete(node.listen(8468))
loop.run_until_complete(node.bootstrap([(rootNodeAddress, rootNodePort)])) #<<<<<< It was 8468
try:
loop.run_forever()
except KeyboardInterrupt:
pass
finally:
node.stop()
loop.close()
rootNodeAddress = os.getenv('ROOTNODEADDRESS')
rootNodePort = int(os.getenv('ROOTNODEPORT'))
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
loop = asyncio.get_event_loop()
loop.set_debug(True)
persistentStorage = PersistentStorage(mongourl=mongoURL,
db=db,
collection=collection)
storageNode = Server(storage=persistentStorage)
loop.run_until_complete(storageNode.listen(8468))
loop.run_until_complete(
storageNode.bootstrap([(rootNodeAddress, rootNodePort)]))
try:
loop.run_forever()
except KeyboardInterrupt:
pass
finally:
storageNode.stop()
loop.close()
db = os.getenv('MONGODBNAME')
collection = os.getenv('MONGOCOLLECTION')
#rootNodeAddress = os.getenv('ROOTNODEADDRESS')
rootNodePort = int(os.getenv('ROOTNODEPORT'))
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
loop = asyncio.get_event_loop()
loop.set_debug(True)
persistentStorage = PersistentStorage(mongourl=mongoURL,
db=db,
collection=collection)
rootNode = Server(node_id=b'ROOT', storage=persistentStorage)
loop.run_until_complete(rootNode.listen(rootNodePort))
try:
loop.run_forever()
except KeyboardInterrupt:
pass
finally:
rootNode.stop()
loop.close()
import logging
import asyncio
from kademlia.network import Server
handler = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
loop = asyncio.get_event_loop()
loop.set_debug(True)
server = Server()
loop.run_until_complete(server.listen(8468))
try:
loop.run_forever()
except KeyboardInterrupt:
pass
finally:
server.stop()
loop.close()
class Node:
"""This is the object that should be created to start listening
as an active node on the network.
Attributes:
connecting_port: The port to connect to
ip: The ip to connect to
block_height: The height of the blockchain of the node
nodes: If other nodes are connected or not
node: The actual node
loop: The loop tied to the node
verified_block: Boolean that check if a block_hash is correct or not
"""
def __init__(self, connecting_port, ip):
"""
Create a node instance. This will start listening on the given port.
Args:
connecting_port: The port to connect to
ip: The ip to connect to
"""
self.connecting_port = connecting_port
self.opening_port = randint(8800, 9000)
self.TARGET_MAX = 0x0000FFFFFFFFFFFFFFFFFFFFFFF000000000000000000000000000000000000
self.target = 1
self.ip = ip
self.block_height = 0
self.nodes_connected = None
self.node = None
self.loop = None
self.verified_block = False
if self.connecting_port is None:
bootstrap_node = Thread(target=self.bootstrap)
bootstrap_node.start()
else:
self.start_node()
self.get_blockheight()
def bootstrap(self):
"""Bootstrap the server by connecting to other known nodes in the network."""
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
print("Node operating at " + str(self.opening_port))
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
self.loop.set_debug(True)
self.node = Server()
self.loop.run_until_complete(self.node.listen(self.opening_port))
try:
self.loop.run_forever()
except KeyboardInterrupt:
pass
finally:
self.node.stop()
self.loop.close()
def start_node(self):
"""Each blackbox instance is a node which other nodes can connect to"""
print("Node operating at " + str(self.opening_port))
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
self.loop = asyncio.get_event_loop()
self.loop.set_debug(True)
self.node = Server()
self.loop.run_until_complete(self.node.listen(self.opening_port))
bootstrap_node = (self.ip, int(self.connecting_port))
self.loop.run_until_complete(self.node.bootstrap([bootstrap_node]))
self.loop.run_until_complete(self.node.set("nodes", True))
def get_blockheight(self):
"""When other nodes join the network they need to start mining at the most
up-to-date block"""
block_height = self.loop.run_until_complete(
self.node.get("block_height"))
if self.block_height is None or block_height is None or 0:
self.get_blockheight()
return
if (block_height > self.block_height):
self.block_height = block_height
def set_hash(self, block_hash):
"""When a new block is found, sets in the DHT the associated block_hash and block_height
Args:
block_hash: The hash of the block found by the participating node
"""
key = "blk" + str(self.block_height)
if self.connecting_port is None and block_hash is not None:
asyncio.run_coroutine_threadsafe(
self.node.set("block_height", self.block_height + 1),
self.loop)
asyncio.run_coroutine_threadsafe(self.node.set(key, block_hash),
self.loop)
elif block_hash is not None and self.block_height is not 0:
self.loop.run_until_complete(
self.node.set("block_height", self.block_height + 1))
self.loop.run_until_complete(self.node.set(key, block_hash))
def get_last_block(self):
key = "blk" + str(self.block_height)
if self.connecting_port is None:
return asyncio.run_coroutine_threadsafe(self.node.get(key),
self.loop)
return self.loop.run_until_complete(self.node.get(key))
def broadcast(self, hash_broadcasted, difficulty):
"""
Args:
hash_broadcasted: The hash that a node broadcasted to the network
difficulty: The current difficulty on the network
"""
if self.connecting_port is None:
hash_broadcasted = hash_broadcasted.result()
if hash_broadcasted is not None:
self.target = self.TARGET_MAX / difficulty
if int(hash_broadcasted, 16) < int(self.target):
if self.connecting_port is None:
asyncio.run_coroutine_threadsafe(
self.node.set("verified_block", True), self.loop)
else:
self.loop.run_until_complete(
self.node.set("verified_block", True))
return True
return False
def verify(self, difficulty):
"""Verify if a block_hash found by a node is a correct one
Args:
difficulty: The current difficulty of the network
"""
key = "blk" + str(self.block_height)
if self.connecting_port is None:
self.nodes_connected = asyncio.run_coroutine_threadsafe(
self.node.get("nodes"), self.loop)
if self.nodes_connected.result() is not True:
asyncio.run_coroutine_threadsafe(
self.node.set("verified_block", True), self.loop)
return True
else:
hash_broadcasted = asyncio.run_coroutine_threadsafe(
self.node.get(key), self.loop)
return (self.broadcast(hash_broadcasted, difficulty))
else:
hash_broadcasted = self.loop.run_until_complete(self.node.get(key))
return (self.broadcast(hash_broadcasted, difficulty))
class IPFSNode():
def __init__(self, bootstrapHost: str = 'bootstrap'):
self.server = Server()
self.server.listen(8469)
self.has_list = []
# Hack to get the "deafult" IP
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('1.1.1.1', 80))
self.local_ip = s.getsockname()[0]
s.close()
#self.local_ip = socket.gethostbyname(socket.gethostname())
bootstrap_ip = socket.gethostbyname(bootstrapHost)
self.bootstrap_node = (bootstrap_ip, 8469)
self.loop = asyncio.get_event_loop()
self.loop.set_debug(True)
self.loop.run_until_complete(
self.server.bootstrap([self.bootstrap_node]))
neighbors = self.server.bootstrappableNeighbors()
for node in neighbors:
print("DHT Peer found! {0}:{1}".format(node[0], node[1]))
print("Starting TCP transfer server")
self.tcpThread = threading.Thread(target=self.startTCPServer)
self.tcpThread.start()
def startTCPServer(self):
host = '0.0.0.0'
port = 9528
self.running = True
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((host, port))
s.listen(10)
while self.running:
conn, addr = s.accept()
print("Connection from " + str(addr))
sys.stdout.flush()
while 1:
data = conn.recv(4096)
if not data:
break
# Check if data is correct format
data = data.decode('utf-8')
if (len(data) == 133
or len(data) == 134) and data[0:5] == 'hash=':
requestedHash = data[5:134]
print("looking for hash " + requestedHash)
sys.stdout.flush()
#Find requested hash in our data
found = False
for has_chunk in self.has_list:
if has_chunk[1] in requestedHash:
conn.send(has_chunk[0]) #Send the found data
found = True
break
if not found:
conn.send(b"notfound")
else:
conn.send(b"invalid_request")
print("Request: " + data)
sys.stdout.flush()
conn.close()
s.close()
def chunkFile(self, data: str):
chunks = []
for i in range(0, len(data), block_size):
chunk = data[i:i + block_size]
hsh = hashlib.sha512(chunk).hexdigest()
chunks.append({"data": chunk, "hash": hsh, "size": len(chunk)})
return chunks
def setDHTKey(self, key: str, val: str):
self.loop.run_until_complete(self.server.set(key, val))
def getDHTKey(self, key: str) -> str:
return self.loop.run_until_complete(self.server.get(key))
def addFile(self, filepath: str, compression: bool):
with open(filepath, 'rb') as f:
data = f.read()
fileHash = hashlib.sha512(data).hexdigest()
fileLen = len(data)
if compression: #Compress data if necessary
data = zlib.compress(data, level=7)
fileChunks = self.chunkFile(data)
fileName = os.path.basename(filepath)
ipfsList = List(fileHash, fileName, fileLen, compression)
ipfsBlobs = []
for chunk in fileChunks:
ipfsBlob = Blob()
ipfsBlob.setData(chunk['data'], chunk['hash'])
ipfsBlobs.append(ipfsBlob)
ipfsList.addLink(chunk['hash'], chunk['size'])
#Add data to our has list
self.has_list.append((chunk['data'], chunk['hash']))
print(ipfsList)
if debug:
print(ipfsList.getData())
print("\n\n\n")
for blob in ipfsBlobs:
print(blob.getData())
if self.server:
#Need to add data onto DHT network
self.setDHTKey(fileHash,
str(ipfsList)) #Add master file record to DHT
for blob in ipfsBlobs: #Add items onto DHT
block = blob.getData()
if len(
block['data']
) > 1024: #If the block is bigger than 1k, add to local TCP server
self.setDHTKey(block['hash'], 'ip=' + self.local_ip)
else: #Otherwise store directly on DHT
self.setDHTKey(blob.getData()['hash'],
blob.getData()['data'])
return fileHash
def TCPGet(self, host, hsh):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host, 9528))
if not isinstance(hsh, bytes):
hsh = hsh.encode()
s.send(b'hash=' + hsh + b'\n')
data = s.recv(8192)
s.close()
return data
def getFile(self, hsh: str) -> (bytes, dict):
masterFileRecord = self.getDHTKey(hsh) #Get metadata
metadata = None
#Convert from string dictionary to python dictionary
if masterFileRecord and len(masterFileRecord) > 1:
masterFileRecord = masterFileRecord.replace("'", "\"").replace(
'True', 'true').replace('False',
'false') #transform into valid JSON
metadata = json.loads(masterFileRecord)
else:
raise Exception("Unable to locate file record on network!")
fileContents = b''
for link in metadata['links']:
DHTData = self.getDHTKey(link['hash'])
data = None
if not DHTData:
raise Exception("Unable to get part of file with hash " +
link['hash'])
if DHTData[0:3] == 'ip=': #Need to get data via TCP not DHT
data = self.TCPGet(DHTData[3:], link['hash'])
else:
data = DHTData
if len(data) != link['size']:
raise Exception(
"Hash value ({}) has invalid or corrupted length".format(
link['hash']))
fileContents += data
if metadata['compression']:
fileContents = zlib.decompress(fileContents)
return (fileContents, metadata)
def __del__(self):
self.server.stop()
self.loop.close()
self.tcpThread.join()
import asyncio
import sys
from kademlia.network import Server
from kademlia.routing import RoutingTable
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
loop = asyncio.get_event_loop()
loop.set_debug(True)
server = Server()
server.listen(8469)
bootstrap_node = ('127.0.0.1', 8468)
loop.run_until_complete(server.bootstrap([bootstrap_node]))
#loop.run_until_complete(server.set("key", "a"*1024*4))
print(server.bootstrappableNeighbors())
result = loop.run_until_complete(server.get('key'))
server.stop()
loop.close()
print("Get result:", result)
class KademliaServer:
def __init__(self, ip, port):
self.ip = ip
self.port = port
self.loop = None
def start_server(self, bootstrap_nodes):
handler = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s'
'- %(message)s')
handler.setFormatter(formatter)
# DEBUG
if DEBUG:
log = logging.getLogger('kademlia')
log.addHandler(handler)
log.setLevel(logging.DEBUG)
self.loop = asyncio.get_event_loop()
if DEBUG:
self.loop.set_debug(True)
self.server = Server()
self.loop.run_until_complete(self.server.listen(self.port))
self.loop.run_until_complete(self.server.bootstrap(bootstrap_nodes))
return self.loop
def close_server(self):
self.server.stop()
async def register(self, username):
result = await self.server.get(username)
if result is None:
value = {
"followers": [],
"following": {},
"redirect": {},
"msg_nr": 0,
"ip": self.ip,
"port": self.port
}
value_to_set = json.dumps(value)
await self.server.set(username, value_to_set)
return value
else:
raise Exception("Username already exists")
async def login(self, username):
result = await self.server.get(username)
result = json.loads(result)
if result is not None:
value = {
"followers": result['followers'],
"following": result['following'],
"redirect": result['redirect'],
"msg_nr": result['msg_nr'],
"ip": self.ip,
"port": self.port
}
value_to_set = json.dumps(value)
await self.server.set(username, value_to_set)
return value
else:
raise Exception("User doesn't exist! Please register")
async def get_user_ip(self, username):
result = await self.server.get(username)
result = json.loads(result)
if result is None:
raise Exception("User doesn't exist!")
else:
return (result["ip"], result["port"])
async def get_user_ip_msgnr(self, username):
result = await self.server.get(username)
result = json.loads(result)
if result is None:
raise Exception("User doesn't exist!")
else:
return (result["ip"], result["port"], result["msg_nr"])
async def get_location_and_followers(self, usernames):
res = {}
for username in usernames:
result = await self.server.get(username)
result = json.loads(result)
if result is not None:
res[username] = (result["ip"],
result["port"],
result["followers"])
return res
async def get_user_followers(self, username):
followers = {}
result = await self.get_user(username)
for follower in result["followers"]:
result = await self.server.get(follower)
result = json.loads(result)
if result is not None:
followers[follower] = (result["ip"],
result["port"],
result["followers"])
return followers
async def get_users_following_user(self, user):
user = await self.get_user(user)
user_followers = user['followers']
return user_followers
async def get_outdated_user_following(self, following):
res = []
for follw, info in following.items():
user_knowledge = info[0]
result = await self.server.get(follw)
result = json.loads(result)
if (result is not None) and result['msg_nr'] > user_knowledge:
res.append((follw, result["ip"], result["port"],
result["msg_nr"]))
return res
async def get_user_following(self, state):
following = {}
result = await self.get_user(username)
for follw in result["following"]:
result = await self.server.get(follw)
result = json.loads(result)
if result is not None:
following[follw] = (result["ip"],
result["port"])
return following
async def get_user(self, username):
result = await self.server.get(username)
result = json.loads(result)
return result
async def set_user(self, username, value):
await self.server.set(username, value)