def __init__(self, ip, port):
self.ip = ip
self.port = int(port)
self.server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.server.bind((self.ip, self.port))
self.server_process = Process(target=self._start)
self.routing_table = RoutingTable()
def __init__(self, sourceNode, storage, ksize):
RPCProtocol.__init__(self)
self.router = RoutingTable(self, ksize, sourceNode)
self.storage = storage
self.sourceNode = sourceNode
self.log = Logger(system=self)
self.transactionSeq = 0
def __init__(self):
self.udp_ip = cfg.UdpServer['ip']
self.udp_port = cfg.UdpServer['port']
self.flask_ip = cfg.FlaskServer['ip']
self.flask_port = cfg.FlaskServer['port']
self.username = cfg.Application['name']
self.dir = cfg.Application['dir']
self.bs_ip = cfg.BoostrapServer['ip']
self.bs_port = cfg.BoostrapServer['port']
self.routing_table = RoutingTable()
self.cli = CLI()
self.udp_server = UDPServer(self.udp_ip, self.udp_port)
self.rest_server = RESTServer(self.flask_ip, self.flask_port)
class UDPServer:
def __init__(self, ip, port):
self.ip = ip
self.port = int(port)
self.server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.server.bind((self.ip, self.port))
self.server_process = Process(target=self._start)
self.routing_table = RoutingTable()
def run(self):
self.server_process.start()
def terminate(self):
self.server_process.terminate()
def _start(self):
executor = ThreadPoolExecutor(max_workers=3)
while True:
msg, addr = self.server.recvfrom(CONST.BUFFER_SIZE)
executor.submit(self._process_request, msg=msg, addr=addr)
def _process_request(self, msg, addr):
msg = msg.decode("utf-8")
tokens = msg.split()
if tokens[1] == "JOIN":
self.routing_table.add(tokens[2], tokens[3])
response = query_builder("JOINOK", ["0"])
udp_send_recv(addr[0], addr[1], response, recieve=False)
elif tokens[1] == "LEAVE":
for node in self.routing_table.get():
if node[1] == tokens[3]:
self.routing_table.remove(node)
break
response = query_builder("LEAVEOK", ["0"])
udp_send_recv(addr[0], addr[1], response, recieve=False)
elif tokens[1] == "SER":
hops = int(tokens[5])
files_found, file_names = search_file(tokens[4])
if files_found > 0:
response = query_builder("SEROK", [files_found, cfg.FlaskServer['ip'], cfg.FlaskServer['port'], hops,
file_names])
# time.sleep(random.randint(1, 100) / 100)
udp_send_recv(tokens[2], tokens[3], response, recieve=False)
elif hops > 0:
request = query_builder("SER", [tokens[2], tokens[3], tokens[4], hops - 1])
for node in self.routing_table.get():
udp_send_recv(node[0], node[1], request, recieve=False)
elif tokens[1] == "SEROK":
pretty_print_message_to_cli(
"Files Found: " + " ".join(tokens[6:]) + " | Flask IP: " + tokens[3] + " | Flask Port: " + tokens[4])
def __init__(self, rpc_protocol=None, routing_table=None, data_storage=None):
self.node_id = Key.generateID()
if data_storage is None:
self.data_storage = dict()
else:
self.data_storage = data_storage
if rpc_protocol is None:
self.protocol = DHTProtocol(self, self.node_id)
else:
self.protocol = rpc_protocol
if routing_table is None:
self.routing_table = RoutingTable(self.node_id)
else:
self.routing_table = routing_table
self.heartbeat_loop = LoopingCall(self._heartbeatLoop)
def clickAction(e):
nodeList = []
obj = gw.getElementAt(e.getX(), e.getY())
if isinstance(obj, ArpanetNode):
nodeList = [obj]
elif isinstance(obj, GLine):
active = obj.getColor() == "Black"
start = obj.getStartPoint()
end = obj.getEndPoint()
n1 = gw.getElementAt(start.getX(), start.getY())
n2 = gw.getElementAt(end.getX(), end.getY())
if active:
obj.setColor("LightGray")
n1.removeNeighbor(n2)
n2.removeNeighbor(n1)
else:
obj.setColor("Black")
n1.addNeighbor(n2)
n2.addNeighbor(n1)
elif obj == allButton:
nodeList = arpanet.getNodes()
elif isinstance(obj, GLabel):
node = arpanet.findNode(obj.getLabel())
name = node.getName()
if node.isActive():
node.setActive(False)
obj.setColor("LightGray")
node.setRoutingTable(RoutingTable(name))
monitors[name].update()
else:
node.setActive(True)
obj.setColor("Black")
monitors[name].update()
for node in nodeList:
name = node.getName()
myTable = node.getRoutingTable()
if node.isActive():
for neighbor in node.getNeighbors():
if neighbor.isActive():
neighbor.getRoutingTable().update(name, myTable)
for name in monitors:
monitors[name].update()
def createArpanetGraph(gw):
graph = ArpanetGraph(gw)
for name, x, y, dx, dy in NODE_TABLE:
node = ArpanetNode(name)
node.setRoutingTable(RoutingTable(name))
graph.addNode(node)
gw.add(node, x, y)
label = GLabel(name)
label.setFont(LABEL_FONT)
gw.add(label, x + dx, y + dy)
graph.connect("BBN", "CMU")
graph.connect("BBN", "MIT")
graph.connect("BBN", "UTAH")
graph.connect("CMU", "NRL")
graph.connect("CMU", "UTAH")
graph.connect("HARV", "MIT")
graph.connect("HARV", "NRL")
graph.connect("MIT", "HARV")
graph.connect("NRL", "RAND")
graph.connect("RAND", "UCLA")
graph.connect("SRI", "STAN")
graph.connect("SRI", "UTAH")
graph.connect("STAN", "UCLA")
return graph
def __init__(self, source_node, storage, ksize):
RPCProtocol.__init__(self)
self.router = RoutingTable(self, ksize, source_node)
self.storage = storage
self.source_node = source_node
class KademliaProtocol(RPCProtocol):
def __init__(self, source_node, storage, ksize):
RPCProtocol.__init__(self)
self.router = RoutingTable(self, ksize, source_node)
self.storage = storage
self.source_node = source_node
def get_refresh_ids(self):
ids = []
for bucket in self.router.lonely_buckets():
rid = random.randint(*bucket.range).to_bytes(20, byteorder='big')
ids.append(rid)
return ids
def rpc_stun(self, sender):
return sender
def rpc_ping(self, sender, nodeid):
source = Node(nodeid, sender[0], sender[1])
self.welcome_if_new(source)
return self.source_node.id
def rpc_store(self, sender, nodeid, dkey, key, name, value, hash=True):
source = Node(nodeid, sender[0], sender[1])
self.welcome_if_new(source)
log.debug("got a store request from %s, storing '%s'='%s'", sender,
dkey.hex(), value)
self.storage.set(dkey, key, name, value, hash)
return True
def rpc_delete(self, sender, nodeid, key):
source = Node(nodeid, sender[0], sender[1])
self.welcome_if_new(source)
log.debug("got a delete request from %s, deleting '%s'", sender,
key.hex())
self.storage.delete(key)
return True
def rpc_delete_tag(self, sender, nodeid, dkey, key, value):
source = Node(nodeid, sender[0], sender[1])
self.welcome_if_new(source)
log.debug("got a delete request from %s, deleting '%s'", sender,
dkey.hex())
self.storage.delete_tag(dkey, key, value)
return True
def rpc_find_node(self, sender, nodeid, key):
log.info("finding neighbors of %i in local table",
int(nodeid.hex(), 16))
source = Node(nodeid, sender[0], sender[1])
self.welcome_if_new(source)
node = Node(key)
neighbors = self.router.find_neighbors(node, exclude=source)
return list(map(tuple, neighbors))
def rpc_find_value(self, sender, nodeid, key):
source = Node(nodeid, sender[0], sender[1])
self.welcome_if_new(source)
value = self.storage.get(key, None)
if value is None:
return self.rpc_find_node(sender, nodeid, key)
return {'value': value}
async def call_find_node(self, node_to_ask, node_to_find):
address = (node_to_ask.ip, node_to_ask.port)
result = await self.find_node(address, self.source_node.id,
node_to_find.id)
return self.handle_call_response(result, node_to_ask)
async def call_find_value(self, node_to_ask, node_to_find):
address = (node_to_ask.ip, node_to_ask.port)
result = await self.find_value(address, self.source_node.id,
node_to_find.id)
return self.handle_call_response(result, node_to_ask)
async def call_ping(self, node_to_ask):
address = (node_to_ask.ip, node_to_ask.port)
result = await self.ping(address, self.source_node.id)
return self.handle_call_response(result, node_to_ask)
async def call_store(self,
node_to_ask,
dkey,
key,
name=None,
value=None,
hash=True):
address = (node_to_ask.ip, node_to_ask.port)
result = await self.store(address, self.source_node.id, dkey, key,
name, value, hash)
return self.handle_call_response(result, node_to_ask)
async def call_delete(self, node_to_ask, key):
address = (node_to_ask.ip, node_to_ask.port)
result = await self.delete(address, self.source_node.id, key)
return self.handle_call_response(result, node_to_ask)
async def call_delete_tag(self, node_to_ask, dkey, key, value):
address = (node_to_ask.ip, node_to_ask.port)
result = await self.delete_tag(address, self.source_node.id, dkey, key,
value)
return self.handle_call_response(result, node_to_ask)
def welcome_if_new(self, node):
if not self.router.is_new_node(node):
return
log.info("never seen %s before, adding to router", node)
for key, value in self.storage:
keynode = Node(digest(key))
neighbors = self.router.find_neighbors(keynode)
if neighbors:
last = neighbors[-1].distance_to(keynode)
new_node_close = node.distance_to(keynode) < last
first = neighbors[0].distance_to(keynode)
this_closest = self.source_node.distance_to(keynode) < first
if not neighbors or (new_node_close and this_closest):
asyncio.ensure_future(self.call_store(node, key, value))
self.router.add_contact(node)
def handle_call_response(self, result, node):
#print("result >>>>")
#print(result)
if not result[0]:
#log.warning("no response from %s, removing from router", node)
self.router.remove_contact(node)
return result
#log.info("got successful response from %s", node)
self.welcome_if_new(node)
return result
class Node:
def __init__(self):
self.udp_ip = cfg.UdpServer['ip']
self.udp_port = cfg.UdpServer['port']
self.flask_ip = cfg.FlaskServer['ip']
self.flask_port = cfg.FlaskServer['port']
self.username = cfg.Application['name']
self.dir = cfg.Application['dir']
self.bs_ip = cfg.BoostrapServer['ip']
self.bs_port = cfg.BoostrapServer['port']
self.routing_table = RoutingTable()
self.cli = CLI()
self.udp_server = UDPServer(self.udp_ip, self.udp_port)
self.rest_server = RESTServer(self.flask_ip, self.flask_port)
def run(self):
# generate random files to be shared
self.generate_files(random.randint(2, 5))
self.reg_in_bs()
self.connect_to_network()
# starting udp server in a new process
self.udp_server.run()
# starting rest server in a new process
self.rest_server.run()
# starting cli in the main process
self.cli.run()
self.udp_server.terminate()
self.rest_server.terminate()
self.unreg_from_bs()
self.disconnect_from_network()
shutil.rmtree(self.dir)
def reg_in_bs(self):
query = query_builder("REG",
data=[self.udp_ip, self.udp_port, self.username])
data = udp_send_recv(self.bs_ip, self.bs_port, query)
try:
res_type, data = query_parser(data)
except Exception as e:
print("Error:", str(e))
sys.exit("Exiting, Couldn't connect to BS")
else:
if res_type == "REGOK":
for i in range(0, len(data), 2):
self.routing_table.add(data[i], data[i + 1])
else:
print("Error: Invalid response from BS")
sys.exit("Exiting, Couldn't connect to BS")
def unreg_from_bs(self):
query = query_builder("UNREG",
data=[self.udp_ip, self.udp_port, self.username])
res = udp_send_recv(self.bs_ip, self.bs_port, query)
try:
res_type, res = query_parser(res)
except Exception as e:
pass
def connect_to_network(self):
for ip, port in self.routing_table.get():
query = query_builder("JOIN", data=[self.udp_ip, self.udp_port])
data = udp_send_recv(ip, port, query)
try:
res_type, data = query_parser(data)
except Exception as e:
print("Error:", str(e))
self.routing_table.remove((ip, port))
else:
if res_type == "JOINOK":
pass
def disconnect_from_network(self):
for ip, port in self.routing_table.get():
query = query_builder("LEAVE", data=[self.udp_ip, self.udp_port])
data = udp_send_recv(ip, port, query)
try:
res_type, data = query_parser(data)
except Exception as e:
print("Error:", str(e))
else:
if res_type == "LEAVEOK":
pass
def generate_files(self, num_files):
if os.path.isdir(self.dir):
print("path already exist")
else:
os.mkdir(self.dir)
file_names = []
with open("File Names.txt", 'r') as in_file:
for line in in_file:
file_names.append(line.strip())
random.shuffle(file_names)
for i in range(num_files):
generate_random_file(self.dir, file_names[i],
random.randint(2, 10))
class KademliaProtocol(RPCProtocol):
def __init__(self, sourceNode, storage, ksize):
RPCProtocol.__init__(self)
self.router = RoutingTable(self, ksize, sourceNode)
self.storage = storage
self.sourceNode = sourceNode
self.log = Logger(system=self)
self.transactionSeq = 0
def datagramReceived(self, datagram, address):
if self.noisy:
log.msg("received datagram from %s" % repr(address))
try:
msg = bdecode(datagram)
msgID = msg["t"]
msgType = msg["y"]
if msgType == "q":
f = getattr(self, "rpc_%s" % msg["q"], None)
if f is None or not callable(f):
self.transport.write(
bencode({
"t": msgID,
"y": "e",
"e": [204, "Method Unknown"]
}), address)
else:
self._acceptRequest(msgID, [msg["q"], [msg["a"]]], address)
elif msgType == "r":
self._acceptResponse(msgID, msg["r"], address)
elif msgType == "e":
# Ignore error messages
pass
else:
# otherwise, don't know the format, don't do anything
log.msg("Received unknown message from %s, ignoring" %
repr(address))
self.transport.write(
bencode({
"t": msgID,
"y": "e",
"e": [203, "Protocol Error, invalid arguments"]
}), address)
except KeyError:
log.msg("Invalid message data from %s, ignoring" % repr(address))
self.transport.write(
bencode({
"y": "e",
"e": [201, "Generic Error"]
}), address)
except BTFailure:
log.msg("Not a valid bencoded string from %s, ignoring" %
repr(address))
self.transport.write(
bencode({
"y": "e",
"e": [203, "Protocol Error, malformed packet"]
}), address)
def _sendResponse(self, response, msgID, address):
if self.noisy:
log.msg("sending response for msg id %s to %s" %
(b64encode(msgID), repr(address)))
response["t"] = msgID
self.transport.write(bencode(response), address)
def sendMessage(self, address, message):
msgID = pack(">I", self.transactionSeq)
self.transactionSeq += 1
message["t"] = msgID
self.transport.write(bencode(message), address)
d = defer.Deferred()
timeout = reactor.callLater(self._waitTimeout, self._timeout, msgID)
self._outstanding[msgID] = (d, timeout)
return d
def getRefreshIDs(self):
"""
Get ids to search for to keep old buckets up to date.
"""
ids = []
for bucket in self.router.getLonelyBuckets():
ids.append(random.randint(*bucket.range))
return ids
@staticmethod
def _response_error(error_code, error_message):
return {"y": "e", "e": [error_code, error_message]}
def rpc_ping(self, sender, args):
try:
node_id = args["id"]
source = Node(node_id, sender[0], sender[1])
self.welcomeIfNewNode(source)
return {"y": "r", "r": {"id": self.sourceNode.id}}
except KeyError:
return self._response_error(203,
"Protocol Error, invalid arguments")
def rpc_announce_peer(self, sender, args):
try:
node_id = args["id"]
info_hash = args["info_hash"]
port = args["port"]
token = args["token"]
source = Node(node_id, sender[0], sender[1])
self.welcomeIfNewNode(source)
self.log.debug("got a store request from %s, storing value" %
str(sender))
if verify_token(sender[0], sender[1], token):
values = self.storage.get(info_hash, [])
values.append((sender[0], port))
# Redeclare value by info_hash
self.storage[info_hash] = values
return {"y": "r", "r": {"id": self.sourceNode.id}}
else:
return self._response_error(203, "Protocol Error, bad token")
except KeyError:
return self._response_error(203,
"Protocol Error, invalid arguments")
def rpc_find_node(self, sender, args):
try:
node_id = args["id"]
target = args["target"]
self.log.info("finding neighbors of %i in local table" %
long(node_id.encode('hex'), 16))
source = Node(node_id, sender[0], sender[1])
self.welcomeIfNewNode(source)
node = Node(target)
return {
"y": "r",
"r": {
"id":
self.sourceNode.id,
"nodes":
encode_nodes(
self.router.findNeighbors(node, exclude=source))
}
}
except KeyError:
return self._response_error(203,
"Protocol Error, invalid arguments")
def rpc_get_peers(self, sender, args):
try:
node_id = args["id"]
info_hash = args["info_hash"]
source = Node(node_id, sender[0], sender[1])
self.welcomeIfNewNode(source)
values = self.storage.get(info_hash, None)
if values is not None:
# We must calculate unique token for sender
return {
"y": "r",
"r": {
"id": self.sourceNode.id,
"token": generate_token(sender[0], sender[1]),
"values": encode_values(values)
}
}
else:
return self.rpc_find_node(sender, {
"id": node_id,
"target": info_hash
})
except KeyError:
return self._response_error(203,
"Protocol Error, invalid arguments")
def callFindNode(self, nodeToAsk, nodeToFind):
address = (nodeToAsk.ip, nodeToAsk.port)
d = self.find_node(address, self.sourceNode.id, nodeToFind.id)
return d.addCallback(self.handleCallResponse,
nodeToAsk,
responseMessage="find_node")
def callGetPeers(self, nodeToAsk, key):
address = (nodeToAsk.ip, nodeToAsk.port)
d = self.get_peers(address, self.sourceNode.id, key.id)
return d.addCallback(self.handleCallResponse,
nodeToAsk,
responseMessage="get_peers")
def callPing(self, nodeToAsk):
address = (nodeToAsk.ip, nodeToAsk.port)
d = self.ping(address, self.sourceNode.id)
return d.addCallback(self.handleCallResponse,
nodeToAsk,
responseMessage="ping")
def callAnnouncePeer(self, nodeToAsk, key, value, token):
address = (nodeToAsk.ip, nodeToAsk.port)
d = self.announce_peer(address, self.sourceNode.id, key.id, value,
token)
return d.addCallback(self.handleCallResponse,
nodeToAsk,
responseMessage="announce_peer")
# BitTorrent protocol messages implementation
def ping(self, address, nodeId):
return self.sendMessage(address, {
"y": "q",
"q": "ping",
"a": {
"id": nodeId
}
})
def find_node(self, address, nodeId, targetId):
return self.sendMessage(address, {
"y": "q",
"q": "find_node",
"a": {
"id": nodeId,
"target": targetId
}
})
def get_peers(self, address, nodeId, info_hash):
return self.sendMessage(
address, {
"y": "q",
"q": "get_peers",
"a": {
"id": nodeId,
"info_hash": info_hash
}
})
def announce_peer(self, address, nodeId, info_hash, port, token):
return self.sendMessage(
address, {
"y": "q",
"q": "announce_peer",
"a": {
"id": nodeId,
"implied_port": 0,
"info_hash": info_hash,
"port": port,
"token": token
}
})
def welcomeIfNewNode(self, node):
if self.router.isNewNode(node):
self.router.addContact(node)
def handleCallResponse(self, result, node, responseMessage):
"""
If we get a response, add the node to the routing table. If
we get no response, make sure it's removed from the routing table.
"""
if result[0]:
self.log.info("got response from %s, adding to router" % node)
self.welcomeIfNewNode(node)
else:
self.log.debug("no response from %s, removing from router" % node)
self.router.removeContact(node)
# TODO: Its looks like a software crutch, need some solution to avoid it.
return result + (node, )
class DHTNode(RPCHandler):
heartbeat_interval = 20
keepalive_timeout = 45
rpc_prefix = 'rpc_'
def __init__(self, rpc_protocol=None, routing_table=None, data_storage=None):
self.node_id = Key.generateID()
if data_storage is None:
self.data_storage = dict()
else:
self.data_storage = data_storage
if rpc_protocol is None:
self.protocol = DHTProtocol(self, self.node_id)
else:
self.protocol = rpc_protocol
if routing_table is None:
self.routing_table = RoutingTable(self.node_id)
else:
self.routing_table = routing_table
self.heartbeat_loop = LoopingCall(self._heartbeatLoop)
def addContact(self, contact):
contact.touch()
self.routing_table.addContact(contact)
def removeContact(self, contact_id):
self.routing_table.removeContact(contact_id)
def findCloseNodes(self, node_id, size):
return self.routing_table.findCloseNodes(node_id, size)
def joinNetwork(self, known_nodes=None):
join_defer = Deferred()
if known_nodes is None:
join_defer.errback(Exception("empty list"))
return join_defer
known_nodes = known_nodes[:]
for (address, port) in known_nodes[:]:
def add_node(node_id):
# Add node to routing table
new_contact = Contact(node_id, address, port)
new_contact.touch()
self.addContact(new_contact)
return
def rm_addr(arg):
# Remove addr from list after processing
known_nodes.remove((address, port))
# Run second stage after all address get processed
if len(known_nodes) == 0:
find_self_key()
return arg
df = self.callGetID(Contact(None, address, port))
df.addCallback(add_node)
df.addBoth(rm_addr)
def find_self_key():
if len(self.routing_table) == 0:
join_defer.errback(Exception("empty routing table"))
else:
def return_join(arg):
print "find to join"
join_defer.callback(None)
return
df = self.findNodes(self.node_id)
df.addBoth(return_join)
return
join_defer.addBoth(self._start_loops)
return join_defer
def _find(self, key):
key = Key(key)
parallel_limit = 3
find_defer = Deferred()
init_list = self.findCloseNodes(self.node_id, 8)
if len(init_list) == 0:
find_defer.callback(list())
return find_defer
contacts = dict()
in_progress = list()
with_errors = dict()
def get_next_contact():
"""
Find next contact for processing.
"""
k_sorted = sorted(contacts.keys(), key=lambda id: key.distance(id))
for k in k_sorted:
if contacts[k]['processed'] == False and not k in in_progress:
return contacts[k]['contact']
return
def check_stop_condition(chk_range=8):
"""
If first chk_range elements are processed returns True.
"""
k_sorted = sorted(contacts.keys(), key=lambda id: key.distance(id))
k_sorted = k_sorted[:chk_range]
for k in k_sorted:
if contacts[k]['processed'] == False:
return False
return True
def make_result():
"""
Make list of contacts from head of 'contacts'
"""
expr = (c['contact'] for _, c in sorted(contacts.items(), key=lambda (id, c): key.distance(id))[:8])
return list(expr)
def start_process(node_id):
"""
Start processing of node with given 'node_id'
"""
in_progress.append(node_id)
c = contacts[node_id]['contact']
def stop_process(c_list):
in_progress.remove(node_id)
contacts[node_id]['processed'] = True
process_new_contacts(c_list)
# return c_list
return
def err_process(err):
in_progress.remove(node_id)
del contacts[node_id]
with_errors[node_id] = c
process_new_contacts()
# return err
return
df = self.callFindNode(c, key.toString())
df.addCallbacks(stop_process, err_process)
return
def process_new_contacts(c_list=[]):
# Add new contacts from given list to 'contacts'
for c in c_list:
if c.node_id.toString() in with_errors or c.node_id == self.node_id:
continue
self.addContact(c)
if not c.node_id.toString() in contacts:
contacts[c.node_id.toString()] = {'contact':c, 'processed':False}
if check_stop_condition(8):
find_defer.callback(make_result())
else:
if len(in_progress) <= parallel_limit:
n_contact = get_next_contact()
if n_contact is None:
return
start_process(n_contact.node_id.toString())
return
process_new_contacts(init_list)
return find_defer
def storeValue(self, key, value):
store_defer = Deferred()
node_df = self.findNodes(key)
def node_callback(l):
if len(l) == 0:
store_defer.callback(None)
for cnt in l[:]:
def bothback_store(arg, c = cnt):
l.remove(c)
if len(l) == 0:
store_defer.callback(None)
return arg
store_df = self.callStore(cnt, key, value)
store_df.addBoth(bothback_store)
node_df.addCallback(node_callback)
return store_defer
def findNodes(self, key):
def err_back(f):
f.printDetailedTraceback()
return list()
df = self._find(key)
df.addErrback(err_back)
return df
def findValue(self, key):
find_defer = Deferred()
node_df = self.findNodes(key)
def callback_nodes(l):
result = list()
if len(l) == 0:
find_defer.callback(result)
for cnt in l:
def callback_val(val, c=cnt):
if not val is None:
val['contact_id'] = c.node_id.toString()
val['contact_addr'] = c.address
val['contact_port'] = c.port
result.append(val)
return
def bothback_val(arg, c=cnt):
l.remove(c)
if len(l) == 0:
find_defer.callback(result)
return arg
val_df = self.callGetValue(cnt, key)
val_df.addCallback(callback_val)
val_df.addBoth(bothback_val)
return
def lookup_in_storage(l):
if key in self.data_storage:
l.append({'key':key, 'value': self._getFromStorage(key), 'contact_id': self.node_id.toString(), 'contact_addr':"", 'contact_port':0})
return l
node_df.addCallback(callback_nodes)
find_defer.addCallback(lookup_in_storage)
return find_defer
def handleCall(self, msg, address):
df = Deferred()
try:
c = Contact(msg.node_id, address[0], address[1])
self.addContact(c)
m = getattr(self, self.rpc_prefix + msg.method, None)
if callable(m):
result = m(*msg.params, **{'contact':c})
df.callback(result)
else:
raise Exception("unknown rpc {}".format(msg.method))
except Exception as e:
df.errback(e)
return df
def callRPC(self, contact, method, params=()):
contact.touch()
def callback(arg):
contact.touch()
return arg
def errback(f):
try:
f.raiseException()
except RPCTimeout:
if not contact.node_id is None:
self.removeContact(contact.node_id)
raise
self.addContact(contact)
df = self.protocol.callRPC((contact.address, contact.port), method, params)
df.addCallbacks(callback, errback)
return df
def callPing(self, contact):
return self.callRPC(contact, 'ping')
def rpc_ping(self, contact):
return 'pong'
def callGetID(self, contact):
return self.callRPC(contact, 'get_id')
def rpc_get_id(self, contact):
return self.node_id.toString()
def callStore(self, contact, key, value):
return self.callRPC(contact, 'store', (key, value))
def rpc_store(self, key, value, contact=None):
self._putToStorage(key, value)
return 'OK'
def callFindNode(self, contact, key):
def make_contact_list(result):
l = list()
for node_id, address, port in result:
c = Contact(node_id, address, port)
self.addContact(c)
l.append(c)
return l
df = self.callRPC(contact, 'find_node', (key,))
df.addCallback(make_contact_list)
return df
def rpc_find_node(self, key, contact):
key = Key(key)
contacts = self.findCloseNodes(key,8)
if contact in contacts:
contacts.remove(contact)
result = list()
for c in contacts:
result.append((c.node_id.toString(), c.address, c.port))
return result
def callGetValue(self, contact, key):
return self.callRPC(contact, 'find_value', (key,))
def rpc_find_value(self, key, contact):
if key in self.data_storage:
return {'key':key, 'value': self._getFromStorage(key)}
else:
return None
def _heartbeatLoop(self):
to_check = self.routing_table.getAllContacts()
now = time.time()
for c in to_check:
if now-c.last_accessed > self.keepalive_timeout:
self.callPing(c)
return
def _putToStorage(self, key, value):
now = time.time()
self.data_storage[key] = {'value':value, 'timestamp':now}
def _getFromStorage(self, key):
return self.data_storage[key]['value']
def _start_loops(self, arg=None):
if not self.heartbeat_loop.running:
self.heartbeat_loop.start(self.heartbeat_interval)
return arg
def _stop_loops(self, arg=None):
self.heartbeat_loop.stop()
return arg
class UDPServer:
def __init__(self, ip, port):
self.ip = ip
self.port = int(port)
self.server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.server.bind((self.ip, self.port))
self.server_process = Process(target=self._start)
self.routing_table = RoutingTable()
self.lock = Lock()
def run(self):
self.server_process.start()
def terminate(self):
self.server_process.terminate()
def _start(self):
executor = ThreadPoolExecutor(max_workers=3)
while True:
msg, addr = self.server.recvfrom(CONST.BUFFER_SIZE)
executor.submit(self._process_request, msg=msg, addr=addr)
def _process_request(self, msg, addr):
msg = msg.decode("utf-8")
tokens = msg.split()
if tokens[1] == "JOIN":
self.routing_table.add(tokens[2], tokens[3])
response = query_builder("JOINOK", ["0"])
udp_send_recv(addr[0], addr[1], response, recieve=False)
elif tokens[1] == "LEAVE":
for node in self.routing_table.get():
if node[1] == tokens[3]:
self.routing_table.remove(node)
break
response = query_builder("LEAVEOK", ["0"])
udp_send_recv(addr[0], addr[1], response, recieve=False)
elif tokens[1] == "SER":
hops = int(tokens[-1])
file_name = " ".join(tokens[4:-1])
files_found, file_names = search_file(file_name)
if files_found > 0:
response = query_builder("SEROK", [
files_found, cfg.FlaskServer['ip'],
cfg.FlaskServer['port'], hops, file_names
])
udp_send_recv(tokens[2], tokens[3], response, recieve=False)
elif hops > 0:
request = query_builder(
"SER", [tokens[2], tokens[3], file_name, hops - 1])
for node in self.routing_table.get():
udp_send_recv(node[0], node[1], request, recieve=False)
elif tokens[1] == "SEROK":
dir = cfg.Application['dir']
films = " ".join(tokens[6:]).strip()
self.lock.acquire()
f = open(f"{dir}/film_details.txt", "a")
file1 = open(f"{dir}/film_details.txt")
for film in films.split(','):
found = False
for line in file1.readlines():
if film in line:
found = True
if not found:
print(f"\t* {film}")
f.write(f"{film}|{tokens[3]}|{tokens[4]}\n")
f.close()
file1.close()
self.lock.release()