def fetch_transfer(self, mode, fileName, path, userName, msgLen, overwrite): self.isBusy() self.TRANSFER_BUSY = True tid = self.generate_tid() data = [mode, fileName, path, userName, msgLen, overwrite, tid] self.CLIENT.send( (packets.fetchSmallPacket(packets.Packets.REQ_TRANSFER, data)).encode()) attempts = 0 while attempts < 60: for i in range(len(self.TRANSFERS)): if str(tid) in self.TRANSFERS[i]: ret = self.TRANSFERS[i].get(str(tid)) del self.TRANSFERS[i] return ret attempts += 1 time.sleep(0.5) utils.log("[SERVER] Could not fetch a transfer subnode from the node.", True)
def __init__(self, mode, host, port, fileName, path, userName, msgLen, overwrite, db): self.MODE = mode self.MID = utils.fetch_mid() self.DB = db self.CLIENT = None self.ADDR = None self.FILENAME = fileName self.PATH = path self.USER = userName self.LENGTH = int(msgLen) self.HOST = host self.OVERWRITE = overwrite self.PORT = int(port) threading.Thread.__init__(self) try: self.SOCK = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.SOCK.settimeout(10) self.SOCK.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.SOCK.bind((self.HOST, self.PORT)) self.SOCK.listen(1) utils.log( "[NODE] Opened a NodeFileHandler socket on %s" % repr(self) + " waiting on connection...", True) except: utils.log( "[NODE] Failed to open a NodeFileReceiver socket on " + repr(self), True)
def exec_receive(self): fa = fileops.FileAdder(self.USER, self.FILENAME, self.PATH) tbytes = 0 count = 0 breakPercent = self.LENGTH * 0.1 try: self.DB.queryFileAddition(self.USER, self.MID, self.PATH, self.LENGTH, self.FILENAME) while tbytes < self.LENGTH: recv = self.CLIENT.recv(32768) fa.write(recv) tbytes += len(recv) count += len(recv) if (count >= breakPercent): count = 0 self.DB.queryNodeProgress(self.USER, self.MID, self.FILENAME, (tbytes / self.LENGTH) * 100) self.DB.queryNodeProgress(self.USER, self.MID, self.FILENAME, 100) utils.log("Successfully received file %s" % repr(fa), True) except ConnectionResetError: utils.log("[NODE] NodeFileReceiver socket closed.", True) self.SOCK.close() fa.close()
def rm_file(user, path, filename): if os.path.isfile('Data' + '/' + user + '/' + path + '/' + filename): os.remove('Data' + '/' + user + '/' + path + '/' + filename) else: utils.log("[NODE] The path specified is not a file, can not complete.", True) raise FileNotFoundError
def __init__(self, user, filename, path): self.FILEOBJ = None self.USER = user self.FILENAME = filename self.PATH = path try: originalName = self.FILENAME count = 0 while os.path.isfile('Data' + '/' + self.USER + '/' + self.PATH + '/' + self.FILENAME): count += 1 split = os.path.splitext(originalName) self.FILENAME = split[0] + '_' + str(count) + split[1] if not os.path.exists('Data' + '/' + user + '/' + path): os.makedirs('Data' + '/' + user + '/' + path) self.FILEOBJ = open( 'Data' + '/' + self.USER + '/' + self.PATH + '/' + self.FILENAME, "wb") utils.log( "[NODE] FileAdder added a new file with filename %s" % self.FILENAME, True) except Exception as ex: utils.log( "[NODE] FileAdder failed to open a new file at %s with filename %s with exception %s" % (self.PATH, self.FILENAME, ex), True)
def send_space_resp(self): space = utils.fetch_avail_space() self.SOCK.send((packets.fetchSmallPacket(packets.Packets.RESP_SPACE, space).encode())) utils.log( "[NODE] Received a space request, sending space response: %s" % space, True)
def add_folder(user, path): if not os.path.exists('Data' + '/' + user + '/' + path): os.makedirs('Data' + '/' + user + '/' + path) else: utils.log( "[NODE] User %s tried to add a folder that already exist." % user, True) raise FileExistsError
def recv_del_resp(self, data): self.DEL_BUSY = False success = json.loads(data.decode())[1] if success == True: utils.log( "[SERVER] Node reports the file deletion was successful.", True) else: utils.log("[SERVER] Node reports the file deletion failed.", True)
def run(self): if self.MODE == "SEND": self.exec_send() return if self.MODE == "RECV": self.exec_receive() return utils.log( "[SERVER] Invalid mode specified to ServerFileHandler. Returning.", True)
def rm_folder(user, path): if os.path.exists('Data' + '/' + user + '/' + path): try: shutil.rmtree('Data' + '/' + user + '/' + path) except Exception as ex: utils.log( "[NODE] User %s got exception %s while trying to remove folder" % (user, ex), True) else: utils.log( "[NODE] User %s tried to remove a folder that does not exist." % user, True) raise FileNotFoundError
def run(self): try: self.CLIENT, self.ADDR = self.SOCK.accept() utils.log( "[NODE] Received a file transmitter from %s" % repr(self.ADDR), True) except: utils.log( "[NODE] Failed to receive a connection on file transmitter.", True) return if self.CLIENT == None or self.ADDR == None: utils.log("[NODE] Socket connection does not exist...", True) return if self.MODE == "SEND": self.exec_send() return if self.MODE == "RECV": self.exec_receive() return utils.log( "[NODE] Invalid mode was specified in NodeFileHandler. Returning...", True)
def upload(): user = flask.session['username'] file = flask.request.files['file'] if utils.doesTmpFileExist(user, file.filename): return json.dumps({'status': 'fileintransfer'}) if (len(db.queryGatherSpecificFile(user, file.filename)) > 0): regExMatch = re.search('[_]\d+[.]', file.filename) if (regExMatch): s = regExMatch.span() file.filename = file.filename[:s[0]] + "_" + str( int(file.filename[s[0] + 1:s[1] - 1]) + 1) else: ss = file.filename.split(".") if (len(ss) >= 2): file.filename = ss[len(ss) - 2] + "_1" + "." + ss[len(ss) - 1] else: file.filename = file.filename + "_1" utils.log( "[WEB SERVER] A naming collision occured, altered name to %s" % file.filename, True) file.save("tmp/" + user + "_" + file.filename) size = os.path.getsize("tmp/" + user + "_" + file.filename) overwrite = "True" if not os.path.exists('jobs'): os.makedirs('jobs') data = json.dumps({ "Type": "Upload", "Filename": file.filename, "Folder": "", "User": user, "Size": size, "Overwrite": overwrite }) for i in range(2000): if not os.path.isfile('jobs/%s.mjob' % str(i)): f = open('jobs/%s.mjob' % str(i), "w") f.write(data) return json.dumps({'status': 'success'}) return json.dumps({'status': 'serveroor'})
def __init__(self, ip, port): self.TRANSFERS = [] self.ACTIVE = False self.IP = ip self.PORT = int(port) self.SOCK = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: self.ACTIVE = True self.SOCK.connect((self.IP, self.PORT)) utils.log( "[NODE] Successfully established a connection to %s:%s" % (self.IP, self.PORT), True) except: utils.log( "[NODE] Failed to establish a connection to %s:%s" % (self.IP, self.PORT), True) self.send_handshake()
def commit(self, query, fields): conn = self.conn_pool.get_connection() try: cursor = conn.cursor() cursor.execute(query, fields) conn.commit() utils.log( "[DB] Successfully executed the query: %s, with the fields: %s." % (query, fields), True) except Exception as ex: utils.log( "[DB] Failed to execute the query: %s, with the fields: %s, because of the exception: %s" % (query, fields, ex), True) raise ex cursor.close() conn.close()
def exec_send(self): try: f = open("Data/%s/%s/%s" % (self.USER, self.PATH, self.FILENAME), "rb") data = f.read(32768) while data: self.CLIENT.send(data) data = f.read(32768) utils.log( "[NODE] Successfully sent the file %s to server to be relayed to %s" % (self.FILENAME, self.USER), True) except Exception as ex: utils.log("[NODE] Could not complete transfer, exception: %s" % ex, True) return self.SOCK.close()
def exec_receive(self): tbytes = 0 try: with open("tmp/%s_%s" % (self.USER, self.FILENAME), "wb") as f: while tbytes < self.LENGTH: bytesWritten = f.write(self.SOCK.recv(32768)) tbytes += bytesWritten utils.log( "[SERVER] Successfully received file %s from %s" % (self.FILENAME, self.HOST), True) try: self.SOCK.close() except: pass except Exception as ex: utils.log( "[NODE] ServerFileHandler exception occured while reciving data, reason: %s" % ex, True)
def run(self): while True: try: recv = self.CLIENT.recv(1024) rtype = packets.Packets(json.loads(recv.decode())[0]) if (rtype == packets.Packets.HANDSHAKE): self.recv_handshake(recv) if (rtype == packets.Packets.RESP_SPACE): self.recv_space(recv) if (rtype == packets.Packets.RESP_TRANSFER): self.recv_transfer_resp(recv) if rtype == packets.Packets.RESP_DEL: self.recv_del_resp(recv) except ConnectionResetError: utils.log( "[SERVER] Connection to node %s lost." % repr(self.ADDRESS), True) break except ConnectionAbortedError: break
def __init__(self, mode, host, port, fileName, path, userName, msgLen, overwrite): self.MODE = mode self.SOCK = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.FILENAME = fileName self.PATH = path self.USER = userName self.LENGTH = int(msgLen) self.HOST = host self.OVERWRITE = overwrite self.PORT = int(port) threading.Thread.__init__(self) try: self.SOCK.connect((self.HOST, self.PORT)) utils.log("[SERVER] Connected to transfer node at %s" % repr(self), True) except: utils.log( "[SERVER] Could not connect to transfer node at %s" % repr(self), True)
def find_node(self, attempts): # always keep cleanse_nodes to purge dead threads # at the top of the functions self.cleanse_nodes() if len(self.THREADS) == 0: while attempts > 0: if len(self.THREADS) > 0: break attempts -= 1 time.sleep(0.5) space = {} for i in range(len(self.THREADS)): self.THREADS[i].send_space_req() while self.THREADS[i].SPACE == 0 or self.THREADS[i].MID == None: time.sleep(0.1) space[self.THREADS[i].MID] = self.THREADS[i].SPACE self.THREADS[i].SPACE = 0 space = { k: v for k, v in sorted(space.items(), key=lambda item: item[1]) } utils.log( "[SERVER] Found the most suitable node as %s" % list(space.keys())[len(space) - 1], True) for i in range(len(self.THREADS)): if list(space.keys())[len(space) - 1] == self.THREADS[i].MID: return self.THREADS[i] return None
def exec_send(self): tbytes = 0 try: with open("tmp/%s_%s" % (self.USER, self.FILENAME), "rb") as f: while tbytes < self.LENGTH: bytesSent = self.SOCK.send(f.read(32768)) tbytes += bytesSent os.remove("tmp/%s_%s" % (self.USER, self.FILENAME)) utils.log( "[SERVER] Successfully sent file %s to %s" % (self.FILENAME, repr(self)), True) try: self.SOCK.close() except: pass except Exception as ex: utils.log( "[SERVER] ServerFileHandler exception occured while sending data to %s, reason: %s" % (repr(self), ex), True)
def __init__(self, host, port, peers): self.HOST = host self.PORT = int(port) self.PEERS = int(peers) self.ALIVE = True try: self.SOCK = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.SOCK.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.SOCK.bind((self.HOST, self.PORT)) self.SOCK.listen(self.PEERS) self.NHT = NodeHandler(self.SOCK) self.NHT.daemon = True self.NHT.start() utils.log( "[SERVER] Opened a NodeHandler & socket on " + repr(self), True) except: utils.log("[SERVER] Failed to open a socket on " + repr(self), True) self.ALIVE = False
def recv_handshake(self, data): self.MID = json.loads(data.decode())[1] utils.log( "[SERVER] Got a new node, handshake with %s resulted in MID: %s" % (repr(self.ADDRESS), self.MID), True)
def recv_space(self, data): self.SPACE_BUSY = False self.SPACE = json.loads(data.decode())[1] utils.log( "[SERVER] Received a response with available node space: %s" % self.SPACE, True)
def recv_transfer_resp(self, data): self.TRANSFER_BUSY = False self.TRANSFERS.append(json.loads(data.decode())[1]) utils.log( "[SERVER] Received a response with an availble transfer node: %s" % json.loads(data.decode())[1], True)
def __del__(self): self.SOCK.close() utils.log( "[SERVER] Shutting down file send operation to transfer node %s" % repr(self), True)
def __del__(self): self.SOCK.close() utils.log("[NODE] %s shutting down" % repr(self), True)
def __del__(self): try: del self.conn_pool utils.log("[DB] Removing DB pool object and class.", True) except: pass
from includes import nodeclient from includes import nodefilehandler from includes import packets from includes import fileops from includes import utils from includes import database import json import time utils.log("[NODE] Node starting...", True) utils.initialConfig("node") utils.initialConfig("db") config = utils.fetchConfig("node.mconf") nc = nodeclient.NodeClient(config['Remote'], config['RemotePort']) db = database.Database() while True: try: recv = nc.SOCK.recv(1024) except ConnectionResetError: utils.log("[NODE] The remote socket closed the connection.", True) break try: rtype = packets.Packets(json.loads(recv.decode())[0]) if rtype == packets.Packets.REQ_SPACE: nc.send_space_resp() if rtype == packets.Packets.REQ_TRANSFER:
config = utils.fetchConfig("server.mconf") ns = nodeserver.NodeServer(config['IP'], config['Port'], config['MaxNodes']) if not os.path.exists('jobs'): os.makedirs('jobs') if not os.path.exists('tmp'): os.makedirs('tmp') while True: for file in os.listdir('jobs'): try: if file.endswith(".mjob") and len(ns.NHT.THREADS) != 0: utils.log( "[SERVER] Found a new job. Relaying to appropriate channel.", True) data = None with open("jobs/%s" % file, "r") as f: data = json.loads(f.readline()) os.remove("jobs/%s" % file) if data and data['Type'] == "Upload": with open("tmp/%s_%s" % (data['User'], data['Filename']), "rb") as f: node = ns.NHT.find_node(60) node_ip = node.fetch_ip() node_port = node.fetch_transfer(