def validate_path(path): if not path or path == parameters.sep: # root return '/' with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.connect((Constants.NAMENODE_IP, Constants.CLIENT_TO_NAMENODE)) sock.send(str(Codes.validate_path).encode('utf-8')) sock.recv(1024) sock.send(path.encode('utf-8')) is_valid = (sock.recv(1024).decode('utf-8')) try: is_valid = int(is_valid) except: logger.print_debug_info("Couldn't cast to int") return False if is_valid: sock.send('ok'.encode('utf-8')) true_path = sock.recv(1024).decode('utf-8') else: true_path = None return true_path
def run(self): code = int(self.sock.recv(1024).decode('utf-8')) self.sock.send('ok'.encode('utf-8')) logger.print_debug_info(code) if code == Codes.print: full_path = self.sock.recv(1024).decode('utf-8') CommandHandler.handle_print_to(self.sock, full_path) elif code == Codes.upload: full_path = self.sock.recv(1024).decode('utf-8') self.sock.send('ok'.encode('utf-8')) CommandHandler.handle_upload_from(self.sock, full_path) elif code == Codes.download_all: CommandHandler.handle_download_all(self.address[0]) elif code == Codes.make_dir: full_path = self.sock.recv(1024).decode('utf-8') CommandHandler.handle_mkdir(full_path) else: print("StorageListener: no command correspond to code", code) # regarding shutdown: # Shut down one or both halves of the connection. # If how is SHUT_RD, further receives are disallowed. # If how is SHUT_WR, further sends are disallowed. # If how is SHUT_RDWR, further sends and receives are disallowed. self.sock.close()
def handle_make_file(full_path: str): # to properly join full_path = full_path.strip(os.sep) with open(os.path.join(Constants.STORAGE_PATH, full_path), "wb+"): logger.print_debug_info( os.path.join(Constants.STORAGE_PATH, full_path)) return
def recreate_storage_dirs(): if os.path.exists(Constants.STORAGE_PATH): try: # os.removedirs(Constants.STORAGE_PATH) os.system("rm -rf {}/*".format(Constants.STORAGE_PATH)) except: logger.print_debug_info("removedirs fail") # then create this dir empty logger.print_debug_info(Constants.STORAGE_PATH, "(re)created")
def handle_print_to(socket: socket.socket, full_path: str): # to properly join full_path = full_path.strip(os.sep) # sending file to a client logger.print_debug_info("Sending", full_path) with open(os.path.join(Constants.STORAGE_PATH, full_path), 'rb') as file: data = file.read(1024) while data: socket.send(data) data = file.read(1024)
def handle_upload_from(socket: socket.socket, full_path: str): # to properly join full_path = full_path.strip(os.sep) # receiving file from a client logger.print_debug_info("Receiving", full_path) with open(os.path.join(Constants.STORAGE_PATH, full_path), 'wb+') as file: data = socket.recv(1024) while data: if data: file.write(data) else: return data = socket.recv(1024)
def ping_thread(): sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock.settimeout(Constants.PING_TIMEOUT) while True: time.sleep(10) for ip in clean_nodes.nodes.copy(): try: sock.sendto('ping'.encode('utf-8'), (ip, Constants.STORAGE_PING)) sock.recvfrom(1024) except: # socket.timeout or socket.gaierror: with clean_nodes.lock: clean_nodes.nodes.discard(ip) logger.print_debug_info('timeout storagenode', ip)
def handle_download_all(ip: tuple): """address is (host, port) tuple We do separate request for each make_dir & upload request for simplicity of code. """ @logger.log def ask(code, full_path: str): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((ip, Constants.STORAGE_TO_STORAGE)) sock.send(str(code).encode('utf-8')) sock.recv(1024) # ack if code == Codes.upload: # we need off distribution on upload sock.send(full_path.encode('utf-8')) sock.recv(1024) CommandHandler.handle_print_to(sock, full_path) else: sock.send(full_path.encode('utf-8')) sock.close() home_path_length = len(Constants.STORAGE_PATH) + 1 # print("home_path_length", home_path_length) # for all files for dir_name, subdir_list, file_list in os.walk( Constants.STORAGE_PATH): # print("not processed", dir_name) dir_name = dir_name[home_path_length:] ask(Codes.make_dir, dir_name) # print("processed", dir_name) if len(dir_name) > 0: # dir_name = os.path.join(*dir_name) logger.print_debug_info("replicate dir {}".format(dir_name)) # print("dir_name is", dir_name) ask(Codes.make_dir, dir_name) else: dir_name = '' for file in file_list: logger.print_debug_info("downloading all: current file is", file) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((ip, Constants.STORAGE_TO_STORAGE)) full_path = os.path.join(dir_name, file) # print("fullpath is ", full_path) ask(Codes.upload, full_path)
def run(self): code = int(self.sock.recv(4).decode('utf-8')) self.sock.send('ok'.encode('utf-8')) logger.print_debug_info(code) if code == Codes.make_file: full_path = self.sock.recv(1024).decode('utf-8') CommandHandler.handle_make_file(full_path) elif code == Codes.rm: full_path = self.sock.recv(1024).decode('utf-8') CommandHandler.handle_rm(full_path) elif code == Codes.copy: source = self.sock.recv(1024).decode('utf-8') self.sock.send('ok'.encode('utf-8')) destination = self.sock.recv(1024).decode('utf-8') CommandHandler.handle_copy(source, destination) elif code == Codes.move: source = self.sock.recv(1024).decode('utf-8') self.sock.send('ok'.encode('utf-8')) destination = self.sock.recv(1024).decode('utf-8') CommandHandler.handle_move(source, destination) elif code == Codes.make_dir: full_path = self.sock.recv(1024).decode('utf-8') CommandHandler.handle_mkdir(full_path) elif code == Codes.rmdir: full_path = self.sock.recv(1024).decode('utf-8') CommandHandler.handle_rmdir(full_path) elif code == Codes.init: if os.path.exists(Constants.STORAGE_PATH): try: # os.removedirs(Constants.STORAGE_PATH) os.system("rm -rf {}/*".format(Constants.STORAGE_PATH)) except: logger.print_debug_info("rmdirs failed") # then create this dir empty elif code == Codes.upload: logger.print_debug_info("hey") full_path = self.sock.recv(1024).decode('utf-8') self.sock.send('ok'.encode('utf-8')) CommandHandler.handle_upload_from(self.sock, full_path) logger.print_debug_info("hey2") CommandHandler.distribute(full_path) logger.log("he3") elif code == Codes.print: full_path = self.sock.recv(1024).decode('utf-8') CommandHandler.handle_print_to(self.sock, full_path) else: print("NamenodeListener: no command correspond to code", code) self.sock.close()
def distribute(full_path: str): """Distributing a file which we just uploaded from a client""" # real_path = os.path.join(Constants.STORAGE_PATH, full_path) # ask namenode for all storage's ips storages_ip = CommandHandler._get_all_storages_ip() # send file to everybody if storages_ip is None: return for ip in storages_ip: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((ip, Constants.STORAGE_TO_STORAGE)) # Check if another storage doesn't have a file sock.send(str(Codes.upload).encode('utf-8')) sock.recv(1024) if not CommandHandler._has_file(sock, full_path): logger.print_debug_info("Distributing", full_path) sock.send(full_path.encode('utf-8')) sock.recv(1024) CommandHandler.handle_print_to(sock, full_path) sock.close()
def insert(self, path, size=-1, ip_address_pool=None): path = path.split('/') path = list(filter(lambda path: path != '', path)) curr = self.__root for node_name in path[:-1]: child = curr.get_child(node_name) if not child: child = self.DirNode(node_name, curr) curr = child if path[-1] in curr.children.keys(): logger.print_debug_info("Item already exists: ignoring") if size < 0: # Creating dir child = self.DirNode(path[-1], curr) else: child = self.FileNode(path[-1], curr, size, ip_address_pool) return child
def init_sync(): # First we delete everything we have # because we don't resurrect old nodes. recreate_storage_dirs() storage_ip = get_sync_storage_ip() logger.print_debug_info("storage_ip", storage_ip) if storage_ip == '-1': return sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # connect and ask for all files sock.connect((storage_ip, Constants.STORAGE_TO_STORAGE)) sock.send(str(Codes.download_all).encode('utf-8')) # From this point we are going to receive a lot mkdir, and upload requests # from another storage and in this socket we are just waiting # When we receive ack then we are consistent with storage with 'storage_ip' sock.recv(1024) # Notify we are clear logger.print_debug_info("Sending i_clear") notify_i_clear() sock.close()
def waiter(sock: socket.socket, is_namenode): logger.print_debug_info("Wait on accept.", "is_namenode = ", is_namenode) while True: socket, addr = sock.accept() if is_namenode: logger.print_debug_info("This is Namenode connection", addr) NamenodeListener(socket).start() else: logger.print_debug_info("This is Storage connection", addr) StorageListener(socket, addr).start()
def new_nodes_listener_thread(): soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM) soc.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) soc.bind(('', Constants.NEW_NODES_PORT)) soc.listen() while True: # get_all_storage_ips = 12 # init_new_storage = 13 # i_clear = 15 con, addr = soc.accept() code = int(con.recv(1024).decode('utf-8')) # ack con.send('ack'.encode('utf-8')) storagename = con.recv(1024).decode('utf-8') if code == Codes.i_clear: logger.print_debug_info("Node {} says it's clean".format(storagename)) dirty_nodes.nodes.discard(storagename) clean_nodes.nodes.add(storagename) elif code == Codes.init_new_storage: if len(clean_nodes.nodes) > 0: dirty_nodes.nodes.add(storagename) copy_from = random_address() con.send(copy_from.encode('utf-8')) logger.print_debug_info("New storage node {} connected " "and will copy from {}".format(storagename, copy_from)) else: # sanity check assert len(dirty_nodes.nodes) == 0 con.send('-1'.encode('utf-8')) clean_nodes.nodes.add(storagename) logger.print_debug_info("New storage node {} connected, system init".format(storagename)) elif code == Codes.get_all_storage_ips: if len(clean_nodes.nodes) > 0: res = ';'.join(clean_nodes.nodes) else: res = '-1' con.send(res.encode('utf-8')) logger.print_debug_info("Node {} requested ip list. Sending {}".format(storagename, res)) con.close()
def multicast(cmd, arg1='', arg2=''): """ threaded multicast to all storage servers :param cmd: :param arg1: :param arg2: :return: """ logger.print_debug_info("code", cmd) logger.print_debug_info("dirty_nodes:", dirty_nodes) logger.print_debug_info("clean_nodes:", clean_nodes) # wait for all the dirty nodes to become clean while len(dirty_nodes.nodes) > 0: pass for ip in clean_nodes.nodes: thread = Thread(target=send_args, args=[ip, Constants.NAMENODE_TO_STORAGE, cmd, arg1, arg2]) thread.start()
new_nodes_listener.start() new_nodes_listener() ping() tree = FSTree() # Main thread while True: soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM) soc.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) soc.bind(('', Constants.CLIENT_TO_NAMENODE)) soc.listen() while True: logger.print_debug_info("Waiting for a new client connection") con, addr = soc.accept() # addr is a tuple logger.print_debug_info('New client connection') code = int(con.recv(1024).decode('utf-8')) logger.print_debug_info('Received code ' + str(code)) con.send('ok'.encode('utf-8')) if code == Codes.init: # init del tree tree = FSTree() multicast(Codes.init) logger.print_debug_info('init success') elif code == Codes.make_file: # make_file (not dir)