def __init__(self, config: Config): self.HOST_NAME = '0.0.0.0' self.SERVER_PORT = config.getInteger('queen_port', 4040) self.CLIENT_PORT = config.getInteger('client_port', 4041) self.KDFS_CONFIG = config self.KDFS_START_IP = config.get('nodes_start_ip', '192.168.0.0') self.KDFS_END_IP = config.get('nodes_end_ip', '192.168.5.255') self.CHUNK_SIZE = config.getInteger('chunk_size', 1024) self.MAX_TIMEOUT = config.getInteger('max_timeout', 60) # get client ip address self.CLIENT_IP = self.KDFS_CONFIG.get('client_ip', '127.0.0.1') # check if this node server is queen if config.getBoolean('is_queen', False): from server.KDFSQueen import KDFSQueen self.IS_QUEEN = True self.QUEEN = KDFSQueen(config) self.MAX_LISTEN = config.getInteger('queen_max_nodes', 1) # check for ports is already use if ServerUtils.checkPortInUse( self.CLIENT_PORT, self.CLIENT_IP) or ServerUtils.checkPortInUse( self.SERVER_PORT, self.HOST_NAME): KDFSProtocol.echo( "One of \"{}:{}\" or \"{}:{}\" ports are already use!".format( self.HOST_NAME, self.SERVER_PORT, self.CLIENT_IP, self.CLIENT_PORT), is_err=True) return # run local socket client in another thread! # self.LOCALSERVERTHREAD = threading.Thread(target=self._runLocalServer) self.LOCALSERVERTHREAD = multiprocessing.Process( target=self._runLocalServer) self.LOCALSERVERTHREAD.start() # run global socket server self._runGlobalServer()
def __init__(self, config: Config): self.GLOBAL_PORT = config.getInteger('queen_port', 4040) self.GLOBAL_CHUNK_SIZE = config.getInteger('chunk_size', 1024) start_ip = config.get('nodes_start_ip', '192.168.0.0') end_ip = config.get('nodes_end_ip', '192.168.5.255') self.GLOBAL_CONFIG = config # scan all up hosts and validate kdfs node self.scanNodes(start_ip, end_ip)
def sendAndReceiveServerIdentify(config: Config, ip: str): socketi = ServerUtils.socketConnect( ip, config.getInteger('queen_port', 4040), 2) try: # send identify command chunk_size = config.getInteger('chunk_size', 1024) KDFSProtocol.sendMessage( socketi, chunk_size, KDFSProtocol.sendCommandFormatter('identify', {}, send_queen=True)) # get response of command, if exist! response = KDFSProtocol.receiveMessage(socketi, chunk_size)['data'] # print('(debug) node identify:',response) return response except Exception as e: return None # raise finally: # close socket if socketi is not None: socketi.close() return None
class MinimalCommands: def __init__(self): # get kdfs config self.config = Config(ServerUtils.CONFIG_PATH) self.storage_path = self.config.get('storage', './') # ------------------------------------------------ def _getAbsPath(self, pathi: str): return os.path.abspath("{}/{}".format(self.storage_path, pathi)) # ------------------------------------------------ def listCommand(self, params: dict): response = [] # create absolute path with storage path absPath = self._getAbsPath(params['path']) # check for exist directory if not os.path.exists(absPath) or not os.path.isdir(absPath): return ('', "no such directory to retrive") # get list of files and dirs files = os.listdir(absPath) # iterate all files for filei in files: newPath = os.path.join(absPath, filei) sizei = os.stat(newPath).st_size # translate size humanly size = minimalUitls.convertBytesToHumanly(sizei) # append to response response.append({ 'name': filei, 'size': size, 'type': 'dir' if os.path.isdir(newPath) else 'file', 'path': os.path.join(params['path'], filei) }) return (response, '') # ------------------------------------------------ def findCommand(self, params: dict): response = [] # print("minimal:",params) # create absolute path with storage path absPath = self._getAbsPath(params['path']) # check for exist directory if not os.path.exists(absPath) or not os.path.isdir(absPath): return ('', "no such directory to retrive") # get list of files and dirs (can recursion) files = minimalUitls.getAllFilesList( absPath, is_recursion=(params['type'] == 'rec'), just_files=(params['type'] == 'file')) # get max items maxItems = self.config.getInteger('list_limit', 10) # print('final files:',files) # iterate all files for filei in files: # print('file to find:',filei) # get abs path newPath = os.path.join(absPath, filei) findCount = 0 # if search in filename if params['mode'] == 'name': # search regex on filename findCount = len( minimalUitls.searchTextByRegex(filei, params['regex'])) elif params['mode'] == 'content': # check if directory if os.path.isdir(newPath): continue # check if file is binary if minimalUitls.checkIsBinaryFile(newPath): continue # open file and read line by line try: with open(newPath, 'r', encoding='utf-8') as f: for line in f: # search regex on every line findCount += len( minimalUitls.searchTextByRegex( line, params['regex'])) except: pass # append to response if findCount > 0 and len(response) <= maxItems: response.append({ 'name': filei, 'count': findCount, 'type': 'dir' if os.path.isdir(newPath) else 'file', 'path': os.path.join(params['path'], filei) }) # print('response_minimal:',params,response) return (response, '') # ------------------------------------------------ def existCommand(self, params: dict): # create absolute path with storage path absPath = self._getAbsPath(params['path']) # check for exist path if os.path.exists(absPath): return ("true", '') else: return ('false', '') # ------------------------------------------------ def statCommand(self, params: dict): response = {} # create absolute path with storage path absPath = self._getAbsPath(params['path']) # check for exist directory or file if not os.path.exists(absPath): return ('', "no such file or directory to retrive") # get stat of absolute path (mode, ino, dev, nlink, uid, gid, sizei, atime, mtime, ctime) = os.stat(absPath) # fileStat = os.stat(newPath) # translate size humanly size = minimalUitls.convertBytesToHumanly(sizei) # append to response response = { 'name': os.path.basename(absPath), 'size': size, 'type': 'dir' if os.path.isdir(absPath) else 'file', 'kdfs_path': params['path'], 'local_path': absPath, 'last_access': time.ctime(atime), 'last_modify': time.ctime(mtime), 'owner': pwd.getpwuid(uid).pw_name } return (response, '') # ------------------------------------------------ def identifyCommand(self, params: dict = {}): res: dict = { 'macaddr': ServerUtils.getMacAddress(), 'version': self.config.getInteger('version', 0), 'os': platform.system(), 'arch': platform.machine(), 'hostname': platform.node(), 'node_type': 'queen' if self.config.getBoolean('is_queen', False) else 'node' } return (res, '') # ------------------------------------------------ def upgradeCommand(self, params: dict): # if packet number is 1,then just return 'yes' or 'no' if params['packnumber'] == 1: # check if server can accept upgrades or not if self.config.getBoolean('accept_upgrades', True): KDFSProtocol.echo( "Accept kdfs version {} upgrading...".format( params['version']), 'upgrade') # check if upgrades folder is not exsit if not os.path.exists(ServerUtils.UPGRADE_PATH): os.makedirs(ServerUtils.UPGRADE_PATH) # resturn accept response return ('yes', '') else: KDFSProtocol.echo("Refuse any upgrading", 'upgrade') # return refuse response return ('no', '') # if get file in next packet number, save it else: try: # save to file with open(ServerUtils.UPGRADE_ZIP_PATH.format( params['version']), mode='wb') as f: f.write(bytearray(params['data'])) # extract to min source with tarfile.open( ServerUtils.UPGRADE_ZIP_PATH.format(params['version']), 'r:gz') as kdfsTar: kdfsTar.extractall("./") # update kdfs version number self.config.updateItem('version', params['version']) KDFSProtocol.echo( "saved kdfs version {} upgraded file in local".format( params['version']), 'upgrade') # if os is linux, then run bash script if ServerUtils.detectOS() == 'linux': os.system("sh ./upgrade.sh &") else: pass # TODO: return ('success', '') except Exception as e: return (e, '') # ------------------------------------------------ def notifyCommand(self, params: dict): response = Notification(self.config.get('app_name', 'KDFS'), params['text'], 30).send() if response: return ('success', '') else: return ('failed', '') # ------------------------------------------------ def copyCommand(self, params: dict): # create absolute path with storage path absPath = self._getAbsPath(params['path']) # print('abspath:',absPath) # if type of copy is 'src' if params['mode'] == 'src': # check for exist directory or file if not os.path.exists(absPath): return ('', "no such file or directory to retrive") # if path is for file if os.path.isfile(absPath): content = KDFSProtocol.fileCompress(absPath) return (content, '') # if path is for directory elif os.path.isdir(absPath): content = KDFSProtocol.directoryCompress(absPath) return (content, '') else: return ('', "undefined path") # if type of copy is 'dest' elif params['mode'] == 'dest': if params['packnumber'] == 1: # check for exist parent directory of dest path if not os.path.exists(os.path.dirname(absPath)): return ('', "no such parent directory to retrive") return ('success', '') else: # print("data file (copy):",params['data']) # save to file filename = KDFSProtocol.saveTempCompressedFile(params['data']) # extract to min source with tarfile.open(filename, 'r:gz') as kdfsTar: kdfsTar.extractall(absPath) return ('success', '')
exit(0) # get user command command = sys.argv[1] params = sys.argv[2:] # print('argvs:',command,params) # check for arguments if command == 'help': help() else: # start time of process command startTime = currentMilliseconds() # set loading print('Please Wait...',end="\r") try: # create a socket object socketi = ServerUtils.socketConnect(KDFSConfig.get('client_ip','127.0.0.1'),KDFSConfig.getInteger('client_port',4041),KDFSConfig.getInteger('max_timeout',60)) # send command to server chunk_size = KDFSConfig.getInteger('chunk_size',1024) KDFSProtocol.sendMessage(socketi,chunk_size,KDFSProtocol.sendCommandFormatter(command,params)) # print("(debug):",socketi,chunk_size,command,params) # get response of command response = KDFSProtocol.receiveMessage(socketi,chunk_size) # print("(debug) receive response : ",response) # end time of process command endTime = currentMilliseconds() # show command with params on output print("\r({}) >> [{} sec] {} {}\n\n".format(platform.node(),(endTime-startTime)/1000, command,' '.join(params))) # check for errors if response is None or response['data'] is None: raiseError("Can not retrive response from server!",True) exit(1)