def execute(self, command, *commandArgs): commandWithOptionList = command.split() commandName = commandWithOptionList[0] commandOptions = '' if len(commandWithOptionList) > 1: for option in commandWithOptionList[1::]: commandOptions = commandOptions + " " + option args = '' for arg in commandArgs: args = args + " " + arg if commandWithOptionList[0].startswith("rm"): return self.__rm(commandOptions, args) elif commandWithOptionList[0].startswith("test"): return self.__test(commandOptions, args) elif commandWithOptionList[0].startswith("put"): return self.__put(commandOptions, args) elif commandWithOptionList[0].startswith("cat"): return self.__cat(commandOptions, args) elif commandWithOptionList[0].startswith("mv"): return self.__mv(commandOptions, args) elif commandWithOptionList[0].startswith("ls"): return self.__ls(commandOptions, args) else: glogger.warn('command not supported') return -1
def __removeLockFile(self): rVal = '' try: rVal = hdfsCommandExecutor.execute('cat', self.lockBaseDir + "/" + self.lockname) except HdfsException as ex: errorMessage = self.clientId + " failed to release lock\n" + str(ex) + "\n" raise HdfsLockReleaseException(errorMessage) msgs = rVal.rstrip().splitlines() if len(msgs) == 1 and msgs[0] == self.clientId: retryCount = 0 count = 3; # max try while retryCount < count: try: glogger.debug(self.clientId + " trying to remove lock file") status = hdfsCommandExecutor.execute('rm', self.lockBaseDir + "/" + self.lockname) if status == 0: return True except HdfsException as ex: glogger.warn(self.clientId + " could not remove lock file\n" + str(ex) + "\n") retryCount = retryCount + 1 errorMessage = self.clientId + " failed to release lock after trying " + str(count) + " times\n" raise HdfsLockReleaseException(errorMessage) else: glogger.debug(self.clientId + " found lock file data: " + str(msgs)) errorMessage = "no lock file found for " + self.clientId + "\n" raise HdfsNoLockFoundException(errorMessage)
def __createLockFile(self): tmpFolder = self.tmpDir + "/" + self.clientId command = 'mkdir -p ' + tmpFolder status = utilities.runCommandSilently(command) if status != 0: glogger.warn(self.clientId + " could not create directory " + tmpFolder) return -1 tmpLockFile = tmpFolder + "/" + self.lockname command = 'touch ' + tmpLockFile + ' | echo "' + self.clientId + '" > ' + tmpLockFile status = utilities.runCommandSilently(command) if status != 0: glogger.warn(self.clientId + " could not create file " + tmpLockFile) command = "rm -rf " + tmpFolder utilities.runCommandSilently(command) return -1 try: status = hdfsCommandExecutor.execute('put', tmpLockFile, self.lockBaseDir) if status == 0: glogger.debug(self.clientId + " has created lock file successfully") except HdfsDataNodeDownException as ex: status = -1 glogger.warn(self.clientId + " could not create lock file\n" + str(ex) + "\n") try: glogger.debug(self.clientId + " removing partial lock file") HdfsLock.releaseForcefully(self.lockname) glogger.debug(self.clientId + " removed partial lock file") except HdfsException as e: glogger.warn(self.clientId + " could not remove partial lock file\n" + str(e) + "\n") except HdfsException as e: status = -1 glogger.warn(self.clientId + " could not create lock file\n" + str(e) + "\n") command = "rm -rf " + tmpFolder utilities.runCommandSilently(command) return status
def __removeLockFile(self): rVal = '' try: rVal = hdfsCommandExecutor.execute( 'cat', self.lockBaseDir + "/" + self.lockname) except HdfsException as ex: errorMessage = self.clientId + " failed to release lock\n" + str( ex) + "\n" raise HdfsLockReleaseException(errorMessage) msgs = rVal.rstrip().splitlines() if len(msgs) == 1 and msgs[0] == self.clientId: retryCount = 0 count = 3 # max try while retryCount < count: try: glogger.debug(self.clientId + " trying to remove lock file") status = hdfsCommandExecutor.execute( 'rm', self.lockBaseDir + "/" + self.lockname) if status == 0: return True except HdfsException as ex: glogger.warn(self.clientId + " could not remove lock file\n" + str(ex) + "\n") retryCount = retryCount + 1 errorMessage = self.clientId + " failed to release lock after trying " + str( count) + " times\n" raise HdfsLockReleaseException(errorMessage) else: glogger.debug(self.clientId + " found lock file data: " + str(msgs)) errorMessage = "no lock file found for " + self.clientId + "\n" raise HdfsNoLockFoundException(errorMessage)
def __hasAlreadyAcquiredLock(self): try: status = hdfsCommandExecutor.execute( 'test -e', self.lockBaseDir + "/" + self.lockname) if status == 0: lockFileData = hdfsCommandExecutor.execute( 'cat', self.lockBaseDir + "/" + self.lockname) lsResult = hdfsCommandExecutor.execute( 'ls', self.lockBaseDir + "/" + self.lockname) if not self.__isValidLockFile( lockFileData) or not self.__lockingProcessExists( lockFileData) or self.__isLockExpired(lsResult): try: HdfsLock.releaseForcefully(self.lockname) except HdfsException, e: glogger.warn(self.clientId + ' could not clear old lock') else: msgs = lockFileData.rstrip().splitlines() if msgs[0] == self.clientId: return True except HdfsFileNotFoundException: pass except HdfsException as ex: errorMessage = self.clientId + " could not check for already lock acquired\n" + str( ex) + "\n" raise HdfsLockInternalException(errorMessage) return False
def __isLockFree(self): try: status = hdfsCommandExecutor.execute('test -e', self.lockBaseDir + "/" + self.lockname) if status == 0: lockFileData = hdfsCommandExecutor.execute('cat', self.lockBaseDir + "/" + self.lockname) lsResult = hdfsCommandExecutor.execute('ls', self.lockBaseDir + "/" + self.lockname) if not self.__isValidLockFile(lockFileData) or not self.__lockingProcessExists(lockFileData) or self.__isLockExpired(lsResult): try: HdfsLock.releaseForcefully(self.lockname) return True except HdfsException, e: glogger.warn(self.clientId + ' could not clear old lock') return False elif status == 1: glogger.debug(self.clientId + " found lock free") return True
def __hasAlreadyAcquiredLock(self): try: status = hdfsCommandExecutor.execute('test -e', self.lockBaseDir + "/" + self.lockname) if status == 0: lockFileData = hdfsCommandExecutor.execute('cat', self.lockBaseDir + "/" + self.lockname) lsResult = hdfsCommandExecutor.execute('ls', self.lockBaseDir + "/" + self.lockname) if not self.__isValidLockFile(lockFileData) or not self.__lockingProcessExists(lockFileData) or self.__isLockExpired(lsResult): try: HdfsLock.releaseForcefully(self.lockname) except HdfsException, e: glogger.warn(self.clientId + ' could not clear old lock') else: msgs = lockFileData.rstrip().splitlines() if msgs[0] == self.clientId: return True except HdfsFileNotFoundException: pass except HdfsException as ex: errorMessage = self.clientId + " could not check for already lock acquired\n" + str(ex) + "\n" raise HdfsLockInternalException(errorMessage) return False
def __isLockFree(self): try: status = hdfsCommandExecutor.execute( 'test -e', self.lockBaseDir + "/" + self.lockname) if status == 0: lockFileData = hdfsCommandExecutor.execute( 'cat', self.lockBaseDir + "/" + self.lockname) lsResult = hdfsCommandExecutor.execute( 'ls', self.lockBaseDir + "/" + self.lockname) if not self.__isValidLockFile( lockFileData) or not self.__lockingProcessExists( lockFileData) or self.__isLockExpired(lsResult): try: HdfsLock.releaseForcefully(self.lockname) return True except HdfsException, e: glogger.warn(self.clientId + ' could not clear old lock') return False elif status == 1: glogger.debug(self.clientId + " found lock free") return True
status = hdfsCommandExecutor.execute('test -e', self.lockBaseDir + "/" + self.lockname) if status == 0: lockFileData = hdfsCommandExecutor.execute('cat', self.lockBaseDir + "/" + self.lockname) lsResult = hdfsCommandExecutor.execute('ls', self.lockBaseDir + "/" + self.lockname) if not self.__isValidLockFile(lockFileData) or not self.__lockingProcessExists(lockFileData) or self.__isLockExpired(lsResult): try: HdfsLock.releaseForcefully(self.lockname) return True except HdfsException, e: glogger.warn(self.clientId + ' could not clear old lock') return False elif status == 1: glogger.debug(self.clientId + " found lock free") return True except HdfsException, ex: glogger.warn(self.clientId + " could not check for lock to be free\n" + str(ex) + "\n") return False def __isValidLockFile(self, lockFileData): msgs = lockFileData.rstrip().splitlines() if len(msgs) != 1: glogger.debug('found invalid lock file') return False tokens = msgs[0].split(',') if len(tokens) != 2: glogger.debug('found invalid lock file') return False return True
if not self.__isValidLockFile( lockFileData) or not self.__lockingProcessExists( lockFileData) or self.__isLockExpired(lsResult): try: HdfsLock.releaseForcefully(self.lockname) return True except HdfsException, e: glogger.warn(self.clientId + ' could not clear old lock') return False elif status == 1: glogger.debug(self.clientId + " found lock free") return True except HdfsException, ex: glogger.warn(self.clientId + " could not check for lock to be free\n" + str(ex) + "\n") return False def __isValidLockFile(self, lockFileData): msgs = lockFileData.rstrip().splitlines() if len(msgs) != 1: glogger.debug('found invalid lock file') return False tokens = msgs[0].split(',') if len(tokens) != 2: glogger.debug('found invalid lock file') return False return True def __lockingProcessExists(self, lockFileData):