def __removeLockFile(self):
rVal = ''
try:
rVal = hdfsCommandExecutor.execute('cat', self.lockBaseDir + "/" + self.lockname)
except HdfsException as ex:
errorMessage = self.clientId + " failed to release lock\n" + str(ex) + "\n"
raise HdfsLockReleaseException(errorMessage)
msgs = rVal.rstrip().splitlines()
if len(msgs) == 1 and msgs[0] == self.clientId:
retryCount = 0
count = 3; # max try
while retryCount < count:
try:
glogger.debug(self.clientId + " trying to remove lock file")
status = hdfsCommandExecutor.execute('rm', self.lockBaseDir + "/" + self.lockname)
if status == 0:
return True
except HdfsException as ex:
glogger.warn(self.clientId + " could not remove lock file\n" + str(ex) + "\n")
retryCount = retryCount + 1
errorMessage = self.clientId + " failed to release lock after trying " + str(count) + " times\n"
raise HdfsLockReleaseException(errorMessage)
else:
glogger.debug(self.clientId + " found lock file data: " + str(msgs))
errorMessage = "no lock file found for " + self.clientId + "\n"
raise HdfsNoLockFoundException(errorMessage)
def __removeLockFile(self):
rVal = ''
try:
rVal = hdfsCommandExecutor.execute(
'cat', self.lockBaseDir + "/" + self.lockname)
except HdfsException as ex:
errorMessage = self.clientId + " failed to release lock\n" + str(
ex) + "\n"
raise HdfsLockReleaseException(errorMessage)
msgs = rVal.rstrip().splitlines()
if len(msgs) == 1 and msgs[0] == self.clientId:
retryCount = 0
count = 3
# max try
while retryCount < count:
try:
glogger.debug(self.clientId +
" trying to remove lock file")
status = hdfsCommandExecutor.execute(
'rm', self.lockBaseDir + "/" + self.lockname)
if status == 0:
return True
except HdfsException as ex:
glogger.warn(self.clientId +
" could not remove lock file\n" + str(ex) +
"\n")
retryCount = retryCount + 1
errorMessage = self.clientId + " failed to release lock after trying " + str(
count) + " times\n"
raise HdfsLockReleaseException(errorMessage)
else:
glogger.debug(self.clientId + " found lock file data: " +
str(msgs))
errorMessage = "no lock file found for " + self.clientId + "\n"
raise HdfsNoLockFoundException(errorMessage)
def __hasAlreadyAcquiredLock(self):
try:
status = hdfsCommandExecutor.execute(
'test -e', self.lockBaseDir + "/" + self.lockname)
if status == 0:
lockFileData = hdfsCommandExecutor.execute(
'cat', self.lockBaseDir + "/" + self.lockname)
lsResult = hdfsCommandExecutor.execute(
'ls', self.lockBaseDir + "/" + self.lockname)
if not self.__isValidLockFile(
lockFileData) or not self.__lockingProcessExists(
lockFileData) or self.__isLockExpired(lsResult):
try:
HdfsLock.releaseForcefully(self.lockname)
except HdfsException, e:
glogger.warn(self.clientId +
' could not clear old lock')
else:
msgs = lockFileData.rstrip().splitlines()
if msgs[0] == self.clientId:
return True
except HdfsFileNotFoundException:
pass
except HdfsException as ex:
errorMessage = self.clientId + " could not check for already lock acquired\n" + str(
ex) + "\n"
raise HdfsLockInternalException(errorMessage)
return False
def __createLockFile(self):
tmpFolder = self.tmpDir + "/" + self.clientId
command = 'mkdir -p ' + tmpFolder
status = utilities.runCommandSilently(command)
if status != 0:
glogger.warn(self.clientId + " could not create directory " + tmpFolder)
return -1
tmpLockFile = tmpFolder + "/" + self.lockname
command = 'touch ' + tmpLockFile + ' | echo "' + self.clientId + '" > ' + tmpLockFile
status = utilities.runCommandSilently(command)
if status != 0:
glogger.warn(self.clientId + " could not create file " + tmpLockFile)
command = "rm -rf " + tmpFolder
utilities.runCommandSilently(command)
return -1
try:
status = hdfsCommandExecutor.execute('put', tmpLockFile, self.lockBaseDir)
if status == 0:
glogger.debug(self.clientId + " has created lock file successfully")
except HdfsDataNodeDownException as ex:
status = -1
glogger.warn(self.clientId + " could not create lock file\n" + str(ex) + "\n")
try:
glogger.debug(self.clientId + " removing partial lock file")
HdfsLock.releaseForcefully(self.lockname)
glogger.debug(self.clientId + " removed partial lock file")
except HdfsException as e:
glogger.warn(self.clientId + " could not remove partial lock file\n" + str(e) + "\n")
except HdfsException as e:
status = -1
glogger.warn(self.clientId + " could not create lock file\n" + str(e) + "\n")
command = "rm -rf " + tmpFolder
utilities.runCommandSilently(command)
return status
def __isLockFree(self):
try:
status = hdfsCommandExecutor.execute('test -e', self.lockBaseDir + "/" + self.lockname)
if status == 0:
lockFileData = hdfsCommandExecutor.execute('cat', self.lockBaseDir + "/" + self.lockname)
lsResult = hdfsCommandExecutor.execute('ls', self.lockBaseDir + "/" + self.lockname)
if not self.__isValidLockFile(lockFileData) or not self.__lockingProcessExists(lockFileData) or self.__isLockExpired(lsResult):
try:
HdfsLock.releaseForcefully(self.lockname)
return True
except HdfsException, e:
glogger.warn(self.clientId + ' could not clear old lock')
return False
elif status == 1:
glogger.debug(self.clientId + " found lock free")
return True
def __hasAlreadyAcquiredLock(self):
try:
status = hdfsCommandExecutor.execute('test -e', self.lockBaseDir + "/" + self.lockname)
if status == 0:
lockFileData = hdfsCommandExecutor.execute('cat', self.lockBaseDir + "/" + self.lockname)
lsResult = hdfsCommandExecutor.execute('ls', self.lockBaseDir + "/" + self.lockname)
if not self.__isValidLockFile(lockFileData) or not self.__lockingProcessExists(lockFileData) or self.__isLockExpired(lsResult):
try:
HdfsLock.releaseForcefully(self.lockname)
except HdfsException, e:
glogger.warn(self.clientId + ' could not clear old lock')
else:
msgs = lockFileData.rstrip().splitlines()
if msgs[0] == self.clientId:
return True
except HdfsFileNotFoundException:
pass
except HdfsException as ex:
errorMessage = self.clientId + " could not check for already lock acquired\n" + str(ex) + "\n"
raise HdfsLockInternalException(errorMessage)
return False
def releaseForcefully(lockFileName):
lockFilePath = HdfsLock.lockBaseDirectory + '/' + lockFileName
glogger.debug("trying to release lock " + lockFilePath + " forcefully")
try:
status = hdfsCommandExecutor.execute('rm', lockFilePath)
if status == 0:
glogger.debug("released lock " + lockFilePath + " successfully")
except HdfsFileNotFoundException:
glogger.debug("No lock found with name = " + lockFilePath)
except HdfsException as ex:
errorMessage = "could not release lock = " + lockFilePath + "\n" + str(ex) + "\n"
raise HdfsLockReleaseException(errorMessage)
def __isLockFree(self):
try:
status = hdfsCommandExecutor.execute(
'test -e', self.lockBaseDir + "/" + self.lockname)
if status == 0:
lockFileData = hdfsCommandExecutor.execute(
'cat', self.lockBaseDir + "/" + self.lockname)
lsResult = hdfsCommandExecutor.execute(
'ls', self.lockBaseDir + "/" + self.lockname)
if not self.__isValidLockFile(
lockFileData) or not self.__lockingProcessExists(
lockFileData) or self.__isLockExpired(lsResult):
try:
HdfsLock.releaseForcefully(self.lockname)
return True
except HdfsException, e:
glogger.warn(self.clientId +
' could not clear old lock')
return False
elif status == 1:
glogger.debug(self.clientId + " found lock free")
return True
def releaseForcefully(lockFileName):
lockFilePath = HdfsLock.lockBaseDirectory + '/' + lockFileName
glogger.debug("trying to release lock " + lockFilePath + " forcefully")
try:
status = hdfsCommandExecutor.execute('rm', lockFilePath)
if status == 0:
glogger.debug("released lock " + lockFilePath +
" successfully")
except HdfsFileNotFoundException:
glogger.debug("No lock found with name = " + lockFilePath)
except HdfsException as ex:
errorMessage = "could not release lock = " + lockFilePath + "\n" + str(
ex) + "\n"
raise HdfsLockReleaseException(errorMessage)
def __createLockFile(self):
tmpFolder = self.tmpDir + "/" + self.clientId
command = 'mkdir -p ' + tmpFolder
status = utilities.runCommandSilently(command)
if status != 0:
glogger.warn(self.clientId + " could not create directory " +
tmpFolder)
return -1
tmpLockFile = tmpFolder + "/" + self.lockname
command = 'touch ' + tmpLockFile + ' | echo "' + self.clientId + '" > ' + tmpLockFile
status = utilities.runCommandSilently(command)
if status != 0:
glogger.warn(self.clientId + " could not create file " +
tmpLockFile)
command = "rm -rf " + tmpFolder
utilities.runCommandSilently(command)
return -1
try:
status = hdfsCommandExecutor.execute('put', tmpLockFile,
self.lockBaseDir)
if status == 0:
glogger.debug(self.clientId +
" has created lock file successfully")
except HdfsDataNodeDownException as ex:
status = -1
glogger.warn(self.clientId + " could not create lock file\n" +
str(ex) + "\n")
try:
glogger.debug(self.clientId + " removing partial lock file")
HdfsLock.releaseForcefully(self.lockname)
glogger.debug(self.clientId + " removed partial lock file")
except HdfsException as e:
glogger.warn(self.clientId +
" could not remove partial lock file\n" + str(e) +
"\n")
except HdfsException as e:
status = -1
glogger.warn(self.clientId + " could not create lock file\n" +
str(e) + "\n")
command = "rm -rf " + tmpFolder
utilities.runCommandSilently(command)
return status