def checkDirExists(self, directory): """ _checkDirExists_ Check if directory exists (will throw if it exists as a file) Only used for castor and local file """ command = "rfstat %s 2> /dev/null | grep Protection" % directory print("Check dir existence : %s" % command) try: exitCode, output = runCommandWithOutput(command) except Exception as ex: msg = "Error: Exception while invoking command:\n" msg += "%s\n" % command msg += "Exception: %s\n" % str(ex) msg += "Fatal error, abort stageout..." raise StageOutError(msg) if exitCode != 0: return False else: regExpParser = re.compile('^Protection[ ]+: d') if regExpParser.match(output) is None: raise StageOutError("Output path is not a directory !") else: return True
def testCallable_StageOutError(self, mock_time, mock_executeCommand, mock_createStageOutCommand, mock_createOutputDirectory, mock_createTargetName, mock_createSourceName): mock_createSourceName.return_value = "sourcePFN" mock_createTargetName.return_value = "targetPFN" mock_createStageOutCommand.return_value = "command" mock_createOutputDirectory.side_effect = [ StageOutError("error"), StageOutError("error"), None ] mock_executeCommand.side_effect = [ StageOutError("error"), StageOutError("error"), None ] self.StageOutImpl("protocol", "inputPFN", "targetPFN") mock_createSourceName.assert_called_with("protocol", "inputPFN") mock_createTargetName.assert_called_with("protocol", "targetPFN") mock_createOutputDirectory.assert_called_with("targetPFN") mock_createStageOutCommand.assert_called_with("sourcePFN", "targetPFN", None, None) mock_executeCommand.assert_called_with("command") calls = [call(600), call(600), call(600), call(600)] mock_time.sleep.assert_has_calls(calls)
def testCallable_StageOutErrorFail(self, mock_time, mock_createOutputDirectory, mock_createTargetName, mock_createSourceName): mock_createSourceName.return_value = "sourcePFN" mock_createTargetName.return_value = "targetPFN" mock_createOutputDirectory.side_effect = [StageOutError("error"), StageOutError("error"), StageOutError("Last error")] with self.assertRaises(Exception) as context: self.StageOutImpl("protocol", "inputPFN", "targetPFN") self.assertTrue('Last error' in context.exception) mock_createSourceName.assert_called_with("protocol", "inputPFN") mock_createTargetName.assert_called_with("protocol", "targetPFN") mock_createOutputDirectory.assert_called_with("targetPFN") calls = [call(600), call(600)] mock_time.sleep.assert_has_calls(calls)
def doTransfer(self, fromPfn, toPfn, stageOut, pnn, command, options, protocol, checksum): """ performs a transfer. stageOut tells you which way to go. returns the new pfn or raises on failure. StageOutError (and inherited exceptions) are for expected errors such as temporary connection failures. Anything else will be handled as an unexpected error and skip retrying with this plugin if stageOut is true: The fromPfn is the LOCAL FILE NAME on the node, without file:// the toPfn is the target PFN, mapped from the LFN using the TFC or overrrides if stageOut is false: The toPfn is the LOCAL FILE NAME on the node, without file:// the fromPfn is the target PFN, mapped from the LFN using the TFC or overrrides this behavior is because most transfer commands will switch their direction simply by swapping the order of the arguments. the stageOut flag is provided however, because sometimes you want to pass different command line args """ try: import dcap except ImportError as ie: raise StageOutError("Python dCap wrappers not found on this host.") ourCommand = \ self.generateCommandFromPreAndPostParts(\ ["dc_stageout"], [fromPfn, toPfn], options) self.runCommandFailOnNonZero(ourCommand) return toPfn
def testCheckDirExists_stageOutError(self, mock_runCommandWithOutput): mock_runCommandWithOutput.return_value = StageOutError("Error") with self.assertRaises(StageOutError) as context: self.RFCPCERNImpl.checkDirExists("test2") self.assertTrue('Error: Exception while invoking command:' in context.exception) mock_runCommandWithOutput.assert_called_with( "rfstat test2 2> /dev/null | grep Protection")
def runCommandFailOnNonZero(self, command): logging.info("Executing %s", command) (exitCode, output) = runCommand(command) if exitCode: logging.error("Error in file transfer:") logging.error(" Command executed was: %s", command) logging.error(" Output was: %s", output) logging.error(" Exit code was: %s", exitCode) raise StageOutError("Transfer failure") return (exitCode, output)
def testCreateRemoveFileCommand_StageOutError(self, mock_createRemoveFileCommand, mock_parseCastorPath, mock_isEOS): mock_isEOS.return_value = False mock_createRemoveFileCommand.return_value = "test" mock_parseCastorPath.side_effect = StageOutError("Errors!") self.assertEqual( "test", self.RFCPCERNImpl.createRemoveFileCommand("file/path")) mock_isEOS.assert_called_once_with("file/path")
def executeCommand(self, command): """ _execute_ Execute the command provided, throw a StageOutError if it exits non zero """ try: exitCode, output = runCommandWithOutput(command) msg = "Command exited with status: %s\nOutput message: %s" % (exitCode, output) logging.info(msg) except Exception as ex: raise StageOutError(str(ex), Command=command, ExitCode=60311) if exitCode: msg = "Command exited non-zero, ExitCode:%s\nOutput: %s " % (exitCode, output) logging.error("Exception During Stage Out:\n%s", msg) raise StageOutError(msg, Command=command, ExitCode=exitCode) return
class StageOutImpl: """ _StageOutImpl_ Define the interface that needs to be implemented by stage out plugins Object attributes: - *numRetries* : Number of automated retry attempts if the command fails default is 3 attempts - *retryPause* : Time in seconds to wait between retries. default is 10 minutes """ def __init__(self, stagein=False): self.numRetries = 3 self.retryPause = 600 self.stageIn = stagein # tuple of exit codes of copy when dest directory does not exist self.directoryErrorCodes = tuple() def deferDirectoryCreation(self): """ Can we defer directory creation, hoping it exists, only to create on a given error condition """ return len(self.directoryErrorCodes) != 0 def executeCommand(self, command): """ _execute_ Execute the command provided, throw a StageOutError if it exits non zero """ try: exitCode = runCommand(command) msg = "Command exited with status: %s" % (exitCode) print msg except Exception, ex: raise StageOutError(str(ex), Command = command, ExitCode = 60311) if exitCode in self.directoryErrorCodes: raise StageOutInvalidPath() elif exitCode: msg = "Command exited non-zero" print "ERROR: Exception During Stage Out:\n" print msg raise StageOutError(msg, Command = command, ExitCode = exitCode) return
def executeCommand(self, command): """ _execute_ Execute the command provided, throw a StageOutError if it exits non zero """ try: exitCode, output = runCommandWithOutput(command) msg = "%s : Command exited with status: %s\n Output message: %s" % ( time.strftime("%Y-%m-%dT%H:%M:%S"), exitCode, output) print(msg) except Exception as ex: raise StageOutError(str(ex), Command=command, ExitCode=60311) if exitCode: msg = "%s : Command exited non-zero ExitCode:%s\nOutput: (%s) " % ( time.strftime("%Y-%m-%dT%H:%M:%S"), exitCode, output) print("ERROR: Exception During Stage Out:\n%s" % msg) raise StageOutError(msg, Command=command, ExitCode=exitCode) return
def executeCommand(self, command): """ _execute_ Execute the command provided, throw a StageOutError if it exits non zero """ try: exitCode = runCommand(command) msg = "Command exited with status: %s" % (exitCode) print msg except Exception as ex: raise StageOutError(str(ex), Command=command, ExitCode=60311) if exitCode in self.directoryErrorCodes: raise StageOutInvalidPath() elif exitCode: msg = "Command exited non-zero" print "ERROR: Exception During Stage Out:\n" print msg raise StageOutError(msg, Command=command, ExitCode=exitCode) return
def doTransfer(self, fromPfn, toPfn, stageOut, seName, command, options, protocol, checksum): toPfn = self.createSourceName(protocol, toPfn) fromPfn = self.createSourceName(protocol, fromPfn) (_, reportFile) = tempfile.mkstemp() ourCommand = \ self.generateCommandFromPreAndPostParts(\ ['srmcp','-report=%s'%reportFile, '-retry_num=0'], [fromPfn, toPfn], options) self.runCommandWarnOnNonZero(ourCommand) if not self.stageOut: remotePFN, localPFN = fromPfn, toPfn.replace("file://", "", 1) else: remotePFN, localPFN = toPfn, fromPfn.replace("file://", "", 1) targetPnfsPath = self.createPnfsPath(remotePFN) if _CheckExitCodeOption: p1 = Popen(["rfstat", remotePFN], stdout=PIPE) p3 = Popen(['cut', '-f3', '-d" "'], stdin=p1.stdout, stdout=PIPE) exitCode = p3.communicate()[0] if exitCode: raise StageOutError("srmcp failed! Error code: %s" % exitCode) localSize = os.path.getsize(localPFN) logging.info("Local Size %s" % localSize) # filesize() { cat "`dirname $1`/.(use)(2)(`basename $1`)'" | grep l= | sed -e's/.*;l=\([0-9]*\).*/\\1/'; } # the following replaces the above targetDirName = os.path.dirname(targetPnfsPath) targetBaseName = os.path.basename(targetPnfsPath) p1 = Popen( ["cat", "%s/.(use)(2)(%s)" % (targetDirName, targetBaseName)], stdout=PIPE) p2 = Popen(["grep", "l="], stdout=PIPE, stdin=p1.stdout) p3 = Popen(["sed", "-e's/.*;l=\([0-9]*\).*/\\1/'"], stdout=PIPE, stdin=p2.stdout) remoteSize = p3.communicate()[0] logging.info("Localsize: %s Remotesize: %s" % (localSize, remoteSize)) if int(localSize) != int(remoteSize): try: self.doDelete(toPfn, None, None, None, None) except: pass raise StageOutFailure("File sizes don't match") return toPfn
def executeCommand(self, command): """ _execute_ Execute the command provided, throw a StageOutError if it exits non zero """ try: exitCode = runCommand(command) msg = "Command exited with status: %s" % (exitCode) print msg except Exception, ex: raise StageOutError(str(ex), Command = command, ExitCode = 60311)
def execute(command): """ _execute_ Execute the command provided, throw a StageOutError if it exits non zero """ try: exitCode, output = runCommandWithOutput(command) msg = "Command exited with status: %s, Output: (%s)" % (exitCode, output) print(msg) except Exception as ex: msg = "Command threw exception: %s" % str(ex) print("ERROR: Exception During Stage Out:\n%s" % msg) raise StageOutError(msg, Command=command, ExitCode=60311) if exitCode: msg = "Command exited non-zero: ExitCode:%s \nOutput (%s)" % (exitCode, output) print("ERROR: Exception During Stage Out:\n%s" % msg) raise StageOutError(msg, Command=command, ExitCode=exitCode) return
def execute(command): """ _execute_ Execute the command provided, throw a StageOutError if it exits non zero """ try: exitCode = runCommand(command) msg = "Command exited with status: %s" % (exitCode) print(msg) except Exception as ex: msg = "Command threw exception" print("ERROR: Exception During Stage Out:\n") print(msg) raise StageOutError(msg, Command=command, ExitCode=60311) if exitCode: msg = "Command exited non-zero" print("ERROR: Exception During Stage Out:\n") print(msg) raise StageOutError(msg, Command=command, ExitCode=exitCode) return
def execute(command): """ _execute_ Execute the command provided, throw a StageOutError if it exits non zero """ try: exitCode = runCommand(command) msg = "Command exited with status: %s" % (exitCode) print msg except Exception, ex: print "ERROR: Exception During Stage Out:\n" raise StageOutError(msg, Command=command, ExitCode=60311)
def createStageOutCommand(self, sourcePFN, targetPFN, options=None, checksums=None): """ _createStageOutCommand_ Build a dccp command with a pnfs mkdir to generate the directory """ try: import dcap except ImportError, ie: raise StageOutError("Python dCap wrappers not found on this host.")
def doDelete(self, pfn, seName, command, options, protocol): """ _removeFile_ Removes the pfn. """ command = "%s %s" % (self._rmScript, pfn) exitCode, output = runCommand(command) if exitCode != 0: logging.error("Error removing file from LStore") logging.error(output) raise StageOutError("remove file failure command %s, error %s" % (command, output))
def doTransfer(self, fromPfn, toPfn, stageOut, pnn, command, options, protocol, checksum): toPfn = self.createSourceName(protocol, toPfn) fromPfn = self.createSourceName(protocol, fromPfn) (_, reportFile) = tempfile.mkstemp() ourCommand = \ self.generateCommandFromPreAndPostParts(\ ['srmcp','-report=%s'%reportFile, '-retry_num=0'], [fromPfn, toPfn], options) self.runCommandWarnOnNonZero(ourCommand) if not self.stageOut: remotePFN, localPFN = fromPfn, toPfn.replace("file://", "", 1) else: remotePFN, localPFN = toPfn, fromPfn.replace("file://", "", 1) targetPnfsPath = self.createPnfsPath(remotePFN) if _CheckExitCodeOption: p1 = Popen(["rfstat", remotePFN], stdout=PIPE) p3 = Popen(['cut', '-f3', '-d" "'], stdin=p1.stdout, stdout=PIPE) exitCode = p3.communicate()[0] if exitCode: raise StageOutError("srmcp failed! Error code: %s" % exitCode) localSize = os.path.getsize(localPFN) logging.info("Local Size %s" % localSize) # filesize() { `srm-get-metadata -retry_num=0 %s 2>/dev/null | grep 'size :[0-9]' | cut -f2 -d":"`} # the following replaces the above p1 = Popen(["srm-get-metadata", '-retry_num=0', remotePFN], stdout=PIPE) p2 = Popen(["grep", "size :[0-9]"], stdout=PIPE, stdin=p1.stdout) p3 = Popen(["sed", "-f2", '-d":"'], stdout=PIPE, stdin=p2.stdout) remoteSize = p3.communicate()[0] logging.info("Localsize: %s Remotesize: %s" % (localSize, remoteSize)) if int(localSize) != int(remoteSize): try: self.doDelete(toPfn, None, None, None, None) except: pass raise StageOutFailure("File sizes don't match") return toPfn
def parseCastorPath(self, complexCastorPath): """ _parseCastorPath_ Castor filenames can be full URLs with control statements for the rfcp Some other castor command line tools do not understand that syntax, so we need to retrieve the path and other parameters from the URL """ simpleCastorPath = None # full castor PFNs if simpleCastorPath == None: regExpParser = re.compile('/+castor/cern.ch/(.*)') match = regExpParser.match(complexCastorPath) if ( match != None ): simpleCastorPath = '/castor/cern.ch/' + match.group(1) # rfio style URLs if simpleCastorPath == None: regExpParser = re.compile('rfio:.*/+castor/cern.ch/([^?]+).*') match = regExpParser.match(complexCastorPath) if ( match != None ): simpleCastorPath = '/castor/cern.ch/' + match.group(1) # xrootd/castor style URLs if simpleCastorPath == None: regExpParser = re.compile('root:.*/+castor/cern.ch/([^?]+).*') match = regExpParser.match(complexCastorPath) if ( match != None ): simpleCastorPath = '/castor/cern.ch/' + match.group(1) # if that does not work raise an error if simpleCastorPath == None: raise StageOutError("Can't parse castor path out of URL !") # remove multi-slashes from path while ( simpleCastorPath.find('//') > -1 ): simpleCastorPath = simpleCastorPath.replace('//','/') return simpleCastorPath
def checkDirExists(self, directory): """ _checkDirExists_ Check if directory exists (will throw if it exists as a file) Only used for castor and local file """ command = "rfstat %s 2> /dev/null | grep Protection" % directory print "Check dir existence : %s" % command try: exitCode, output = runCommandWithOutput(command) except Exception, ex: msg = "Error: Exception while invoking command:\n" msg += "%s\n" % command msg += "Exception: %s\n" % str(ex) msg += "Fatal error, abort stageout..." raise StageOutError(msg)
def doTransfer(self, fromPfn, toPfn, stageOut, seName, command, options, protocol, checksum): """ if stageOut is true: The fromPfn is the LOCAL FILE NAME on the node, without file:// the toPfn is the target PFN, mapped from the LFN using the TFC or overrrides if stageOut is false: The toPfn is the LOCAL FILE NAME on the node, without file:// the fromPfn is the source PFN, mapped from the LFN using the TFC or overrrides """ # Figures out the src and dst files if stageOut: dstPath = toPfn else: dstPath = fromPfn # Creates the directory if stageOut: self.createOutputDirectory(os.path.dirname(dstPath)) else: os.makedirs(os.path.dirname(dstPath)) # Does the copy if stageOut: command = "%s %s %s" % (self._cpScript, fromPfn, toPfn) else: command = "%s %s %s" % (self._downloadScript, fromPfn, toPfn) exitCode, output = runCommand(command) print(output) if exitCode != 0: logging.error("Error in file transfer:") logging.error(output) raise StageOutError("Transfer failure, command %s, error %s" % (command, output)) # Returns the path return dstPath
class RFCPCERNImpl(StageOutImpl): """ _RFCPCERNImpl_ """ def __init__(self, stagein=False): StageOutImpl.__init__(self, stagein) self.numRetries = 5 self.retryPause = 300 def createSourceName(self, protocol, pfn): """ _createSourceName_ uses pfn """ return "%s" % pfn def createOutputDirectory(self, targetPFN): """ _createOutputDirectory_ create dir with group permission """ # EOS stageout auto-creates directories if self.isEOS(targetPFN): return # Only create dir on remote storage if self.stageIn: return targetDir = os.path.dirname(self.parseCastorPath(targetPFN)) # targetDir does not exist => create it if not self.checkDirExists(targetDir): # only use the fileclass code path if we run on t0export serviceClass = os.environ.get('STAGE_SVCCLASS', None) if serviceClass == 't0export': # determine file class from PFN fileclass = None # check for correct naming convention in PFN regExpParser = re.compile('/castor/cern.ch/cms/store/([^/]*data)/([^/]+)/([^/]+)/([^/]+)/') match = regExpParser.match(targetDir) if ( match != None ): # RAW data files use cms_raw, all others cms_production if match.group(4) == 'RAW': fileclass = 'cms_raw' else: fileclass = 'cms_production' fileclassDir = '/castor/cern.ch/cms/store/%s/%s/%s/%s' % match.group(1,2,3,4) # fileclassDir does not exist => create it if not self.checkDirExists(fileclassDir): self.createDir(fileclassDir) if ( fileclass != None ): self.setFileClass(fileclassDir,fileclass) # now create targetDir self.createDir(targetDir) return def createStageOutCommand(self, sourcePFN, targetPFN, options = None, checksums = None): """ _createStageOutCommand_ Build the stageout command: rfcp for castor and xrdcp for eos If adler32 checksum is provided, use it for the transfer """ result = "" if self.stageIn: remotePFN, localPFN = sourcePFN, targetPFN else: remotePFN, localPFN = targetPFN, sourcePFN result += "LOCAL_SIZE=`stat -c%%s \"%s\"`\n" % localPFN result += "echo \"Local File Size is: $LOCAL_SIZE\"\n" isRemoteEOS = self.isEOS(remotePFN) useChecksum = ( checksums != None and checksums.has_key('adler32') and not self.stageIn ) removeCommand = self.createRemoveFileCommand(targetPFN) if isRemoteEOS: result += "xrdcp -f -s " if useChecksum: checksums['adler32'] = "%08x" % int(checksums['adler32'], 16) # non-functional in 3.3.1 xrootd clients due to bug #result += "-ODeos.targetsize=$LOCAL_SIZE\&eos.checksum=%s " % checksums['adler32'] # therefor embed information into target URL targetPFN += "?eos.targetsize=$LOCAL_SIZE&eos.checksum=%s" % checksums['adler32'] else: if useChecksum: targetFile = self.parseCastorPath(targetPFN) result += "nstouch %s\n" % targetFile result += "nssetchecksum -n adler32 -k %s %s\n" % (checksums['adler32'], targetFile) result += "rfcp " if options != None: result += " %s " % options result += " \"%s\" " % sourcePFN result += " \"%s\" \n" % targetPFN if self.stageIn: result += "LOCAL_SIZE=`stat -c%%s \"%s\"`\n" % localPFN result += "echo \"Local File Size is: $LOCAL_SIZE\"\n" if isRemoteEOS: (_,host,path,_) = self.splitPFN(remotePFN) result += "REMOTE_SIZE=`xrd '%s' stat '%s' | sed -r 's/.* Size: ([0-9]+) .*/\\1/'`\n" % (host, path) result += "echo \"Remote File Size is: $REMOTE_SIZE\"\n" if useChecksum: result += "echo \"Local File Checksum is: %s\"\n" % checksums['adler32'] result += "REMOTE_XS=`xrd '%s' getchecksum '%s' | sed -r 's/.* eos ([0-9a-fA-F]{8}).*/\\1/'`\n" % (host, path) result += "echo \"Remote File Checksum is: $REMOTE_XS\"\n" result += "if [ $REMOTE_SIZE ] && [ $REMOTE_XS ] && [ $LOCAL_SIZE == $REMOTE_SIZE ] && [ '%s' == $REMOTE_XS ]; then exit 0; " % checksums['adler32'] result += "else echo \"Error: Size or Checksum Mismatch between local and SE\"; %s ; exit 60311 ; fi" % removeCommand else: result += "if [ $REMOTE_SIZE ] && [ $LOCAL_SIZE == $REMOTE_SIZE ]; then exit 0; " result += "else echo \"Error: Size Mismatch between local and SE\"; %s ; exit 60311 ; fi" % removeCommand else: result += "REMOTE_SIZE=`rfstat '%s' | grep Size | cut -f2 -d: | tr -d ' '`\n" % remotePFN result += "echo \"Remote File Size is: $REMOTE_SIZE\"\n" result += "if [ $REMOTE_SIZE ] && [ $LOCAL_SIZE == $REMOTE_SIZE ]; then exit 0; else echo \"Error: Size Mismatch between local and SE\"; %s ; exit 60311 ; fi" % removeCommand return result def createRemoveFileCommand(self, pfn): """ _createRemoveFileCommand_ Alternate between EOS, CASTOR and local. """ if self.isEOS(pfn): (_,host,path,_) = self.splitPFN(pfn) return "xrd %s rm %s" % (host, path) try: simplePFN = self.parseCastorPath(pfn) return "stager_rm -a -M %s ; nsrm %s" % (simplePFN, simplePFN) except StageOutError: # Not castor pass return StageOutImpl.createRemoveFileCommand(self, pfn) def removeFile(self, pfnToRemove): """ _removeFile_ """ if self.isEOS(pfnToRemove): (_,host,path,_) = self.splitPFN(pfnToRemove) command = "xrd %s rm %s" % (host, path) else: simplePFN = self.parseCastorPath(pfnToRemove) command = "stager_rm -a -M %s ; nsrm %s" % (simplePFN, simplePFN) execute(command) return def checkDirExists(self, directory): """ _checkDirExists_ Check if directory exists (will throw if it exists as a file) Only used for castor and local file """ command = "rfstat %s 2> /dev/null | grep Protection" % directory print "Check dir existence : %s" % command try: exitCode, output = runCommandWithOutput(command) except Exception, ex: msg = "Error: Exception while invoking command:\n" msg += "%s\n" % command msg += "Exception: %s\n" % str(ex) msg += "Fatal error, abort stageout..." raise StageOutError(msg) if exitCode != 0: return False else: regExpParser = re.compile('^Protection[ ]+: d') if ( regExpParser.match(output) == None): raise StageOutError("Output path is not a directory !") else: return True
if os.WIFEXITED(err): err = os.WEXITSTATUS(err) elif os.WIFSIGNALED(err): err = os.WTERMSIG(err) elif os.WIFSTOPPED(err): err = os.WSTOPSIG(err) return err, output def execute(command): """ _execute_ Execute the command provided, throw a StageOutError if it exits non zero """ try: exitCode = runCommand(command) msg = "Command exited with status: %s" % (exitCode) print msg except Exception, ex: print "ERROR: Exception During Stage Out:\n" raise StageOutError(msg, Command=command, ExitCode=60311) if exitCode: msg = "Command exited non-zero" print "ERROR: Exception During Stage Out:\n" print msg raise StageOutError(msg, Command=command, ExitCode=exitCode) return
def stageFile(self, fileToStage, stageOut=True): """ _stageFile_ Use call to invoke transfers (either in or out) input: fileToStage: a dict containing at least: LFN: the LFN for one end of the transfer, will be mapped to a PFN before the transfer PFN: (annoyingly) Not a PFN, but is the local filename for the file when it's transferred to/from stageOut: boolean for if the file is staged in or out output: dict from fileToStage with PFN, PNN, StageOutCommand added I'm not entirely sure that StageOutCommand makes sense, but I don't want to break old code -AMM 6/30/2010 """ log.info("Working on file: %s" % fileToStage['LFN']) lfn = fileToStage['LFN'] localFileName = fileToStage['PFN'] self.firstException = None log.info("Beginning %s" % ('StageOut' if stageOut else 'StageIn')) # generate list of stageout methods we will try stageOutMethods = [self.defaultMethod] stageOutMethods.extend(self.fallbacks) # loop over all the different methods. This unifies regular and fallback stuff. Nice. methodCounter = 0 for currentMethod in stageOutMethods: methodCounter += 1 # the PFN that is received here is mapped from the LFN log.info("Getting transfer details for %s LFN %s" % (currentMethod, lfn)) (pnn, command, options, pfn, protocol) = \ self.getTransferDetails(lfn, currentMethod) log.info("Using PNN: %s" % pnn) log.info("Command: %s" % command) log.info("Options: %s" % options) log.info("Protocol: %s" % protocol) log.info("Mapped LFN: %s" % lfn) log.info(" to PFN: %s" % pfn) log.info("LocalFileName: %s" % localFileName) newPfn = self._doTransfer(currentMethod, methodCounter, localFileName, pfn, stageOut) if newPfn: log.info("Transfer succeeded: %s" % fileToStage) fileToStage['PFN'] = newPfn fileToStage['PNN'] = pnn fileToStage['StageOutCommand'] = command self.completedFiles[fileToStage['LFN']] = fileToStage return fileToStage else: # transfer method didn't work, go to next one continue # if we're here, then nothing worked. transferfail. log.error("Error in stageout") if self.firstException: raise self.firstException else: raise StageOutError( "Error in stageout, this has been logged in the logs")
def createOutputDirectory(self, targetPFN): """ _createOutputDirectory_ create dir with group permission """ # check how the targetPFN looks like and parse out the target dir targetdir = None if targetdir == None: regExpParser = re.compile('/+castor/(.*)') match = regExpParser.match(targetPFN) if (match != None): targetdir = os.path.dirname(targetPFN) if targetdir == None: regExpParser = re.compile('rfio:/+castor/(.*)') match = regExpParser.match(targetPFN) if (match != None): targetdir = os.path.dirname('/castor/' + match.group(1)) if targetdir == None: regExpParser = re.compile('rfio:.*path=/+castor/(.*)') match = regExpParser.match(targetPFN) if (match != None): targetdir = os.path.dirname('/castor/' + match.group(1)) # raise exception if we have no rule that can parse the target dir if targetdir == None: raise StageOutError("Cannot parse directory out of targetPFN") # remove multi-slashes from path while (targetdir.find('//') > -1): targetdir = targetdir.replace('//', '/') # # determine file class from LFN # fileclass = None # temp or unmerged files use cms_no_tape if (fileclass == None): regExpParser = re.compile('.*/castor/cern.ch/cms/store/temp/') if (regExpParser.match(targetdir) != None): fileclass = 'cms_no_tape' if (fileclass == None): regExpParser = re.compile('.*/castor/cern.ch/cms/store/unmerged/') if (regExpParser.match(targetdir) != None): fileclass = 'cms_no_tape' # RAW data files use cms_raw if (fileclass == None): regExpParser = re.compile( '.*/castor/cern.ch/cms/store/data/[^/]+/[^/]+/RAW/') if (regExpParser.match(targetdir) != None): fileclass = 'cms_raw' # otherwise we assume another type of production file if (fileclass == None): fileclass = 'cms_production' print "Setting fileclass to : %s" % fileclass # check if directory exists rfstatCmd = "rfstat %s 2> /dev/null | grep Protection" % targetdir print "Check dir existence : %s" % rfstatCmd try: rfstatExitCode, rfstatOutput = runCommandWithOutput(rfstatCmd) except Exception as ex: msg = "Error: Exception while invoking command:\n" msg += "%s\n" % rfstatCmd msg += "Exception: %s\n" % str(ex) msg += "Fatal error, abort stageout..." raise StageOutError(msg) # does not exist => create it if rfstatExitCode: if (fileclass != None): self.createDir(targetdir, '000') self.setFileClass(targetdir, fileclass) self.changeDirMode(targetdir, self.permissions) else: self.createDir(targetdir, self.permissions) else: # check if this is a directory regExpParser = re.compile('Protection.*: d') if (regExpParser.match(rfstatOutput) == None): raise StageOutError("Output path is not a directory !") else: # check if directory is writable regExpParser = re.compile('Protection.*: d---------') if (regExpParser.match(rfstatOutput) != None and fileclass != None): self.setFileClass(targetdir, fileclass) self.makeDirWritable(targetdir) return
def createStageOutCommand(self, sourcePFN, targetPFN, options = None, checksums = None): """ _createStageOutCommand_ Build an srmcp command """ result = "#!/bin/sh\n" result += "REPORT_FILE=`pwd`/srm.report.$$\n" result += "srmcp -2 -report=$REPORT_FILE -retry_num=0" if options != None: result += " %s " % options result += " %s " % sourcePFN result += " %s" % targetPFN result += " 2>&1 | tee srm.output.$$ \n" if _CheckExitCodeOption: result += """ EXIT_STATUS=`cat $REPORT_FILE | cut -f3 -d" "` echo "srmcp exit status: $EXIT_STATUS" if [[ "X$EXIT_STATUS" == "X" ]] && [[ `grep -c SRM_INVALID_PATH srm.output.$$` != 0 ]]; then exit 1 # dir does not eixst elif [[ $EXIT_STATUS != 0 ]]; then echo "Non-zero srmcp Exit status!!!" echo "Cleaning up failed file:" %s exit 60311 fi """ % self.createRemoveFileCommand(targetPFN) if self.stageIn: remotePFN, localPFN = sourcePFN, targetPFN.replace("file://", "", 1) else: remotePFN, localPFN = targetPFN, sourcePFN.replace("file://", "", 1) #targetPFN = remotePFN remotePath = None SFN = '?SFN=' sfn_idx = remotePFN.find(SFN) if sfn_idx >= 0: remotePath = remotePFN[sfn_idx+5:] r = re.compile('srm://([A-Za-z\-\.0-9]*)(:[0-9]*)?(/.*)') m = r.match(remotePFN) if not m: raise StageOutError("Unable to determine path from PFN for " \ "target %s." % remotePFN) if remotePath == None: remotePath = m.groups()[2] remoteHost = m.groups()[0] # for filePath in (sourcePFN, targetPFN): # if filePath.startswith("file://"): # localPFN = filePath.replace("file://", "") # elif filePath.startswith("srm://"): # remotePFN = filePath # targetPFN = filePath # targetPath = None # SFN = '?SFN=' # sfn_idx = filePath.find(SFN) # if sfn_idx >= 0: # targetPath = filePath[sfn_idx+5:] # r = re.compile('srm://([A-Za-z\-\.0-9]*)(:[0-9]*)?(/.*)') # m = r.match(filePath) # if not m: # raise StageOutError("Unable to determine path from PFN for " \ # "target %s." % filePath) # if targetPath == None: # targetPath = m.groups()[2] # targetHost = m.groups()[0] result += "FILE_SIZE=`stat -c %s" result += " %s `\n" % localPFN result += "echo \"Local File Size is: $FILE_SIZE\"\n" metadataCheck = \ """ for ((a=1; a <= 10 ; a++)) do SRM_SIZE=`srmls -recursion_depth=0 -retry_num=0 %s 2>/dev/null | grep '%s' | grep -v '%s' | awk '{print $1;}'` echo "SRM Size is $SRM_SIZE" if [[ $SRM_SIZE > 0 ]]; then if [[ $SRM_SIZE == $FILE_SIZE ]]; then exit 0 else echo "Error: Size Mismatch between local and SE" echo "Cleaning up failed file:" %s exit 60311 fi else sleep 2 fi done echo "Cleaning up failed file:" %s exit 60311 """ % (remotePFN, remotePath, remoteHost, self.createRemoveFileCommand(targetPFN), self.createRemoveFileCommand(targetPFN)) result += metadataCheck return result
def doTransfer(self, fromPfn, toPfn, stageOut, pnn, command, options, protocol, checksum): toPfn = self.createSourceName(protocol, toPfn) fromPfn = self.createSourceName(protocol, fromPfn) # TODO tee the output to another file # attempt copy for x in range(self.numRetries): (_, reportFile) = tempfile.mkstemp() ourCommand = \ self.generateCommandFromPreAndPostParts(\ ['srmcp','-2','-report=%s'%reportFile, '-retry_num=0'], [fromPfn, toPfn], options) self.runCommandWarnOnNonZero(ourCommand) if not stageOut: remotePFN, localPFN = fromPfn, toPfn.replace("file://", "", 1) else: remotePFN, localPFN = toPfn, fromPfn.replace("file://", "", 1) if _CheckExitCodeOption: p1 = Popen(["cat", reportFile], stdout=PIPE) p3 = Popen(['cut', '-f3', '-d', ' '], stdin=p1.stdout, stdout=PIPE) exitCode = p3.communicate()[0].rstrip() logging.info("srmcp exit status: %s" % exitCode) p2 = Popen(['grep', '-c', 'SRM_INVALID_PATH', reportFile], stdout=PIPE) invalidPathCount = p2.communicate()[0] logging.info("got this for SRM_INVALID_PATH: %s" % invalidPathCount) if (invalidPathCount and (exitCode == '')): logging.warn( "Directory doesn't exist in srmv2 stageout...creating and retrying" ) self.createOutputDirectory(toPfn, stageOut) continue elif (str(exitCode) != "0"): logging.error("Couldn't stage out! Error code: %s" % exitCode) self.doDelete(toPfn, None, None, None, None) raise StageOutFailure("srmcp failed! Error code: %s" % exitCode) else: logging.info( "Tentatively succeeded transfer, will check metadata") break localSize = os.path.getsize(localPFN) logging.info("Local Size %s" % localSize) remotePath = None SFN = '?SFN=' sfn_idx = remotePFN.find(SFN) if sfn_idx >= 0: remotePath = remotePFN[sfn_idx + 5:] r = re.compile('srm://([A-Za-z\-\.0-9]*)(:[0-9]*)?(/.*)') m = r.match(remotePFN) if not m: raise StageOutError("Unable to determine path from PFN for " \ "target %s." % remotePFN) if remotePath == None: remotePath = m.groups()[2] remoteHost = m.groups()[0] # filesize() { `srm-get-metadata -retry_num=0 %s 2>/dev/null | grep 'size :[0-9]' | cut -f2 -d":"`} # the following replaces the above logging.info("remote path: %s" % remotePath) logging.info("remote host: %s" % remoteHost) p1 = Popen(["srmls", '-recursion_depth=0', '-retry_num=0', remotePFN], stdout=PIPE) p2 = Popen(["grep", remotePath], stdout=PIPE, stdin=p1.stdout) p3 = Popen(["grep", '-v', remoteHost], stdout=PIPE, stdin=p2.stdout) p4 = Popen(["awk", "{print $1;}"], stdout=PIPE, stdin=p3.stdout) remoteSize = p4.communicate()[0] logging.info("Localsize: %s Remotesize: %s" % (localSize, remoteSize)) if int(localSize) != int(remoteSize): try: self.doDelete(toPfn, None, None, None, None) except: pass raise StageOutFailure("File sizes don't match") return toPfn
def createStageOutCommand(self, sourcePFN, targetPFN, options = None, checksums = None): """ _createStageOutCommand_ Build an srmcp command srmcp options used (so hard to find documentation for it...): -2 enables srm protocol version 2 -report path to the report file -retry_num number of retries before before client gives up -request_lifetime request lifetime in seconds """ result = "#!/bin/sh\n" result += "REPORT_FILE=`pwd`/srm.report.$$\n" result += "srmcp -2 -report=$REPORT_FILE -retry_num=0 -request_lifetime=2400" if options != None: result += " %s " % options result += " %s " % sourcePFN result += " %s" % targetPFN result += " 2>&1 | tee srm.output.$$ \n" if _CheckExitCodeOption: result += """ EXIT_STATUS=`cat $REPORT_FILE | cut -f3 -d" "` echo "srmcp exit status: $EXIT_STATUS" if [[ "X$EXIT_STATUS" == "X" ]] && [[ `grep -c SRM_INVALID_PATH srm.output.$$` != 0 ]]; then exit 1 # dir does not eixst elif [[ $EXIT_STATUS != 0 ]]; then echo "Non-zero srmcp Exit status!!!" echo "Cleaning up failed file:" %s exit 60311 fi """ % self.createRemoveFileCommand(targetPFN) if self.stageIn: remotePFN, localPFN = sourcePFN, targetPFN.replace("file://", "", 1) else: remotePFN, localPFN = targetPFN, sourcePFN.replace("file://", "", 1) remotePath = None SFN = '?SFN=' sfn_idx = remotePFN.find(SFN) if sfn_idx >= 0: remotePath = remotePFN[sfn_idx+5:] r = re.compile('srm://([A-Za-z\-\.0-9]*)(:[0-9]*)?(/.*)') m = r.match(remotePFN) if not m: raise StageOutError("Unable to determine path from PFN for " \ "target %s." % remotePFN) if remotePath == None: remotePath = m.groups()[2] remoteHost = m.groups()[0] result += "FILE_SIZE=`stat -c %s" result += " %s `\n" % localPFN result += "echo \"Local File Size is: $FILE_SIZE\"\n" metadataCheck = \ """ SRM_OUTPUT=`srmls -recursion_depth=0 -retry_num=1 %s 2>/dev/null` SRM_SIZE=`echo $SRM_OUTPUT | grep '%s' | grep -v '%s' | awk '{print $1;}'` echo "SRM Size is $SRM_SIZE" if [[ $SRM_SIZE == $FILE_SIZE ]]; then exit 0 else echo $SRM_OUTPUT echo "ERROR: Size Mismatch between local and SE. Cleaning up failed file..." %s exit 60311 fi """ % (remotePFN, remotePath, remoteHost, self.createRemoveFileCommand(targetPFN)) result += metadataCheck return result