def checkDirExists(self, directory):
"""
_checkDirExists_
Check if directory exists (will throw if it exists as a file)
Only used for castor and local file
"""
command = "rfstat %s 2> /dev/null | grep Protection" % directory
print("Check dir existence : %s" % command)
try:
exitCode, output = runCommandWithOutput(command)
except Exception as ex:
msg = "Error: Exception while invoking command:\n"
msg += "%s\n" % command
msg += "Exception: %s\n" % str(ex)
msg += "Fatal error, abort stageout..."
raise StageOutError(msg)
if exitCode != 0:
return False
else:
regExpParser = re.compile('^Protection[ ]+: d')
if regExpParser.match(output) is None:
raise StageOutError("Output path is not a directory !")
else:
return True
def testCallable_StageOutError(self, mock_time, mock_executeCommand,
mock_createStageOutCommand,
mock_createOutputDirectory,
mock_createTargetName,
mock_createSourceName):
mock_createSourceName.return_value = "sourcePFN"
mock_createTargetName.return_value = "targetPFN"
mock_createStageOutCommand.return_value = "command"
mock_createOutputDirectory.side_effect = [
StageOutError("error"),
StageOutError("error"), None
]
mock_executeCommand.side_effect = [
StageOutError("error"),
StageOutError("error"), None
]
self.StageOutImpl("protocol", "inputPFN", "targetPFN")
mock_createSourceName.assert_called_with("protocol", "inputPFN")
mock_createTargetName.assert_called_with("protocol", "targetPFN")
mock_createOutputDirectory.assert_called_with("targetPFN")
mock_createStageOutCommand.assert_called_with("sourcePFN", "targetPFN",
None, None)
mock_executeCommand.assert_called_with("command")
calls = [call(600), call(600), call(600), call(600)]
mock_time.sleep.assert_has_calls(calls)
def testCallable_StageOutErrorFail(self, mock_time, mock_createOutputDirectory, mock_createTargetName,
mock_createSourceName):
mock_createSourceName.return_value = "sourcePFN"
mock_createTargetName.return_value = "targetPFN"
mock_createOutputDirectory.side_effect = [StageOutError("error"), StageOutError("error"),
StageOutError("Last error")]
with self.assertRaises(Exception) as context:
self.StageOutImpl("protocol", "inputPFN", "targetPFN")
self.assertTrue('Last error' in context.exception)
mock_createSourceName.assert_called_with("protocol", "inputPFN")
mock_createTargetName.assert_called_with("protocol", "targetPFN")
mock_createOutputDirectory.assert_called_with("targetPFN")
calls = [call(600), call(600)]
mock_time.sleep.assert_has_calls(calls)
def doTransfer(self, fromPfn, toPfn, stageOut, pnn, command, options,
protocol, checksum):
"""
performs a transfer. stageOut tells you which way to go. returns the new pfn or
raises on failure. StageOutError (and inherited exceptions) are for expected errors
such as temporary connection failures. Anything else will be handled as an unexpected
error and skip retrying with this plugin
if stageOut is true:
The fromPfn is the LOCAL FILE NAME on the node, without file://
the toPfn is the target PFN, mapped from the LFN using the TFC or overrrides
if stageOut is false:
The toPfn is the LOCAL FILE NAME on the node, without file://
the fromPfn is the target PFN, mapped from the LFN using the TFC or overrrides
this behavior is because most transfer commands will switch their direction
simply by swapping the order of the arguments. the stageOut flag is provided
however, because sometimes you want to pass different command line args
"""
try:
import dcap
except ImportError as ie:
raise StageOutError("Python dCap wrappers not found on this host.")
ourCommand = \
self.generateCommandFromPreAndPostParts(\
["dc_stageout"],
[fromPfn, toPfn],
options)
self.runCommandFailOnNonZero(ourCommand)
return toPfn
def testCheckDirExists_stageOutError(self, mock_runCommandWithOutput):
mock_runCommandWithOutput.return_value = StageOutError("Error")
with self.assertRaises(StageOutError) as context:
self.RFCPCERNImpl.checkDirExists("test2")
self.assertTrue('Error: Exception while invoking command:' in
context.exception)
mock_runCommandWithOutput.assert_called_with(
"rfstat test2 2> /dev/null | grep Protection")
def runCommandFailOnNonZero(self, command):
logging.info("Executing %s", command)
(exitCode, output) = runCommand(command)
if exitCode:
logging.error("Error in file transfer:")
logging.error(" Command executed was: %s", command)
logging.error(" Output was: %s", output)
logging.error(" Exit code was: %s", exitCode)
raise StageOutError("Transfer failure")
return (exitCode, output)
def testCreateRemoveFileCommand_StageOutError(self,
mock_createRemoveFileCommand,
mock_parseCastorPath,
mock_isEOS):
mock_isEOS.return_value = False
mock_createRemoveFileCommand.return_value = "test"
mock_parseCastorPath.side_effect = StageOutError("Errors!")
self.assertEqual(
"test", self.RFCPCERNImpl.createRemoveFileCommand("file/path"))
mock_isEOS.assert_called_once_with("file/path")
def executeCommand(self, command):
"""
_execute_
Execute the command provided, throw a StageOutError if it exits
non zero
"""
try:
exitCode, output = runCommandWithOutput(command)
msg = "Command exited with status: %s\nOutput message: %s" % (exitCode, output)
logging.info(msg)
except Exception as ex:
raise StageOutError(str(ex), Command=command, ExitCode=60311)
if exitCode:
msg = "Command exited non-zero, ExitCode:%s\nOutput: %s " % (exitCode, output)
logging.error("Exception During Stage Out:\n%s", msg)
raise StageOutError(msg, Command=command, ExitCode=exitCode)
return
class StageOutImpl:
"""
_StageOutImpl_
Define the interface that needs to be implemented by stage out
plugins
Object attributes:
- *numRetries* : Number of automated retry attempts if the command fails
default is 3 attempts
- *retryPause* : Time in seconds to wait between retries.
default is 10 minutes
"""
def __init__(self, stagein=False):
self.numRetries = 3
self.retryPause = 600
self.stageIn = stagein
# tuple of exit codes of copy when dest directory does not exist
self.directoryErrorCodes = tuple()
def deferDirectoryCreation(self):
"""
Can we defer directory creation, hoping it exists,
only to create on a given error condition
"""
return len(self.directoryErrorCodes) != 0
def executeCommand(self, command):
"""
_execute_
Execute the command provided, throw a StageOutError if it exits
non zero
"""
try:
exitCode = runCommand(command)
msg = "Command exited with status: %s" % (exitCode)
print msg
except Exception, ex:
raise StageOutError(str(ex), Command = command, ExitCode = 60311)
if exitCode in self.directoryErrorCodes:
raise StageOutInvalidPath()
elif exitCode:
msg = "Command exited non-zero"
print "ERROR: Exception During Stage Out:\n"
print msg
raise StageOutError(msg, Command = command, ExitCode = exitCode)
return
def executeCommand(self, command):
"""
_execute_
Execute the command provided, throw a StageOutError if it exits
non zero
"""
try:
exitCode, output = runCommandWithOutput(command)
msg = "%s : Command exited with status: %s\n Output message: %s" % (
time.strftime("%Y-%m-%dT%H:%M:%S"), exitCode, output)
print(msg)
except Exception as ex:
raise StageOutError(str(ex), Command=command, ExitCode=60311)
if exitCode:
msg = "%s : Command exited non-zero ExitCode:%s\nOutput: (%s) " % (
time.strftime("%Y-%m-%dT%H:%M:%S"), exitCode, output)
print("ERROR: Exception During Stage Out:\n%s" % msg)
raise StageOutError(msg, Command=command, ExitCode=exitCode)
return
def executeCommand(self, command):
"""
_execute_
Execute the command provided, throw a StageOutError if it exits
non zero
"""
try:
exitCode = runCommand(command)
msg = "Command exited with status: %s" % (exitCode)
print msg
except Exception as ex:
raise StageOutError(str(ex), Command=command, ExitCode=60311)
if exitCode in self.directoryErrorCodes:
raise StageOutInvalidPath()
elif exitCode:
msg = "Command exited non-zero"
print "ERROR: Exception During Stage Out:\n"
print msg
raise StageOutError(msg, Command=command, ExitCode=exitCode)
return
def doTransfer(self, fromPfn, toPfn, stageOut, seName, command, options,
protocol, checksum):
toPfn = self.createSourceName(protocol, toPfn)
fromPfn = self.createSourceName(protocol, fromPfn)
(_, reportFile) = tempfile.mkstemp()
ourCommand = \
self.generateCommandFromPreAndPostParts(\
['srmcp','-report=%s'%reportFile,
'-retry_num=0'],
[fromPfn, toPfn],
options)
self.runCommandWarnOnNonZero(ourCommand)
if not self.stageOut:
remotePFN, localPFN = fromPfn, toPfn.replace("file://", "", 1)
else:
remotePFN, localPFN = toPfn, fromPfn.replace("file://", "", 1)
targetPnfsPath = self.createPnfsPath(remotePFN)
if _CheckExitCodeOption:
p1 = Popen(["rfstat", remotePFN], stdout=PIPE)
p3 = Popen(['cut', '-f3', '-d" "'], stdin=p1.stdout, stdout=PIPE)
exitCode = p3.communicate()[0]
if exitCode:
raise StageOutError("srmcp failed! Error code: %s" % exitCode)
localSize = os.path.getsize(localPFN)
logging.info("Local Size %s" % localSize)
# filesize() { cat "`dirname $1`/.(use)(2)(`basename $1`)'" | grep l= | sed -e's/.*;l=\([0-9]*\).*/\\1/'; }
# the following replaces the above
targetDirName = os.path.dirname(targetPnfsPath)
targetBaseName = os.path.basename(targetPnfsPath)
p1 = Popen(
["cat",
"%s/.(use)(2)(%s)" % (targetDirName, targetBaseName)],
stdout=PIPE)
p2 = Popen(["grep", "l="], stdout=PIPE, stdin=p1.stdout)
p3 = Popen(["sed", "-e's/.*;l=\([0-9]*\).*/\\1/'"],
stdout=PIPE,
stdin=p2.stdout)
remoteSize = p3.communicate()[0]
logging.info("Localsize: %s Remotesize: %s" % (localSize, remoteSize))
if int(localSize) != int(remoteSize):
try:
self.doDelete(toPfn, None, None, None, None)
except:
pass
raise StageOutFailure("File sizes don't match")
return toPfn
def executeCommand(self, command):
"""
_execute_
Execute the command provided, throw a StageOutError if it exits
non zero
"""
try:
exitCode = runCommand(command)
msg = "Command exited with status: %s" % (exitCode)
print msg
except Exception, ex:
raise StageOutError(str(ex), Command = command, ExitCode = 60311)
def execute(command):
"""
_execute_
Execute the command provided, throw a StageOutError if it exits
non zero
"""
try:
exitCode, output = runCommandWithOutput(command)
msg = "Command exited with status: %s, Output: (%s)" % (exitCode,
output)
print(msg)
except Exception as ex:
msg = "Command threw exception: %s" % str(ex)
print("ERROR: Exception During Stage Out:\n%s" % msg)
raise StageOutError(msg, Command=command, ExitCode=60311)
if exitCode:
msg = "Command exited non-zero: ExitCode:%s \nOutput (%s)" % (exitCode,
output)
print("ERROR: Exception During Stage Out:\n%s" % msg)
raise StageOutError(msg, Command=command, ExitCode=exitCode)
return
def execute(command):
"""
_execute_
Execute the command provided, throw a StageOutError if it exits
non zero
"""
try:
exitCode = runCommand(command)
msg = "Command exited with status: %s" % (exitCode)
print(msg)
except Exception as ex:
msg = "Command threw exception"
print("ERROR: Exception During Stage Out:\n")
print(msg)
raise StageOutError(msg, Command=command, ExitCode=60311)
if exitCode:
msg = "Command exited non-zero"
print("ERROR: Exception During Stage Out:\n")
print(msg)
raise StageOutError(msg, Command=command, ExitCode=exitCode)
return
def execute(command):
"""
_execute_
Execute the command provided, throw a StageOutError if it exits
non zero
"""
try:
exitCode = runCommand(command)
msg = "Command exited with status: %s" % (exitCode)
print msg
except Exception, ex:
print "ERROR: Exception During Stage Out:\n"
raise StageOutError(msg, Command=command, ExitCode=60311)
def createStageOutCommand(self,
sourcePFN,
targetPFN,
options=None,
checksums=None):
"""
_createStageOutCommand_
Build a dccp command with a pnfs mkdir to generate the directory
"""
try:
import dcap
except ImportError, ie:
raise StageOutError("Python dCap wrappers not found on this host.")
def doDelete(self, pfn, seName, command, options, protocol):
"""
_removeFile_
Removes the pfn.
"""
command = "%s %s" % (self._rmScript, pfn)
exitCode, output = runCommand(command)
if exitCode != 0:
logging.error("Error removing file from LStore")
logging.error(output)
raise StageOutError("remove file failure command %s, error %s" %
(command, output))
def doTransfer(self, fromPfn, toPfn, stageOut, pnn, command, options,
protocol, checksum):
toPfn = self.createSourceName(protocol, toPfn)
fromPfn = self.createSourceName(protocol, fromPfn)
(_, reportFile) = tempfile.mkstemp()
ourCommand = \
self.generateCommandFromPreAndPostParts(\
['srmcp','-report=%s'%reportFile,
'-retry_num=0'],
[fromPfn, toPfn],
options)
self.runCommandWarnOnNonZero(ourCommand)
if not self.stageOut:
remotePFN, localPFN = fromPfn, toPfn.replace("file://", "", 1)
else:
remotePFN, localPFN = toPfn, fromPfn.replace("file://", "", 1)
targetPnfsPath = self.createPnfsPath(remotePFN)
if _CheckExitCodeOption:
p1 = Popen(["rfstat", remotePFN], stdout=PIPE)
p3 = Popen(['cut', '-f3', '-d" "'], stdin=p1.stdout, stdout=PIPE)
exitCode = p3.communicate()[0]
if exitCode:
raise StageOutError("srmcp failed! Error code: %s" % exitCode)
localSize = os.path.getsize(localPFN)
logging.info("Local Size %s" % localSize)
# filesize() { `srm-get-metadata -retry_num=0 %s 2>/dev/null | grep 'size :[0-9]' | cut -f2 -d":"`}
# the following replaces the above
p1 = Popen(["srm-get-metadata", '-retry_num=0', remotePFN],
stdout=PIPE)
p2 = Popen(["grep", "size :[0-9]"], stdout=PIPE, stdin=p1.stdout)
p3 = Popen(["sed", "-f2", '-d":"'], stdout=PIPE, stdin=p2.stdout)
remoteSize = p3.communicate()[0]
logging.info("Localsize: %s Remotesize: %s" % (localSize, remoteSize))
if int(localSize) != int(remoteSize):
try:
self.doDelete(toPfn, None, None, None, None)
except:
pass
raise StageOutFailure("File sizes don't match")
return toPfn
def parseCastorPath(self, complexCastorPath):
"""
_parseCastorPath_
Castor filenames can be full URLs with control
statements for the rfcp
Some other castor command line tools do not understand
that syntax, so we need to retrieve the path and other
parameters from the URL
"""
simpleCastorPath = None
# full castor PFNs
if simpleCastorPath == None:
regExpParser = re.compile('/+castor/cern.ch/(.*)')
match = regExpParser.match(complexCastorPath)
if ( match != None ):
simpleCastorPath = '/castor/cern.ch/' + match.group(1)
# rfio style URLs
if simpleCastorPath == None:
regExpParser = re.compile('rfio:.*/+castor/cern.ch/([^?]+).*')
match = regExpParser.match(complexCastorPath)
if ( match != None ):
simpleCastorPath = '/castor/cern.ch/' + match.group(1)
# xrootd/castor style URLs
if simpleCastorPath == None:
regExpParser = re.compile('root:.*/+castor/cern.ch/([^?]+).*')
match = regExpParser.match(complexCastorPath)
if ( match != None ):
simpleCastorPath = '/castor/cern.ch/' + match.group(1)
# if that does not work raise an error
if simpleCastorPath == None:
raise StageOutError("Can't parse castor path out of URL !")
# remove multi-slashes from path
while ( simpleCastorPath.find('//') > -1 ):
simpleCastorPath = simpleCastorPath.replace('//','/')
return simpleCastorPath
def checkDirExists(self, directory):
"""
_checkDirExists_
Check if directory exists (will throw if it exists as a file)
Only used for castor and local file
"""
command = "rfstat %s 2> /dev/null | grep Protection" % directory
print "Check dir existence : %s" % command
try:
exitCode, output = runCommandWithOutput(command)
except Exception, ex:
msg = "Error: Exception while invoking command:\n"
msg += "%s\n" % command
msg += "Exception: %s\n" % str(ex)
msg += "Fatal error, abort stageout..."
raise StageOutError(msg)
def doTransfer(self, fromPfn, toPfn, stageOut, seName, command, options,
protocol, checksum):
"""
if stageOut is true:
The fromPfn is the LOCAL FILE NAME on the node, without file://
the toPfn is the target PFN, mapped from the LFN using the TFC or overrrides
if stageOut is false:
The toPfn is the LOCAL FILE NAME on the node, without file://
the fromPfn is the source PFN, mapped from the LFN using the TFC or overrrides
"""
# Figures out the src and dst files
if stageOut:
dstPath = toPfn
else:
dstPath = fromPfn
# Creates the directory
if stageOut:
self.createOutputDirectory(os.path.dirname(dstPath))
else:
os.makedirs(os.path.dirname(dstPath))
# Does the copy
if stageOut:
command = "%s %s %s" % (self._cpScript, fromPfn, toPfn)
else:
command = "%s %s %s" % (self._downloadScript, fromPfn, toPfn)
exitCode, output = runCommand(command)
print(output)
if exitCode != 0:
logging.error("Error in file transfer:")
logging.error(output)
raise StageOutError("Transfer failure, command %s, error %s" %
(command, output))
# Returns the path
return dstPath
class RFCPCERNImpl(StageOutImpl):
"""
_RFCPCERNImpl_
"""
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
self.numRetries = 5
self.retryPause = 300
def createSourceName(self, protocol, pfn):
"""
_createSourceName_
uses pfn
"""
return "%s" % pfn
def createOutputDirectory(self, targetPFN):
"""
_createOutputDirectory_
create dir with group permission
"""
# EOS stageout auto-creates directories
if self.isEOS(targetPFN):
return
# Only create dir on remote storage
if self.stageIn:
return
targetDir = os.path.dirname(self.parseCastorPath(targetPFN))
# targetDir does not exist => create it
if not self.checkDirExists(targetDir):
# only use the fileclass code path if we run on t0export
serviceClass = os.environ.get('STAGE_SVCCLASS', None)
if serviceClass == 't0export':
# determine file class from PFN
fileclass = None
# check for correct naming convention in PFN
regExpParser = re.compile('/castor/cern.ch/cms/store/([^/]*data)/([^/]+)/([^/]+)/([^/]+)/')
match = regExpParser.match(targetDir)
if ( match != None ):
# RAW data files use cms_raw, all others cms_production
if match.group(4) == 'RAW':
fileclass = 'cms_raw'
else:
fileclass = 'cms_production'
fileclassDir = '/castor/cern.ch/cms/store/%s/%s/%s/%s' % match.group(1,2,3,4)
# fileclassDir does not exist => create it
if not self.checkDirExists(fileclassDir):
self.createDir(fileclassDir)
if ( fileclass != None ):
self.setFileClass(fileclassDir,fileclass)
# now create targetDir
self.createDir(targetDir)
return
def createStageOutCommand(self, sourcePFN, targetPFN, options = None, checksums = None):
"""
_createStageOutCommand_
Build the stageout command: rfcp for castor and xrdcp for eos
If adler32 checksum is provided, use it for the transfer
"""
result = ""
if self.stageIn:
remotePFN, localPFN = sourcePFN, targetPFN
else:
remotePFN, localPFN = targetPFN, sourcePFN
result += "LOCAL_SIZE=`stat -c%%s \"%s\"`\n" % localPFN
result += "echo \"Local File Size is: $LOCAL_SIZE\"\n"
isRemoteEOS = self.isEOS(remotePFN)
useChecksum = ( checksums != None and checksums.has_key('adler32') and not self.stageIn )
removeCommand = self.createRemoveFileCommand(targetPFN)
if isRemoteEOS:
result += "xrdcp -f -s "
if useChecksum:
checksums['adler32'] = "%08x" % int(checksums['adler32'], 16)
# non-functional in 3.3.1 xrootd clients due to bug
#result += "-ODeos.targetsize=$LOCAL_SIZE\&eos.checksum=%s " % checksums['adler32']
# therefor embed information into target URL
targetPFN += "?eos.targetsize=$LOCAL_SIZE&eos.checksum=%s" % checksums['adler32']
else:
if useChecksum:
targetFile = self.parseCastorPath(targetPFN)
result += "nstouch %s\n" % targetFile
result += "nssetchecksum -n adler32 -k %s %s\n" % (checksums['adler32'], targetFile)
result += "rfcp "
if options != None:
result += " %s " % options
result += " \"%s\" " % sourcePFN
result += " \"%s\" \n" % targetPFN
if self.stageIn:
result += "LOCAL_SIZE=`stat -c%%s \"%s\"`\n" % localPFN
result += "echo \"Local File Size is: $LOCAL_SIZE\"\n"
if isRemoteEOS:
(_,host,path,_) = self.splitPFN(remotePFN)
result += "REMOTE_SIZE=`xrd '%s' stat '%s' | sed -r 's/.* Size: ([0-9]+) .*/\\1/'`\n" % (host, path)
result += "echo \"Remote File Size is: $REMOTE_SIZE\"\n"
if useChecksum:
result += "echo \"Local File Checksum is: %s\"\n" % checksums['adler32']
result += "REMOTE_XS=`xrd '%s' getchecksum '%s' | sed -r 's/.* eos ([0-9a-fA-F]{8}).*/\\1/'`\n" % (host, path)
result += "echo \"Remote File Checksum is: $REMOTE_XS\"\n"
result += "if [ $REMOTE_SIZE ] && [ $REMOTE_XS ] && [ $LOCAL_SIZE == $REMOTE_SIZE ] && [ '%s' == $REMOTE_XS ]; then exit 0; " % checksums['adler32']
result += "else echo \"Error: Size or Checksum Mismatch between local and SE\"; %s ; exit 60311 ; fi" % removeCommand
else:
result += "if [ $REMOTE_SIZE ] && [ $LOCAL_SIZE == $REMOTE_SIZE ]; then exit 0; "
result += "else echo \"Error: Size Mismatch between local and SE\"; %s ; exit 60311 ; fi" % removeCommand
else:
result += "REMOTE_SIZE=`rfstat '%s' | grep Size | cut -f2 -d: | tr -d ' '`\n" % remotePFN
result += "echo \"Remote File Size is: $REMOTE_SIZE\"\n"
result += "if [ $REMOTE_SIZE ] && [ $LOCAL_SIZE == $REMOTE_SIZE ]; then exit 0; else echo \"Error: Size Mismatch between local and SE\"; %s ; exit 60311 ; fi" % removeCommand
return result
def createRemoveFileCommand(self, pfn):
"""
_createRemoveFileCommand_
Alternate between EOS, CASTOR and local.
"""
if self.isEOS(pfn):
(_,host,path,_) = self.splitPFN(pfn)
return "xrd %s rm %s" % (host, path)
try:
simplePFN = self.parseCastorPath(pfn)
return "stager_rm -a -M %s ; nsrm %s" % (simplePFN, simplePFN)
except StageOutError:
# Not castor
pass
return StageOutImpl.createRemoveFileCommand(self, pfn)
def removeFile(self, pfnToRemove):
"""
_removeFile_
"""
if self.isEOS(pfnToRemove):
(_,host,path,_) = self.splitPFN(pfnToRemove)
command = "xrd %s rm %s" % (host, path)
else:
simplePFN = self.parseCastorPath(pfnToRemove)
command = "stager_rm -a -M %s ; nsrm %s" % (simplePFN, simplePFN)
execute(command)
return
def checkDirExists(self, directory):
"""
_checkDirExists_
Check if directory exists (will throw if it exists as a file)
Only used for castor and local file
"""
command = "rfstat %s 2> /dev/null | grep Protection" % directory
print "Check dir existence : %s" % command
try:
exitCode, output = runCommandWithOutput(command)
except Exception, ex:
msg = "Error: Exception while invoking command:\n"
msg += "%s\n" % command
msg += "Exception: %s\n" % str(ex)
msg += "Fatal error, abort stageout..."
raise StageOutError(msg)
if exitCode != 0:
return False
else:
regExpParser = re.compile('^Protection[ ]+: d')
if ( regExpParser.match(output) == None):
raise StageOutError("Output path is not a directory !")
else:
return True
if os.WIFEXITED(err):
err = os.WEXITSTATUS(err)
elif os.WIFSIGNALED(err):
err = os.WTERMSIG(err)
elif os.WIFSTOPPED(err):
err = os.WSTOPSIG(err)
return err, output
def execute(command):
"""
_execute_
Execute the command provided, throw a StageOutError if it exits
non zero
"""
try:
exitCode = runCommand(command)
msg = "Command exited with status: %s" % (exitCode)
print msg
except Exception, ex:
print "ERROR: Exception During Stage Out:\n"
raise StageOutError(msg, Command=command, ExitCode=60311)
if exitCode:
msg = "Command exited non-zero"
print "ERROR: Exception During Stage Out:\n"
print msg
raise StageOutError(msg, Command=command, ExitCode=exitCode)
return
def stageFile(self, fileToStage, stageOut=True):
"""
_stageFile_
Use call to invoke transfers (either in or out)
input:
fileToStage: a dict containing at least:
LFN: the LFN for one end of the transfer, will be
mapped to a PFN before the transfer
PFN: (annoyingly) Not a PFN, but is the local filename
for the file when it's transferred to/from
stageOut: boolean for if the file is staged in or out
output:
dict from fileToStage with PFN, PNN, StageOutCommand added
I'm not entirely sure that StageOutCommand makes sense, but I don't want to break old code
-AMM 6/30/2010
"""
log.info("Working on file: %s" % fileToStage['LFN'])
lfn = fileToStage['LFN']
localFileName = fileToStage['PFN']
self.firstException = None
log.info("Beginning %s" % ('StageOut' if stageOut else 'StageIn'))
# generate list of stageout methods we will try
stageOutMethods = [self.defaultMethod]
stageOutMethods.extend(self.fallbacks)
# loop over all the different methods. This unifies regular and fallback stuff. Nice.
methodCounter = 0
for currentMethod in stageOutMethods:
methodCounter += 1
# the PFN that is received here is mapped from the LFN
log.info("Getting transfer details for %s LFN %s" %
(currentMethod, lfn))
(pnn, command, options, pfn, protocol) = \
self.getTransferDetails(lfn, currentMethod)
log.info("Using PNN: %s" % pnn)
log.info("Command: %s" % command)
log.info("Options: %s" % options)
log.info("Protocol: %s" % protocol)
log.info("Mapped LFN: %s" % lfn)
log.info(" to PFN: %s" % pfn)
log.info("LocalFileName: %s" % localFileName)
newPfn = self._doTransfer(currentMethod, methodCounter,
localFileName, pfn, stageOut)
if newPfn:
log.info("Transfer succeeded: %s" % fileToStage)
fileToStage['PFN'] = newPfn
fileToStage['PNN'] = pnn
fileToStage['StageOutCommand'] = command
self.completedFiles[fileToStage['LFN']] = fileToStage
return fileToStage
else:
# transfer method didn't work, go to next one
continue
# if we're here, then nothing worked. transferfail.
log.error("Error in stageout")
if self.firstException:
raise self.firstException
else:
raise StageOutError(
"Error in stageout, this has been logged in the logs")
def createOutputDirectory(self, targetPFN):
"""
_createOutputDirectory_
create dir with group permission
"""
# check how the targetPFN looks like and parse out the target dir
targetdir = None
if targetdir == None:
regExpParser = re.compile('/+castor/(.*)')
match = regExpParser.match(targetPFN)
if (match != None):
targetdir = os.path.dirname(targetPFN)
if targetdir == None:
regExpParser = re.compile('rfio:/+castor/(.*)')
match = regExpParser.match(targetPFN)
if (match != None):
targetdir = os.path.dirname('/castor/' + match.group(1))
if targetdir == None:
regExpParser = re.compile('rfio:.*path=/+castor/(.*)')
match = regExpParser.match(targetPFN)
if (match != None):
targetdir = os.path.dirname('/castor/' + match.group(1))
# raise exception if we have no rule that can parse the target dir
if targetdir == None:
raise StageOutError("Cannot parse directory out of targetPFN")
# remove multi-slashes from path
while (targetdir.find('//') > -1):
targetdir = targetdir.replace('//', '/')
#
# determine file class from LFN
#
fileclass = None
# temp or unmerged files use cms_no_tape
if (fileclass == None):
regExpParser = re.compile('.*/castor/cern.ch/cms/store/temp/')
if (regExpParser.match(targetdir) != None):
fileclass = 'cms_no_tape'
if (fileclass == None):
regExpParser = re.compile('.*/castor/cern.ch/cms/store/unmerged/')
if (regExpParser.match(targetdir) != None):
fileclass = 'cms_no_tape'
# RAW data files use cms_raw
if (fileclass == None):
regExpParser = re.compile(
'.*/castor/cern.ch/cms/store/data/[^/]+/[^/]+/RAW/')
if (regExpParser.match(targetdir) != None):
fileclass = 'cms_raw'
# otherwise we assume another type of production file
if (fileclass == None):
fileclass = 'cms_production'
print "Setting fileclass to : %s" % fileclass
# check if directory exists
rfstatCmd = "rfstat %s 2> /dev/null | grep Protection" % targetdir
print "Check dir existence : %s" % rfstatCmd
try:
rfstatExitCode, rfstatOutput = runCommandWithOutput(rfstatCmd)
except Exception as ex:
msg = "Error: Exception while invoking command:\n"
msg += "%s\n" % rfstatCmd
msg += "Exception: %s\n" % str(ex)
msg += "Fatal error, abort stageout..."
raise StageOutError(msg)
# does not exist => create it
if rfstatExitCode:
if (fileclass != None):
self.createDir(targetdir, '000')
self.setFileClass(targetdir, fileclass)
self.changeDirMode(targetdir, self.permissions)
else:
self.createDir(targetdir, self.permissions)
else:
# check if this is a directory
regExpParser = re.compile('Protection.*: d')
if (regExpParser.match(rfstatOutput) == None):
raise StageOutError("Output path is not a directory !")
else:
# check if directory is writable
regExpParser = re.compile('Protection.*: d---------')
if (regExpParser.match(rfstatOutput) != None
and fileclass != None):
self.setFileClass(targetdir, fileclass)
self.makeDirWritable(targetdir)
return
def createStageOutCommand(self, sourcePFN, targetPFN, options = None, checksums = None):
"""
_createStageOutCommand_
Build an srmcp command
"""
result = "#!/bin/sh\n"
result += "REPORT_FILE=`pwd`/srm.report.$$\n"
result += "srmcp -2 -report=$REPORT_FILE -retry_num=0"
if options != None:
result += " %s " % options
result += " %s " % sourcePFN
result += " %s" % targetPFN
result += " 2>&1 | tee srm.output.$$ \n"
if _CheckExitCodeOption:
result += """
EXIT_STATUS=`cat $REPORT_FILE | cut -f3 -d" "`
echo "srmcp exit status: $EXIT_STATUS"
if [[ "X$EXIT_STATUS" == "X" ]] && [[ `grep -c SRM_INVALID_PATH srm.output.$$` != 0 ]]; then
exit 1 # dir does not eixst
elif [[ $EXIT_STATUS != 0 ]]; then
echo "Non-zero srmcp Exit status!!!"
echo "Cleaning up failed file:"
%s
exit 60311
fi
""" % self.createRemoveFileCommand(targetPFN)
if self.stageIn:
remotePFN, localPFN = sourcePFN, targetPFN.replace("file://", "", 1)
else:
remotePFN, localPFN = targetPFN, sourcePFN.replace("file://", "", 1)
#targetPFN = remotePFN
remotePath = None
SFN = '?SFN='
sfn_idx = remotePFN.find(SFN)
if sfn_idx >= 0:
remotePath = remotePFN[sfn_idx+5:]
r = re.compile('srm://([A-Za-z\-\.0-9]*)(:[0-9]*)?(/.*)')
m = r.match(remotePFN)
if not m:
raise StageOutError("Unable to determine path from PFN for " \
"target %s." % remotePFN)
if remotePath == None:
remotePath = m.groups()[2]
remoteHost = m.groups()[0]
# for filePath in (sourcePFN, targetPFN):
# if filePath.startswith("file://"):
# localPFN = filePath.replace("file://", "")
# elif filePath.startswith("srm://"):
# remotePFN = filePath
# targetPFN = filePath
# targetPath = None
# SFN = '?SFN='
# sfn_idx = filePath.find(SFN)
# if sfn_idx >= 0:
# targetPath = filePath[sfn_idx+5:]
# r = re.compile('srm://([A-Za-z\-\.0-9]*)(:[0-9]*)?(/.*)')
# m = r.match(filePath)
# if not m:
# raise StageOutError("Unable to determine path from PFN for " \
# "target %s." % filePath)
# if targetPath == None:
# targetPath = m.groups()[2]
# targetHost = m.groups()[0]
result += "FILE_SIZE=`stat -c %s"
result += " %s `\n" % localPFN
result += "echo \"Local File Size is: $FILE_SIZE\"\n"
metadataCheck = \
"""
for ((a=1; a <= 10 ; a++))
do
SRM_SIZE=`srmls -recursion_depth=0 -retry_num=0 %s 2>/dev/null | grep '%s' | grep -v '%s' | awk '{print $1;}'`
echo "SRM Size is $SRM_SIZE"
if [[ $SRM_SIZE > 0 ]]; then
if [[ $SRM_SIZE == $FILE_SIZE ]]; then
exit 0
else
echo "Error: Size Mismatch between local and SE"
echo "Cleaning up failed file:"
%s
exit 60311
fi
else
sleep 2
fi
done
echo "Cleaning up failed file:"
%s
exit 60311
""" % (remotePFN, remotePath, remoteHost, self.createRemoveFileCommand(targetPFN), self.createRemoveFileCommand(targetPFN))
result += metadataCheck
return result
def doTransfer(self, fromPfn, toPfn, stageOut, pnn, command, options,
protocol, checksum):
toPfn = self.createSourceName(protocol, toPfn)
fromPfn = self.createSourceName(protocol, fromPfn)
# TODO tee the output to another file
# attempt copy
for x in range(self.numRetries):
(_, reportFile) = tempfile.mkstemp()
ourCommand = \
self.generateCommandFromPreAndPostParts(\
['srmcp','-2','-report=%s'%reportFile,
'-retry_num=0'],
[fromPfn, toPfn],
options)
self.runCommandWarnOnNonZero(ourCommand)
if not stageOut:
remotePFN, localPFN = fromPfn, toPfn.replace("file://", "", 1)
else:
remotePFN, localPFN = toPfn, fromPfn.replace("file://", "", 1)
if _CheckExitCodeOption:
p1 = Popen(["cat", reportFile], stdout=PIPE)
p3 = Popen(['cut', '-f3', '-d', ' '],
stdin=p1.stdout,
stdout=PIPE)
exitCode = p3.communicate()[0].rstrip()
logging.info("srmcp exit status: %s" % exitCode)
p2 = Popen(['grep', '-c', 'SRM_INVALID_PATH', reportFile],
stdout=PIPE)
invalidPathCount = p2.communicate()[0]
logging.info("got this for SRM_INVALID_PATH: %s" %
invalidPathCount)
if (invalidPathCount and (exitCode == '')):
logging.warn(
"Directory doesn't exist in srmv2 stageout...creating and retrying"
)
self.createOutputDirectory(toPfn, stageOut)
continue
elif (str(exitCode) != "0"):
logging.error("Couldn't stage out! Error code: %s" %
exitCode)
self.doDelete(toPfn, None, None, None, None)
raise StageOutFailure("srmcp failed! Error code: %s" %
exitCode)
else:
logging.info(
"Tentatively succeeded transfer, will check metadata")
break
localSize = os.path.getsize(localPFN)
logging.info("Local Size %s" % localSize)
remotePath = None
SFN = '?SFN='
sfn_idx = remotePFN.find(SFN)
if sfn_idx >= 0:
remotePath = remotePFN[sfn_idx + 5:]
r = re.compile('srm://([A-Za-z\-\.0-9]*)(:[0-9]*)?(/.*)')
m = r.match(remotePFN)
if not m:
raise StageOutError("Unable to determine path from PFN for " \
"target %s." % remotePFN)
if remotePath == None:
remotePath = m.groups()[2]
remoteHost = m.groups()[0]
# filesize() { `srm-get-metadata -retry_num=0 %s 2>/dev/null | grep 'size :[0-9]' | cut -f2 -d":"`}
# the following replaces the above
logging.info("remote path: %s" % remotePath)
logging.info("remote host: %s" % remoteHost)
p1 = Popen(["srmls", '-recursion_depth=0', '-retry_num=0', remotePFN],
stdout=PIPE)
p2 = Popen(["grep", remotePath], stdout=PIPE, stdin=p1.stdout)
p3 = Popen(["grep", '-v', remoteHost], stdout=PIPE, stdin=p2.stdout)
p4 = Popen(["awk", "{print $1;}"], stdout=PIPE, stdin=p3.stdout)
remoteSize = p4.communicate()[0]
logging.info("Localsize: %s Remotesize: %s" % (localSize, remoteSize))
if int(localSize) != int(remoteSize):
try:
self.doDelete(toPfn, None, None, None, None)
except:
pass
raise StageOutFailure("File sizes don't match")
return toPfn
def createStageOutCommand(self, sourcePFN, targetPFN, options = None, checksums = None):
"""
_createStageOutCommand_
Build an srmcp command
srmcp options used (so hard to find documentation for it...):
-2 enables srm protocol version 2
-report path to the report file
-retry_num number of retries before before client gives up
-request_lifetime request lifetime in seconds
"""
result = "#!/bin/sh\n"
result += "REPORT_FILE=`pwd`/srm.report.$$\n"
result += "srmcp -2 -report=$REPORT_FILE -retry_num=0 -request_lifetime=2400"
if options != None:
result += " %s " % options
result += " %s " % sourcePFN
result += " %s" % targetPFN
result += " 2>&1 | tee srm.output.$$ \n"
if _CheckExitCodeOption:
result += """
EXIT_STATUS=`cat $REPORT_FILE | cut -f3 -d" "`
echo "srmcp exit status: $EXIT_STATUS"
if [[ "X$EXIT_STATUS" == "X" ]] && [[ `grep -c SRM_INVALID_PATH srm.output.$$` != 0 ]]; then
exit 1 # dir does not eixst
elif [[ $EXIT_STATUS != 0 ]]; then
echo "Non-zero srmcp Exit status!!!"
echo "Cleaning up failed file:"
%s
exit 60311
fi
""" % self.createRemoveFileCommand(targetPFN)
if self.stageIn:
remotePFN, localPFN = sourcePFN, targetPFN.replace("file://", "", 1)
else:
remotePFN, localPFN = targetPFN, sourcePFN.replace("file://", "", 1)
remotePath = None
SFN = '?SFN='
sfn_idx = remotePFN.find(SFN)
if sfn_idx >= 0:
remotePath = remotePFN[sfn_idx+5:]
r = re.compile('srm://([A-Za-z\-\.0-9]*)(:[0-9]*)?(/.*)')
m = r.match(remotePFN)
if not m:
raise StageOutError("Unable to determine path from PFN for " \
"target %s." % remotePFN)
if remotePath == None:
remotePath = m.groups()[2]
remoteHost = m.groups()[0]
result += "FILE_SIZE=`stat -c %s"
result += " %s `\n" % localPFN
result += "echo \"Local File Size is: $FILE_SIZE\"\n"
metadataCheck = \
"""
SRM_OUTPUT=`srmls -recursion_depth=0 -retry_num=1 %s 2>/dev/null`
SRM_SIZE=`echo $SRM_OUTPUT | grep '%s' | grep -v '%s' | awk '{print $1;}'`
echo "SRM Size is $SRM_SIZE"
if [[ $SRM_SIZE == $FILE_SIZE ]]; then
exit 0
else
echo $SRM_OUTPUT
echo "ERROR: Size Mismatch between local and SE. Cleaning up failed file..."
%s
exit 60311
fi
""" % (remotePFN, remotePath, remoteHost, self.createRemoveFileCommand(targetPFN))
result += metadataCheck
return result
