def testInit(self, mock_os):
mock_os.environ.get.side_effect = lambda x: True if x == "GLITE_LOCATION" or x == "GRID_ENV_LOCATION" else False
mock_os.path.isfile.return_value = True
mock_os.path.normpath.side_effect = lambda x: "home" + x
mock_os.path.join.side_effect = lambda a, b: os.path.join("", b)
testLCGImpl = LCGImpl(True)
self.assertTrue(testLCGImpl.stageIn)
setups = []
setups += ['source home/../etc/profile.d/grid-env.sh; ']
setups += ['source home/grid-env.sh; ']
setups += ['date "+%Y-%m-%dT%H:%M:%S"; ']
self.assertTrue(testLCGImpl.stageIn)
for setup in setups:
self.assertIn(setup, testLCGImpl.setups)
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
# Create and hold onto a srm implementation in case we need it
self.srmImpl = LCGImpl(stagein)
class FNALImpl(StageOutImpl):
"""
_FNALImpl_
Implement interface for dcache xrootd command
"""
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
# Create and hold onto a srm implementation in case we need it
self.srmImpl = LCGImpl(stagein)
def storageMethod(self, PFN):
"""
Return xrdcp or srm
"""
method = 'local' # default
if PFN.startswith("root://"):
method = 'xrdcp'
if PFN.startswith("srm://"):
method = 'srm'
print("Using method %s for PFN %s" % (method, PFN))
return method
def createOutputDirectory(self, targetPFN):
"""
_createOutputDirectory_
Create a dir for the target pfn by translating it to
a /dcache or /lustre name and calling mkdir
we don't need to convert it, just mkdir.
"""
method = self.storageMethod(targetPFN)
if method == 'xrdcp': # xrdcp autocreates parent directories
return
elif method == 'srm':
self.srmImpl.createOutputDirectory(targetPFN)
elif method == 'local':
targetdir = os.path.dirname(targetPFN)
command = "#!/bin/sh\n"
command += "if [ ! -e \"%s\" ]; then\n" % targetdir
command += " mkdir -p %s\n" % targetdir
command += "fi\n"
self.executeCommand(command)
def createSourceName(self, protocol, pfn):
"""
_createSourceName_
generate the target PFN
"""
method = self.storageMethod(pfn)
if method == 'srm':
return self.srmImpl.createSourceName(protocol, pfn)
return pfn
def createStageOutCommand(self, sourcePFN, targetPFN, options=None, checksums=None):
"""
_createStageOutCommand_
Build a mkdir to generate the directory
"""
if getattr(self, 'stageIn', False):
return self.buildStageInCommand(sourcePFN, targetPFN, options)
method = self.storageMethod(targetPFN)
sourceMethod = self.storageMethod(sourcePFN)
if ((method == 'srm' and sourceMethod == 'xrdcp') or
(method == 'xrdcp' and sourceMethod == 'srm')):
print("Incompatible methods for source and target")
print("\tSource: method %s for PFN %s" % (sourceMethod, sourcePFN))
print("\tTarget: method %s for PFN %s" % (method, targetPFN))
return 1
if method == 'srm' or sourceMethod == 'srm':
return self.srmImpl.createStageOutCommand(sourcePFN, targetPFN, options)
if method == 'xrdcp' or sourceMethod == 'xrdcp':
original_size = os.stat(sourcePFN)[6]
print("Local File Size is: %s" % original_size)
useChecksum = (checksums != None and 'adler32' in checksums and not self.stageIn)
if useChecksum:
checksums['adler32'] = "%08x" % int(checksums['adler32'], 16)
# non-functional in 3.3.1 xrootd clients due to bug
# result += "-ODeos.targetsize=$LOCAL_SIZE\&eos.checksum=%s " % checksums['adler32']
# therefor embed information into target URL
targetPFN += "\?eos.targetsize=%s\&eos.checksum=%s" % (original_size, checksums['adler32'])
print("Local File Checksum is: %s\"\n" % checksums['adler32'])
# always overwrite the output
result = "xrdcp-old -d 0 -f "
if options != None:
result += " %s " % options
result += " %s " % sourcePFN
result += " %s " % targetPFN
result += """
EXIT_STATUS=$?
if [[ $EXIT_STATUS != 0 ]]; then
echo "ERROR: xrdcp exited with $EXIT_STATUS"
fi
exit $EXIT_STATUS
"""
return result
def buildStageInCommand(self, sourcePFN, targetPFN, options=None):
"""
_buildStageInCommand_
Create normal xrdcp commad for staging in files.
"""
result = "/usr/bin/xrdcp -d 0 "
if options != None:
result += " %s " % options
result += " %s " % sourcePFN
result += " %s " % targetPFN
result += """
EXIT_STATUS=$?
if [[ $EXIT_STATUS != 0 ]]; then
echo "ERROR: xrdcp exited with $EXIT_STATUS"
fi
exit $EXIT_STATUS
"""
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
"""
method = self.storageMethod(pfnToRemove)
if method == 'xrdcp':
(_, host, path, _) = self.splitPFN(pfnToRemove)
command = "xrd %s rm %s" % (host, path)
print("Executing: %s" % command)
self.executeCommand(command)
elif method == 'srm':
return self.srmImpl.removeFile(pfnToRemove)
else:
command = "/bin/rm %s" % stripPrefixTOUNIX(pfnToRemove)
print("Executing: %s" % command)
self.executeCommand(command)
class FNALImpl(StageOutImpl):
"""
_FNALImpl_
Implement interface for dcache door based dccp command
"""
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
# Create and hold onto a srm implementation in case we need it
self.srmImpl = LCGImpl(stagein)
def storageMethod(self, PFN):
"""
Figure out which paths use DCAP, lustre, or SRM for access
"""
method = 'dccp'
for path in checkPaths:
if PFN.find(path) != -1:
method = 'lustre'
# Over ride a few paths for srm
for path in srmPaths:
if PFN.find(path) != -1:
method = 'srm'
print "Using method:", method
return method
def createOutputDirectory(self, targetPFN):
"""
_createOutputDirectory_
Create a dir for the target pfn by translating it to
a /pnfs name and calling mkdir
PFN will be of the form:
dcap://cmsdca.fnal.gov:22125/pnfs/fnal.gov/usr/cms/WAX/11/store/blah
We need to convert that into /pnfs/cms/WAX/11/store/blah, as it
will be seen from the worker node
Unmerged PFN will be of the form:
/lustre/unmerged
we don't need to convert it, just mkdir.
"""
method = self.storageMethod(targetPFN)
if method == 'srm':
self.srmImpl.createOutputDirectory(targetPFN)
elif method == 'dccp':
# only create dir on remote storage
if targetPFN.find('/pnfs/') == -1:
return
pfnSplit = targetPFN.split("WAX/11/store/", 1)[1]
filePath = "/pnfs/cms/WAX/11/store/%s" % pfnSplit
directory = os.path.dirname(filePath)
command = "#!/bin/sh\n"
command += " . /opt/d-cache/dcap/bin/setenv-cmsprod.sh\n"
command += "if [ ! -e \"%s\" ]; then\n" % directory
command += " mkdir -p %s\n" % directory
command += "fi\n"
self.executeCommand(command)
else:
for i in range(checkPathsCount):
if targetPFN.find(checkPaths[i]) != -1:
pfnSplit = targetPFN.split(checkPaths[i], 1)[1]
filePath = "%s%s" % (checkPathsReplace[i],pfnSplit)
targetdir= os.path.dirname(filePath)
# checkdircmd="/bin/ls %s > /dev/null " % targetdir
# print "Check dir existence : %s" %checkdircmd
# checkdirexitCode = 0
# try:
# checkdirexitCode = self.executeCommand(checkdircmd)
# except Exception, ex:
# msg = "Warning: Exception while invoking command:\n"
# msg += "%s\n" % checkdircmd
# msg += "Exception: %s\n" % str(ex)
# msg += "Go on anyway..."
# print msg
# pass
# if checkdirexitCode != 0:
# mkdircmd = "/bin/mkdir -m 775 -p %s" % targetdir
# print "=> creating the dir : %s" %mkdircmd
# try:
# self.executeCommand(mkdircmd)
# except Exception, ex:
# msg = "Warning: Exception while invoking command:\n"
# msg += "%s\n" % mkdircmd
# msg += "Exception: %s\n" % str(ex)
# msg += "Go on anyway..."
# print msg
# pass
# else:
# print "=> dir already exists... do nothing."
command = "#!/bin/sh\n"
command += "if [ ! -e \"%s\" ]; then\n" % targetdir
command += " mkdir -p %s\n" % targetdir
command += "fi\n"
self.executeCommand(command)
def createSourceName(self, protocol, pfn):
"""
createTargetName
generate the target PFN
"""
if not pfn.startswith("srm"):
return pfn
method = self.storageMethod(pfn)
if method == 'srm':
return self.srmImpl.createSourceName(protocol, pfn)
elif method == 'dccp':
print "Translating PFN: %s\n To use dcache door" % pfn
dcacheDoor = commands.getoutput(
"/opt/d-cache/dcap/bin/setenv-cmsprod.sh; /opt/d-cache/dcap/bin/select_RdCapDoor.sh")
pfn = pfn.split("/store/")[1]
pfn = "%s%s" % (dcacheDoor, pfn)
print "Created Target PFN with dCache Door: ", pfn
else:
print "Translating PFN: %s\n To use lustre" % pfn
for i in range(checkPathsCount):
if pfn.find(checkPaths[i]) != -1:
pfnSplit = pfn.split(checkPaths[i], 1)[1]
pfn = "%s%s" % (checkPathsReplace[i],pfnSplit)
return pfn
def createStageOutCommand(self, sourcePFN, targetPFN, options = None, checksums = None):
"""
_createStageOutCommand_
Build a dccp command with a pnfs mkdir to generate the directory
"""
method = self.storageMethod(targetPFN)
sourceMethod = self.storageMethod(sourcePFN)
if method == 'srm' or sourceMethod == 'srm':
return self.srmImpl.createStageOutCommand(sourcePFN, targetPFN, options)
if getattr(self, 'stageIn', False):
return self.buildStageInCommand(sourcePFN, targetPFN, options)
if method == 'dccp':
optionsStr = ""
if options != None:
optionsStr = str(options)
dirname = os.path.dirname(targetPFN)
result = "#!/bin/sh\n"
result += ". /opt/d-cache/dcap/bin/setenv-cmsprod.sh\n"
result += "dccp -o 86400 -d 0 -X -role=cmsprod %s %s %s" % ( optionsStr, sourcePFN, targetPFN)
result += \
"""
EXIT_STATUS=$?
echo "dccp exit status: $EXIT_STATUS"
if [[ $EXIT_STATUS != 0 ]]; then
echo "Non-zero dccp Exit status!!!"
echo "Cleaning up failed file:"
/bin/rm -fv %s
exit 60311
fi
""" % pnfsPfn2(targetPFN)
# //
# // CRC check
#//
result += \
"""
/opt/d-cache/dcap/bin/check_dCachefilecksum.sh %s %s
EXIT_STATUS=$?
echo "CRC Check Exit status: $EXIT_STATUS"
if [[ $EXIT_STATUS != 0 ]]; then
echo "Non-zero CRC Check Exit status!!!"
echo "Cleaning up failed file:"
/bin/rm -fv %s
exit 60311
fi
""" % (pnfsPfn2(targetPFN), sourcePFN, pnfsPfn2(targetPFN))
print "Executing:\n", result
return result
else:
original_size = os.stat(sourcePFN)[6]
print "Local File Size is: %s" % original_size
result = "/bin/cp "
if options != None:
result += " %s " % options
result += " %s " % sourcePFN
result += " %s " % targetPFN
result += "; sync; DEST_SIZE=`/bin/ls -l %s | /bin/awk '{print $5}'` ; if [ $DEST_SIZE ] && [ '%s' == $DEST_SIZE ]; then exit 0; else echo \"Error: Size Mismatch between local and SE\"; exit 60311 ; fi " % (targetPFN,original_size)
return result
def buildStageInCommand(self, sourcePFN, targetPFN, options = None):
"""
_buildStageInCommand_
Create normal dccp commad for staging in files.
"""
dcapLocation = 0
for i in range(checkPathsCount):
if sourcePFN.find(checkPaths[i]) != -1:
dcapLocation = 1
# Even if matched above, some paths are not lustre
for path in srmPaths:
if sourcePFN.find(path) != -1:
dcapLocation = 0
if dcapLocation == 0:
optionsStr = ""
if options != None:
optionsStr = str(options)
dirname = os.path.dirname(targetPFN)
result = "#!/bin/sh\n"
result += "dccp %s %s %s" % (optionsStr, pnfsPfn2(sourcePFN), targetPFN)
result += \
"""
EXIT_STATUS=$?
echo "dccp exit status: $EXIT_STATUS"
if [[ $EXIT_STATUS != 0 ]]; then
echo "Non-zero dccp Exit status!!!"
echo "Cleaning up failed file:"
/bin/rm -fv %s
exit 60311
fi
""" % targetPFN
# //
# // Size Check
#//
result += \
"""
DEST_SIZE=`dcsize %s | cut -d" " -f1`
FILE_SIZE=`dcsize %s | cut -d" " -f1`
if [[ $DEST_SIZE == "" || $FILE_SIZE == "" ]]; then
echo "dcsize command is not available or produced an invalid result."
echo "Trying stat command:"
DEST_SIZE=`stat -c %s %s`
FILE_SIZE=`stat -c %s %s`
fi
if [[ $DEST_SIZE == "" || $FILE_SIZE == "" ]]; then
echo "stat command is not available or produced an invalid result."
echo "Trying ls command:"
DEST_SIZE=`/bin/ls -l %s | awk '{ print $5 }'`
FILE_SIZE=`/bin/ls -l %s | awk '{ print $5 }'`
fi
if [ $FILE_SIZE != $DEST_SIZE ]; then
echo "Source and destination files do not have same file size."
echo "Cleaning up failed file:"
/bin/rm -fv %s
exit 60311
fi
""" % (pnfsPfn2(targetPFN), pnfsPfn2(sourcePFN),
'%s', pnfsPfn2(targetPFN), '%s', pnfsPfn2(sourcePFN),
pnfsPfn2(targetPFN), pnfsPfn2(sourcePFN),
pnfsPfn2(targetPFN))
print "Executing:\n", result
return result
else:
for i in range(checkPathsCount):
if sourcePFN.find(checkPaths[i]) != -1:
pfnSplit = sourcePFN.split(checkPaths[i], 1)[1]
filePath = "%s%s" % (checkPathsReplace[i],pfnSplit)
original_size = os.stat(filePath)[6]
print "Local File Size is: %s" % original_size
result = "/bin/cp "
if options != None:
result += " %s " % options
result += " %s " % filePath
result += " %s " % targetPFN
result += "; sync; DEST_SIZE=`/bin/ls -l %s | /bin/awk '{print $5}'` ; if [ $DEST_SIZE ] && [ '%s' == $DEST_SIZE ]; then exit 0; else echo \"Error: Size Mismatch between local and SE\"; exit 60311 ; fi " % (targetPFN,original_size)
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
"""
method = self.storageMethod(pfnToRemove)
if method == 'srm':
return self.srmImpl.removeFile(pfnToRemove)
elif method == 'dccp':
pfnSplit = pfnToRemove.split("/11/store/", 1)[1]
filePath = "/pnfs/cms/WAX/11/store/%s" % pfnSplit
command = "rm -fv %s" %filePath
self.executeCommand(command)
else:
for i in range(checkPathsCount):
if pfnToRemove.find(checkPaths[i]) != -1:
pfnSplit = pfnToRemove.split(checkPaths[i], 1)[1]
pfnToRemove = "%s%s" % (checkPathsReplace[i],pfnSplit)
command = "/bin/rm %s" % pfnToRemove
self.executeCommand(command)
class LCGImplTest(unittest.TestCase):
def setUp(self):
self.LCGImpl = LCGImpl()
self.LCGImpl.setups = self.setups = "test setups"
self.timeoutOptions = self.LCGImpl.timeoutOptions = \
'--srm-timeout 600 --sendreceive-timeout 600 --connect-timeout 300'
@mock.patch('WMCore.Storage.Backends.LCGImpl.os')
def testInit(self, mock_os):
mock_os.environ.get.side_effect = lambda x: True if x == "GLITE_LOCATION" or x == "GRID_ENV_LOCATION" else False
mock_os.path.isfile.return_value = True
mock_os.path.normpath.side_effect = lambda x: "home" + x
mock_os.path.join.side_effect = lambda a, b: os.path.join("", b)
testLCGImpl = LCGImpl(True)
self.assertTrue(testLCGImpl.stageIn)
setups = []
setups += ['source home/../etc/profile.d/grid-env.sh; ']
setups += ['source home/grid-env.sh; ']
setups += ['date "+%Y-%m-%dT%H:%M:%S"; ']
self.assertTrue(testLCGImpl.stageIn)
for setup in setups:
self.assertIn(setup, testLCGImpl.setups)
def testCreateSourceName_startswithSlash(self):
self.assertEqual("file://name",
self.LCGImpl.createSourceName("protocol", "//name"))
@mock.patch('WMCore.Storage.Backends.LCGImpl.os.path')
def testCreateSourceName_isfile(self, mock_path):
mock_path.isfile.return_value = True
mock_path.abspath.return_value = "/some/path"
self.assertEqual("file:/some/path",
self.LCGImpl.createSourceName("protocol", "name"))
def testCreateSourceName_simple(self):
self.assertEqual("name",
self.LCGImpl.createSourceName("protocol", "name"))
def testCreateStageOutCommand_stageInFile(self):
self.LCGImpl.stageIn = True
result = self.LCGImpl.createStageOutCommand("srm://sourcePFN",
"file://targetPFN")
expectedResults = self.getStageOutCommandResults(
"srm://sourcePFN", "file://targetPFN", "srm://sourcePFN",
"//targetPFN", "/bin/rm -f //targetPFN", False)
self.assertEqual(expectedResults, result)
def testCreateStageOutCommand_stageISrmCvmfs(self):
self.LCGImpl.stageIn = True
result = self.LCGImpl.createStageOutCommand("srm://sourcePFN",
"srm://targetPFN",
options="cvmfs")
createRemoveFileCommandResult = "%s lcg-del -b -l -D srmv2 %s --vo cms srm://targetPFN" % \
(self.setups, self.timeoutOptions)
expectedResults = self.getStageOutCommandResults(
"srm://sourcePFN", "srm://targetPFN", "srm://sourcePFN",
"srm://targetPFN", createRemoveFileCommandResult, True)
self.assertEqual(expectedResults, result)
@mock.patch(
'WMCore.Storage.Backends.LCGImpl.StageOutImpl.createRemoveFileCommand')
def testCreateStageOutCommand_options(self, mock_createRemoveFileCommand):
mock_createRemoveFileCommand.return_value = "command"
result = self.LCGImpl.createStageOutCommand("file://sourcePFN",
"targetPFN",
options="test")
expectedResults = self.getStageOutCommandResults("file://sourcePFN",
"targetPFN",
"targetPFN",
"//sourcePFN",
"command",
False,
options="test")
self.assertEqual(expectedResults, result)
@mock.patch(
'WMCore.Storage.Backends.LCGImpl.StageOutImpl.createRemoveFileCommand')
def testCreateStageOutCommand_checksum(self, mock_createRemoveFileCommand):
mock_createRemoveFileCommand.return_value = "command"
result = self.LCGImpl.createStageOutCommand(
"file://sourcePFN", "targetPFN", checksums={"adler32": "32"})
expectedResults = self.getStageOutCommandResults("file://sourcePFN",
"targetPFN",
"targetPFN",
"//sourcePFN",
"command",
False,
checksums="00000032")
self.assertEqual(expectedResults, result)
def getStageOutCommandResults(self,
sourcePFN,
targetPFN,
remotePFN,
localPFN,
createRemoveFileCommandResult,
useCVMFS,
options=None,
checksums=None):
result = "#!/bin/sh\n"
copyCommand = "lcg-cp -b -D srmv2 --vo cms --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 300 --verbose"
if options != None:
copyCommand += " %s " % options
copyCommand += " %s " % sourcePFN
copyCommand += " %s 2> stageout.log" % targetPFN
if useCVMFS:
result += "(\n"
result += "echo Modifying PATH and LD_LIBRARY_PATH to remove /cvmfs/cms.cern.ch elements\n"
result += "export PATH=`echo $PATH | sed -e 's+:*/cvmfs/cms.cern.ch/[^:]*++'g`\n"
result += "export LD_LIBRARY_PATH=`echo $LD_LIBRARY_PATH | sed -e 's+:*/cvmfs/cms.cern.ch/[^:]*++'g`\n"
result += "echo Sourcing CVMFS UI setup script\n"
result += ". /cvmfs/grid.cern.ch/emi3ui-latest/etc/profile.d/setup-ui-example.sh\n"
result += copyCommand
else:
result += self.setups
result += copyCommand
result += """
EXIT_STATUS=$?
cat stageout.log
echo -e "\nlcg-cp exit status: $EXIT_STATUS"
if [[ $EXIT_STATUS != 0 ]]; then
echo "ERROR: lcg-cp exited with $EXIT_STATUS"
echo "Cleaning up failed file:"
%s
exit $EXIT_STATUS
fi
""" % createRemoveFileCommandResult
result += "FILE_SIZE=`stat -c %s"
result += " %s`\n" % localPFN
result += "echo \"Local File Size is: $FILE_SIZE\"\n"
if checksums:
checksumCommand = \
"""
if [[ "X$SRM_CHECKSUM" != "X" ]]; then
if [[ "$SRM_CHECKSUM" == "%s" ]]; then
exit 0
else
echo "ERROR: Checksum Mismatch between local and SE"
echo "Cleaning up failed file"
%s
exit 60311
fi
fi
exit 0
""" % (checksums, createRemoveFileCommandResult)
else:
checksumCommand = "exit 0"
metadataCheck = \
"""
LCG_OUTPUT=`lcg-ls -l -b -D srmv2 %s %s 2>/dev/null`
SRM_SIZE=`echo "$LCG_OUTPUT" | awk 'NR==1{print $5}'`
SRM_CHECKSUM=`echo "$LCG_OUTPUT" | sed -nr 's/^.*\s([a-f0-9]{8})\s*\([aA][dD][lL][eE][rR]32\)\s*$/\\1/p'`
echo "Remote File Size is: $SRM_SIZE"
echo "Remote Checksum is: $SRM_CHECKSUM"
if [[ $SRM_SIZE == $FILE_SIZE ]]; then
%s
else
echo $LCG_OUTPUT
echo "ERROR: Size Mismatch between local and SE. Cleaning up failed file..."
%s
exit 60311
fi
""" % (self.timeoutOptions, remotePFN, checksumCommand, createRemoveFileCommandResult)
result += metadataCheck
if useCVMFS:
result += ")\n"
return result
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.executeCommand')
def testRemoveFile(self, mock_executeCommand):
self.LCGImpl.removeFile("file")
mock_executeCommand.assert_called_with(
"%s lcg-del -b -l -D srmv2 %s --vo cms file" %
(self.setups, self.timeoutOptions))
def setUp(self):
self.LCGImpl = LCGImpl()
self.LCGImpl.setups = self.setups = "test setups"
self.timeoutOptions = self.LCGImpl.timeoutOptions = \
'--srm-timeout 600 --sendreceive-timeout 600 --connect-timeout 300'
class FNALImpl(StageOutImpl):
"""
_FNALImpl_
Implement interface for dcache xrootd command
"""
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
# Create and hold onto a srm implementation in case we need it
self.srmImpl = LCGImpl(stagein)
def storageMethod(self, PFN):
"""
Return xrdcp or srm
"""
method = 'srm'
if PFN.startswith("root://"):
method = 'xrdcp'
print "Using method:", method
return method
def createOutputDirectory(self, targetPFN):
"""
_createOutputDirectory_
Create a dir for the target pfn by translating it to
a /dcache or /lustre name and calling mkdir
we don't need to convert it, just mkdir.
"""
method = self.storageMethod(targetPFN)
if method == 'srm':
self.srmImpl.createOutputDirectory(targetPFN)
else:
pfnSplit = stripPrefixTOUNIX(targetPFN)
targetdir= os.path.dirname(pfnSplit)
command = "#!/bin/sh\n"
command += "if [ ! -e \"%s\" ]; then\n" % targetdir
command += " mkdir -p %s\n" % targetdir
command += "fi\n"
self.executeCommand(command)
def createSourceName(self, protocol, pfn):
"""
createTargetName
generate the target PFN
"""
method = self.storageMethod(pfn)
if method == 'srm':
return self.srmImpl.createSourceName(protocol, pfn)
elif method == 'xrdcp':
print "Translated PFN: %s\n To use xrdcp" % pfn
return pfn
def createStageOutCommand(self, sourcePFN, targetPFN, options = None, checksums = None):
"""
_createStageOutCommand_
Build a mkdir to generate the directory
"""
method = self.storageMethod(targetPFN)
sourceMethod = self.storageMethod(sourcePFN)
if method == 'srm' or sourceMethod == 'srm':
return self.srmImpl.createStageOutCommand(sourcePFN, targetPFN, options)
if getattr(self, 'stageIn', False):
return self.buildStageInCommand(sourcePFN, targetPFN, options)
if method == 'xrdcp':
original_size = os.stat(sourcePFN)[6]
print "Local File Size is: %s" % original_size
pfnWithoutChecksum = stripPrefixTOUNIX(targetPFN)
useChecksum = (checksums != None and checksums.has_key('adler32') and not self.stageIn)
if useChecksum:
checksums['adler32'] = "%08x" % int(checksums['adler32'], 16)
# non-functional in 3.3.1 xrootd clients due to bug
#result += "-ODeos.targetsize=$LOCAL_SIZE\&eos.checksum=%s " % checksums['adler32']
# therefor embed information into target URL
targetPFN += "\?eos.targetsize=%s\&eos.checksum=%s" % (original_size, checksums['adler32'])
print "Local File Checksum is: %s\"\n" % checksums['adler32']
# remove the file first befor it writes.
# there is eos bug when disk partition is full.
result = "/bin/rm -f %s" % pfnWithoutChecksum
result += "; /usr/bin/xrdcp -d 0 "
if options != None:
result += " %s " % options
result += " %s " % sourcePFN
result += " %s " % targetPFN
result += "; if [ $? -eq 0 ] ; then exit 0; else echo \"Error: xrdcp exited with $?\"; exit 60311 ; fi "
return result
def buildStageInCommand(self, sourcePFN, targetPFN, options = None):
"""
_buildStageInCommand_
Create normal dccp commad for staging in files.
"""
# Even if matched above, some paths are not lustre
for path in srmPaths:
if sourcePFN.find(path) != -1:
dcapLocation = 0
result = "/usr/bin/xrdcp -d 0 "
if options != None:
result += " %s " % options
result += " %s " % sourcePFN
result += " %s " % targetPFN
result += "; if [ $? -eq 0 ] ; then exit 0; else echo \"Error: xrdcp exited with $?\"; exit 60311 ; fi "
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
"""
method = self.storageMethod(pfnToRemove)
if method == 'srm':
return self.srmImpl.removeFile(pfnToRemove)
elif method == 'xrdcp':
command = "/bin/rm %s" % stripPrefixTOUNIX(pfnToRemove)
self.executeCommand(command)
class LCGImplTest(unittest.TestCase):
def setUp(self):
self.LCGImpl = LCGImpl()
self.LCGImpl.setups = self.setups = "test setups"
self.timeoutOptions = self.LCGImpl.timeoutOptions = \
'--srm-timeout 600 --sendreceive-timeout 600 --connect-timeout 300'
@mock.patch('WMCore.Storage.Backends.LCGImpl.os')
def testInit(self, mock_os):
mock_os.environ.get.side_effect = lambda x: True if x == "GLITE_LOCATION" or x == "GRID_ENV_LOCATION" else False
mock_os.path.isfile.return_value = True
mock_os.path.normpath.side_effect = lambda x: "home" + x
mock_os.path.join.side_effect = lambda a, b: os.path.join("", b)
testLCGImpl = LCGImpl(True)
self.assertTrue(testLCGImpl.stageIn)
setups = []
setups += ['source home/../etc/profile.d/grid-env.sh; ']
setups += ['source home/grid-env.sh; ']
setups += ['date "+%Y-%m-%dT%H:%M:%S"; ']
self.assertTrue(testLCGImpl.stageIn)
for setup in setups:
self.assertIn(setup, testLCGImpl.setups)
def testCreateSourceName_startswithSlash(self):
self.assertEqual("file://name", self.LCGImpl.createSourceName("protocol", "//name"))
@mock.patch('WMCore.Storage.Backends.LCGImpl.os.path')
def testCreateSourceName_isfile(self, mock_path):
mock_path.isfile.return_value = True
mock_path.abspath.return_value = "/some/path"
self.assertEqual("file:/some/path", self.LCGImpl.createSourceName("protocol", "name"))
def testCreateSourceName_simple(self):
self.assertEqual("name", self.LCGImpl.createSourceName("protocol", "name"))
def testCreateStageOutCommand_stageInFile(self):
self.LCGImpl.stageIn = True
result = self.LCGImpl.createStageOutCommand("srm://sourcePFN", "file://targetPFN")
expectedResults = self.getStageOutCommandResults("srm://sourcePFN", "file://targetPFN", "srm://sourcePFN",
"//targetPFN", "/bin/rm -f //targetPFN", False)
self.assertEqual(expectedResults, result)
def testCreateStageOutCommand_stageISrmCvmfs(self):
self.LCGImpl.stageIn = True
result = self.LCGImpl.createStageOutCommand("srm://sourcePFN", "srm://targetPFN", options="cvmfs")
createRemoveFileCommandResult = "%s lcg-del -b -l -D srmv2 %s --vo cms srm://targetPFN" % \
(self.setups, self.timeoutOptions)
expectedResults = self.getStageOutCommandResults("srm://sourcePFN", "srm://targetPFN", "srm://sourcePFN",
"srm://targetPFN", createRemoveFileCommandResult, True)
self.assertEqual(expectedResults, result)
@mock.patch('WMCore.Storage.Backends.LCGImpl.StageOutImpl.createRemoveFileCommand')
def testCreateStageOutCommand_options(self, mock_createRemoveFileCommand):
mock_createRemoveFileCommand.return_value = "command"
result = self.LCGImpl.createStageOutCommand("file://sourcePFN", "targetPFN", options="test")
expectedResults = self.getStageOutCommandResults("file://sourcePFN", "targetPFN",
"targetPFN", "//sourcePFN", "command", False, options="test")
self.assertEqual(expectedResults, result)
@mock.patch('WMCore.Storage.Backends.LCGImpl.StageOutImpl.createRemoveFileCommand')
def testCreateStageOutCommand_checksum(self, mock_createRemoveFileCommand):
mock_createRemoveFileCommand.return_value = "command"
result = self.LCGImpl.createStageOutCommand("file://sourcePFN", "targetPFN", checksums={"adler32": "32"})
expectedResults = self.getStageOutCommandResults("file://sourcePFN", "targetPFN",
"targetPFN", "//sourcePFN", "command", False,
checksums="00000032")
self.assertEqual(expectedResults, result)
def getStageOutCommandResults(self, sourcePFN, targetPFN, remotePFN, localPFN,
createRemoveFileCommandResult, useCVMFS, options=None, checksums=None):
result = "#!/bin/sh\n"
copyCommand = "lcg-cp -b -D srmv2 --vo cms --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 300 --verbose"
if options != None:
copyCommand += " %s " % options
copyCommand += " %s " % sourcePFN
copyCommand += " %s 2> stageout.log" % targetPFN
if useCVMFS:
result += "(\n"
result += "echo Modifying PATH and LD_LIBRARY_PATH to remove /cvmfs/cms.cern.ch elements\n"
result += "export PATH=`echo $PATH | sed -e 's+:*/cvmfs/cms.cern.ch/[^:]*++'g`\n"
result += "export LD_LIBRARY_PATH=`echo $LD_LIBRARY_PATH | sed -e 's+:*/cvmfs/cms.cern.ch/[^:]*++'g`\n"
result += "echo Sourcing CVMFS UI setup script\n"
result += ". /cvmfs/grid.cern.ch/emi3ui-latest/etc/profile.d/setup-ui-example.sh\n"
result += copyCommand
else:
result += self.setups
result += copyCommand
result += """
EXIT_STATUS=$?
cat stageout.log
echo -e "\nlcg-cp exit status: $EXIT_STATUS"
if [[ $EXIT_STATUS != 0 ]]; then
echo "Non-zero lcg-cp Exit status!!!"
echo "Cleaning up failed file:"
%s
exit 60311
fi
""" % createRemoveFileCommandResult
result += "FILE_SIZE=`stat -c %s"
result += " %s`\n" % localPFN
result += "echo \"Local File Size is: $FILE_SIZE\"\n"
if checksums:
checksumCommand = \
"""
if [[ "X$SRM_CHECKSUM" != "X" ]]; then
if [[ "$SRM_CHECKSUM" == "%s" ]]; then
exit 0
else
echo "ERROR: Checksum Mismatch between local and SE"
echo "Cleaning up failed file"
%s
exit 60311
fi
else
exit 0
fi
""" % (checksums, createRemoveFileCommandResult)
else:
checksumCommand = "exit 0"
metadataCheck = \
"""
LCG_OUTPUT=`lcg-ls -l -b -D srmv2 %s %s 2>/dev/null`
SRM_SIZE=`echo "$LCG_OUTPUT" | awk 'NR==1{print $5}'`
SRM_CHECKSUM=`echo "$LCG_OUTPUT" | sed -nr 's/^.*\s([a-f0-9]{8})\s*\([aA][dD][lL][eE][rR]32\)\s*$/\\1/p'`
echo "Remote File Size is: $SRM_SIZE"
echo "Remote Checksum is: $SRM_CHECKSUM"
if [[ $SRM_SIZE == $FILE_SIZE ]]; then
%s
else
echo $LCG_OUTPUT
echo "ERROR: Size Mismatch between local and SE. Cleaning up failed file..."
%s
exit 60311
fi
""" % (self.timeoutOptions, remotePFN, checksumCommand, createRemoveFileCommandResult)
result += metadataCheck
if useCVMFS:
result += ")\n"
return result
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.executeCommand')
def testRemoveFile(self, mock_executeCommand):
self.LCGImpl.removeFile("file")
mock_executeCommand.assert_called_with("%s lcg-del -b -l -D srmv2 %s --vo cms file" %
(self.setups, self.timeoutOptions))
def setUp(self):
self.LCGImpl = LCGImpl()
self.LCGImpl.setups = self.setups = "test setups"
self.timeoutOptions = self.LCGImpl.timeoutOptions = \
'--srm-timeout 600 --sendreceive-timeout 600 --connect-timeout 300'
class FNALImpl(StageOutImpl):
"""
_FNALImpl_
Implement interface for dcache door based dccp command
"""
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
# Create and hold onto a srm implementation in case we need it
self.srmImpl = LCGImpl(stagein)
def storageMethod(self, PFN):
"""
Figure out which paths use DCAP, lustre, or SRM for access
"""
method = 'dccp'
for path in checkPaths:
if PFN.find(path) != -1:
method = 'lustre'
# Over ride a few paths for srm
for path in srmPaths:
if PFN.find(path) != -1:
method = 'srm'
print "Using method:", method
return method
def createOutputDirectory(self, targetPFN):
"""
_createOutputDirectory_
Create a dir for the target pfn by translating it to
a /pnfs name and calling mkdir
PFN will be of the form:
dcap://cmsdca.fnal.gov:22125/pnfs/fnal.gov/usr/cms/WAX/11/store/blah
We need to convert that into /pnfs/cms/WAX/11/store/blah, as it
will be seen from the worker node
Unmerged PFN will be of the form:
/lustre/unmerged
we don't need to convert it, just mkdir.
"""
method = self.storageMethod(targetPFN)
if method == 'srm':
self.srmImpl.createOutputDirectory(targetPFN)
elif method == 'dccp':
# only create dir on remote storage
if targetPFN.find('/pnfs/') == -1:
return
pfnSplit = targetPFN.split("WAX/11/store/", 1)[1]
filePath = "/pnfs/cms/WAX/11/store/%s" % pfnSplit
directory = os.path.dirname(filePath)
command = "#!/bin/sh\n"
command += " . /opt/d-cache/dcap/bin/setenv-cmsprod.sh\n"
command += "if [ ! -e \"%s\" ]; then\n" % directory
command += " mkdir -p %s\n" % directory
command += "fi\n"
self.executeCommand(command)
else:
for i in range(checkPathsCount):
if targetPFN.find(checkPaths[i]) != -1:
pfnSplit = targetPFN.split(checkPaths[i], 1)[1]
filePath = "%s%s" % (checkPathsReplace[i],pfnSplit)
targetdir= os.path.dirname(filePath)
# checkdircmd="/bin/ls %s > /dev/null " % targetdir
# print "Check dir existence : %s" %checkdircmd
# checkdirexitCode = 0
# try:
# checkdirexitCode = self.executeCommand(checkdircmd)
# except Exception, ex:
# msg = "Warning: Exception while invoking command:\n"
# msg += "%s\n" % checkdircmd
# msg += "Exception: %s\n" % str(ex)
# msg += "Go on anyway..."
# print msg
# pass
# if checkdirexitCode != 0:
# mkdircmd = "/bin/mkdir -m 775 -p %s" % targetdir
# print "=> creating the dir : %s" %mkdircmd
# try:
# self.executeCommand(mkdircmd)
# except Exception, ex:
# msg = "Warning: Exception while invoking command:\n"
# msg += "%s\n" % mkdircmd
# msg += "Exception: %s\n" % str(ex)
# msg += "Go on anyway..."
# print msg
# pass
# else:
# print "=> dir already exists... do nothing."
command = "#!/bin/sh\n"
command += "if [ ! -e \"%s\" ]; then\n" % targetdir
command += " mkdir -p %s\n" % targetdir
command += "fi\n"
self.executeCommand(command)
def createSourceName(self, protocol, pfn):
"""
createTargetName
generate the target PFN
"""
if not pfn.startswith("srm"):
return pfn
method = self.storageMethod(pfn)
if method == 'srm':
return self.srmImpl.createSourceName(protocol, pfn)
elif method == 'dccp':
print "Translating PFN: %s\n To use dcache door" % pfn
dcacheDoor = commands.getoutput(
"/opt/d-cache/dcap/bin/setenv-cmsprod.sh; /opt/d-cache/dcap/bin/select_RdCapDoor.sh")
pfn = pfn.split("/store/")[1]
pfn = "%s%s" % (dcacheDoor, pfn)
print "Created Target PFN with dCache Door: ", pfn
else:
print "Translating PFN: %s\n To use lustre" % pfn
for i in range(checkPathsCount):
if pfn.find(checkPaths[i]) != -1:
pfnSplit = pfn.split(checkPaths[i], 1)[1]
pfn = "%s%s" % (checkPathsReplace[i],pfnSplit)
return pfn
def createStageOutCommand(self, sourcePFN, targetPFN, options = None, checksums = None):
"""
_createStageOutCommand_
Build a dccp command with a pnfs mkdir to generate the directory
"""
method = self.storageMethod(targetPFN)
sourceMethod = self.storageMethod(sourcePFN)
if method == 'srm' or sourceMethod == 'srm':
return self.srmImpl.createStageOutCommand(sourcePFN, targetPFN, options)
if getattr(self, 'stageIn', False):
return self.buildStageInCommand(sourcePFN, targetPFN, options)
if method == 'dccp':
optionsStr = ""
if options != None:
optionsStr = str(options)
dirname = os.path.dirname(targetPFN)
result = "#!/bin/sh\n"
result += ". /opt/d-cache/dcap/bin/setenv-cmsprod.sh\n"
result += "dccp -o 86400 -d 0 -X -role=cmsprod %s %s %s" % ( optionsStr, sourcePFN, targetPFN)
result += \
"""
EXIT_STATUS=$?
echo "dccp exit status: $EXIT_STATUS"
if [[ $EXIT_STATUS != 0 ]]; then
echo "Non-zero dccp Exit status!!!"
echo "Cleaning up failed file:"
/bin/rm -fv %s
exit 60311
fi
""" % pnfsPfn2(targetPFN)
# //
# // CRC check
#//
result += \
"""
/opt/d-cache/dcap/bin/check_dCachefilecksum.sh %s %s
EXIT_STATUS=$?
echo "CRC Check Exit status: $EXIT_STATUS"
if [[ $EXIT_STATUS != 0 ]]; then
echo "Non-zero CRC Check Exit status!!!"
echo "Cleaning up failed file:"
/bin/rm -fv %s
exit 60311
fi
""" % (pnfsPfn2(targetPFN), sourcePFN, pnfsPfn2(targetPFN))
print "Executing:\n", result
return result
else:
original_size = os.stat(sourcePFN)[6]
print "Local File Size is: %s" % original_size
result = "/bin/cp "
if options != None:
result += " %s " % options
result += " %s " % sourcePFN
result += " %s " % targetPFN
result += "; sync; DEST_SIZE=`/bin/ls -l %s | /bin/awk '{print $5}'` ; if [ $DEST_SIZE ] && [ '%s' == $DEST_SIZE ]; then exit 0; else echo \"Error: Size Mismatch between local and SE\"; exit 60311 ; fi " % (targetPFN,original_size)
return result
def buildStageInCommand(self, sourcePFN, targetPFN, options = None):
"""
_buildStageInCommand_
Create normal dccp commad for staging in files.
"""
dcapLocation = 0
for i in range(checkPathsCount):
if sourcePFN.find(checkPaths[i]) != -1:
dcapLocation = 1
# Even if matched above, some paths are not lustre
for path in srmPaths:
if sourcePFN.find(path) != -1:
dcapLocation = 0
if dcapLocation == 0:
optionsStr = ""
if options != None:
optionsStr = str(options)
dirname = os.path.dirname(targetPFN)
result = "#!/bin/sh\n"
result += "dccp %s %s %s" % (optionsStr, pnfsPfn2(sourcePFN), targetPFN)
result += \
"""
EXIT_STATUS=$?
echo "dccp exit status: $EXIT_STATUS"
if [[ $EXIT_STATUS != 0 ]]; then
echo "Non-zero dccp Exit status!!!"
echo "Cleaning up failed file:"
/bin/rm -fv %s
exit 60311
fi
""" % targetPFN
# //
# // Size Check
#//
result += \
"""
DEST_SIZE=`dcsize %s | cut -d" " -f1`
FILE_SIZE=`dcsize %s | cut -d" " -f1`
if [[ $DEST_SIZE == "" || $FILE_SIZE == "" ]]; then
echo "dcsize command is not available or produced an invalid result."
echo "Trying stat command:"
DEST_SIZE=`stat -c %s %s`
FILE_SIZE=`stat -c %s %s`
fi
if [[ $DEST_SIZE == "" || $FILE_SIZE == "" ]]; then
echo "stat command is not available or produced an invalid result."
echo "Trying ls command:"
DEST_SIZE=`/bin/ls -l %s | awk '{ print $5 }'`
FILE_SIZE=`/bin/ls -l %s | awk '{ print $5 }'`
fi
if [ $FILE_SIZE != $DEST_SIZE ]; then
echo "Source and destination files do not have same file size."
echo "Cleaning up failed file:"
/bin/rm -fv %s
exit 60311
fi
""" % (pnfsPfn2(targetPFN), pnfsPfn2(sourcePFN),
'%s', pnfsPfn2(targetPFN), '%s', pnfsPfn2(sourcePFN),
pnfsPfn2(targetPFN), pnfsPfn2(sourcePFN),
pnfsPfn2(targetPFN))
print "Executing:\n", result
return result
else:
for i in range(checkPathsCount):
if sourcePFN.find(checkPaths[i]) != -1:
pfnSplit = sourcePFN.split(checkPaths[i], 1)[1]
filePath = "%s%s" % (checkPathsReplace[i],pfnSplit)
original_size = os.stat(filePath)[6]
print "Local File Size is: %s" % original_size
result = "/bin/cp "
if options != None:
result += " %s " % options
result += " %s " % filePath
result += " %s " % targetPFN
result += "; sync; DEST_SIZE=`/bin/ls -l %s | /bin/awk '{print $5}'` ; if [ $DEST_SIZE ] && [ '%s' == $DEST_SIZE ]; then exit 0; else echo \"Error: Size Mismatch between local and SE\"; exit 60311 ; fi " % (targetPFN,original_size)
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
"""
method = self.storageMethod(pfnToRemove)
if method == 'srm':
return self.srmImpl.removeFile(pfnToRemove)
elif method == 'dccp':
pfnSplit = pfnToRemove.split("/11/store/", 1)[1]
filePath = "/pnfs/cms/WAX/11/store/%s" % pfnSplit
command = "rm -fv %s" %filePath
self.executeCommand(command)
else:
for i in range(checkPathsCount):
if pfnToRemove.find(checkPaths[i]) != -1:
pfnSplit = pfnToRemove.split(checkPaths[i], 1)[1]
pfnToRemove = "%s%s" % (checkPathsReplace[i],pfnSplit)
command = "/bin/rm %s" % pfnToRemove
self.executeCommand(command)