class FNALImpl(StageOutImpl): """ _FNALImpl_ Implement interface for dcache xrootd command """ def __init__(self, stagein=False): StageOutImpl.__init__(self, stagein) # Create and hold onto a srm implementation in case we need it self.srmImpl = LCGImpl(stagein) def storageMethod(self, PFN): """ Return xrdcp or srm """ method = 'local' # default if PFN.startswith("root://"): method = 'xrdcp' if PFN.startswith("srm://"): method = 'srm' print("Using method %s for PFN %s" % (method, PFN)) return method def createOutputDirectory(self, targetPFN): """ _createOutputDirectory_ Create a dir for the target pfn by translating it to a /dcache or /lustre name and calling mkdir we don't need to convert it, just mkdir. """ method = self.storageMethod(targetPFN) if method == 'xrdcp': # xrdcp autocreates parent directories return elif method == 'srm': self.srmImpl.createOutputDirectory(targetPFN) elif method == 'local': targetdir = os.path.dirname(targetPFN) command = "#!/bin/sh\n" command += "if [ ! -e \"%s\" ]; then\n" % targetdir command += " mkdir -p %s\n" % targetdir command += "fi\n" self.executeCommand(command) def createSourceName(self, protocol, pfn): """ _createSourceName_ generate the target PFN """ method = self.storageMethod(pfn) if method == 'srm': return self.srmImpl.createSourceName(protocol, pfn) return pfn def createStageOutCommand(self, sourcePFN, targetPFN, options=None, checksums=None): """ _createStageOutCommand_ Build a mkdir to generate the directory """ if getattr(self, 'stageIn', False): return self.buildStageInCommand(sourcePFN, targetPFN, options) method = self.storageMethod(targetPFN) sourceMethod = self.storageMethod(sourcePFN) if ((method == 'srm' and sourceMethod == 'xrdcp') or (method == 'xrdcp' and sourceMethod == 'srm')): print("Incompatible methods for source and target") print("\tSource: method %s for PFN %s" % (sourceMethod, sourcePFN)) print("\tTarget: method %s for PFN %s" % (method, targetPFN)) return 1 if method == 'srm' or sourceMethod == 'srm': return self.srmImpl.createStageOutCommand(sourcePFN, targetPFN, options) if method == 'xrdcp' or sourceMethod == 'xrdcp': original_size = os.stat(sourcePFN)[6] print("Local File Size is: %s" % original_size) useChecksum = (checksums != None and 'adler32' in checksums and not self.stageIn) if useChecksum: checksums['adler32'] = "%08x" % int(checksums['adler32'], 16) # non-functional in 3.3.1 xrootd clients due to bug # result += "-ODeos.targetsize=$LOCAL_SIZE\&eos.checksum=%s " % checksums['adler32'] # therefor embed information into target URL targetPFN += "\?eos.targetsize=%s\&eos.checksum=%s" % (original_size, checksums['adler32']) print("Local File Checksum is: %s\"\n" % checksums['adler32']) # always overwrite the output result = "xrdcp-old -d 0 -f " if options != None: result += " %s " % options result += " %s " % sourcePFN result += " %s " % targetPFN result += """ EXIT_STATUS=$? if [[ $EXIT_STATUS != 0 ]]; then echo "ERROR: xrdcp exited with $EXIT_STATUS" fi exit $EXIT_STATUS """ return result def buildStageInCommand(self, sourcePFN, targetPFN, options=None): """ _buildStageInCommand_ Create normal xrdcp commad for staging in files. """ result = "/usr/bin/xrdcp -d 0 " if options != None: result += " %s " % options result += " %s " % sourcePFN result += " %s " % targetPFN result += """ EXIT_STATUS=$? if [[ $EXIT_STATUS != 0 ]]; then echo "ERROR: xrdcp exited with $EXIT_STATUS" fi exit $EXIT_STATUS """ return result def removeFile(self, pfnToRemove): """ _removeFile_ CleanUp pfn provided """ method = self.storageMethod(pfnToRemove) if method == 'xrdcp': (_, host, path, _) = self.splitPFN(pfnToRemove) command = "xrd %s rm %s" % (host, path) print("Executing: %s" % command) self.executeCommand(command) elif method == 'srm': return self.srmImpl.removeFile(pfnToRemove) else: command = "/bin/rm %s" % stripPrefixTOUNIX(pfnToRemove) print("Executing: %s" % command) self.executeCommand(command)
class FNALImpl(StageOutImpl): """ _FNALImpl_ Implement interface for dcache door based dccp command """ def __init__(self, stagein=False): StageOutImpl.__init__(self, stagein) # Create and hold onto a srm implementation in case we need it self.srmImpl = LCGImpl(stagein) def storageMethod(self, PFN): """ Figure out which paths use DCAP, lustre, or SRM for access """ method = 'dccp' for path in checkPaths: if PFN.find(path) != -1: method = 'lustre' # Over ride a few paths for srm for path in srmPaths: if PFN.find(path) != -1: method = 'srm' print "Using method:", method return method def createOutputDirectory(self, targetPFN): """ _createOutputDirectory_ Create a dir for the target pfn by translating it to a /pnfs name and calling mkdir PFN will be of the form: dcap://cmsdca.fnal.gov:22125/pnfs/fnal.gov/usr/cms/WAX/11/store/blah We need to convert that into /pnfs/cms/WAX/11/store/blah, as it will be seen from the worker node Unmerged PFN will be of the form: /lustre/unmerged we don't need to convert it, just mkdir. """ method = self.storageMethod(targetPFN) if method == 'srm': self.srmImpl.createOutputDirectory(targetPFN) elif method == 'dccp': # only create dir on remote storage if targetPFN.find('/pnfs/') == -1: return pfnSplit = targetPFN.split("WAX/11/store/", 1)[1] filePath = "/pnfs/cms/WAX/11/store/%s" % pfnSplit directory = os.path.dirname(filePath) command = "#!/bin/sh\n" command += " . /opt/d-cache/dcap/bin/setenv-cmsprod.sh\n" command += "if [ ! -e \"%s\" ]; then\n" % directory command += " mkdir -p %s\n" % directory command += "fi\n" self.executeCommand(command) else: for i in range(checkPathsCount): if targetPFN.find(checkPaths[i]) != -1: pfnSplit = targetPFN.split(checkPaths[i], 1)[1] filePath = "%s%s" % (checkPathsReplace[i],pfnSplit) targetdir= os.path.dirname(filePath) # checkdircmd="/bin/ls %s > /dev/null " % targetdir # print "Check dir existence : %s" %checkdircmd # checkdirexitCode = 0 # try: # checkdirexitCode = self.executeCommand(checkdircmd) # except Exception, ex: # msg = "Warning: Exception while invoking command:\n" # msg += "%s\n" % checkdircmd # msg += "Exception: %s\n" % str(ex) # msg += "Go on anyway..." # print msg # pass # if checkdirexitCode != 0: # mkdircmd = "/bin/mkdir -m 775 -p %s" % targetdir # print "=> creating the dir : %s" %mkdircmd # try: # self.executeCommand(mkdircmd) # except Exception, ex: # msg = "Warning: Exception while invoking command:\n" # msg += "%s\n" % mkdircmd # msg += "Exception: %s\n" % str(ex) # msg += "Go on anyway..." # print msg # pass # else: # print "=> dir already exists... do nothing." command = "#!/bin/sh\n" command += "if [ ! -e \"%s\" ]; then\n" % targetdir command += " mkdir -p %s\n" % targetdir command += "fi\n" self.executeCommand(command) def createSourceName(self, protocol, pfn): """ createTargetName generate the target PFN """ if not pfn.startswith("srm"): return pfn method = self.storageMethod(pfn) if method == 'srm': return self.srmImpl.createSourceName(protocol, pfn) elif method == 'dccp': print "Translating PFN: %s\n To use dcache door" % pfn dcacheDoor = commands.getoutput( "/opt/d-cache/dcap/bin/setenv-cmsprod.sh; /opt/d-cache/dcap/bin/select_RdCapDoor.sh") pfn = pfn.split("/store/")[1] pfn = "%s%s" % (dcacheDoor, pfn) print "Created Target PFN with dCache Door: ", pfn else: print "Translating PFN: %s\n To use lustre" % pfn for i in range(checkPathsCount): if pfn.find(checkPaths[i]) != -1: pfnSplit = pfn.split(checkPaths[i], 1)[1] pfn = "%s%s" % (checkPathsReplace[i],pfnSplit) return pfn def createStageOutCommand(self, sourcePFN, targetPFN, options = None, checksums = None): """ _createStageOutCommand_ Build a dccp command with a pnfs mkdir to generate the directory """ method = self.storageMethod(targetPFN) sourceMethod = self.storageMethod(sourcePFN) if method == 'srm' or sourceMethod == 'srm': return self.srmImpl.createStageOutCommand(sourcePFN, targetPFN, options) if getattr(self, 'stageIn', False): return self.buildStageInCommand(sourcePFN, targetPFN, options) if method == 'dccp': optionsStr = "" if options != None: optionsStr = str(options) dirname = os.path.dirname(targetPFN) result = "#!/bin/sh\n" result += ". /opt/d-cache/dcap/bin/setenv-cmsprod.sh\n" result += "dccp -o 86400 -d 0 -X -role=cmsprod %s %s %s" % ( optionsStr, sourcePFN, targetPFN) result += \ """ EXIT_STATUS=$? echo "dccp exit status: $EXIT_STATUS" if [[ $EXIT_STATUS != 0 ]]; then echo "Non-zero dccp Exit status!!!" echo "Cleaning up failed file:" /bin/rm -fv %s exit 60311 fi """ % pnfsPfn2(targetPFN) # // # // CRC check #// result += \ """ /opt/d-cache/dcap/bin/check_dCachefilecksum.sh %s %s EXIT_STATUS=$? echo "CRC Check Exit status: $EXIT_STATUS" if [[ $EXIT_STATUS != 0 ]]; then echo "Non-zero CRC Check Exit status!!!" echo "Cleaning up failed file:" /bin/rm -fv %s exit 60311 fi """ % (pnfsPfn2(targetPFN), sourcePFN, pnfsPfn2(targetPFN)) print "Executing:\n", result return result else: original_size = os.stat(sourcePFN)[6] print "Local File Size is: %s" % original_size result = "/bin/cp " if options != None: result += " %s " % options result += " %s " % sourcePFN result += " %s " % targetPFN result += "; sync; DEST_SIZE=`/bin/ls -l %s | /bin/awk '{print $5}'` ; if [ $DEST_SIZE ] && [ '%s' == $DEST_SIZE ]; then exit 0; else echo \"Error: Size Mismatch between local and SE\"; exit 60311 ; fi " % (targetPFN,original_size) return result def buildStageInCommand(self, sourcePFN, targetPFN, options = None): """ _buildStageInCommand_ Create normal dccp commad for staging in files. """ dcapLocation = 0 for i in range(checkPathsCount): if sourcePFN.find(checkPaths[i]) != -1: dcapLocation = 1 # Even if matched above, some paths are not lustre for path in srmPaths: if sourcePFN.find(path) != -1: dcapLocation = 0 if dcapLocation == 0: optionsStr = "" if options != None: optionsStr = str(options) dirname = os.path.dirname(targetPFN) result = "#!/bin/sh\n" result += "dccp %s %s %s" % (optionsStr, pnfsPfn2(sourcePFN), targetPFN) result += \ """ EXIT_STATUS=$? echo "dccp exit status: $EXIT_STATUS" if [[ $EXIT_STATUS != 0 ]]; then echo "Non-zero dccp Exit status!!!" echo "Cleaning up failed file:" /bin/rm -fv %s exit 60311 fi """ % targetPFN # // # // Size Check #// result += \ """ DEST_SIZE=`dcsize %s | cut -d" " -f1` FILE_SIZE=`dcsize %s | cut -d" " -f1` if [[ $DEST_SIZE == "" || $FILE_SIZE == "" ]]; then echo "dcsize command is not available or produced an invalid result." echo "Trying stat command:" DEST_SIZE=`stat -c %s %s` FILE_SIZE=`stat -c %s %s` fi if [[ $DEST_SIZE == "" || $FILE_SIZE == "" ]]; then echo "stat command is not available or produced an invalid result." echo "Trying ls command:" DEST_SIZE=`/bin/ls -l %s | awk '{ print $5 }'` FILE_SIZE=`/bin/ls -l %s | awk '{ print $5 }'` fi if [ $FILE_SIZE != $DEST_SIZE ]; then echo "Source and destination files do not have same file size." echo "Cleaning up failed file:" /bin/rm -fv %s exit 60311 fi """ % (pnfsPfn2(targetPFN), pnfsPfn2(sourcePFN), '%s', pnfsPfn2(targetPFN), '%s', pnfsPfn2(sourcePFN), pnfsPfn2(targetPFN), pnfsPfn2(sourcePFN), pnfsPfn2(targetPFN)) print "Executing:\n", result return result else: for i in range(checkPathsCount): if sourcePFN.find(checkPaths[i]) != -1: pfnSplit = sourcePFN.split(checkPaths[i], 1)[1] filePath = "%s%s" % (checkPathsReplace[i],pfnSplit) original_size = os.stat(filePath)[6] print "Local File Size is: %s" % original_size result = "/bin/cp " if options != None: result += " %s " % options result += " %s " % filePath result += " %s " % targetPFN result += "; sync; DEST_SIZE=`/bin/ls -l %s | /bin/awk '{print $5}'` ; if [ $DEST_SIZE ] && [ '%s' == $DEST_SIZE ]; then exit 0; else echo \"Error: Size Mismatch between local and SE\"; exit 60311 ; fi " % (targetPFN,original_size) return result def removeFile(self, pfnToRemove): """ _removeFile_ CleanUp pfn provided """ method = self.storageMethod(pfnToRemove) if method == 'srm': return self.srmImpl.removeFile(pfnToRemove) elif method == 'dccp': pfnSplit = pfnToRemove.split("/11/store/", 1)[1] filePath = "/pnfs/cms/WAX/11/store/%s" % pfnSplit command = "rm -fv %s" %filePath self.executeCommand(command) else: for i in range(checkPathsCount): if pfnToRemove.find(checkPaths[i]) != -1: pfnSplit = pfnToRemove.split(checkPaths[i], 1)[1] pfnToRemove = "%s%s" % (checkPathsReplace[i],pfnSplit) command = "/bin/rm %s" % pfnToRemove self.executeCommand(command)
class LCGImplTest(unittest.TestCase): def setUp(self): self.LCGImpl = LCGImpl() self.LCGImpl.setups = self.setups = "test setups" self.timeoutOptions = self.LCGImpl.timeoutOptions = \ '--srm-timeout 600 --sendreceive-timeout 600 --connect-timeout 300' @mock.patch('WMCore.Storage.Backends.LCGImpl.os') def testInit(self, mock_os): mock_os.environ.get.side_effect = lambda x: True if x == "GLITE_LOCATION" or x == "GRID_ENV_LOCATION" else False mock_os.path.isfile.return_value = True mock_os.path.normpath.side_effect = lambda x: "home" + x mock_os.path.join.side_effect = lambda a, b: os.path.join("", b) testLCGImpl = LCGImpl(True) self.assertTrue(testLCGImpl.stageIn) setups = [] setups += ['source home/../etc/profile.d/grid-env.sh; '] setups += ['source home/grid-env.sh; '] setups += ['date "+%Y-%m-%dT%H:%M:%S"; '] self.assertTrue(testLCGImpl.stageIn) for setup in setups: self.assertIn(setup, testLCGImpl.setups) def testCreateSourceName_startswithSlash(self): self.assertEqual("file://name", self.LCGImpl.createSourceName("protocol", "//name")) @mock.patch('WMCore.Storage.Backends.LCGImpl.os.path') def testCreateSourceName_isfile(self, mock_path): mock_path.isfile.return_value = True mock_path.abspath.return_value = "/some/path" self.assertEqual("file:/some/path", self.LCGImpl.createSourceName("protocol", "name")) def testCreateSourceName_simple(self): self.assertEqual("name", self.LCGImpl.createSourceName("protocol", "name")) def testCreateStageOutCommand_stageInFile(self): self.LCGImpl.stageIn = True result = self.LCGImpl.createStageOutCommand("srm://sourcePFN", "file://targetPFN") expectedResults = self.getStageOutCommandResults( "srm://sourcePFN", "file://targetPFN", "srm://sourcePFN", "//targetPFN", "/bin/rm -f //targetPFN", False) self.assertEqual(expectedResults, result) def testCreateStageOutCommand_stageISrmCvmfs(self): self.LCGImpl.stageIn = True result = self.LCGImpl.createStageOutCommand("srm://sourcePFN", "srm://targetPFN", options="cvmfs") createRemoveFileCommandResult = "%s lcg-del -b -l -D srmv2 %s --vo cms srm://targetPFN" % \ (self.setups, self.timeoutOptions) expectedResults = self.getStageOutCommandResults( "srm://sourcePFN", "srm://targetPFN", "srm://sourcePFN", "srm://targetPFN", createRemoveFileCommandResult, True) self.assertEqual(expectedResults, result) @mock.patch( 'WMCore.Storage.Backends.LCGImpl.StageOutImpl.createRemoveFileCommand') def testCreateStageOutCommand_options(self, mock_createRemoveFileCommand): mock_createRemoveFileCommand.return_value = "command" result = self.LCGImpl.createStageOutCommand("file://sourcePFN", "targetPFN", options="test") expectedResults = self.getStageOutCommandResults("file://sourcePFN", "targetPFN", "targetPFN", "//sourcePFN", "command", False, options="test") self.assertEqual(expectedResults, result) @mock.patch( 'WMCore.Storage.Backends.LCGImpl.StageOutImpl.createRemoveFileCommand') def testCreateStageOutCommand_checksum(self, mock_createRemoveFileCommand): mock_createRemoveFileCommand.return_value = "command" result = self.LCGImpl.createStageOutCommand( "file://sourcePFN", "targetPFN", checksums={"adler32": "32"}) expectedResults = self.getStageOutCommandResults("file://sourcePFN", "targetPFN", "targetPFN", "//sourcePFN", "command", False, checksums="00000032") self.assertEqual(expectedResults, result) def getStageOutCommandResults(self, sourcePFN, targetPFN, remotePFN, localPFN, createRemoveFileCommandResult, useCVMFS, options=None, checksums=None): result = "#!/bin/sh\n" copyCommand = "lcg-cp -b -D srmv2 --vo cms --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 300 --verbose" if options != None: copyCommand += " %s " % options copyCommand += " %s " % sourcePFN copyCommand += " %s 2> stageout.log" % targetPFN if useCVMFS: result += "(\n" result += "echo Modifying PATH and LD_LIBRARY_PATH to remove /cvmfs/cms.cern.ch elements\n" result += "export PATH=`echo $PATH | sed -e 's+:*/cvmfs/cms.cern.ch/[^:]*++'g`\n" result += "export LD_LIBRARY_PATH=`echo $LD_LIBRARY_PATH | sed -e 's+:*/cvmfs/cms.cern.ch/[^:]*++'g`\n" result += "echo Sourcing CVMFS UI setup script\n" result += ". /cvmfs/grid.cern.ch/emi3ui-latest/etc/profile.d/setup-ui-example.sh\n" result += copyCommand else: result += self.setups result += copyCommand result += """ EXIT_STATUS=$? cat stageout.log echo -e "\nlcg-cp exit status: $EXIT_STATUS" if [[ $EXIT_STATUS != 0 ]]; then echo "ERROR: lcg-cp exited with $EXIT_STATUS" echo "Cleaning up failed file:" %s exit $EXIT_STATUS fi """ % createRemoveFileCommandResult result += "FILE_SIZE=`stat -c %s" result += " %s`\n" % localPFN result += "echo \"Local File Size is: $FILE_SIZE\"\n" if checksums: checksumCommand = \ """ if [[ "X$SRM_CHECKSUM" != "X" ]]; then if [[ "$SRM_CHECKSUM" == "%s" ]]; then exit 0 else echo "ERROR: Checksum Mismatch between local and SE" echo "Cleaning up failed file" %s exit 60311 fi fi exit 0 """ % (checksums, createRemoveFileCommandResult) else: checksumCommand = "exit 0" metadataCheck = \ """ LCG_OUTPUT=`lcg-ls -l -b -D srmv2 %s %s 2>/dev/null` SRM_SIZE=`echo "$LCG_OUTPUT" | awk 'NR==1{print $5}'` SRM_CHECKSUM=`echo "$LCG_OUTPUT" | sed -nr 's/^.*\s([a-f0-9]{8})\s*\([aA][dD][lL][eE][rR]32\)\s*$/\\1/p'` echo "Remote File Size is: $SRM_SIZE" echo "Remote Checksum is: $SRM_CHECKSUM" if [[ $SRM_SIZE == $FILE_SIZE ]]; then %s else echo $LCG_OUTPUT echo "ERROR: Size Mismatch between local and SE. Cleaning up failed file..." %s exit 60311 fi """ % (self.timeoutOptions, remotePFN, checksumCommand, createRemoveFileCommandResult) result += metadataCheck if useCVMFS: result += ")\n" return result @mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.executeCommand') def testRemoveFile(self, mock_executeCommand): self.LCGImpl.removeFile("file") mock_executeCommand.assert_called_with( "%s lcg-del -b -l -D srmv2 %s --vo cms file" % (self.setups, self.timeoutOptions))
class FNALImpl(StageOutImpl): """ _FNALImpl_ Implement interface for dcache xrootd command """ def __init__(self, stagein=False): StageOutImpl.__init__(self, stagein) # Create and hold onto a srm implementation in case we need it self.srmImpl = LCGImpl(stagein) def storageMethod(self, PFN): """ Return xrdcp or srm """ method = 'srm' if PFN.startswith("root://"): method = 'xrdcp' print "Using method:", method return method def createOutputDirectory(self, targetPFN): """ _createOutputDirectory_ Create a dir for the target pfn by translating it to a /dcache or /lustre name and calling mkdir we don't need to convert it, just mkdir. """ method = self.storageMethod(targetPFN) if method == 'srm': self.srmImpl.createOutputDirectory(targetPFN) else: pfnSplit = stripPrefixTOUNIX(targetPFN) targetdir= os.path.dirname(pfnSplit) command = "#!/bin/sh\n" command += "if [ ! -e \"%s\" ]; then\n" % targetdir command += " mkdir -p %s\n" % targetdir command += "fi\n" self.executeCommand(command) def createSourceName(self, protocol, pfn): """ createTargetName generate the target PFN """ method = self.storageMethod(pfn) if method == 'srm': return self.srmImpl.createSourceName(protocol, pfn) elif method == 'xrdcp': print "Translated PFN: %s\n To use xrdcp" % pfn return pfn def createStageOutCommand(self, sourcePFN, targetPFN, options = None, checksums = None): """ _createStageOutCommand_ Build a mkdir to generate the directory """ method = self.storageMethod(targetPFN) sourceMethod = self.storageMethod(sourcePFN) if method == 'srm' or sourceMethod == 'srm': return self.srmImpl.createStageOutCommand(sourcePFN, targetPFN, options) if getattr(self, 'stageIn', False): return self.buildStageInCommand(sourcePFN, targetPFN, options) if method == 'xrdcp': original_size = os.stat(sourcePFN)[6] print "Local File Size is: %s" % original_size pfnWithoutChecksum = stripPrefixTOUNIX(targetPFN) useChecksum = (checksums != None and checksums.has_key('adler32') and not self.stageIn) if useChecksum: checksums['adler32'] = "%08x" % int(checksums['adler32'], 16) # non-functional in 3.3.1 xrootd clients due to bug #result += "-ODeos.targetsize=$LOCAL_SIZE\&eos.checksum=%s " % checksums['adler32'] # therefor embed information into target URL targetPFN += "\?eos.targetsize=%s\&eos.checksum=%s" % (original_size, checksums['adler32']) print "Local File Checksum is: %s\"\n" % checksums['adler32'] # remove the file first befor it writes. # there is eos bug when disk partition is full. result = "/bin/rm -f %s" % pfnWithoutChecksum result += "; /usr/bin/xrdcp -d 0 " if options != None: result += " %s " % options result += " %s " % sourcePFN result += " %s " % targetPFN result += "; if [ $? -eq 0 ] ; then exit 0; else echo \"Error: xrdcp exited with $?\"; exit 60311 ; fi " return result def buildStageInCommand(self, sourcePFN, targetPFN, options = None): """ _buildStageInCommand_ Create normal dccp commad for staging in files. """ # Even if matched above, some paths are not lustre for path in srmPaths: if sourcePFN.find(path) != -1: dcapLocation = 0 result = "/usr/bin/xrdcp -d 0 " if options != None: result += " %s " % options result += " %s " % sourcePFN result += " %s " % targetPFN result += "; if [ $? -eq 0 ] ; then exit 0; else echo \"Error: xrdcp exited with $?\"; exit 60311 ; fi " return result def removeFile(self, pfnToRemove): """ _removeFile_ CleanUp pfn provided """ method = self.storageMethod(pfnToRemove) if method == 'srm': return self.srmImpl.removeFile(pfnToRemove) elif method == 'xrdcp': command = "/bin/rm %s" % stripPrefixTOUNIX(pfnToRemove) self.executeCommand(command)
class LCGImplTest(unittest.TestCase): def setUp(self): self.LCGImpl = LCGImpl() self.LCGImpl.setups = self.setups = "test setups" self.timeoutOptions = self.LCGImpl.timeoutOptions = \ '--srm-timeout 600 --sendreceive-timeout 600 --connect-timeout 300' @mock.patch('WMCore.Storage.Backends.LCGImpl.os') def testInit(self, mock_os): mock_os.environ.get.side_effect = lambda x: True if x == "GLITE_LOCATION" or x == "GRID_ENV_LOCATION" else False mock_os.path.isfile.return_value = True mock_os.path.normpath.side_effect = lambda x: "home" + x mock_os.path.join.side_effect = lambda a, b: os.path.join("", b) testLCGImpl = LCGImpl(True) self.assertTrue(testLCGImpl.stageIn) setups = [] setups += ['source home/../etc/profile.d/grid-env.sh; '] setups += ['source home/grid-env.sh; '] setups += ['date "+%Y-%m-%dT%H:%M:%S"; '] self.assertTrue(testLCGImpl.stageIn) for setup in setups: self.assertIn(setup, testLCGImpl.setups) def testCreateSourceName_startswithSlash(self): self.assertEqual("file://name", self.LCGImpl.createSourceName("protocol", "//name")) @mock.patch('WMCore.Storage.Backends.LCGImpl.os.path') def testCreateSourceName_isfile(self, mock_path): mock_path.isfile.return_value = True mock_path.abspath.return_value = "/some/path" self.assertEqual("file:/some/path", self.LCGImpl.createSourceName("protocol", "name")) def testCreateSourceName_simple(self): self.assertEqual("name", self.LCGImpl.createSourceName("protocol", "name")) def testCreateStageOutCommand_stageInFile(self): self.LCGImpl.stageIn = True result = self.LCGImpl.createStageOutCommand("srm://sourcePFN", "file://targetPFN") expectedResults = self.getStageOutCommandResults("srm://sourcePFN", "file://targetPFN", "srm://sourcePFN", "//targetPFN", "/bin/rm -f //targetPFN", False) self.assertEqual(expectedResults, result) def testCreateStageOutCommand_stageISrmCvmfs(self): self.LCGImpl.stageIn = True result = self.LCGImpl.createStageOutCommand("srm://sourcePFN", "srm://targetPFN", options="cvmfs") createRemoveFileCommandResult = "%s lcg-del -b -l -D srmv2 %s --vo cms srm://targetPFN" % \ (self.setups, self.timeoutOptions) expectedResults = self.getStageOutCommandResults("srm://sourcePFN", "srm://targetPFN", "srm://sourcePFN", "srm://targetPFN", createRemoveFileCommandResult, True) self.assertEqual(expectedResults, result) @mock.patch('WMCore.Storage.Backends.LCGImpl.StageOutImpl.createRemoveFileCommand') def testCreateStageOutCommand_options(self, mock_createRemoveFileCommand): mock_createRemoveFileCommand.return_value = "command" result = self.LCGImpl.createStageOutCommand("file://sourcePFN", "targetPFN", options="test") expectedResults = self.getStageOutCommandResults("file://sourcePFN", "targetPFN", "targetPFN", "//sourcePFN", "command", False, options="test") self.assertEqual(expectedResults, result) @mock.patch('WMCore.Storage.Backends.LCGImpl.StageOutImpl.createRemoveFileCommand') def testCreateStageOutCommand_checksum(self, mock_createRemoveFileCommand): mock_createRemoveFileCommand.return_value = "command" result = self.LCGImpl.createStageOutCommand("file://sourcePFN", "targetPFN", checksums={"adler32": "32"}) expectedResults = self.getStageOutCommandResults("file://sourcePFN", "targetPFN", "targetPFN", "//sourcePFN", "command", False, checksums="00000032") self.assertEqual(expectedResults, result) def getStageOutCommandResults(self, sourcePFN, targetPFN, remotePFN, localPFN, createRemoveFileCommandResult, useCVMFS, options=None, checksums=None): result = "#!/bin/sh\n" copyCommand = "lcg-cp -b -D srmv2 --vo cms --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 300 --verbose" if options != None: copyCommand += " %s " % options copyCommand += " %s " % sourcePFN copyCommand += " %s 2> stageout.log" % targetPFN if useCVMFS: result += "(\n" result += "echo Modifying PATH and LD_LIBRARY_PATH to remove /cvmfs/cms.cern.ch elements\n" result += "export PATH=`echo $PATH | sed -e 's+:*/cvmfs/cms.cern.ch/[^:]*++'g`\n" result += "export LD_LIBRARY_PATH=`echo $LD_LIBRARY_PATH | sed -e 's+:*/cvmfs/cms.cern.ch/[^:]*++'g`\n" result += "echo Sourcing CVMFS UI setup script\n" result += ". /cvmfs/grid.cern.ch/emi3ui-latest/etc/profile.d/setup-ui-example.sh\n" result += copyCommand else: result += self.setups result += copyCommand result += """ EXIT_STATUS=$? cat stageout.log echo -e "\nlcg-cp exit status: $EXIT_STATUS" if [[ $EXIT_STATUS != 0 ]]; then echo "Non-zero lcg-cp Exit status!!!" echo "Cleaning up failed file:" %s exit 60311 fi """ % createRemoveFileCommandResult result += "FILE_SIZE=`stat -c %s" result += " %s`\n" % localPFN result += "echo \"Local File Size is: $FILE_SIZE\"\n" if checksums: checksumCommand = \ """ if [[ "X$SRM_CHECKSUM" != "X" ]]; then if [[ "$SRM_CHECKSUM" == "%s" ]]; then exit 0 else echo "ERROR: Checksum Mismatch between local and SE" echo "Cleaning up failed file" %s exit 60311 fi else exit 0 fi """ % (checksums, createRemoveFileCommandResult) else: checksumCommand = "exit 0" metadataCheck = \ """ LCG_OUTPUT=`lcg-ls -l -b -D srmv2 %s %s 2>/dev/null` SRM_SIZE=`echo "$LCG_OUTPUT" | awk 'NR==1{print $5}'` SRM_CHECKSUM=`echo "$LCG_OUTPUT" | sed -nr 's/^.*\s([a-f0-9]{8})\s*\([aA][dD][lL][eE][rR]32\)\s*$/\\1/p'` echo "Remote File Size is: $SRM_SIZE" echo "Remote Checksum is: $SRM_CHECKSUM" if [[ $SRM_SIZE == $FILE_SIZE ]]; then %s else echo $LCG_OUTPUT echo "ERROR: Size Mismatch between local and SE. Cleaning up failed file..." %s exit 60311 fi """ % (self.timeoutOptions, remotePFN, checksumCommand, createRemoveFileCommandResult) result += metadataCheck if useCVMFS: result += ")\n" return result @mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.executeCommand') def testRemoveFile(self, mock_executeCommand): self.LCGImpl.removeFile("file") mock_executeCommand.assert_called_with("%s lcg-del -b -l -D srmv2 %s --vo cms file" % (self.setups, self.timeoutOptions))