Exemplo n.º 1
0
 def testInit(self, mock_os):
     mock_os.environ.get.side_effect = lambda x: True if x == "GLITE_LOCATION" or x == "GRID_ENV_LOCATION" else False
     mock_os.path.isfile.return_value = True
     mock_os.path.normpath.side_effect = lambda x: "home" + x
     mock_os.path.join.side_effect = lambda a, b: os.path.join("", b)
     testLCGImpl = LCGImpl(True)
     self.assertTrue(testLCGImpl.stageIn)
     setups = []
     setups += ['source home/../etc/profile.d/grid-env.sh; ']
     setups += ['source home/grid-env.sh; ']
     setups += ['date "+%Y-%m-%dT%H:%M:%S"; ']
     self.assertTrue(testLCGImpl.stageIn)
     for setup in setups:
         self.assertIn(setup, testLCGImpl.setups)
Exemplo n.º 2
0
    def __init__(self, stagein=False):

        StageOutImpl.__init__(self, stagein)

        # Create and hold onto a srm implementation in case we need it
        self.srmImpl = LCGImpl(stagein)
Exemplo n.º 3
0
class FNALImpl(StageOutImpl):
    """
    _FNALImpl_

    Implement interface for dcache xrootd command

    """

    def __init__(self, stagein=False):

        StageOutImpl.__init__(self, stagein)

        # Create and hold onto a srm implementation in case we need it
        self.srmImpl = LCGImpl(stagein)

    def storageMethod(self, PFN):
        """
        Return xrdcp or srm
        """

        method = 'local'  # default
        if PFN.startswith("root://"):
            method = 'xrdcp'
        if PFN.startswith("srm://"):
            method = 'srm'
        print("Using method %s for PFN %s" % (method, PFN))
        return method

    def createOutputDirectory(self, targetPFN):
        """
        _createOutputDirectory_

        Create a dir for the target pfn by translating it to
        a /dcache or /lustre name and calling mkdir
        we don't need to convert it, just mkdir.
        """
        method = self.storageMethod(targetPFN)

        if method == 'xrdcp':  # xrdcp autocreates parent directories
            return
        elif method == 'srm':
            self.srmImpl.createOutputDirectory(targetPFN)
        elif method == 'local':
            targetdir = os.path.dirname(targetPFN)
            command = "#!/bin/sh\n"
            command += "if [ ! -e \"%s\" ]; then\n" % targetdir
            command += " mkdir -p %s\n" % targetdir
            command += "fi\n"
            self.executeCommand(command)

    def createSourceName(self, protocol, pfn):
        """
        _createSourceName_

        generate the target PFN
        """
        method = self.storageMethod(pfn)

        if method == 'srm':
            return self.srmImpl.createSourceName(protocol, pfn)
        return pfn

    def createStageOutCommand(self, sourcePFN, targetPFN, options=None, checksums=None):
        """
        _createStageOutCommand_

        Build a mkdir to generate the directory
        """

        if getattr(self, 'stageIn', False):
            return self.buildStageInCommand(sourcePFN, targetPFN, options)

        method = self.storageMethod(targetPFN)
        sourceMethod = self.storageMethod(sourcePFN)

        if ((method == 'srm' and sourceMethod == 'xrdcp') or
                (method == 'xrdcp' and sourceMethod == 'srm')):
            print("Incompatible methods for source and target")
            print("\tSource: method %s for PFN %s" % (sourceMethod, sourcePFN))
            print("\tTarget: method %s for PFN %s" % (method, targetPFN))
            return 1

        if method == 'srm' or sourceMethod == 'srm':
            return self.srmImpl.createStageOutCommand(sourcePFN, targetPFN, options)

        if method == 'xrdcp' or sourceMethod == 'xrdcp':
            original_size = os.stat(sourcePFN)[6]
            print("Local File Size is: %s" % original_size)

            useChecksum = (checksums != None and 'adler32' in checksums and not self.stageIn)
            if useChecksum:
                checksums['adler32'] = "%08x" % int(checksums['adler32'], 16)
                # non-functional in 3.3.1 xrootd clients due to bug
                # result += "-ODeos.targetsize=$LOCAL_SIZE\&eos.checksum=%s " % checksums['adler32']

                # therefor embed information into target URL
                targetPFN += "\?eos.targetsize=%s\&eos.checksum=%s" % (original_size, checksums['adler32'])
                print("Local File Checksum is: %s\"\n" % checksums['adler32'])

            # always overwrite the output

            result = "xrdcp-old -d 0 -f "
            if options != None:
                result += " %s " % options
            result += " %s " % sourcePFN
            result += " %s " % targetPFN
            result += """
            EXIT_STATUS=$?
            if [[ $EXIT_STATUS != 0 ]]; then
                echo "ERROR: xrdcp exited with $EXIT_STATUS"
            fi
            exit $EXIT_STATUS
            """
            return result

    def buildStageInCommand(self, sourcePFN, targetPFN, options=None):
        """
        _buildStageInCommand_

        Create normal xrdcp commad for staging in files.
        """
        result = "/usr/bin/xrdcp -d 0 "
        if options != None:
            result += " %s " % options
        result += " %s " % sourcePFN
        result += " %s " % targetPFN
        result += """
        EXIT_STATUS=$?
        if [[ $EXIT_STATUS != 0 ]]; then
            echo "ERROR: xrdcp exited with $EXIT_STATUS"
        fi
        exit $EXIT_STATUS
        """
        return result

    def removeFile(self, pfnToRemove):
        """
        _removeFile_

        CleanUp pfn provided
        """

        method = self.storageMethod(pfnToRemove)

        if method == 'xrdcp':
            (_, host, path, _) = self.splitPFN(pfnToRemove)
            command = "xrd %s rm %s" % (host, path)
            print("Executing: %s" % command)
            self.executeCommand(command)
        elif method == 'srm':
            return self.srmImpl.removeFile(pfnToRemove)
        else:
            command = "/bin/rm %s" % stripPrefixTOUNIX(pfnToRemove)
            print("Executing: %s" % command)
            self.executeCommand(command)
Exemplo n.º 4
0
class FNALImpl(StageOutImpl):
    """
    _FNALImpl_

    Implement interface for dcache door based dccp command

    """

    def __init__(self, stagein=False):

        StageOutImpl.__init__(self, stagein)

        # Create and hold onto a srm implementation in case we need it
        self.srmImpl = LCGImpl(stagein)


    def storageMethod(self, PFN):
        """
        Figure out which paths use DCAP, lustre, or SRM for access
        """

        method = 'dccp'
        for path in checkPaths:
            if PFN.find(path) != -1:
                method = 'lustre'

        # Over ride a few paths for srm
        for path in srmPaths:
            if PFN.find(path) != -1:
                method = 'srm'
        print "Using method:", method
        return method


    def createOutputDirectory(self, targetPFN):
        """
        _createOutputDirectory_

        Create a dir for the target pfn by translating it to
        a /pnfs name and calling mkdir

        PFN will be of the form:
        dcap://cmsdca.fnal.gov:22125/pnfs/fnal.gov/usr/cms/WAX/11/store/blah

        We need to convert that into /pnfs/cms/WAX/11/store/blah, as it
        will be seen from the worker node

        Unmerged PFN will be of the form:
        /lustre/unmerged

        we don't need to convert it, just mkdir.


        """
        method =  self.storageMethod(targetPFN)

        if method == 'srm':
            self.srmImpl.createOutputDirectory(targetPFN)
        elif method == 'dccp':
            # only create dir on remote storage
            if targetPFN.find('/pnfs/') == -1:
                return

            pfnSplit = targetPFN.split("WAX/11/store/", 1)[1]
            filePath = "/pnfs/cms/WAX/11/store/%s" % pfnSplit
            directory = os.path.dirname(filePath)
            command = "#!/bin/sh\n"
            command += " . /opt/d-cache/dcap/bin/setenv-cmsprod.sh\n"
            command += "if [ ! -e \"%s\" ]; then\n" % directory
            command += "  mkdir -p %s\n" % directory
            command += "fi\n"
            self.executeCommand(command)
        else:
            for i in range(checkPathsCount):
                if targetPFN.find(checkPaths[i]) != -1:
                    pfnSplit = targetPFN.split(checkPaths[i], 1)[1]
                    filePath = "%s%s" % (checkPathsReplace[i],pfnSplit)
                    targetdir= os.path.dirname(filePath)
                    # checkdircmd="/bin/ls %s > /dev/null " % targetdir
                    # print "Check dir existence : %s" %checkdircmd
                    # checkdirexitCode = 0
                    # try:
                    #     checkdirexitCode = self.executeCommand(checkdircmd)
                    # except Exception, ex:
                    #     msg = "Warning: Exception while invoking command:\n"
                    #     msg += "%s\n" % checkdircmd
                    #     msg += "Exception: %s\n" % str(ex)
                    #     msg += "Go on anyway..."
                    #     print msg
                    #     pass
                    # if checkdirexitCode != 0:
                    #     mkdircmd = "/bin/mkdir -m 775 -p %s" % targetdir
                    #     print "=> creating the dir : %s" %mkdircmd
                    #     try:
                    #         self.executeCommand(mkdircmd)
                    #     except Exception, ex:
                    #         msg = "Warning: Exception while invoking command:\n"
                    #         msg += "%s\n" % mkdircmd
                    #         msg += "Exception: %s\n" % str(ex)
                    #         msg += "Go on anyway..."
                    #         print msg
                    #         pass
                    # else:
                    #     print "=> dir already exists... do nothing."
                    command = "#!/bin/sh\n"
                    command += "if [ ! -e \"%s\" ]; then\n" % targetdir
                    command += "  mkdir -p %s\n" % targetdir
                    command += "fi\n"
                    self.executeCommand(command)



    def createSourceName(self, protocol, pfn):
        """
        createTargetName

        generate the target PFN

        """
        if not pfn.startswith("srm"):
            return pfn

        method =  self.storageMethod(pfn)

        if method == 'srm':
            return self.srmImpl.createSourceName(protocol, pfn)
        elif method == 'dccp':
            print "Translating PFN: %s\n To use dcache door" % pfn
            dcacheDoor = commands.getoutput(
                "/opt/d-cache/dcap/bin/setenv-cmsprod.sh; /opt/d-cache/dcap/bin/select_RdCapDoor.sh")
            pfn = pfn.split("/store/")[1]
            pfn = "%s%s" % (dcacheDoor, pfn)
            print "Created Target PFN with dCache Door: ", pfn
        else:
            print "Translating PFN: %s\n To use lustre" % pfn
            for i in range(checkPathsCount):
                if pfn.find(checkPaths[i]) != -1:
                    pfnSplit = pfn.split(checkPaths[i], 1)[1]
                    pfn = "%s%s" % (checkPathsReplace[i],pfnSplit)
        return pfn



    def createStageOutCommand(self, sourcePFN, targetPFN, options = None, checksums = None):
        """
        _createStageOutCommand_

        Build a dccp command with a pnfs mkdir to generate the directory

        """

        method =  self.storageMethod(targetPFN)
        sourceMethod = self.storageMethod(sourcePFN)

        if method == 'srm' or sourceMethod == 'srm':
            return self.srmImpl.createStageOutCommand(sourcePFN, targetPFN, options)

        if getattr(self, 'stageIn', False):
            return self.buildStageInCommand(sourcePFN, targetPFN, options)

        if method == 'dccp':
            optionsStr = ""
            if options != None:
                optionsStr = str(options)
            dirname = os.path.dirname(targetPFN)
            result = "#!/bin/sh\n"
            result += ". /opt/d-cache/dcap/bin/setenv-cmsprod.sh\n"
            result += "dccp -o 86400  -d 0 -X -role=cmsprod %s %s %s" % ( optionsStr, sourcePFN, targetPFN)

            result += \
"""
EXIT_STATUS=$?
echo "dccp exit status: $EXIT_STATUS"
if [[ $EXIT_STATUS != 0 ]]; then
   echo "Non-zero dccp Exit status!!!"
   echo "Cleaning up failed file:"
   /bin/rm -fv %s
   exit 60311
fi
"""  % pnfsPfn2(targetPFN)

            #  //
            # //  CRC check
            #//
            result += \
"""
/opt/d-cache/dcap/bin/check_dCachefilecksum.sh %s %s
EXIT_STATUS=$?
echo "CRC Check Exit status: $EXIT_STATUS"
if [[ $EXIT_STATUS != 0 ]]; then
   echo "Non-zero CRC Check Exit status!!!"
   echo "Cleaning up failed file:"
   /bin/rm -fv %s
   exit 60311
fi

""" % (pnfsPfn2(targetPFN), sourcePFN, pnfsPfn2(targetPFN))

            print "Executing:\n", result
            return result

        else:
            original_size = os.stat(sourcePFN)[6]
            print "Local File Size is: %s" % original_size
            result = "/bin/cp "
            if options != None:
                result += " %s " % options
            result += " %s " % sourcePFN
            result += " %s " % targetPFN
            result += "; sync; DEST_SIZE=`/bin/ls -l %s | /bin/awk '{print $5}'` ; if [ $DEST_SIZE ] && [ '%s' == $DEST_SIZE ]; then exit 0; else echo \"Error: Size Mismatch between local and SE\"; exit 60311 ; fi " % (targetPFN,original_size)
            return result



    def buildStageInCommand(self, sourcePFN, targetPFN, options = None):
        """
        _buildStageInCommand_

        Create normal dccp commad for staging in files.
        """

        dcapLocation = 0
        for i in range(checkPathsCount):
            if sourcePFN.find(checkPaths[i]) != -1:
                dcapLocation = 1

        # Even if matched above, some paths are not lustre
        for path in srmPaths:
            if sourcePFN.find(path) != -1:
                dcapLocation = 0

        if dcapLocation == 0:
            optionsStr = ""
            if options != None:
                optionsStr = str(options)
            dirname = os.path.dirname(targetPFN)
            result = "#!/bin/sh\n"
            result += "dccp %s %s %s" % (optionsStr, pnfsPfn2(sourcePFN), targetPFN)
            result += \
"""
EXIT_STATUS=$?
echo "dccp exit status: $EXIT_STATUS"
if [[ $EXIT_STATUS != 0 ]]; then
   echo "Non-zero dccp Exit status!!!"
   echo "Cleaning up failed file:"
   /bin/rm -fv %s
   exit 60311
fi
"""  % targetPFN

            #  //
            # //  Size Check
            #//
            result += \
"""
DEST_SIZE=`dcsize %s | cut -d" " -f1`
FILE_SIZE=`dcsize %s | cut -d" " -f1`
if [[ $DEST_SIZE == "" || $FILE_SIZE == "" ]]; then
    echo "dcsize command is not available or produced an invalid result."
    echo "Trying stat command:"
    DEST_SIZE=`stat -c %s %s`
    FILE_SIZE=`stat -c %s %s`
fi
if [[ $DEST_SIZE == "" || $FILE_SIZE == "" ]]; then
    echo "stat command is not available or produced an invalid result."
    echo "Trying ls command:"
    DEST_SIZE=`/bin/ls -l %s | awk '{ print $5 }'`
    FILE_SIZE=`/bin/ls -l %s | awk '{ print $5 }'`
fi
if [ $FILE_SIZE != $DEST_SIZE ]; then
    echo "Source and destination files do not have same file size."
    echo "Cleaning up failed file:"
    /bin/rm -fv %s
    exit 60311
fi
""" % (pnfsPfn2(targetPFN), pnfsPfn2(sourcePFN),
       '%s', pnfsPfn2(targetPFN), '%s', pnfsPfn2(sourcePFN),
       pnfsPfn2(targetPFN), pnfsPfn2(sourcePFN),
       pnfsPfn2(targetPFN))

            print "Executing:\n", result
            return result

        else:
            for i in range(checkPathsCount):
                if sourcePFN.find(checkPaths[i]) != -1:
                    pfnSplit = sourcePFN.split(checkPaths[i], 1)[1]
                    filePath = "%s%s" % (checkPathsReplace[i],pfnSplit)
            original_size = os.stat(filePath)[6]
            print "Local File Size is: %s" % original_size
            result = "/bin/cp "
            if options != None:
                result += " %s " % options
            result += " %s " % filePath
            result += " %s " % targetPFN
            result += "; sync; DEST_SIZE=`/bin/ls -l %s | /bin/awk '{print $5}'` ; if [ $DEST_SIZE ] && [ '%s' == $DEST_SIZE ]; then exit 0; else echo \"Error: Size Mismatch between local and SE\"; exit 60311 ; fi " % (targetPFN,original_size)
            return result


    def removeFile(self, pfnToRemove):
        """
        _removeFile_

        CleanUp pfn provided

        """

        method =  self.storageMethod(pfnToRemove)

        if method == 'srm':
            return self.srmImpl.removeFile(pfnToRemove)
        elif method == 'dccp':
            pfnSplit = pfnToRemove.split("/11/store/", 1)[1]
            filePath = "/pnfs/cms/WAX/11/store/%s" % pfnSplit
            command = "rm -fv %s" %filePath
            self.executeCommand(command)
        else:
            for i in range(checkPathsCount):
                if pfnToRemove.find(checkPaths[i]) != -1:
                    pfnSplit = pfnToRemove.split(checkPaths[i], 1)[1]
                    pfnToRemove = "%s%s" % (checkPathsReplace[i],pfnSplit)
            command = "/bin/rm %s" % pfnToRemove
            self.executeCommand(command)
Exemplo n.º 5
0
class LCGImplTest(unittest.TestCase):
    def setUp(self):
        self.LCGImpl = LCGImpl()
        self.LCGImpl.setups = self.setups = "test setups"
        self.timeoutOptions = self.LCGImpl.timeoutOptions = \
            '--srm-timeout 600 --sendreceive-timeout 600 --connect-timeout 300'

    @mock.patch('WMCore.Storage.Backends.LCGImpl.os')
    def testInit(self, mock_os):
        mock_os.environ.get.side_effect = lambda x: True if x == "GLITE_LOCATION" or x == "GRID_ENV_LOCATION" else False
        mock_os.path.isfile.return_value = True
        mock_os.path.normpath.side_effect = lambda x: "home" + x
        mock_os.path.join.side_effect = lambda a, b: os.path.join("", b)
        testLCGImpl = LCGImpl(True)
        self.assertTrue(testLCGImpl.stageIn)
        setups = []
        setups += ['source home/../etc/profile.d/grid-env.sh; ']
        setups += ['source home/grid-env.sh; ']
        setups += ['date "+%Y-%m-%dT%H:%M:%S"; ']
        self.assertTrue(testLCGImpl.stageIn)
        for setup in setups:
            self.assertIn(setup, testLCGImpl.setups)

    def testCreateSourceName_startswithSlash(self):
        self.assertEqual("file://name",
                         self.LCGImpl.createSourceName("protocol", "//name"))

    @mock.patch('WMCore.Storage.Backends.LCGImpl.os.path')
    def testCreateSourceName_isfile(self, mock_path):
        mock_path.isfile.return_value = True
        mock_path.abspath.return_value = "/some/path"
        self.assertEqual("file:/some/path",
                         self.LCGImpl.createSourceName("protocol", "name"))

    def testCreateSourceName_simple(self):
        self.assertEqual("name",
                         self.LCGImpl.createSourceName("protocol", "name"))

    def testCreateStageOutCommand_stageInFile(self):
        self.LCGImpl.stageIn = True
        result = self.LCGImpl.createStageOutCommand("srm://sourcePFN",
                                                    "file://targetPFN")
        expectedResults = self.getStageOutCommandResults(
            "srm://sourcePFN", "file://targetPFN", "srm://sourcePFN",
            "//targetPFN", "/bin/rm -f //targetPFN", False)
        self.assertEqual(expectedResults, result)

    def testCreateStageOutCommand_stageISrmCvmfs(self):
        self.LCGImpl.stageIn = True
        result = self.LCGImpl.createStageOutCommand("srm://sourcePFN",
                                                    "srm://targetPFN",
                                                    options="cvmfs")
        createRemoveFileCommandResult = "%s lcg-del -b -l -D srmv2 %s --vo cms srm://targetPFN" % \
                                        (self.setups, self.timeoutOptions)
        expectedResults = self.getStageOutCommandResults(
            "srm://sourcePFN", "srm://targetPFN", "srm://sourcePFN",
            "srm://targetPFN", createRemoveFileCommandResult, True)
        self.assertEqual(expectedResults, result)

    @mock.patch(
        'WMCore.Storage.Backends.LCGImpl.StageOutImpl.createRemoveFileCommand')
    def testCreateStageOutCommand_options(self, mock_createRemoveFileCommand):
        mock_createRemoveFileCommand.return_value = "command"
        result = self.LCGImpl.createStageOutCommand("file://sourcePFN",
                                                    "targetPFN",
                                                    options="test")
        expectedResults = self.getStageOutCommandResults("file://sourcePFN",
                                                         "targetPFN",
                                                         "targetPFN",
                                                         "//sourcePFN",
                                                         "command",
                                                         False,
                                                         options="test")
        self.assertEqual(expectedResults, result)

    @mock.patch(
        'WMCore.Storage.Backends.LCGImpl.StageOutImpl.createRemoveFileCommand')
    def testCreateStageOutCommand_checksum(self, mock_createRemoveFileCommand):
        mock_createRemoveFileCommand.return_value = "command"
        result = self.LCGImpl.createStageOutCommand(
            "file://sourcePFN", "targetPFN", checksums={"adler32": "32"})
        expectedResults = self.getStageOutCommandResults("file://sourcePFN",
                                                         "targetPFN",
                                                         "targetPFN",
                                                         "//sourcePFN",
                                                         "command",
                                                         False,
                                                         checksums="00000032")
        self.assertEqual(expectedResults, result)

    def getStageOutCommandResults(self,
                                  sourcePFN,
                                  targetPFN,
                                  remotePFN,
                                  localPFN,
                                  createRemoveFileCommandResult,
                                  useCVMFS,
                                  options=None,
                                  checksums=None):
        result = "#!/bin/sh\n"

        copyCommand = "lcg-cp -b -D srmv2 --vo cms --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 300 --verbose"
        if options != None:
            copyCommand += " %s " % options
        copyCommand += " %s " % sourcePFN
        copyCommand += " %s 2> stageout.log" % targetPFN
        if useCVMFS:
            result += "(\n"
            result += "echo Modifying PATH and LD_LIBRARY_PATH to remove /cvmfs/cms.cern.ch elements\n"
            result += "export PATH=`echo $PATH | sed -e 's+:*/cvmfs/cms.cern.ch/[^:]*++'g`\n"
            result += "export LD_LIBRARY_PATH=`echo $LD_LIBRARY_PATH | sed -e 's+:*/cvmfs/cms.cern.ch/[^:]*++'g`\n"
            result += "echo Sourcing CVMFS UI setup script\n"
            result += ". /cvmfs/grid.cern.ch/emi3ui-latest/etc/profile.d/setup-ui-example.sh\n"

            result += copyCommand
        else:
            result += self.setups
            result += copyCommand

        result += """
            EXIT_STATUS=$?
            cat stageout.log
            echo -e "\nlcg-cp exit status: $EXIT_STATUS"
            if [[ $EXIT_STATUS != 0 ]]; then
                echo "ERROR: lcg-cp exited with $EXIT_STATUS"
                echo "Cleaning up failed file:"
                %s
                exit $EXIT_STATUS
            fi

            """ % createRemoveFileCommandResult

        result += "FILE_SIZE=`stat -c %s"
        result += " %s`\n" % localPFN
        result += "echo \"Local File Size is:  $FILE_SIZE\"\n"

        if checksums:
            checksumCommand = \
                """
                if [[ "X$SRM_CHECKSUM" != "X" ]]; then
                    if [[ "$SRM_CHECKSUM" == "%s" ]]; then
                        exit 0
                    else
                        echo "ERROR: Checksum Mismatch between local and SE"
                        echo "Cleaning up failed file"
                        %s
                        exit 60311
                    fi
                fi
                exit 0
                """ % (checksums, createRemoveFileCommandResult)
        else:
            checksumCommand = "exit 0"

        metadataCheck = \
            """
            LCG_OUTPUT=`lcg-ls -l -b -D srmv2 %s %s 2>/dev/null`
            SRM_SIZE=`echo "$LCG_OUTPUT" | awk 'NR==1{print $5}'`
            SRM_CHECKSUM=`echo "$LCG_OUTPUT" | sed -nr 's/^.*\s([a-f0-9]{8})\s*\([aA][dD][lL][eE][rR]32\)\s*$/\\1/p'`
            echo "Remote File Size is: $SRM_SIZE"
            echo "Remote Checksum is:  $SRM_CHECKSUM"
            if [[ $SRM_SIZE == $FILE_SIZE ]]; then
                %s
            else
                echo $LCG_OUTPUT
                echo "ERROR: Size Mismatch between local and SE. Cleaning up failed file..."
                %s
                exit 60311
            fi
            """ % (self.timeoutOptions, remotePFN, checksumCommand, createRemoveFileCommandResult)
        result += metadataCheck
        if useCVMFS:
            result += ")\n"
        return result

    @mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.executeCommand')
    def testRemoveFile(self, mock_executeCommand):
        self.LCGImpl.removeFile("file")
        mock_executeCommand.assert_called_with(
            "%s lcg-del -b -l -D srmv2 %s --vo cms file" %
            (self.setups, self.timeoutOptions))
Exemplo n.º 6
0
 def setUp(self):
     self.LCGImpl = LCGImpl()
     self.LCGImpl.setups = self.setups = "test setups"
     self.timeoutOptions = self.LCGImpl.timeoutOptions = \
         '--srm-timeout 600 --sendreceive-timeout 600 --connect-timeout 300'
Exemplo n.º 7
0
class FNALImpl(StageOutImpl):
    """
    _FNALImpl_

    Implement interface for dcache xrootd command

    """

    def __init__(self, stagein=False):

        StageOutImpl.__init__(self, stagein)
        
        # Create and hold onto a srm implementation in case we need it
        self.srmImpl = LCGImpl(stagein)


    def storageMethod(self, PFN):
        """
        Return xrdcp or srm
        """

        method = 'srm'
        if PFN.startswith("root://"):
            method = 'xrdcp'
        print "Using method:", method
        return method


    def createOutputDirectory(self, targetPFN):
        """
        _createOutputDirectory_

        Create a dir for the target pfn by translating it to
        a /dcache or /lustre name and calling mkdir
        we don't need to convert it, just mkdir.       
        """
        method = self.storageMethod(targetPFN)

        if method == 'srm':
            self.srmImpl.createOutputDirectory(targetPFN)
        else:
            pfnSplit = stripPrefixTOUNIX(targetPFN)
            targetdir= os.path.dirname(pfnSplit)
            command = "#!/bin/sh\n"
            command += "if [ ! -e \"%s\" ]; then\n" % targetdir
            command += " mkdir -p %s\n" % targetdir
            command += "fi\n"
            self.executeCommand(command)

    def createSourceName(self, protocol, pfn):
        """
        createTargetName
        
        generate the target PFN
        
        """
        method = self.storageMethod(pfn)

        if method == 'srm':
            return self.srmImpl.createSourceName(protocol, pfn)
        elif method == 'xrdcp':
            print "Translated PFN: %s\n To use xrdcp" % pfn
        return pfn



    def createStageOutCommand(self, sourcePFN, targetPFN, options = None, checksums = None):
        """
        _createStageOutCommand_
        
        Build a mkdir to generate the directory
        
        """

        method = self.storageMethod(targetPFN)
        sourceMethod = self.storageMethod(sourcePFN)

        if method == 'srm' or sourceMethod == 'srm':
            return self.srmImpl.createStageOutCommand(sourcePFN, targetPFN, options)

        if getattr(self, 'stageIn', False):
            return self.buildStageInCommand(sourcePFN, targetPFN, options)

        if method == 'xrdcp':
            original_size = os.stat(sourcePFN)[6]
            print "Local File Size is: %s" % original_size
            pfnWithoutChecksum = stripPrefixTOUNIX(targetPFN)
            
            useChecksum = (checksums != None and checksums.has_key('adler32') and not self.stageIn)
            if useChecksum:
                checksums['adler32'] = "%08x" % int(checksums['adler32'], 16)
                # non-functional in 3.3.1 xrootd clients due to bug
                #result += "-ODeos.targetsize=$LOCAL_SIZE\&eos.checksum=%s " % checksums['adler32']

                # therefor embed information into target URL
                targetPFN += "\?eos.targetsize=%s\&eos.checksum=%s" % (original_size, checksums['adler32'])
                print "Local File Checksum is: %s\"\n" % checksums['adler32']
            
            # remove the file first befor it writes. 
            # there is eos bug when disk partition is full.
            result = "/bin/rm -f %s" % pfnWithoutChecksum
            result += "; /usr/bin/xrdcp -d 0 "
            if options != None:
                result += " %s " % options
            result += " %s " % sourcePFN
            result += " %s " % targetPFN
            result += "; if [ $? -eq 0 ] ; then exit 0; else echo \"Error: xrdcp exited with $?\"; exit 60311 ; fi "
            return result



    def buildStageInCommand(self, sourcePFN, targetPFN, options = None):
        """
        _buildStageInCommand_
        
        Create normal dccp commad for staging in files.
        """

        # Even if matched above, some paths are not lustre
        for path in srmPaths:
            if sourcePFN.find(path) != -1:
                dcapLocation = 0
        
        result = "/usr/bin/xrdcp -d 0 "
        if options != None:
            result += " %s " % options
        result += " %s " % sourcePFN
        result += " %s " % targetPFN
        result += "; if [ $? -eq 0 ] ; then exit 0; else echo \"Error: xrdcp exited with $?\"; exit 60311 ; fi "
        return result


    def removeFile(self, pfnToRemove):
        """
        _removeFile_

        CleanUp pfn provided
        """

        method = self.storageMethod(pfnToRemove)

        if method == 'srm':
            return self.srmImpl.removeFile(pfnToRemove)
        elif method == 'xrdcp':
            command = "/bin/rm %s" % stripPrefixTOUNIX(pfnToRemove)
            self.executeCommand(command)
Exemplo n.º 8
0
class LCGImplTest(unittest.TestCase):
    def setUp(self):
        self.LCGImpl = LCGImpl()
        self.LCGImpl.setups = self.setups = "test setups"
        self.timeoutOptions = self.LCGImpl.timeoutOptions = \
            '--srm-timeout 600 --sendreceive-timeout 600 --connect-timeout 300'

    @mock.patch('WMCore.Storage.Backends.LCGImpl.os')
    def testInit(self, mock_os):
        mock_os.environ.get.side_effect = lambda x: True if x == "GLITE_LOCATION" or x == "GRID_ENV_LOCATION" else False
        mock_os.path.isfile.return_value = True
        mock_os.path.normpath.side_effect = lambda x: "home" + x
        mock_os.path.join.side_effect = lambda a, b: os.path.join("", b)
        testLCGImpl = LCGImpl(True)
        self.assertTrue(testLCGImpl.stageIn)
        setups = []
        setups += ['source home/../etc/profile.d/grid-env.sh; ']
        setups += ['source home/grid-env.sh; ']
        setups += ['date "+%Y-%m-%dT%H:%M:%S"; ']
        self.assertTrue(testLCGImpl.stageIn)
        for setup in setups:
            self.assertIn(setup, testLCGImpl.setups)

    def testCreateSourceName_startswithSlash(self):
        self.assertEqual("file://name", self.LCGImpl.createSourceName("protocol", "//name"))

    @mock.patch('WMCore.Storage.Backends.LCGImpl.os.path')
    def testCreateSourceName_isfile(self, mock_path):
        mock_path.isfile.return_value = True
        mock_path.abspath.return_value = "/some/path"
        self.assertEqual("file:/some/path", self.LCGImpl.createSourceName("protocol", "name"))

    def testCreateSourceName_simple(self):
        self.assertEqual("name", self.LCGImpl.createSourceName("protocol", "name"))

    def testCreateStageOutCommand_stageInFile(self):
        self.LCGImpl.stageIn = True
        result = self.LCGImpl.createStageOutCommand("srm://sourcePFN", "file://targetPFN")
        expectedResults = self.getStageOutCommandResults("srm://sourcePFN", "file://targetPFN", "srm://sourcePFN",
                                                         "//targetPFN", "/bin/rm -f //targetPFN", False)
        self.assertEqual(expectedResults, result)

    def testCreateStageOutCommand_stageISrmCvmfs(self):
        self.LCGImpl.stageIn = True
        result = self.LCGImpl.createStageOutCommand("srm://sourcePFN", "srm://targetPFN", options="cvmfs")
        createRemoveFileCommandResult = "%s lcg-del -b -l -D srmv2 %s --vo cms srm://targetPFN" % \
                                        (self.setups, self.timeoutOptions)
        expectedResults = self.getStageOutCommandResults("srm://sourcePFN", "srm://targetPFN", "srm://sourcePFN",
                                                         "srm://targetPFN", createRemoveFileCommandResult, True)
        self.assertEqual(expectedResults, result)

    @mock.patch('WMCore.Storage.Backends.LCGImpl.StageOutImpl.createRemoveFileCommand')
    def testCreateStageOutCommand_options(self, mock_createRemoveFileCommand):
        mock_createRemoveFileCommand.return_value = "command"
        result = self.LCGImpl.createStageOutCommand("file://sourcePFN", "targetPFN", options="test")
        expectedResults = self.getStageOutCommandResults("file://sourcePFN", "targetPFN",
                                                         "targetPFN", "//sourcePFN", "command", False, options="test")
        self.assertEqual(expectedResults, result)

    @mock.patch('WMCore.Storage.Backends.LCGImpl.StageOutImpl.createRemoveFileCommand')
    def testCreateStageOutCommand_checksum(self, mock_createRemoveFileCommand):
        mock_createRemoveFileCommand.return_value = "command"
        result = self.LCGImpl.createStageOutCommand("file://sourcePFN", "targetPFN", checksums={"adler32": "32"})
        expectedResults = self.getStageOutCommandResults("file://sourcePFN", "targetPFN",
                                                         "targetPFN", "//sourcePFN", "command", False,
                                                         checksums="00000032")
        self.assertEqual(expectedResults, result)

    def getStageOutCommandResults(self, sourcePFN, targetPFN, remotePFN, localPFN,
                                  createRemoveFileCommandResult, useCVMFS, options=None, checksums=None):
        result = "#!/bin/sh\n"

        copyCommand = "lcg-cp -b -D srmv2 --vo cms --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 300 --verbose"
        if options != None:
            copyCommand += " %s " % options
        copyCommand += " %s " % sourcePFN
        copyCommand += " %s 2> stageout.log" % targetPFN
        if useCVMFS:
            result += "(\n"
            result += "echo Modifying PATH and LD_LIBRARY_PATH to remove /cvmfs/cms.cern.ch elements\n"
            result += "export PATH=`echo $PATH | sed -e 's+:*/cvmfs/cms.cern.ch/[^:]*++'g`\n"
            result += "export LD_LIBRARY_PATH=`echo $LD_LIBRARY_PATH | sed -e 's+:*/cvmfs/cms.cern.ch/[^:]*++'g`\n"
            result += "echo Sourcing CVMFS UI setup script\n"
            result += ". /cvmfs/grid.cern.ch/emi3ui-latest/etc/profile.d/setup-ui-example.sh\n"

            result += copyCommand
        else:
            result += self.setups
            result += copyCommand

        result += """
            EXIT_STATUS=$?
            cat stageout.log
            echo -e "\nlcg-cp exit status: $EXIT_STATUS"
            if [[ $EXIT_STATUS != 0 ]]; then
                echo "Non-zero lcg-cp Exit status!!!"
                echo "Cleaning up failed file:"
                %s
                exit 60311
            fi

            """ % createRemoveFileCommandResult

        result += "FILE_SIZE=`stat -c %s"
        result += " %s`\n" % localPFN
        result += "echo \"Local File Size is:  $FILE_SIZE\"\n"

        if checksums:
            checksumCommand = \
                """
                if [[ "X$SRM_CHECKSUM" != "X" ]]; then
                    if [[ "$SRM_CHECKSUM" == "%s" ]]; then
                        exit 0
                    else
                        echo "ERROR: Checksum Mismatch between local and SE"
                        echo "Cleaning up failed file"
                        %s
                        exit 60311
                    fi
                else
                    exit 0
                fi
                """ % (checksums, createRemoveFileCommandResult)
        else:
            checksumCommand = "exit 0"

        metadataCheck = \
            """
            LCG_OUTPUT=`lcg-ls -l -b -D srmv2 %s %s 2>/dev/null`
            SRM_SIZE=`echo "$LCG_OUTPUT" | awk 'NR==1{print $5}'`
            SRM_CHECKSUM=`echo "$LCG_OUTPUT" | sed -nr 's/^.*\s([a-f0-9]{8})\s*\([aA][dD][lL][eE][rR]32\)\s*$/\\1/p'`
            echo "Remote File Size is: $SRM_SIZE"
            echo "Remote Checksum is:  $SRM_CHECKSUM"
            if [[ $SRM_SIZE == $FILE_SIZE ]]; then
                %s
            else
                echo $LCG_OUTPUT
                echo "ERROR: Size Mismatch between local and SE. Cleaning up failed file..."
                %s
                exit 60311
            fi
            """ % (self.timeoutOptions, remotePFN, checksumCommand, createRemoveFileCommandResult)
        result += metadataCheck
        if useCVMFS:
            result += ")\n"
        return result

    @mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.executeCommand')
    def testRemoveFile(self, mock_executeCommand):
        self.LCGImpl.removeFile("file")
        mock_executeCommand.assert_called_with("%s lcg-del -b -l -D srmv2 %s --vo cms file" %
                                               (self.setups, self.timeoutOptions))
Exemplo n.º 9
0
 def setUp(self):
     self.LCGImpl = LCGImpl()
     self.LCGImpl.setups = self.setups = "test setups"
     self.timeoutOptions = self.LCGImpl.timeoutOptions = \
         '--srm-timeout 600 --sendreceive-timeout 600 --connect-timeout 300'
Exemplo n.º 10
0
class FNALImpl(StageOutImpl):
    """
    _FNALImpl_

    Implement interface for dcache door based dccp command

    """

    def __init__(self, stagein=False):

        StageOutImpl.__init__(self, stagein)

        # Create and hold onto a srm implementation in case we need it
        self.srmImpl = LCGImpl(stagein)


    def storageMethod(self, PFN):
        """
        Figure out which paths use DCAP, lustre, or SRM for access
        """

        method = 'dccp'
        for path in checkPaths:
            if PFN.find(path) != -1:
                method = 'lustre'

        # Over ride a few paths for srm
        for path in srmPaths:
            if PFN.find(path) != -1:
                method = 'srm'
        print "Using method:", method
        return method


    def createOutputDirectory(self, targetPFN):
        """
        _createOutputDirectory_

        Create a dir for the target pfn by translating it to
        a /pnfs name and calling mkdir

        PFN will be of the form:
        dcap://cmsdca.fnal.gov:22125/pnfs/fnal.gov/usr/cms/WAX/11/store/blah

        We need to convert that into /pnfs/cms/WAX/11/store/blah, as it
        will be seen from the worker node

        Unmerged PFN will be of the form:
        /lustre/unmerged

        we don't need to convert it, just mkdir.


        """
        method =  self.storageMethod(targetPFN)

        if method == 'srm':
            self.srmImpl.createOutputDirectory(targetPFN)
        elif method == 'dccp':
            # only create dir on remote storage
            if targetPFN.find('/pnfs/') == -1:
                return

            pfnSplit = targetPFN.split("WAX/11/store/", 1)[1]
            filePath = "/pnfs/cms/WAX/11/store/%s" % pfnSplit
            directory = os.path.dirname(filePath)
            command = "#!/bin/sh\n"
            command += " . /opt/d-cache/dcap/bin/setenv-cmsprod.sh\n"
            command += "if [ ! -e \"%s\" ]; then\n" % directory
            command += "  mkdir -p %s\n" % directory
            command += "fi\n"
            self.executeCommand(command)
        else:
            for i in range(checkPathsCount):
                if targetPFN.find(checkPaths[i]) != -1:
                    pfnSplit = targetPFN.split(checkPaths[i], 1)[1]
                    filePath = "%s%s" % (checkPathsReplace[i],pfnSplit)
                    targetdir= os.path.dirname(filePath)
                    # checkdircmd="/bin/ls %s > /dev/null " % targetdir
                    # print "Check dir existence : %s" %checkdircmd
                    # checkdirexitCode = 0
                    # try:
                    #     checkdirexitCode = self.executeCommand(checkdircmd)
                    # except Exception, ex:
                    #     msg = "Warning: Exception while invoking command:\n"
                    #     msg += "%s\n" % checkdircmd
                    #     msg += "Exception: %s\n" % str(ex)
                    #     msg += "Go on anyway..."
                    #     print msg
                    #     pass
                    # if checkdirexitCode != 0:
                    #     mkdircmd = "/bin/mkdir -m 775 -p %s" % targetdir
                    #     print "=> creating the dir : %s" %mkdircmd
                    #     try:
                    #         self.executeCommand(mkdircmd)
                    #     except Exception, ex:
                    #         msg = "Warning: Exception while invoking command:\n"
                    #         msg += "%s\n" % mkdircmd
                    #         msg += "Exception: %s\n" % str(ex)
                    #         msg += "Go on anyway..."
                    #         print msg
                    #         pass
                    # else:
                    #     print "=> dir already exists... do nothing."
                    command = "#!/bin/sh\n"
                    command += "if [ ! -e \"%s\" ]; then\n" % targetdir
                    command += "  mkdir -p %s\n" % targetdir
                    command += "fi\n"
                    self.executeCommand(command)



    def createSourceName(self, protocol, pfn):
        """
        createTargetName

        generate the target PFN

        """
        if not pfn.startswith("srm"):
            return pfn

        method =  self.storageMethod(pfn)

        if method == 'srm':
            return self.srmImpl.createSourceName(protocol, pfn)
        elif method == 'dccp':
            print "Translating PFN: %s\n To use dcache door" % pfn
            dcacheDoor = commands.getoutput(
                "/opt/d-cache/dcap/bin/setenv-cmsprod.sh; /opt/d-cache/dcap/bin/select_RdCapDoor.sh")
            pfn = pfn.split("/store/")[1]
            pfn = "%s%s" % (dcacheDoor, pfn)
            print "Created Target PFN with dCache Door: ", pfn
        else:
            print "Translating PFN: %s\n To use lustre" % pfn
    	    for i in range(checkPathsCount):
              if pfn.find(checkPaths[i]) != -1:
                pfnSplit = pfn.split(checkPaths[i], 1)[1]
                pfn = "%s%s" % (checkPathsReplace[i],pfnSplit)
        return pfn



    def createStageOutCommand(self, sourcePFN, targetPFN, options = None, checksums = None):
        """
        _createStageOutCommand_

        Build a dccp command with a pnfs mkdir to generate the directory

        """

        method =  self.storageMethod(targetPFN)
        sourceMethod = self.storageMethod(sourcePFN)

        if method == 'srm' or sourceMethod == 'srm':
            return self.srmImpl.createStageOutCommand(sourcePFN, targetPFN, options)

        if getattr(self, 'stageIn', False):
            return self.buildStageInCommand(sourcePFN, targetPFN, options)

        if method == 'dccp':
            optionsStr = ""
            if options != None:
                optionsStr = str(options)
            dirname = os.path.dirname(targetPFN)
            result = "#!/bin/sh\n"
            result += ". /opt/d-cache/dcap/bin/setenv-cmsprod.sh\n"
            result += "dccp -o 86400  -d 0 -X -role=cmsprod %s %s %s" % ( optionsStr, sourcePFN, targetPFN)

            result += \
"""
EXIT_STATUS=$?
echo "dccp exit status: $EXIT_STATUS"
if [[ $EXIT_STATUS != 0 ]]; then
   echo "Non-zero dccp Exit status!!!"
   echo "Cleaning up failed file:"
   /bin/rm -fv %s
   exit 60311
fi
"""  % pnfsPfn2(targetPFN)

            #  //
            # //  CRC check
            #//
            result += \
"""
/opt/d-cache/dcap/bin/check_dCachefilecksum.sh %s %s
EXIT_STATUS=$?
echo "CRC Check Exit status: $EXIT_STATUS"
if [[ $EXIT_STATUS != 0 ]]; then
   echo "Non-zero CRC Check Exit status!!!"
   echo "Cleaning up failed file:"
   /bin/rm -fv %s
   exit 60311
fi

""" % (pnfsPfn2(targetPFN), sourcePFN, pnfsPfn2(targetPFN))

            print "Executing:\n", result
            return result

        else:
            original_size = os.stat(sourcePFN)[6]
            print "Local File Size is: %s" % original_size
            result = "/bin/cp "
            if options != None:
                result += " %s " % options
            result += " %s " % sourcePFN
            result += " %s " % targetPFN
            result += "; sync; DEST_SIZE=`/bin/ls -l %s | /bin/awk '{print $5}'` ; if [ $DEST_SIZE ] && [ '%s' == $DEST_SIZE ]; then exit 0; else echo \"Error: Size Mismatch between local and SE\"; exit 60311 ; fi " % (targetPFN,original_size)
            return result



    def buildStageInCommand(self, sourcePFN, targetPFN, options = None):
        """
        _buildStageInCommand_

        Create normal dccp commad for staging in files.
        """

        dcapLocation = 0
        for i in range(checkPathsCount):
          if sourcePFN.find(checkPaths[i]) != -1:
            dcapLocation = 1

        # Even if matched above, some paths are not lustre
        for path in srmPaths:
            if sourcePFN.find(path) != -1:
                dcapLocation = 0

        if dcapLocation == 0:
            optionsStr = ""
            if options != None:
                optionsStr = str(options)
            dirname = os.path.dirname(targetPFN)
            result = "#!/bin/sh\n"
            result += "dccp %s %s %s" % (optionsStr, pnfsPfn2(sourcePFN), targetPFN)
            result += \
"""
EXIT_STATUS=$?
echo "dccp exit status: $EXIT_STATUS"
if [[ $EXIT_STATUS != 0 ]]; then
   echo "Non-zero dccp Exit status!!!"
   echo "Cleaning up failed file:"
   /bin/rm -fv %s
   exit 60311
fi
"""  % targetPFN

            #  //
            # //  Size Check
            #//
            result += \
"""
DEST_SIZE=`dcsize %s | cut -d" " -f1`
FILE_SIZE=`dcsize %s | cut -d" " -f1`
if [[ $DEST_SIZE == "" || $FILE_SIZE == "" ]]; then
    echo "dcsize command is not available or produced an invalid result."
    echo "Trying stat command:"
    DEST_SIZE=`stat -c %s %s`
    FILE_SIZE=`stat -c %s %s`
fi
if [[ $DEST_SIZE == "" || $FILE_SIZE == "" ]]; then
    echo "stat command is not available or produced an invalid result."
    echo "Trying ls command:"
    DEST_SIZE=`/bin/ls -l %s | awk '{ print $5 }'`
    FILE_SIZE=`/bin/ls -l %s | awk '{ print $5 }'`
fi
if [ $FILE_SIZE != $DEST_SIZE ]; then
    echo "Source and destination files do not have same file size."
    echo "Cleaning up failed file:"
    /bin/rm -fv %s
    exit 60311
fi
""" % (pnfsPfn2(targetPFN), pnfsPfn2(sourcePFN),
       '%s', pnfsPfn2(targetPFN), '%s', pnfsPfn2(sourcePFN),
       pnfsPfn2(targetPFN), pnfsPfn2(sourcePFN),
       pnfsPfn2(targetPFN))

            print "Executing:\n", result
            return result

        else:
    	    for i in range(checkPathsCount):
              if sourcePFN.find(checkPaths[i]) != -1:
                pfnSplit = sourcePFN.split(checkPaths[i], 1)[1]
                filePath = "%s%s" % (checkPathsReplace[i],pfnSplit)
            original_size = os.stat(filePath)[6]
            print "Local File Size is: %s" % original_size
            result = "/bin/cp "
            if options != None:
                result += " %s " % options
            result += " %s " % filePath
            result += " %s " % targetPFN
            result += "; sync; DEST_SIZE=`/bin/ls -l %s | /bin/awk '{print $5}'` ; if [ $DEST_SIZE ] && [ '%s' == $DEST_SIZE ]; then exit 0; else echo \"Error: Size Mismatch between local and SE\"; exit 60311 ; fi " % (targetPFN,original_size)
            return result


    def removeFile(self, pfnToRemove):
        """
        _removeFile_

        CleanUp pfn provided

        """

        method =  self.storageMethod(pfnToRemove)

        if method == 'srm':
            return self.srmImpl.removeFile(pfnToRemove)
        elif method == 'dccp':
            pfnSplit = pfnToRemove.split("/11/store/", 1)[1]
            filePath = "/pnfs/cms/WAX/11/store/%s" % pfnSplit
            command = "rm -fv %s" %filePath
            self.executeCommand(command)
        else:
            for i in range(checkPathsCount):
              if pfnToRemove.find(checkPaths[i]) != -1:
                pfnSplit = pfnToRemove.split(checkPaths[i], 1)[1]
                pfnToRemove = "%s%s" % (checkPathsReplace[i],pfnSplit)
            command = "/bin/rm %s" % pfnToRemove
            self.executeCommand(command)