Exemplo n.º 1
0
    def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
        """ The local file (local access to the dCache file) is assumed to have a relative path
        that is the same of the relative path in the 'gpfn'
        loc_... are the variables used to access the file in the locally exported file system
        TODO: document GPFN format
        TODO: document better constraint
        """

        error = PilotErrors()
        pilotErrorDiag = ""

        # get the DQ2 tracing report
        try:
            report = pdict['report']
        except:
            report = {}
        else:
            # set the proper protocol
            report['protocol'] = 'HU'
            # mark the relative start
            report['relativeStart'] = time()
            # the current file
            report['filename'] = lfn
            # guid
            report['guid'] = guid.replace('-','')

        # get a proper envsetup
        envsetup = self.getEnvsetup(get=True)

        if self._setup:
            _setup_str = "source %s; " % self._setup
        else:
            _setup_str = envsetup

        try:
            timeout = pdict['timeout']
        except:
            timeout = 5*3600

        if( gpfn.find('SFN') != -1 ):
            s = gpfn.split('SFN=')
            loc_pfn = s[1]
        else:
            _tmp = gpfn.split('/', 3)
            loc_pfn = '/'+_tmp[3]

        _cmd_str = '%snecp %s %s/%s' % (_setup_str, loc_pfn, path, lfn)
        tolog("NECP executing (timeout %s): %s" % (timeout, _cmd_str))
        report['transferStart'] = time()
        try:
            s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
        except Exception, e:
            tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" % str(e))
            s = 1
            o = str(e)
            telapsed = timeout
Exemplo n.º 2
0
    def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
        """ The local file (local access to the dCache file) is assumed to have a relative path
        that is the same of the relative path in the 'gpfn'
        loc_... are the variables used to access the file in the locally exported file system
        TODO: document GPFN format
        TODO: document better constraint
        """

        error = PilotErrors()
        pilotErrorDiag = ""

        # get the DQ2 tracing report
        try:
            report = pdict['report']
        except:
            report = {}
        else:
            # set the proper protocol
            report['protocol'] = 'HU'
            # mark the relative start
            report['relativeStart'] = time()
            # the current file
            report['filename'] = lfn
            # guid
            report['guid'] = guid.replace('-','')

        # get a proper envsetup
        envsetup = self.getEnvsetup(get=True)

        if self._setup:
            _setup_str = "source %s; " % self._setup
        else:
            _setup_str = envsetup

        try:
            timeout = pdict['timeout']
        except:
            timeout = 5*3600

        if( gpfn.find('SFN') != -1 ):
            s = gpfn.split('SFN=')
            loc_pfn = s[1]
        else:
            _tmp = gpfn.split('/', 3)
            loc_pfn = '/'+_tmp[3]

        _cmd_str = '%snecp %s %s/%s' % (_setup_str, loc_pfn, path, lfn)
        tolog("NECP executing (timeout %s): %s" % (timeout, _cmd_str))
        report['transferStart'] = time()
        try:
            s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
        except Exception, e:
            tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" % str(e))
            s = 1
            o = str(e)
            telapsed = timeout
Exemplo n.º 3
0
    def copy(self, cmd, stagein=False):
        """ Perfom the actual file transfer """

        ec = 0
        pilotErrorDiag = ""
        error = PilotErrors()
        tolog("Executing command: %s" % (cmd))

        try:
            s, telapsed, cout, cerr = timed_command(cmd, self.timeout)
        except Exception, e:
            tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" % str(e))
            s = 1
            o = str(e)
            telapsed = self.timeout
Exemplo n.º 4
0
    def check_space(self, storage_endpoint):
        """ available spave in the SE """

        # build setup string
        envsetup = self.getEnvsetup()

        # build the df command
        execStr = self.__localspace % (envsetup, '', storage_endpoint)
        tolog("Executing command: %s" % (execStr))

        # execute
        try:
            status, telapsed, cout, cerr = timed_command(execStr, self.__timeout)
        except Exception, e:
            self.__pilotErrorDiag = 'timed_command() threw an exception: %s' % str(e)
            tolog(self.__warningStr % self.__pilotErrorDiag)            
            status = 1
            output = str(e)
            telapsed = self.__timeout
Exemplo n.º 5
0
        except Exception, e:
            tolog(
                "Warning: Failed to get RSE: %s (can not add this info to tracing report)"
                % str(e))
        else:
            report['localSite'], report['remoteSite'] = (_RSE, _RSE)
            tolog("RSE: %s" % (_RSE))

        if testLevel == "1":
            source = "thisisjustatest"

        cmd = 'source %s; dccp %s %s' % (self._setup, source, sedir)
        tolog("Executing command: %s" % (cmd))
        report['transferStart'] = time.time()
        try:
            s, telapsed, cout, cerr = timed_command(cmd, timeout)
        except Exception, e:
            tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" %
                  str(e))
            s = 1
            e = sys.exc_info()
            o = str(e[1])  # set error message from exception text
            telapsed = timeout
        else:
            o = cout + cerr
            tolog("Elapsed time: %d" % (telapsed))

        if s != 0:
            o = o.replace('\n', ' ')
            check_syserr(s, o)
            pilotErrorDiag = "Error in copying: %s" % str(o)
Exemplo n.º 6
0
    def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
        """ The local file is assubed to have a relative path that is the same of the relative path in the 'gpfn'
        loc_... are the variables used to access the file in the locally exported file system
        TODO: document GPFN format (SURL from catalog srm://host/path)
        TODO: document better constraint
        """

        error = PilotErrors()
        pilotErrorDiag = ""

        # Get input parameters from pdict
        useCT = pdict.get('usect', True)
        jobId = pdict.get('jobId', '')
        workDir = pdict.get('workDir', '')
        prodDBlockToken = pdict.get('access', '')

        # get the Rucio tracing report
        report = self.getStubTracingReport(pdict['report'], 'castorSVC', lfn,
                                           guid)

        # get a proper envsetup
        envsetup = self.getEnvsetup(get=True)

        # Hard code the configuration dictionary for now, but eventually this should be
        # set dynamically.
        #
        # There are the following configuration sections:
        #  setup - base environment veriables to be set
        #  svcClassMap - dictionary of string matches vs. service class names
        #  svcClassList - list of all service classes in case the svcClassMap matching fails
        #  svcClassDefault - the service class to set if the file appears to be staged no where
        #
        # Information from RAL:
        # [root@srm0661 ~]# listStorageArea -v atlas
        # <Space Token>                            <Description>   <service class>    <type>          <status>
        # 4948ef55-0000-1000-b7dd-9b38bdd87201    "ATLASGROUP"    "atlasStripDeg"     "DURABLE"       "ALLOCATED"
        # 4948ef38-0000-1000-8606-973e4e998e02    "ATLASMCDISK"   "atlasSimStrip"     "DURABLE"       "ALLOCATED"
        # 4948eec6-0000-1000-8ca2-aba0529b4806    "ATLASDATADISK" "atlasStripInput"   "DURABLE"       "ALLOCATED"
        # 4948ee8e-0000-1000-9ac5-81bb9b34ba7b    "ATLASMCTAPE"   "atlasSimRaw"       "PERMANENT"     "ALLOCATED"
        # 4948ee71-0000-1000-b611-a0afad31f6c8    "ATLASDATATAPE" "atlasT0Raw"        "PERMANENT"     "ALLOCATED"
        #                                         "ATLASHOTDISK"  "atlasHotDisk"
        # In addition there is the "atlasFarm" class, which is used when data is staged back from tape
        castorConfig = {
            'setup': {
                'STAGE_HOST': 'catlasstager.ads.rl.ac.uk',
                'STAGER_HOST': 'catlasstager.ads.rl.ac.uk',
                'RFIO_USE_CASTOR_V2': 'YES',
            },
            'svcClassList': (
                'atlasHotDisk',
                'atlasSimStrip',
                'atlasStripInput',
                'atlasFarm',
                'atlasStripDeg',
                'atlasT0Raw',
                'atlasSimRaw',
                'atlasScratchDisk',
            ),
            'svcClassMap': {
                '/atlashotdisk/': 'atlasHotDisk',
                '/atlasmcdisk/': 'atlasStripInput',
                '/atlasdatadisk/': 'atlasStripInput',
                '/atlasgroupdisk/': 'atlasStripDeg',
                '/atlasdatatape/': 'atlasFarm',
                '/atlasmctape/': 'atlasFarm',
                '/atlasscratchdisk/': 'atlasScratchDisk',
                '/atlasProdDisk/': 'atlasScratchDisk',
            },
            'svcClassDefault':
            'atlasFarm',
        }

        # Set all environment variables for castor setup
        for envVar, value in castorConfig['setup'].iteritems():
            os.environ[envVar] = value

        # Strip the gpfn (SURL) back to its bare castor component
        tolog("gpfn is %s" % gpfn)

        if self._setup:
            _setup_str = "source %s; " % self._setup
        else:
            _setup_str = envsetup

        ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
        if ec != 0:
            self.prepareReport('RFCP_FAIL', report)
            return ec, pilotErrorDiag

        loc_pfn = ''
        if (gpfn.find('SFN') != -1):
            s = gpfn.split('SFN=')
            loc_pfn = s[1]
            tolog("Found SFN string. Local file name %s" % loc_pfn)
        else:
            _tmp = gpfn.split('/', 3)
            loc_pfn = '/' + _tmp[3]
            tolog("Splitting SURL on slashes. Got local file name %s" %
                  loc_pfn)

        if not loc_pfn.startswith('/castor/'):
            tolog(
                "WARNING: Problem with local filename: Does not start with '/castor/'."
            )

        # should the root file be copied or read directly by athena?
        directIn, useFileStager = self.getTransferModes()
        if directIn:
            if useCT:
                directIn = False
                tolog(
                    "Direct access mode is switched off (file will be transferred with the copy tool)"
                )
                updateFileState(lfn,
                                workDir,
                                jobId,
                                mode="transfer_mode",
                                state="copy_to_scratch",
                                ftype="input")
            else:
                # determine if the file is a root file according to its name
                rootFile = self.isRootFileName(lfn)

                if prodDBlockToken == 'local' or not rootFile:
                    directIn = False
                    tolog(
                        "Direct access mode has been switched off for this file (will be transferred with the copy tool)"
                    )
                    updateFileState(lfn,
                                    workDir,
                                    jobId,
                                    mode="transfer_mode",
                                    state="copy_to_scratch",
                                    ftype="input")
                elif rootFile:
                    tolog(
                        "Found root file according to file name: %s (will not be transferred in direct reading mode)"
                        % (lfn))
                    report['relativeStart'] = None
                    report['transferStart'] = None
                    self.prepareReport('FOUND_ROOT', report)
                    if useFileStager:
                        updateFileState(lfn,
                                        workDir,
                                        jobId,
                                        mode="transfer_mode",
                                        state="file_stager",
                                        ftype="input")
                    else:
                        updateFileState(lfn,
                                        workDir,
                                        jobId,
                                        mode="transfer_mode",
                                        state="remote_io",
                                        ftype="input")
                    return error.ERR_DIRECTIOFILE, pilotErrorDiag
                else:
                    tolog("Normal file transfer")

        # Now need to find the service class associated with the file.
        # If we find a clear indication of a space token in the file path
        # then this is easy. However, if we don't, then use stager_qry to
        # interrogate each possible service class. If this fails then use
        # atlasFarm in desperation.
        serviceClass = None
        for pathMatch, svcClass in castorConfig['svcClassMap'].iteritems():
            if loc_pfn.find(pathMatch) >= 0:
                tolog('Matched path element %s - service class is %s' %
                      (pathMatch, svcClass))
                serviceClass = svcClass
                break
            else:
                tolog('Path element %s for service class %s - no match' %
                      (pathMatch, svcClass))

        # For testing the fallback, then we need to hobble ourselves by unsetting serviceClass:
        #tolog('Automatic service class was: %s' % serviceClass)
        #tolog('Unsetting service class for fallback testing')
        #serviceClass = None
        if serviceClass == None:
            tolog("Warning: Failed to find service class hint in SURL.")

            for tryMe in castorConfig['svcClassList']:
                os.environ['STAGE_SVCCLASS'] = tryMe
                tolog('Trying service class %s for file' % tryMe)
                err, output = commands.getstatusoutput('stager_qry -M %s' %
                                                       loc_pfn)
                if err != 0:
                    tolog(
                        'WARNING: Unexpected status from stager_qry: %d\n%s' %
                        (err, output))
                else:
                    if output.find('STAGED') >= 0:
                        tolog('Found file in service class %s' % tryMe)
                        serviceClass = tryMe
                        break
                    else:
                        tolog('File not found in service class %s' % tryMe)
            if serviceClass == None:
                tolog(
                    'WARNING: Failed to find file in any expected service class - will set STAGE_SVCCLASS to %s'
                    % castorConfig['svcClassDefault'])
                serviceClass = castorConfig['svcClassDefault']

        tolog('Setting STAGE_SVCCLASS to %s' % serviceClass)
        os.environ['STAGE_SVCCLASS'] = serviceClass

        dest_path = os.path.join(path, lfn)
        _cmd_str = '%s/usr/bin/rfcp %s %s' % (_setup_str, loc_pfn, dest_path)
        tolog("Executing command: %s" % (_cmd_str))
        report['transferStart'] = time()

        # execute
        timeout = 3600
        try:
            s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
        except Exception, e:
            pilotErrorDiag = 'timed_command() threw an exception: %s' % (e)
            tolog("!!WARNING!!1111!! %s" % (pilotErrorDiag))
            s = 1
            o = str(e)
            telapsed = timeout
    def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
        """ copy input file from SE to local dir """

        # try to get the direct reading control variable (False for direct reading mode; file should not be copied)
        useCT = pdict.get('usect', True)
        prodDBlockToken = pdict.get('access', '')

        # get the DQ2 tracing report
        try:
            report = pdict['report']
        except:
            report = {}
        else:
            # set the proper protocol
            report['protocol'] = 'local'
            # mark the relative start
            report['relativeStart'] = time()
            # the current file
            report['filename'] = lfn
            # guid
            report['guid'] = guid.replace('-','')

        if not path:
            tolog('path is empty, using current directory')            
            path = os.getcwd()

        # build setup string
        envsetup = self.getEnvsetup(get=True)

        # should the root file be copied or read directly by athena?
        directIn = False
        dInfo = getDirectAccessDic(readpar('copysetupin'))
        # if copysetupin did not contain direct access info, try the copysetup instead
        if not dInfo:
            dInfo = getDirectAccessDic(readpar('copysetup'))

        tolog("dInfo: %s" % str(dInfo))
        # check if we should use the copytool
        if dInfo:
            directIn = dInfo['directIn']

        if directIn:
            if useCT:
                directIn = False
                tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
            else:
                # determine if the file is a root file according to its name
                rootFile = self.isRootFileName(lfn)

                if prodDBlockToken == 'local' or not rootFile:
                    directIn = False
                    tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
                elif rootFile:
                    tolog("Found root file according to file name: %s (will not be transferred in direct reading mode)" % (lfn))
                    report['relativeStart'] = None
                    report['transferStart'] = None
                    self.prepareReport('FOUND_ROOT', report)
                    return 0, self.__pilotErrorDiag
                else:
                    tolog("Normal file transfer")
        else:
            tolog("not directIn")

        # build the get command
        _params = ""
        if fchecksum and fchecksum != 'None' and fchecksum != 0 and fchecksum != "0" and not self.isDummyChecksum(fchecksum):
            csumtype = self.getChecksumType(fchecksum)
            # special case for md5sum (command only understands 'md5' and 'adler32', and not 'ad' and 'md5sum')
            if csumtype == 'md5sum':
                csumtype = 'md5'

        execStr = self.__localget % (envsetup, _params, gpfn, os.path.join(path, lfn))
        tolog("Executing command: %s" % (execStr))
        
        report['transferStart'] = time()
        try:
            status, telapsed, cout, cerr = timed_command(execStr, self.__timeout)
        except Exception, e:
            self.__pilotErrorDiag = 'timed_command() threw an exception: %s' % str(e)
            tolog(self.__warningStr % self.__pilotErrorDiag)
            status = 1
            output = str(e)
            telapsed = self.__timeout
Exemplo n.º 8
0
    def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
        """
        The local file is assubed to have a relative path that is the same of the relative path in the 'gpfn'
        loc_... are the variables used to access the file in the locally exported file system
        """

        error = PilotErrors()
        pilotErrorDiag = ""

        # Get input parameters from pdict
        useCT = pdict.get('usect', True)
        jobId = pdict.get('jobId', '')
        workDir = pdict.get('workDir', '')
        prodDBlockToken = pdict.get('access', '')

        # get the DQ2 tracing report
        try:
            report = pdict['report']
        except:
            report = {}
        else:
            # set the proper protocol
            report['protocol'] = 'rfcpLFC'
            # mark the relative start
            report['relativeStart'] = time()
            # the current file
            report['filename'] = lfn
            # guid
            report['guid'] = guid.replace('-','')

        tolog("gpfn is %s" % gpfn)

        # get a proper envsetup
        envsetup = self.getEnvsetup(get=True)

        if self._setup:
            _setup_str = "source %s; " % self._setup
        else:
            _setup_str = envsetup

        ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
        if ec != 0:
            self.prepareReport('RFCP_FAIL', report)
            return ec, pilotErrorDiag

        # remove any host and SFN info from PFN path
        loc_pfn = self.extractPathFromPFN(gpfn)

        try:
            if not loc_pfn.startswith(('/dpm', '/castor')):
                tolog("Potential problem with local filename. Does not start with '/dpm' or '/castor/'.")
        except TypeError:
            # Older version of python
            pass

        # should the root file be copied or read directly by athena?
        directIn, useFileStager = self.getTransferModes()
        if directIn:
            if useCT:
                directIn = False
                tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
                updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
            else:
                # determine if the file is a root file according to its name
                rootFile = self.isRootFileName(lfn)

                if prodDBlockToken == 'local' or not rootFile:
                    directIn = False
                    tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
                    updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
                elif rootFile:
                    tolog("Found root file according to file name: %s (will not be transferred in direct reading mode)" % (lfn))
                    report['relativeStart'] = None
                    report['transferStart'] = None
                    self.prepareReport('FOUND_ROOT', report)
                    if useFileStager:
                        updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="file_stager", type="input")
                    else:
                        updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="remote_io", type="input")
                    return error.ERR_DIRECTIOFILE, pilotErrorDiag
                else:
                    tolog("Normal file transfer")

        dest_path = os.path.join(path, lfn)
        #PN
        _cmd_str = '%srfcp %s %s' % (_setup_str, loc_pfn, dest_path)
#        if ".lib." in loc_pfn:
#            _cmd_str = '%srfcp %s %s' % (_setup_str, loc_pfn, dest_path)
#        else:
#            _cmd_str = '%srfcpXXX %s %s' % (_setup_str, loc_pfn, dest_path)
        tolog("Executing command: %s" % (_cmd_str))
        report['transferStart'] = time()

        # execute
        timeout = 3600
        try:
            s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
        except Exception, e:
            pilotErrorDiag = 'timed_command() threw an exception: %s' % (e)
            tolog("!!WARNING!!1111!! %s" % (pilotErrorDiag))
            s = 1
            o = str(e)
            telapsed = timeout
Exemplo n.º 9
0
        except EnvironmentError, e:  # includes OSError (permission) and IOError (write error)
            pilotErrorDiag = "put_data mkdirWperm failed: %s" % str(e)
            tolog("!!WARNING!!2999!! %s" % (pilotErrorDiag))
            self.prepareReport('MKDIRWPERM_FAIL', report)
            return self.put_data_retfail(error.ERR_MKDIR, pilotErrorDiag, surl=dst_gpfn)

        report['transferStart'] = time()
        # execute dccp
        cmd = "%sdccp %s %s" % (_setup_str, source, dst_loc_sedir)
        tolog("Executing command: %s" % (cmd))
        try:
#            t0 = time.time()
#            s, cout = commands.getstatusoutput(cmd)
#            cerr = ""
#            telapsed = time.time() - t0
            s, telapsed, cout, cerr = timed_command(cmd, timeout)
            print "DEBUG: s = ", s, type(s)
            print "DEBUG: telapsed = ", telapsed, type(telapsed)
            print "DEBUG: cout = ", cout, type(cout)
            print "DEBUG: cerr = ", cerr, type(cerr)
            cout = self.filter_text(cout)
            cerr = self.filter_text(cerr)
            if not self.is_number(s):
                s = 1
            if not self.is_number(telapsed):
                telapsed = 0
        except Exception, e:
            # There may be exceptions like permission errors
            tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" % (e))
            s = 1
            e = sys.exc_info()
Exemplo n.º 10
0
except ImportError:
  import commands
  timed_command = None

# Wrapper around commands or timed_command
def run_command(cmd, timeout=0, workdir=None):
  """Run an external command in the workdir directory. Timeout is available only if timed_command is available."""
  olddir = None
  if workdir: 
    olddir = os.getcwd()
    try:
      os.chdir(workdir)
    except OSError, e:
      return 1, "cd to workdir %s failed: %s" % (workdir, e)
  if timed_command:
    ec, elapsed, out, err = timed_command.timed_command(cmd, timeout)
    outerr = out + err
  else:      
    ec, outerr = commands.getstatusoutput(cmd)
  if olddir:
    os.chdir(olddir)
  return ec, outerr

# Test connection to host:port
def ping(host, port="80"):
  """Check if able to connect to HOST on the specified PORT (80 as default). Return True/False and a message."""
  # Simple alive test
  # There are more complex implementation of ping running the command in a subprocess, 
  # as python function, or within a library: 
  #  http://stackoverflow.com/questions/316866/ping-a-site-in-python
  #  http://www.g-loaded.eu/2009/10/30/python-ping/
Exemplo n.º 11
0
    def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
        """
        Fetch a input file
        Use uberftp, invoked as shell command, to copy a file on a Gridftp server to a local directory
        gpfn -- full URL of the source file (gsiftp://... )
        path -- destination directory (local path, absolute or relative to the execution dir, in a local file system).
                It is assumed to be there. get_data returns an error if the path is missing
        fsize -- source file size, if not provided it is evaluated using uberftp as execution agent
        fchecksum -- source file MD5 checsum, if not provided it is evaluated using uberftp as execution agent
        """

        error = PilotErrors()
        pilotErrorDiag = ""

        try:
            timeout = pdict['timeout']
        except:
            timeout = 5*3600

        # hot fix for OSCER, remove troubling port number if present
        gpfn = gpfn.replace(":2811","")

        import socket
        if( gpfn.find('SFN') != -1 ):
            s = gpfn.split('SFN=')
            src_pfn = s[1]
            # in this case the file is local 
            src_host = socket.getfqdn()
        else:
            # it assumes gpfn to be a URL: the host to have a GSIftp server, and it to see the file in the path of the URL
            _tmp = gpfn.split('/', 3)
            src_pfn = '/'+_tmp[3]
            src_host = _tmp[2]
        src_dirname = os.path.dirname(src_pfn)
        src_filename = os.path.basename(src_pfn)
        dest_file = os.path.join(path, src_filename)

        # when the file has been copied we will rename it to the lfn (to remove the __DQ2-string on some files)
        final_dest_file = os.path.join(path, lfn)

        if self._setup:
            _setup_str = "source %s; " % self._setup
        else:
            _setup_str = ''

        # do we have a proxy?
        s, pilotErrorDiag = self.verifyProxy(envsetup=_setup_str)
        if s != 0:
            return s, pilotErrorDiag

        # protect against double slashes that might cause problems
        # [path] /lscratch/usatlas1_912327//Panda_Pilot... -> /lscratch/usatlas1_912327/Panda_Pilot...
        # [src_dirname] //ibrix/data/dq2-cache -> /ibrix/data/dq2-cache
        path = path.replace('//', '/')
        src_dirname = src_dirname.replace('//', '/')

        # download the remote file to the local directory
        # if you end the command string with ';' you'll always receive error (ec=1)
        _cmd_str_remote = '%suberftp %s "lcd %s; cd %s; get %s; ls %s"|grep %s|grep -v uberftp' %\
                          (_setup_str, src_host, path, src_dirname, src_filename, src_filename, src_filename)

        tolog("Executing command: %s" % (_cmd_str_remote))
        try:
            s, telapsed, cout, cerr = timed_command(_cmd_str_remote, timeout)
        except Exception, e:
            tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" % str(e))
            s = 1
            o = str(e)
            telapsed = timeout
Exemplo n.º 12
0
    def put_data(self, src_pfn, destination, fsize=0, fchecksum=0, dsname='', extradirs='', **pdict):
        """
        Copy all output file to the local SE
        Check SiteMover.put_data for a description of the first parameters
        src_pfn (source), destination, fsize, fchecksum
        dsname -- name of the dataset, used as additional path for the destination file
        extradirs -- additional directories
        The function tries to create the destination directory if missing
        """

        error = PilotErrors()
        pilotErrorDiag = ""

        try:
            timeout = pdict['timeout']
        except:
            timeout = 5*3600

        if self._setup:
            _setup_str = "source %s; " % self._setup
        else:
            _setup_str = ''

        # do we have a proxy?
        s, pilotErrorDiag = self.verifyProxy(envsetup=_setup_str, limit=2)
        if s != 0:
            return self.put_data_retfail(s, pilotErrorDiag)

        # hot fix for OSCER, remove troubling port number if present
        src_pfn = src_pfn.replace(":2811","")

        # preparing variables
        if fsize == 0 or fchecksum == 0:
            ec, pilotErrorDiag, fsize, fchecksum = self.getLocalFileInfo(src_pfn)
            if ec != 0:
                return self.put_data_retfail(ec, pilotErrorDiag)

        dst_se = destination
        if dst_se.find('SFN') != -1:  # srm://dcsrm.usatlas.bnl.gov:8443/srm/managerv1?SFN=/pnfs/usatlas.bnl.gov/
            s = dst_se.split('SFN=')
            dst_loc_se = s[1]
            dst_prefix = s[0]
        else:
            _sentries = dst_se.split('/', 3)
            dst_serv = _sentries[0] + '//' + _sentries[2] # 'method://host:port' is it always a ftp server? can it be srm? something else?
            dst_host = _sentries[2] # host and port
            dst_loc_se = '/'+ _sentries[3]
            dst_prefix = dst_serv
        dst_relative_path = os.path.join(extradirs, dsname)
        # dst_loc_se = base directory, not including the destination sub dir
        dst_loc_sedir = os.path.join(dst_loc_se, dst_relative_path)
        # dst_loc_sedir = full path to the final destination directory
        dst_relpath_list = [i for i in dst_relative_path.split('/') if i]
        # >>> a='/root/asd/sss'
        # >>> b=[i for i in a.split('/') if i]
        #  ['root', 'asd', 'sss']
        src_filename = os.path.basename(src_pfn)
        src_dirname = os.path.dirname(src_pfn)

        # Destination file name has to be the same, else all the code of the function has to be checked
        dst_filename = src_filename
        dst_loc_pfn = os.path.join(dst_loc_sedir, dst_filename)
        dst_gpfn = dst_prefix + dst_loc_pfn

        # get the number of directories that don't have to be created [from the dst_relpath_list]
        # TODO: check if the syntax host:port works for uberftp
        existing_dir_count_ev,__err = UberftpSiteMover._checkDir(dst_host, dst_loc_se, dst_relpath_list, _setup_str)
        if existing_dir_count_ev < 0:
            if existing_dir_count_ev == -2:
                pilotErrorDiag = ": %s" % (str(__err))
                return self.put_data_retfail(error.ERR_PUTTIMEOUT, pilotErrorDiag)
            else:
                tolog('!!WARNING!!2999!! Control of existing directories failed: %s' % existing_dir_count_ev)
                # If you try to copy a file and the directory does not exist the copy fails
                # If you try to create an already existing directory you get 'File exists' error
                # If the counting of existing directories failed, try to create all of them
                existing_dir_count = 0
                # to try to copy the file directly (creating no directory)
                # existing_dir_count = len(dst_relpath_list)
        else:
            existing_dir_count = existing_dir_count_ev

        # Preparing and executing the transfer command
        _start_dir = dst_loc_se
        _additional_dir = ''
        _relative_dir = ''
        # eg. _start_dir = /ibrix/data/dq2-cache

        if existing_dir_count < len(dst_relpath_list):
            # if existing_dir_count >= 0:
            for i in dst_relpath_list:
                if existing_dir_count > 0:                   
                    # for every loop iteration, add directory i to _start_dir
                    _start_dir = os.path.join(_start_dir, i)
                    # eg. _start_dir = /ibrix/data/dq2-cache/testpanda.destDB.72c916bd-c490-461b-8665-9fb9990081cf_sub0
                    # eg. i = testpanda.destDB.72c916bd-c490-461b-8665-9fb9990081cf_sub0
                    existing_dir_count -= 1
                else:
                    _relative_dir = os.path.join(_relative_dir, i)
                    _additional_dir += "mkdir %s;" % (_relative_dir)
                    _additional_dir += "chmod %s %s;" % (str(oct(PERMISSIONS_DIR)), _relative_dir)
            if _relative_dir:
                _relative_dir = "cd %s;" % _relative_dir
            _ftp_cmd_0 = "cd %s; %s%s" % (_start_dir, _additional_dir, _relative_dir)
        else:
            # directories are already all there
            _ftp_cmd_0 = "cd %s; " % (os.path.join(_start_dir, dst_relative_path))
        
        # protect against double slashes that might cause problems
        # [src_dirname] //ibrix/data/dq2-cache -> /ibrix/data/dq2-cache
        src_dirname = src_dirname.replace('//', '/')

        # uberftp $host "cd $directory; get $fileName; ls $fileName"|grep $fileName|grep -v uberftp > /tmp/fileSize.txt

        _ftp_cmd_str = '%s lcd %s; put %s; ls %s' %\
                       (_ftp_cmd_0, src_dirname, src_filename, dst_filename)
        _cmd_str = "%swhich uberftp;uberftp %s '%s'" %\
                   (_setup_str, dst_host, _ftp_cmd_str)

        # local dir should exist
        # _cmd_str = Copying uberftp: source /hep/home/gridapp/atlas_app/UberFTP/setup.sh; uberftp tier2-02.ochep.ou.edu
        #            'mkdir /ibrix/data/dq2-cache;
        #            cd /ibrix/data/dq2-cache/testpanda.destDB.72c916bd-c490-461b-8665-9fb9990081cf_sub0;
        #            lcd /lscratch/usatlas1_207555/Panda_Pilot_12498_1174944165/PandaJob_1280039_1174944165;
        #            put 4cd66105-bf50-499d-a81d-9f8c3ba3bf69.evgen.pool.root;
        #            lls 4cd66105-bf50-499d-a81d-9f8c3ba3bf69.evgen.pool.root;
        #            ls 4cd66105-bf50-499d-a81d-9f8c3ba3bf69.evgen.pool.root;'|
        #            grep 4cd66105-bf50-499d-a81d-9f8c3ba3bf69.evgen.pool.root|grep -v uberftp
        tolog("Executing command: %s" % (_cmd_str))
        try:
            s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
        except Exception, e:
            tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" % str(e))
            s = 1
            o = str(e)
            telapsed = timeout
Exemplo n.º 13
0
    def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
        """ copy input file from SE to local dir """

        # Get input parameters from pdict
        useCT = pdict.get('usect', True)
        jobId = pdict.get('jobId', '')
        workDir = pdict.get('workDir', '')
        prodDBlockToken = pdict.get('access', '')

        # get the DQ2 tracing report
        report = self.getStubTracingReport(pdict['report'], 'local', lfn, guid)

        if not path:
            tolog('path is empty, using current directory')            
            path = os.getcwd()

        # build setup string
        envsetup = self.getEnvsetup(get=True)

        # should the root file be copied or read directly by athena?
        directIn, useFileStager = self.getTransferModes()
        if directIn:
            if useCT:
                directIn = False
                tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
                updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
            else:
                # determine if the file is a root file according to its name
                rootFile = self.isRootFileName(lfn)

                if prodDBlockToken == 'local' or not rootFile:
                    directIn = False
                    tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
                    updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
                elif rootFile:
                    tolog("Found root file according to file name: %s (will not be transferred in direct reading mode)" % (lfn))
                    report['relativeStart'] = None
                    report['transferStart'] = None
                    self.__sendReport('FOUND_ROOT', report) 
                    if useFileStager:
                        updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="file_stager", type="input")
                    else:
                        updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="remote_io", type="input")
                    return 0, self.__pilotErrorDiag
                else:
                    tolog("Normal file transfer")
        else:
            tolog("not directIn")



        # THE FOLLOWING CODE UNTIL THE END OF THIS FUNCTION SHOULD BE REPLACED WITH NEW CODE FOR GLOBUS-ONLINE


	def downloadGC():
            """Download Globus Connect file from Globus Server
            """
            arg = ['curl','--connect-timeout','20','--max-time','120','-s','-S','http://confluence.globus.org/download/attachments/14516429/globusconnect','-o','globusconnect']
            tolog("Download Arguments: %s" % (arg))
            subprocess.call(arg)
        
        #Function to retrieve users proxy from Cern myproxy server.
        #The credential should be deposite to the server by the user under the Pilots DN username
        #and authoried to retrieve it by the Pilot without passphrase,
        #to run Globus Online.
        #Code partially extracted from MyproxyUtils.py
        #Uses as parameter userID, which is obtained from job info from Panda.
        
	def getproxy(userID):
            """Gets users myproxy from cern server
            """
            dn = userID
            if dn.count('/CN=') > 1:
               first_index = dn.find('/CN=')
               second_index = dn.find('/CN=', first_index+1)
               dn = dn[0:second_index]
            
            arg = ['myproxy-logon','-s','myproxy.cern.ch','--no_passphrase','-l',dn]
            subprocess.call(arg)
        
        #Fuction uses endpoint as parameter, that the user specifies at job submission.
        #Corresponds to the name of the endpoint for the working node using Globus-
        #Connect. Returns the code for the setup
        
        def createEndpoint(endpoint):
            """Create the endpoint the user specifies and returns the code for setup
            """
            arg = ['gsissh', 'cli.globusonline.org', 'endpoint-add','--gc', endpoint]
            proc = subprocess.Popen(arg, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
            return_code = proc.wait()
            i = 0
            for line in proc.stdout:
                tolog(" %s " % (line.rstrip()))
                i += 1
                if i == 3:
                   code = line
        
            return code

        #Function uses endpoint as parameter to get the code and run the
        #setup in Globus Connect. This connects the endpoint on the working node with
        #Globus Online
        
        def setupGC(endpoint):
            """Installing Globus Connect on working node. Creates endpoint, get the setup code and uses it on setup mode
            """
            code = createEndpoint(endpoint)
            code = code.strip()
            arg = ['sh', 'globusconnect', '-setup', code]
            tolog("Arguments:  %s:" % (arg))
            subprocess.call(arg)

        #Function that runns Globus Connect on the background of the working node
        #to make it a Globus Online Endpoint
        
	def startGC():
            """Start Globus Connect process on working node, on the background
            """
            tolog("-----Running Globus Connect------")
            arg = ['sh', 'globusconnect', '-start']
            subprocess.Popen(arg)

        
        #Function uses source and destinations as parameters.Source is a
        #Globus Online Endpoint and path to file, like this:
        #SITE1:/home/osg_data/panda/0064.file
        #Destination is the Endpoint on the working node ,using Globus Connect,
        #like this:
        #OSGDEV:/direct/usatlas+u/ccontrer/pilot/autopilot/trunk/
        #User defines source and destination at job submission.
        #This function activates the source endpoint, under the consideration
        #it is a gridFTP server. Destination is already activated since it is a
        #Globus Connect endpoint and need no manual activation
        #This function creates one transfer task, so it should be executed
        #for as many source files as the user specifies
        
        
        #For a better dinamic, destination path can/should be obtained from job information (?)
        #and the endpoint is specified by user, as above.

	dest = endpoint + ':' + jobdir

	def stageincmdGO(src, dest):
            """create command for stage-in using Globus Online
            """
            #Activate the source endpoint
            source = src.split(':')[0]
            arg = ['gsissh', 'cli.globusonline.org', 'endpoint-activate','-g', source]
            subprocess.call(arg)
            
            #Create transfer task
            cmd = 'gsissh cli.globusonline.org scp -v %s %s'%(src, dest)
            tolog("Transfer task created: %s" % (cmd))
            return cmd

        #END OF GLOBUS ONLINE FUNCTIONS

    
        # build the get command
        _params = ""
        if fsize != 0 and fsize != "0":
            _params += self.__par_filesize % (fsize,)
        if fchecksum and fchecksum != 'None' and fchecksum != 0 and fchecksum != "0" and not self.isDummyChecksum(fchecksum):
            csumtype = self.getChecksumType(fchecksum)
            # special case for md5sum (command only understands 'md5' and 'adler32', and not 'ad' and 'md5sum')
            if csumtype == 'md5sum':
                csumtype = 'md5'
            _params += self.__par_checksum % ("%s:%s" % (csumtype, fchecksum),)

        # add the guid option
        _params += " --guid %s" % (guid)

        execStr = self.__localget % (envsetup, _params, gpfn, os.path.join(path, lfn))
        tolog("Executing command: %s" % (execStr))
        
        report['transferStart'] = time()
        try:
            status, telapsed, cout, cerr = timed_command(execStr, self.__timeout)
        except Exception, e:
            self.__pilotErrorDiag = 'timed_command() threw an exception: %s' % str(e)
            tolog(self.__warningStr % self.__pilotErrorDiag)
            status = 1
            output = str(e)
            telapsed = self.__timeout
Exemplo n.º 14
0
        if fchecksum != 0:
            # special case for md5sum (command only understands 'md5' and 'adler32', and not 'ad' and 'md5sum')
            if csumtype == 'md5sum':
                _csumtype = 'md5'
            else:
                _csumtype = csumtype
            _params += self.__par_checksum % ("%s:%s" % (_csumtype, fchecksum),)

        # add the guid option
        _params += " --guid %s" % (guid)

        execStr = self.__localput % (envsetup, _params, source, dst_gpfn)
        tolog("Executing command: %s" % (execStr))
        report['transferStart'] = time()
        try:
            status, telapsed, cout, cerr = timed_command(execStr, self.__timeout)
        except Exception, e:
            self.__pilotErrorDiag = 'timed_command() threw an exception: %s' % str(e)
            tolog(self.__warningStr % self.__pilotErrorDiag)
            status = 1
            output = str(e)
            telapsed = self.__timeout
        else:
            output = cout + cerr

        tolog("Elapsed time: %d" % (telapsed))
        tolog("Command output:\n%s" % (output))
        report['validateStart'] = time()

        # validate
        if status:
Exemplo n.º 15
0
    def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
        """ copy input file from SE to local dir """

        # Get input parameters from pdict
        useCT = pdict.get('usect', True)
        jobId = pdict.get('jobId', '')
        workDir = pdict.get('workDir', '')
        prodDBlockToken = pdict.get('access', '')

        # get the DQ2 tracing report
        report = self.getStubTracingReport(pdict['report'], 'local', lfn, guid)

        if not path:
            tolog('path is empty, using current directory')            
            path = os.getcwd()

        # build setup string
        envsetup = self.getEnvsetup(get=True)

        ec, pilotErrorDiag = verifySetupCommand(self.__error, envsetup)
        if ec != 0:
            self.__sendReport('RFCP_FAIL', report)
            return ec, pilotErrorDiag

        # should the root file be copied or read directly by athena?
        directIn, useFileStager = self.getTransferModes()
        if directIn:
            if useCT:
                directIn = False
                tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
                updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
            else:
                # determine if the file is a root file according to its name
                rootFile = self.isRootFileName(lfn)

                if prodDBlockToken == 'local' or not rootFile:
                    directIn = False
                    tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
                    updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
                elif rootFile:
                    tolog("Found root file according to file name: %s (will not be transferred in direct reading mode)" % (lfn))
                    report['relativeStart'] = None
                    report['transferStart'] = None
                    self.__sendReport('FOUND_ROOT', report) 
                    if useFileStager:
                        updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="file_stager", type="input")
                    else:
                        updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="remote_io", type="input")
                    return self.__error.ERR_DIRECTIOFILE, self.__pilotErrorDiag
                else:
                    tolog("Normal file transfer")
        else:
            tolog("not directIn")

        # build the get command
        _params = ""
        if fsize != 0 and fsize != "0":
            _params += self.__par_filesize % (fsize,)
        if fchecksum and fchecksum != 'None' and fchecksum != 0 and fchecksum != "0" and not self.isDummyChecksum(fchecksum):
            csumtype = self.getChecksumType(fchecksum)
            # special case for md5sum (command only understands 'md5' and 'adler32', and not 'ad' and 'md5sum')
            if csumtype == 'md5sum':
                csumtype = 'md5'
            _params += self.__par_checksum % ("%s:%s" % (csumtype, fchecksum),)

        # add the guid option
        _params += " --guid %s" % (guid)

        dest_path = os.path.join(path, lfn)
        #PN
#        if ".lib." in gpfn:
#            localGet = self.__localget
#        else:
#            localGet = self.__localgetBAD
#        execStr = localGet % (envsetup, _params, gpfn, dest_path)
        execStr = self.__localget % (envsetup, _params, gpfn, dest_path)
        tolog("Executing command: %s" % (execStr))

        report['transferStart'] = time()
        try:
            status, telapsed, cout, cerr = timed_command(execStr, self.__timeout)
        except Exception, e:
            self.__pilotErrorDiag = 'timed_command() threw an exception: %s' % str(e)
            tolog(self.__warningStr % self.__pilotErrorDiag)
            status = 1
            output = str(e)
            telapsed = self.__timeout
Exemplo n.º 16
0
    def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
        """
        The local file (local access to the dCache file) is assumed to have a relative path
        that is the same of the relative path in the 'gpfn'
        loc_... are the variables used to access the file in the locally exported file system
        """

        error = PilotErrors()
        pilotErrorDiag = ""

        # Get input parameters from pdict
        useCT = pdict.get('usect', True)
        jobId = pdict.get('jobId', '')
        workDir = pdict.get('workDir', '')
        analJob = pdict.get('analJob', False)
        timeout = pdict.get('timeout', 5 * 3600)
        prodDBlockToken = pdict.get('access', '')

        # get the Rucio tracing report
        report = self.getStubTracingReport(pdict['report'], 'BNLdCache', lfn,
                                           guid)

        # get a proper envsetup
        envsetup = self.getEnvsetup(get=True)

        if self._setup:
            _setup_str = "source %s; " % self._setup
        else:
            _setup_str = envsetup

        ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
        if ec != 0:
            self.prepareReport('RFCP_FAIL', report)
            return ec, pilotErrorDiag

        # remove any host and SFN info from PFN path
        loc_pfn = self.extractPathFromPFN(gpfn)

        copyprefixin = readpar('copyprefixin')
        if copyprefixin != '':
            # Extract the copy prefix
            pfrom, pto = copyprefixin.split('^')
            loc_pfn = pfrom + loc_pfn
            tolog("Added copyprefixin to file: %s" % (loc_pfn))
        else:
            copyprefix = readpar('copyprefix')
            if copyprefix != '':
                # Extract the copy prefix
                pfrom, pto = copyprefix.split('^')
                loc_pfn = pfrom + loc_pfn
                tolog("Added copyprefix to file: %s" % (loc_pfn))

        report['relativeStart'] = time.time()

        pnfsid = self.getPnfsid(loc_pfn, guid)

        # for analysis jobs, skip input file if on tape or if lib file
        if analJob:
            if not self.isLibFile(loc_pfn):
                if pnfsid == None:
                    isStaged = self.isFileStaged(_setup_str, loc_pfn)
                else:
                    _com = "/cacheinfos/isFileInPool?pnfsid=%s" % (pnfsid)
                    isStaged = self.isFileStaged(
                        _setup_str,
                        loc_pfn,
                        url="ddmv02.usatlas.bnl.gov:8000",
                        com=_com)
                if not isStaged:
                    pilotErrorDiag = "File %s is not staged and will be skipped for analysis job" % (
                        loc_pfn)
                    self.prepareReport('FILE_ON_TAPE', report)
                    return error.ERR_FILEONTAPE, pilotErrorDiag
            else:
                tolog("Skipping file stage check for lib file")

        # should the root file be copied or read directly by athena?
        directIn, useFileStager = self.getTransferModes()
        if directIn:
            if useCT:
                directIn = False
                tolog(
                    "Direct access mode is switched off (file will be transferred with the copy tool)"
                )
                updateFileState(lfn,
                                workDir,
                                jobId,
                                mode="transfer_mode",
                                state="copy_to_scratch",
                                ftype="input")
            else:
                # determine if the file is a root file according to its name
                rootFile = self.isRootFileName(lfn)

                if prodDBlockToken == 'local' or not rootFile:
                    directIn = False
                    tolog(
                        "Direct access mode has been switched off for this file (will be transferred with the copy tool)"
                    )
                    updateFileState(lfn,
                                    workDir,
                                    jobId,
                                    mode="transfer_mode",
                                    state="copy_to_scratch",
                                    ftype="input")
                elif rootFile:
                    tolog(
                        "Found root file according to file name: %s (will not be transferred in direct reading mode)"
                        % (lfn))
                    report['relativeStart'] = None
                    report['transferStart'] = None
                    self.prepareReport('FOUND_ROOT', report)
                    if useFileStager:
                        updateFileState(lfn,
                                        workDir,
                                        jobId,
                                        mode="transfer_mode",
                                        state="file_stager",
                                        ftype="input")
                    else:
                        updateFileState(lfn,
                                        workDir,
                                        jobId,
                                        mode="transfer_mode",
                                        state="remote_io",
                                        ftype="input")
                    return error.ERR_DIRECTIOFILE, pilotErrorDiag
                else:
                    tolog("Normal file transfer")

        dest_path = os.path.join(path, lfn)
        if pnfsid == None:
            _cmd_str = '%sdccp %s %s' % (_setup_str, loc_pfn, dest_path)
        else:
            _cmd_str = '%sdccp pnfs://dcdcap.usatlas.bnl.gov:22125/%s %s' % (
                _setup_str, pnfsid, dest_path)

        tolog("Executing command: %s" % (_cmd_str))
        report['transferStart'] = time.time()
        try:
            s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
        except Exception, e:
            tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" %
                  str(e))
            s = 1
            o = str(e)
            telapsed = timeout
Exemplo n.º 17
0
        try:
            _dq2SiteName = self.getDQ2SiteName(surl=dst_gpfn)
        except Exception, e:
            tolog("Warning: Failed to get the DQ2 site name: %s (can not add this info to tracing report)" % str(e))
        else:
            report['localSite'], report['remoteSite'] = (_dq2SiteName, _dq2SiteName)
            tolog("DQ2 site name: %s" % (_dq2SiteName))

        # Directory not created: the copy command creates the direcrory if necessary,
        # all with the right permission

        report['transferStart'] = time()
        try:
            _cmd_str = '%snecp %s %s' % (_setup_str, source, dst_loc_sedir)
            tolog("NECP executing (timeout %s): %s" % (timeout, _cmd_str))
            s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
        except Exception, e:
            # Command may have exceptions like permission errors
            # timed_command hsould block/filter out all exceptions; remove try block?
            s = 1
            e = sys.exc_info()
            o = str(e[1]) #set error message from exception text
            telapsed = timeout
        else:
            o = cout + cerr
            tolog("Elapsed time: %d" % (telapsed))
        report['validateStart'] = time()

        if s != 0:
            check_syserr(s, o)
            pilotErrorDiag = "Error in copying: %s" % (o)
Exemplo n.º 18
0
    def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
        """
        The local file (local access to the dCache file) is assumed to have a relative path
        that is the same of the relative path in the 'gpfn'
        loc_... are the variables used to access the file in the locally exported file system
        """

        error = PilotErrors()
        pilotErrorDiag = ""

        # Get input parameters from pdict
        useCT = pdict.get('usect', True)
        jobId = pdict.get('jobId', '')
        workDir = pdict.get('workDir', '')
        analJob = pdict.get('analJob', False)
        timeout = pdict.get('timeout', 5*3600)
        prodDBlockToken = pdict.get('access', '')

        # get the DQ2 tracing report
        report = self.getStubTracingReport(pdict['report'], 'BNLdCache', lfn, guid)

        # get a proper envsetup
        envsetup = self.getEnvsetup(get=True)

        if self._setup:
            _setup_str = "source %s; " % self._setup
        else:
            _setup_str = envsetup

        ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
        if ec != 0:
            self.__sendReport('RFCP_FAIL', report)
            return ec, pilotErrorDiag

        # remove any host and SFN info from PFN path
        loc_pfn = self.extractPathFromPFN(gpfn)

        copyprefixin = readpar('copyprefixin')
        if copyprefixin != '':
            # Extract the copy prefix
            pfrom, pto = copyprefixin.split('^')
            loc_pfn = pfrom + loc_pfn
            tolog("Added copyprefixin to file: %s" % (loc_pfn))
        else:
            copyprefix = readpar('copyprefix')
            if copyprefix != '':
                # Extract the copy prefix
                pfrom, pto = copyprefix.split('^')
                loc_pfn = pfrom + loc_pfn
                tolog("Added copyprefix to file: %s" % (loc_pfn))

        report['relativeStart'] = time.time()

        pnfsid = self.getPnfsid(loc_pfn, guid)

        # for analysis jobs, skip input file if on tape or if lib file
        if analJob:
            if not self.isLibFile(loc_pfn):
                if pnfsid == None:
                    isStaged = self.isFileStaged(_setup_str, loc_pfn)
                else:
                    _com = "/cacheinfos/isFileInPool?pnfsid=%s" % (pnfsid)
                    isStaged = self.isFileStaged(_setup_str, loc_pfn, url="ddmv02.usatlas.bnl.gov:8000", com=_com)
                if not isStaged:
                    pilotErrorDiag = "File %s is not staged and will be skipped for analysis job" % (loc_pfn)
                    self.__sendReport('FILE_ON_TAPE', report)
                    return error.ERR_FILEONTAPE, pilotErrorDiag
            else:
                tolog("Skipping file stage check for lib file")

        # should the root file be copied or read directly by athena?
        directIn, useFileStager = self.getTransferModes()
        if directIn:
            if useCT:
                directIn = False
                tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
                updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
            else:
                # determine if the file is a root file according to its name
                rootFile = self.isRootFileName(lfn)

                if prodDBlockToken == 'local' or not rootFile:
                    directIn = False
                    tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
                    updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
                elif rootFile:
                    tolog("Found root file according to file name: %s (will not be transferred in direct reading mode)" % (lfn))
                    report['relativeStart'] = None
                    report['transferStart'] = None
                    self.__sendReport('FOUND_ROOT', report)
                    if useFileStager:
                        updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="file_stager", type="input")
                    else:
                        updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="remote_io", type="input")
                    return error.ERR_DIRECTIOFILE, pilotErrorDiag
                else:
                    tolog("Normal file transfer")

        dest_path = os.path.join(path, lfn)
        if pnfsid == None:
            _cmd_str = '%sdccp %s %s' % (_setup_str, loc_pfn, dest_path)
        else:
            _cmd_str = '%sdccp pnfs://dcdcap.usatlas.bnl.gov:22125/%s %s' % (_setup_str, pnfsid, dest_path)
            
        tolog("Executing command: %s" % (_cmd_str))
        report['transferStart'] = time.time()
        try:
            s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
        except Exception, e:
            tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" % str(e))
            s = 1
            o = str(e)
            telapsed = timeout
Exemplo n.º 19
0
    def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
        """
        Moves a DS file the local SE (where was put from DDM) to the working directory.
        Performs the copy and, for systems supporting it, checks size and md5sum correctness
        gpfn: full source URL (e.g. method://[host[:port]/full-dir-path/filename - a SRM URL is OK) 
        path: destination absolute path (in a local file system)
        returns the status of the transfer. In case of failure it should remove the partially copied destination
        """
        # The local file is assumed to have a relative path that is the same of the relative path in the 'gpfn'
        # loc_... are the variables used to access the file in the locally exported file system

        error = PilotErrors()
        pilotErrorDiag = ""

        # try to get the direct reading control variable (False for direct reading mode; file should not be copied)
        useCT = pdict.get('usect', True)
        jobId = pdict.get('jobId', '')
        dsname = pdict.get('dsname', '')
        workDir = pdict.get('workDir', '')
        prodDBlockToken = pdict.get('access', '')

        # get the DQ2 tracing report
        report = self.getStubTracingReport(pdict['report'], 'xrootd', lfn, guid)

        if self._setup:
            _setup_str = "source %s; " % self._setup
        else:
            _setup_str = ''

        ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
        if ec != 0:
            self.__sendReport('RFCP_FAIL', report)
            return ec, pilotErrorDiag

        tolog("xrootdSiteMover get_data using setup: %s" % (_setup_str))

        # remove any host and SFN info from PFN path
        src_loc_pfn = self.extractPathFromPFN(gpfn)

        src_loc_filename = lfn
        # source vars: gpfn, loc_pfn, loc_host, loc_dirname, loc_filename
        # dest vars: path

        if fchecksum != 0 and fchecksum != "":
            csumtype = self.getChecksumType(fchecksum)
        else:
            csumtype = "default"

        # protect against bad pfn's
        src_loc_pfn = src_loc_pfn.replace('///','/')
        src_loc_pfn = src_loc_pfn.replace('//xrootd/','/xrootd/')

        # should the root file be copied or read directly by athena?
        directIn, useFileStager = self.getTransferModes()
        if directIn:
            if useCT:
                directIn = False
                tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
                updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
            else:
                rootFile = self.isRootFile(src_loc_pfn, setup=_setup_str)
                if prodDBlockToken == 'local' or not rootFile:
                    directIn = False
                    tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
                    updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", type="input")
                elif rootFile:
                    tolog("Found root file: %s (will not be transferred in direct reading mode)" % (src_loc_pfn))
                    report['relativeStart'] = None
                    report['transferStart'] = None
                    self.__sendReport('IS_ROOT', report)
                    if useFileStager:
                        updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="file_stager", type="input")
                    else:
                        updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="remote_io", type="input")
                    return error.ERR_DIRECTIOFILE, pilotErrorDiag
                else:
                    tolog("Normal file transfer")
        else:
            tolog("No direct access mode")

        ec = 0
        if fsize == 0 or fchecksum == 0:
            ec, pilotErrorDiag, fsize, fchecksum = self.getLocalFileInfo(src_loc_pfn, csumtype=csumtype)
        if ec != 0:
            self.__sendReport('GET_LOCAL_FILE_INFO_FAIL', report)
            return ec, pilotErrorDiag

        dest_file = os.path.join(path, src_loc_filename)
        
        report['relativeStart'] = time()
        # determine which copy command to use
        cpt = self.getCopytool(_setup_str)

        report['transferStart'] = time()
        cmd = "%s %s %s %s" % (_setup_str, cpt, src_loc_pfn, dest_file)
#PN
#        if ".lib." in src_loc_pfn:
#            cmd = "%s %s %s %s" % (_setup_str, cpt, src_loc_pfn, dest_file)
#        else:
#            cmd = "%s %sXXX %s %s" % (_setup_str, cpt, src_loc_pfn, dest_file)
        tolog("Executing command: %s" % (cmd))

        # execute
        timeout = 3600
        try:
            rc, telapsed, cout, cerr = timed_command(cmd, timeout)
        except Exception, e:
            self.__pilotErrorDiag = 'timed_command() threw an exception: %s' % str(e)
            tolog("!!WARNING!!1111!! %s" % (pilotErrorDiag))
            rc = 1
            rs = str(e)
            telapsed = timeout
Exemplo n.º 20
0
    def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
        """
        Moves a DS file the local SE (where was put from DDM) to the working directory.
        Performs the copy and, for systems supporting it, checks size and md5sum correctness
        gpfn: full source URL (e.g. method://[host[:port]/full-dir-path/filename - a SRM URL is OK)
        path: destination absolute path (in a local file system)
        returns the status of the transfer. In case of failure it should remove the partially copied destination
        """
        # The local file is assumed to have a relative path that is the same of the relative path in the 'gpfn'
        # loc_... are the variables used to access the file in the locally exported file system

        error = PilotErrors()
        pilotErrorDiag = ""

        # try to get the direct reading control variable (False for direct reading mode; file should not be copied)
        useCT = pdict.get('usect', True)
        jobId = pdict.get('jobId', '')
        dsname = pdict.get('dsname', '')
        workDir = pdict.get('workDir', '')
        prodDBlockToken = pdict.get('access', '')

        # get the Rucio tracing report
        report = self.getStubTracingReport(pdict['report'], 'xrootd', lfn,
                                           guid)

        if self._setup:
            _setup_str = "source %s; " % self._setup
        else:
            _setup_str = ''

        ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
        if ec != 0:
            self.prepareReport('RFCP_FAIL', report)
            return ec, pilotErrorDiag

        tolog("xrootdSiteMover get_data using setup: %s" % (_setup_str))

        # remove any host and SFN info from PFN path
        src_loc_pfn = self.extractPathFromPFN(gpfn)

        src_loc_filename = lfn
        # source vars: gpfn, loc_pfn, loc_host, loc_dirname, loc_filename
        # dest vars: path

        if fchecksum != 0 and fchecksum != "":
            csumtype = self.getChecksumType(fchecksum)
        else:
            csumtype = "default"

        # protect against bad pfn's
        src_loc_pfn = src_loc_pfn.replace('///', '/')
        src_loc_pfn = src_loc_pfn.replace('//xrootd/', '/xrootd/')

        # should the root file be copied or read directly by athena?
        directIn, useFileStager = self.getTransferModes()
        if directIn:
            if useCT:
                directIn = False
                tolog(
                    "Direct access mode is switched off (file will be transferred with the copy tool)"
                )
                updateFileState(lfn,
                                workDir,
                                jobId,
                                mode="transfer_mode",
                                state="copy_to_scratch",
                                ftype="input")
            else:
                rootFile = self.isRootFile(src_loc_pfn, setup=_setup_str)
                if prodDBlockToken == 'local' or not rootFile:
                    directIn = False
                    tolog(
                        "Direct access mode has been switched off for this file (will be transferred with the copy tool)"
                    )
                    updateFileState(lfn,
                                    workDir,
                                    jobId,
                                    mode="transfer_mode",
                                    state="copy_to_scratch",
                                    ftype="input")
                elif rootFile:
                    tolog(
                        "Found root file: %s (will not be transferred in direct reading mode)"
                        % (src_loc_pfn))
                    report['relativeStart'] = None
                    report['transferStart'] = None
                    self.prepareReport('IS_ROOT', report)
                    if useFileStager:
                        updateFileState(lfn,
                                        workDir,
                                        jobId,
                                        mode="transfer_mode",
                                        state="file_stager",
                                        ftype="input")
                    else:
                        updateFileState(lfn,
                                        workDir,
                                        jobId,
                                        mode="transfer_mode",
                                        state="remote_io",
                                        ftype="input")
                    return error.ERR_DIRECTIOFILE, pilotErrorDiag
                else:
                    tolog("Normal file transfer")
        else:
            tolog("No direct access mode")

        ec = 0
        if fsize == 0 or fchecksum == 0:
            ec, pilotErrorDiag, fsize, fchecksum = self.getLocalFileInfo(
                src_loc_pfn, csumtype=csumtype)
        if ec != 0:
            self.prepareReport('GET_LOCAL_FILE_INFO_FAIL', report)
            return ec, pilotErrorDiag

        dest_file = os.path.join(path, src_loc_filename)

        report['relativeStart'] = time()
        # determine which copy command to use
        cpt = self.getCopytool(_setup_str)

        report['transferStart'] = time()
        cmd = "%s %s %s %s" % (_setup_str, cpt, src_loc_pfn, dest_file)
        #PN
        #        if ".lib." in src_loc_pfn:
        #            cmd = "%s %s %s %s" % (_setup_str, cpt, src_loc_pfn, dest_file)
        #        else:
        #            cmd = "%s %sXXX %s %s" % (_setup_str, cpt, src_loc_pfn, dest_file)
        tolog("Executing command: %s" % (cmd))

        # execute
        timeout = 3600
        try:
            rc, telapsed, cout, cerr = timed_command(cmd, timeout)
        except Exception, e:
            self.__pilotErrorDiag = 'timed_command() threw an exception: %s' % str(
                e)
            tolog("!!WARNING!!1111!! %s" % (pilotErrorDiag))
            rc = 1
            rs = str(e)
            telapsed = timeout
Exemplo n.º 21
0
    def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
        """
        The local file (local access to the dCache file) is assumed to have a relative path
        that is the same of the relative path in the 'gpfn'
        loc_... are the variables used to access the file in the locally exported file system
        """

        error = PilotErrors()
        pilotErrorDiag = ""

        # Get input parameters from pdict
        jobId = pdict.get('jobId', '')
        workDir = pdict.get('workDir', '')
        analJob = pdict.get('analJob', False)
        timeout = pdict.get('timeout', 5*3600)

        # try to get the direct reading control variable (False for direct reading mode; file should not be copied)
        useCT = pdict.get('usect', True)
        prodDBlockToken = pdict.get('access', '')

        # get the Rucio tracing report
        report = self.getStubTracingReport(pdict['report'], 'dCache', lfn, guid)

        # get a proper envsetup
        envsetup = self.getEnvsetup(get=True)

        if self._setup:
            _setup_str = "source %s; " % self._setup
        else:
            _setup_str = envsetup

        ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
        if ec != 0:
            self.prepareReport('RFCP_FAIL', report)
            return ec, pilotErrorDiag

        # remove any host and SFN info from PFN path
        loc_pfn = self.extractPathFromPFN(gpfn)

        copyprefixin = readpar('copyprefixin')
        if copyprefixin != '':
            # Extract the copy prefix
            pfrom, pto = copyprefixin.split('^')
            loc_pfn = pfrom + loc_pfn
            tolog("Added copyprefixin to file: %s" % (loc_pfn))
        else:
            copyprefix = readpar('copyprefix')
            if copyprefix != '':
                # Extract the copy prefix
                pfrom, pto = copyprefix.split('^')
                loc_pfn = pfrom + loc_pfn
                tolog("Added copyprefix to file: %s" % (loc_pfn))

        report['relativeStart'] = time()
        # for analysis jobs, skip input file if on tape or if lib file
        if analJob:
            if not self.isLibFile(loc_pfn):
                if not self.isFileStaged(_setup_str, loc_pfn):
                    pilotErrorDiag = "File %s is not staged and will be skipped for analysis job" % (loc_pfn)
                    self.prepareReport('FILE_ON_TAPE', report)
                    return error.ERR_FILEONTAPE, pilotErrorDiag
            else:
                tolog("Skipping file stage check for lib file")

        # should the root file be copied or read directly by athena?
        directIn, useFileStager = self.getTransferModes()

        if directIn:
            if useCT:
                directIn = False
                tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
                updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", ftype="input")
            else:
                # determine if the file is a root file according to its name
                rootFile = self.isRootFileName(lfn)

                if prodDBlockToken == 'local' or not rootFile:
                    directIn = False
                    tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
                    updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", ftype="input")
                elif rootFile:
                    tolog("Found root file according to file name: %s (will not be transferred in direct reading mode)" % (lfn))
                    report['relativeStart'] = None
                    report['transferStart'] = None
                    self.prepareReport('FOUND_ROOT', report)
                    if useFileStager:
                        updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="file_stager", ftype="input")
                    else:
                        updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="remote_io", ftype="input")
                    return error.ERR_DIRECTIOFILE, pilotErrorDiag
                else:
                    tolog("Normal file transfer")

        dest_path = os.path.join(path, lfn)
        _cmd_str = '%sdccp %s %s' % (_setup_str, loc_pfn, dest_path)
        tolog("Executing command: %s" % (_cmd_str))
        report['transferStart'] = time()
        telapsed = 0
        try:
#            t0 = time()
#            s, cout = commands.getstatusoutput(_cmd_str)
#            cerr = ""
#            telapsed = time() - t0
            s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
            print "DEBUG: s = ", s, type(s)
            print "DEBUG: telapsed = ", telapsed, type(telapsed)
            print "DEBUG: cout = ", cout, type(cout)
            print "DEBUG: cerr = ", cerr, type(cerr)
            cout = self.filter_text(cout)
            cerr = self.filter_text(cerr)
            if not self.is_number(s):
                s = 1
            if not self.is_number(telapsed):
                telapsed = 0
        except Exception, e:
            tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" % (e))
            s = 1
            o = self.filter_text(e)
            telapsed = timeout

            # write traceback info to stderr
            import traceback
            exc, msg, tb = sys.exc_info()
            traceback.print_tb(tb)

            _pilotErrorDiag = "Unexpected exception: %s" % (get_exc_plus())
            tolog("!!WARNING!!2999!! get_exc_plus: %s" % (_pilotErrorDiag))
            print "!!WARNING!!2999!! get_exc_plus: %s" % (_pilotErrorDiag)
Exemplo n.º 22
0
    def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
        """
        The local file (local access to the dCache file) is assumed to have a relative path
        that is the same of the relative path in the 'gpfn'
        loc_... are the variables used to access the file in the locally exported file system
        """

        error = PilotErrors()
        pilotErrorDiag = ""

        # Get input parameters from pdict
        jobId = pdict.get('jobId', '')
        workDir = pdict.get('workDir', '')
        analJob = pdict.get('analJob', False)
        timeout = pdict.get('timeout', 5 * 3600)

        # try to get the direct reading control variable (False for direct reading mode; file should not be copied)
        useCT = pdict.get('usect', True)
        prodDBlockToken = pdict.get('access', '')

        # get the Rucio tracing report
        report = self.getStubTracingReport(pdict['report'], 'dCache', lfn,
                                           guid)

        # get a proper envsetup
        envsetup = self.getEnvsetup(get=True)

        if self._setup:
            _setup_str = "source %s; " % self._setup
        else:
            _setup_str = envsetup

        ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
        if ec != 0:
            self.prepareReport('RFCP_FAIL', report)
            return ec, pilotErrorDiag

        # remove any host and SFN info from PFN path
        loc_pfn = self.extractPathFromPFN(gpfn)

        copyprefixin = readpar('copyprefixin')
        if copyprefixin != '':
            # Extract the copy prefix
            pfrom, pto = copyprefixin.split('^')
            loc_pfn = pfrom + loc_pfn
            tolog("Added copyprefixin to file: %s" % (loc_pfn))
        else:
            copyprefix = readpar('copyprefix')
            if copyprefix != '':
                # Extract the copy prefix
                pfrom, pto = copyprefix.split('^')
                loc_pfn = pfrom + loc_pfn
                tolog("Added copyprefix to file: %s" % (loc_pfn))

        report['relativeStart'] = time()
        # for analysis jobs, skip input file if on tape or if lib file
        if analJob:
            if not self.isLibFile(loc_pfn):
                if not self.isFileStaged(_setup_str, loc_pfn):
                    pilotErrorDiag = "File %s is not staged and will be skipped for analysis job" % (
                        loc_pfn)
                    self.prepareReport('FILE_ON_TAPE', report)
                    return error.ERR_FILEONTAPE, pilotErrorDiag
            else:
                tolog("Skipping file stage check for lib file")

        # should the root file be copied or read directly by athena?
        directIn, useFileStager = self.getTransferModes()

        if directIn:
            if useCT:
                directIn = False
                tolog(
                    "Direct access mode is switched off (file will be transferred with the copy tool)"
                )
                updateFileState(lfn,
                                workDir,
                                jobId,
                                mode="transfer_mode",
                                state="copy_to_scratch",
                                ftype="input")
            else:
                # determine if the file is a root file according to its name
                rootFile = self.isRootFileName(lfn)

                if prodDBlockToken == 'local' or not rootFile:
                    directIn = False
                    tolog(
                        "Direct access mode has been switched off for this file (will be transferred with the copy tool)"
                    )
                    updateFileState(lfn,
                                    workDir,
                                    jobId,
                                    mode="transfer_mode",
                                    state="copy_to_scratch",
                                    ftype="input")
                elif rootFile:
                    tolog(
                        "Found root file according to file name: %s (will not be transferred in direct reading mode)"
                        % (lfn))
                    report['relativeStart'] = None
                    report['transferStart'] = None
                    self.prepareReport('FOUND_ROOT', report)
                    if useFileStager:
                        updateFileState(lfn,
                                        workDir,
                                        jobId,
                                        mode="transfer_mode",
                                        state="file_stager",
                                        ftype="input")
                    else:
                        updateFileState(lfn,
                                        workDir,
                                        jobId,
                                        mode="transfer_mode",
                                        state="remote_io",
                                        ftype="input")
                    return error.ERR_DIRECTIOFILE, pilotErrorDiag
                else:
                    tolog("Normal file transfer")

        dest_path = os.path.join(path, lfn)
        _cmd_str = '%sdccp %s %s' % (_setup_str, loc_pfn, dest_path)
        tolog("Executing command: %s" % (_cmd_str))
        report['transferStart'] = time()
        telapsed = 0
        try:
            #            t0 = time()
            #            s, cout = commands.getstatusoutput(_cmd_str)
            #            cerr = ""
            #            telapsed = time() - t0
            s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
            print "DEBUG: s = ", s, type(s)
            print "DEBUG: telapsed = ", telapsed, type(telapsed)
            print "DEBUG: cout = ", cout, type(cout)
            print "DEBUG: cerr = ", cerr, type(cerr)
            cout = self.filter_text(cout)
            cerr = self.filter_text(cerr)
            if not self.is_number(s):
                s = 1
            if not self.is_number(telapsed):
                telapsed = 0
        except Exception, e:
            tolog("!!WARNING!!2999!! timed_command() threw an exception: %s" %
                  (e))
            s = 1
            o = self.filter_text(e)
            telapsed = timeout

            # write traceback info to stderr
            import traceback
            exc, msg, tb = sys.exc_info()
            traceback.print_tb(tb)

            _pilotErrorDiag = "Unexpected exception: %s" % (get_exc_plus())
            tolog("!!WARNING!!2999!! get_exc_plus: %s" % (_pilotErrorDiag))
            print "!!WARNING!!2999!! get_exc_plus: %s" % (_pilotErrorDiag)
    def put_data(self, source, destination, fsize=0, fchecksum=0, **pdict):
        """ copy output file from local dir to SE and register into dataset and catalogues """

        # Get input parameters from pdict
        lfn = pdict.get('lfn', '')
        guid = pdict.get('guid', '')
        token = pdict.get('token', '')
        dsname = pdict.get('dsname', '')
        analJob = pdict.get('analJob', False)
        sitename = pdict.get('sitename', '')
        extradirs = pdict.get('extradirs', '')
        prodSourceLabel = pdict.get('prodSourceLabel', '')
        dispatchDBlockTokenForOut = pdict.get('dispatchDBlockTokenForOut', '')

        if sitename == "CERNVM" and dispatchDBlockTokenForOut == "":
            dispatchDBlockTokenForOut = "chirp^cvmappi50.cern.ch^/panda_test^-d chirp"

        # get the DQ2 tracing report
        try:
            report = pdict['report']
        except:
            report = {}
        else:
            # set the proper protocol
            report['protocol'] = 'local'
            # mark the relative start
            report['relativeStart'] = time()
            # the current file
            report['filename'] = lfn
            report['guid'] = guid.replace('-','')
#            report['dataset'] = dsname

        filename = os.path.basename(source)

        # get the local file size and checksum
        csumtype = self.checksum_command
        if fsize == 0 or fchecksum == 0:
            ec, self.__pilotErrorDiag, fsize, fchecksum = self.getLocalFileInfo(source, csumtype=csumtype)
            if ec != 0:
                self.prepareReport('LOCAL_FILE_INFO_FAIL', report)
                return self.put_data_retfail(ec, self.__pilotErrorDiag)

        # do not transfer files larger than 50 MB except for CERNVM
        if sitename != "CERNVM" and int(fsize) > self.__MAX_FILE_SIZE:
            self.__pilotErrorDiag = "File sizes larger than %d B can currently not be tranferred with this site mover: size=%s" % (self.__MAX_FILE_SIZE, fsize)
            tolog("!!WARNING!!2997!! %s" % (self.__pilotErrorDiag))
            return self.put_data_retfail(self.__error.ERR_OUTPUTFILETOOLARGE, self.__pilotErrorDiag)

        # now that the file size is known, add it to the tracing report
        report['filesize'] = fsize

        # build the command
        _params = ""
        if fchecksum != 0:
            # special case for md5sum (command only understands 'md5' and 'adler32', and not 'ad' and 'md5sum')
            if csumtype == 'md5sum':
                _csumtype = 'md5'
            else:
                _csumtype = csumtype

       # This contains the user configuration for chirp server, path, debug
       # Format should be like
       # 'chirp^etpgrid01.garching.physik.uni-muenchen.de^/tanyasandoval^-d chirp'
        dispatchDBlockTokenForOut = pdict.get('dispatchDBlockTokenForOut', '')
        
        csplit = dispatchDBlockTokenForOut.split('^')
        if len(csplit) != 4:
          tolog("Wrong number of fields in chirp string: %s" %
                (dispatchDBlockTokenForOut))
          self.__pilotErrorDiag = "Wrong number of fields in chirp string: %s" % (dispatchDBlockTokenForOut)
          return self.put_data_retfail(self.__error.ERR_STAGEOUTFAILED, self.__pilotErrorDiag)        

       # Remove _sub part from dataset name
        re_sub=re.compile('(.*)_sub\d+')
        resub =re_sub.search(dsname)
        if resub:
            dsname_strip=resub.group(1)
        else:
            dsname_strip=dsname
        
        chirp_server = csplit[1]
        chirp_base = csplit[2]
        chirp_options = csplit[3]
        chirp_path = chirp_base+'/'+dsname_strip+'/'+filename

        tolog("Chirp path: %s" % (chirp_path))

       # Make compound command file to run in chirp
        chirpcom=open('chirp.com','w')
       # Create directories.
        dirs=chirp_path.split('/')
        dir_path=''
        for i in range(1,len(dirs)-1):
          dir_path=dir_path+'/'+dirs[i]
          chirpcom.write('mkdir '+dir_path+'\n')
       # and the cop command too
        chirpcom.write('put %s %s\n'%(source,chirp_path))
        chirpcom.close()  
        execStr = self.__chirp % (chirp_options, chirp_server, 'chirp.com')
        tolog("Executing command: %s" % (execStr))
        try:
            status, telapsed, cout, cerr = timed_command(execStr, self.__timeout)
        except Exception, e:
            self.__pilotErrorDiag = 'timed_command() threw an exception: %s' % str(e)
            tolog(self.__warningStr % self.__pilotErrorDiag)
            status = 1
            output = str(e)
            telapsed = self.__timeout
Exemplo n.º 24
0
        try:
            _dq2SiteName = self.getDQ2SiteName(surl=dst_gpfn)
        except Exception, e:
            tolog("Warning: Failed to get the DQ2 site name: %s (can not add this info to tracing report)" % str(e))
        else:
            report['localSite'], report['remoteSite'] = (_dq2SiteName, _dq2SiteName)
            tolog("DQ2 site name: %s" % (_dq2SiteName))

        # Directory not created: the copy command creates the direcrory if necessary,
        # all with the right permission

        report['transferStart'] = time()
        try:
            _cmd_str = '%snecp %s %s' % (_setup_str, source, dst_loc_sedir)
            tolog("NECP executing (timeout %s): %s" % (timeout, _cmd_str))
            s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
        except Exception, e:
            # Command may have exceptions like permission errors
            # timed_command hsould block/filter out all exceptions; remove try block?
            s = 1
            e = sys.exc_info()
            o = str(e[1]) #set error message from exception text
            telapsed = timeout
        else:
            o = cout + cerr
            tolog("Elapsed time: %d" % (telapsed))
        report['validateStart'] = time()

        if s != 0:
            check_syserr(s, o)
            pilotErrorDiag = "Error in copying: %s" % (o)
Exemplo n.º 25
0
    def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
        """ The local file is assubed to have a relative path that is the same of the relative path in the 'gpfn'
        loc_... are the variables used to access the file in the locally exported file system
        TODO: document GPFN format (SURL from catalog srm://host/path)
        TODO: document better constraint
        """

        error = PilotErrors()
        pilotErrorDiag = ""

        # Get input parameters from pdict
        useCT = pdict.get('usect', True)
        jobId = pdict.get('jobId', '')
        workDir = pdict.get('workDir', '')
        prodDBlockToken = pdict.get('access', '')

        # get the Rucio tracing report
        report = self.getStubTracingReport(pdict['report'], 'castorSVC', lfn, guid)

        # get a proper envsetup
        envsetup = self.getEnvsetup(get=True)

        # Hard code the configuration dictionary for now, but eventually this should be
        # set dynamically.
        #
        # There are the following configuration sections:
        #  setup - base environment veriables to be set
        #  svcClassMap - dictionary of string matches vs. service class names
        #  svcClassList - list of all service classes in case the svcClassMap matching fails
        #  svcClassDefault - the service class to set if the file appears to be staged no where
        #
        # Information from RAL:
        # [root@srm0661 ~]# listStorageArea -v atlas
        # <Space Token>                            <Description>   <service class>    <type>          <status>
        # 4948ef55-0000-1000-b7dd-9b38bdd87201    "ATLASGROUP"    "atlasStripDeg"     "DURABLE"       "ALLOCATED"
        # 4948ef38-0000-1000-8606-973e4e998e02    "ATLASMCDISK"   "atlasSimStrip"     "DURABLE"       "ALLOCATED"
        # 4948eec6-0000-1000-8ca2-aba0529b4806    "ATLASDATADISK" "atlasStripInput"   "DURABLE"       "ALLOCATED"
        # 4948ee8e-0000-1000-9ac5-81bb9b34ba7b    "ATLASMCTAPE"   "atlasSimRaw"       "PERMANENT"     "ALLOCATED"
        # 4948ee71-0000-1000-b611-a0afad31f6c8    "ATLASDATATAPE" "atlasT0Raw"        "PERMANENT"     "ALLOCATED"
        #                                         "ATLASHOTDISK"  "atlasHotDisk"
        # In addition there is the "atlasFarm" class, which is used when data is staged back from tape
        castorConfig = {
            'setup' : {
                'STAGE_HOST' : 'catlasstager.ads.rl.ac.uk',
                'STAGER_HOST' : 'catlasstager.ads.rl.ac.uk',
                'RFIO_USE_CASTOR_V2' : 'YES',
                },
            'svcClassList' :  ('atlasHotDisk', 'atlasSimStrip', 'atlasStripInput', 'atlasFarm', 'atlasStripDeg', 'atlasT0Raw', 'atlasSimRaw', 'atlasScratchDisk', ),
            'svcClassMap' : {
                '/atlashotdisk/' : 'atlasHotDisk',
                '/atlasmcdisk/' : 'atlasStripInput',
                '/atlasdatadisk/' : 'atlasStripInput',
                '/atlasgroupdisk/' : 'atlasStripDeg',
                '/atlasdatatape/' : 'atlasFarm',
                '/atlasmctape/' : 'atlasFarm',
                '/atlasscratchdisk/' : 'atlasScratchDisk',
                '/atlasProdDisk/' : 'atlasScratchDisk',
                },
            'svcClassDefault' : 'atlasFarm',
            }


        # Set all environment variables for castor setup
        for envVar, value in castorConfig['setup'].iteritems():
            os.environ[envVar] = value

        # Strip the gpfn (SURL) back to its bare castor component
        tolog("gpfn is %s" % gpfn)

        if self._setup:
            _setup_str = "source %s; " % self._setup
        else:
            _setup_str = envsetup

        ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
        if ec != 0:
            self.prepareReport('RFCP_FAIL', report)
            return ec, pilotErrorDiag

        loc_pfn = ''
        if( gpfn.find('SFN') != -1 ):
            s = gpfn.split('SFN=')
            loc_pfn = s[1]
            tolog("Found SFN string. Local file name %s" % loc_pfn)
        else:
            _tmp = gpfn.split('/', 3)
            loc_pfn = '/'+_tmp[3]
            tolog("Splitting SURL on slashes. Got local file name %s" % loc_pfn)

        if not loc_pfn.startswith('/castor/'):
            tolog("WARNING: Problem with local filename: Does not start with '/castor/'.")

        # should the root file be copied or read directly by athena?
        directIn, useFileStager = self.getTransferModes()
        if directIn:
            if useCT:
                directIn = False
                tolog("Direct access mode is switched off (file will be transferred with the copy tool)")
                updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", ftype="input")
            else:
                # determine if the file is a root file according to its name
                rootFile = self.isRootFileName(lfn)

                if prodDBlockToken == 'local' or not rootFile:
                    directIn = False
                    tolog("Direct access mode has been switched off for this file (will be transferred with the copy tool)")
                    updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="copy_to_scratch", ftype="input")
                elif rootFile:
                    tolog("Found root file according to file name: %s (will not be transferred in direct reading mode)" % (lfn))
                    report['relativeStart'] = None
                    report['transferStart'] = None
                    self.prepareReport('FOUND_ROOT', report)
                    if useFileStager:
                        updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="file_stager", ftype="input")
                    else:
                        updateFileState(lfn, workDir, jobId, mode="transfer_mode", state="remote_io", ftype="input")
                    return error.ERR_DIRECTIOFILE, pilotErrorDiag
                else:
                    tolog("Normal file transfer")

        # Now need to find the service class associated with the file.
        # If we find a clear indication of a space token in the file path
        # then this is easy. However, if we don't, then use stager_qry to
        # interrogate each possible service class. If this fails then use
        # atlasFarm in desperation.
        serviceClass = None
        for pathMatch, svcClass in castorConfig['svcClassMap'].iteritems():
            if loc_pfn.find(pathMatch) >= 0:
                tolog('Matched path element %s - service class is %s' % (pathMatch, svcClass))
                serviceClass = svcClass
                break
            else:
                tolog('Path element %s for service class %s - no match' % (pathMatch, svcClass))

        # For testing the fallback, then we need to hobble ourselves by unsetting serviceClass:
        #tolog('Automatic service class was: %s' % serviceClass)
        #tolog('Unsetting service class for fallback testing')
        #serviceClass = None
        if serviceClass == None:
            tolog("Warning: Failed to find service class hint in SURL.")

            for tryMe in castorConfig['svcClassList']:
                os.environ['STAGE_SVCCLASS'] = tryMe
                tolog('Trying service class %s for file' % tryMe)
                err, output = commands.getstatusoutput('stager_qry -M %s' % loc_pfn)
                if err != 0:
                    tolog('WARNING: Unexpected status from stager_qry: %d\n%s' % (err, output))
                else:
                    if output.find('STAGED') >= 0:
                        tolog('Found file in service class %s' % tryMe)
                        serviceClass = tryMe
                        break
                    else:
                        tolog('File not found in service class %s' % tryMe)
            if serviceClass == None:
                tolog('WARNING: Failed to find file in any expected service class - will set STAGE_SVCCLASS to %s' % castorConfig['svcClassDefault'])
                serviceClass = castorConfig['svcClassDefault']

        tolog('Setting STAGE_SVCCLASS to %s' % serviceClass)
        os.environ['STAGE_SVCCLASS'] = serviceClass

        dest_path = os.path.join(path, lfn)
        _cmd_str = '%s/usr/bin/rfcp %s %s' % (_setup_str, loc_pfn, dest_path)
        tolog("Executing command: %s" % (_cmd_str))
        report['transferStart'] = time()

        # execute
        timeout = 3600
        try:
            s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
        except Exception, e:
            pilotErrorDiag = 'timed_command() threw an exception: %s' % (e)
            tolog("!!WARNING!!1111!! %s" % (pilotErrorDiag))
            s = 1
            o = str(e)
            telapsed = timeout
    def get_data(self, gpfn, lfn, path, fsize=0, fchecksum=0, guid=0, **pdict):
        """
        The local file is assubed to have a relative path that is the same of the relative path in the 'gpfn'
        loc_... are the variables used to access the file in the locally exported file system
        """

        error = PilotErrors()
        pilotErrorDiag = ""

        # Get input parameters from pdict
        useCT = pdict.get('usect', True)
        jobId = pdict.get('jobId', '')
        workDir = pdict.get('workDir', '')
        prodDBlockToken = pdict.get('access', '')

        # get the DQ2 tracing report
        try:
            report = pdict['report']
        except:
            report = {}
        else:
            # set the proper protocol
            report['protocol'] = 'rfcpLFC'
            # mark the relative start
            report['relativeStart'] = time()
            # the current file
            report['filename'] = lfn
            # guid
            report['guid'] = guid.replace('-', '')

        tolog("gpfn is %s" % gpfn)

        # get a proper envsetup
        envsetup = self.getEnvsetup(get=True)

        if self._setup:
            _setup_str = "source %s; " % self._setup
        else:
            _setup_str = envsetup

        ec, pilotErrorDiag = verifySetupCommand(error, _setup_str)
        if ec != 0:
            self.prepareReport('RFCP_FAIL', report)
            return ec, pilotErrorDiag

        # remove any host and SFN info from PFN path
        loc_pfn = self.extractPathFromPFN(gpfn)

        try:
            if not loc_pfn.startswith(('/dpm', '/castor')):
                tolog(
                    "Potential problem with local filename. Does not start with '/dpm' or '/castor/'."
                )
        except TypeError:
            # Older version of python
            pass

        # should the root file be copied or read directly by athena?
        directIn, useFileStager = self.getTransferModes()
        if directIn:
            if useCT:
                directIn = False
                tolog(
                    "Direct access mode is switched off (file will be transferred with the copy tool)"
                )
                updateFileState(lfn,
                                workDir,
                                jobId,
                                mode="transfer_mode",
                                state="copy_to_scratch",
                                type="input")
            else:
                # determine if the file is a root file according to its name
                rootFile = self.isRootFileName(lfn)

                if prodDBlockToken == 'local' or not rootFile:
                    directIn = False
                    tolog(
                        "Direct access mode has been switched off for this file (will be transferred with the copy tool)"
                    )
                    updateFileState(lfn,
                                    workDir,
                                    jobId,
                                    mode="transfer_mode",
                                    state="copy_to_scratch",
                                    type="input")
                elif rootFile:
                    tolog(
                        "Found root file according to file name: %s (will not be transferred in direct reading mode)"
                        % (lfn))
                    report['relativeStart'] = None
                    report['transferStart'] = None
                    self.prepareReport('FOUND_ROOT', report)
                    if useFileStager:
                        updateFileState(lfn,
                                        workDir,
                                        jobId,
                                        mode="transfer_mode",
                                        state="file_stager",
                                        type="input")
                    else:
                        updateFileState(lfn,
                                        workDir,
                                        jobId,
                                        mode="transfer_mode",
                                        state="remote_io",
                                        type="input")
                    return error.ERR_DIRECTIOFILE, pilotErrorDiag
                else:
                    tolog("Normal file transfer")

        dest_path = os.path.join(path, lfn)
        #PN
        _cmd_str = '%srfcp %s %s' % (_setup_str, loc_pfn, dest_path)
        #        if ".lib." in loc_pfn:
        #            _cmd_str = '%srfcp %s %s' % (_setup_str, loc_pfn, dest_path)
        #        else:
        #            _cmd_str = '%srfcpXXX %s %s' % (_setup_str, loc_pfn, dest_path)
        tolog("Executing command: %s" % (_cmd_str))
        report['transferStart'] = time()

        # execute
        timeout = 3600
        try:
            s, telapsed, cout, cerr = timed_command(_cmd_str, timeout)
        except Exception, e:
            pilotErrorDiag = 'timed_command() threw an exception: %s' % (e)
            tolog("!!WARNING!!1111!! %s" % (pilotErrorDiag))
            s = 1
            o = str(e)
            telapsed = timeout