Beispiel #1
0
  def __createFTSFiles( self ):
    """ create LFNs file for glite-transfer-submit command

    This file consists one line for each fiel to be transferred:

    sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM]

    :param self: self reference
    """
    self.__updateMetadataCache()
    for lfn in self.fileDict:
      lfnStatus = self.fileDict[lfn].get( 'Status' )
      if lfnStatus not in self.noSubmitStatus:
        cksmStr = ""
        # # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself
        if self.__cksmTest and self.__cksmType:
          checkSum = self.catalogMetadata.get( lfn, {} ).get( 'Checksum' )
          if checkSum:
            cksmStr = " %s:%s" % ( self.__cksmType, intAdlerToHex( hexAdlerToInt( checkSum ) ) )
        ftsFile = FTSFile()
        ftsFile.LFN = lfn
        ftsFile.SourceSURL = self.fileDict[lfn].get( 'Source' )
        ftsFile.TargetSURL = self.fileDict[lfn].get( 'Target' )
        ftsFile.SourceSE = self.sourceSE
        ftsFile.TargetSE = self.targetSE
        ftsFile.Status = self.fileDict[lfn].get( 'Status' )
        ftsFile.Checksum = cksmStr
        ftsFile.Size = self.catalogMetadata.get( lfn, {} ).get( 'Size' )
        self.ftsFiles.append( ftsFile )
        self.submittedFiles += 1
    return S_OK()
Beispiel #2
0
  def __createFTSFiles( self ):
    """ create LFNs file for glite-transfer-submit command

    This file consists one line for each fiel to be transferred:

    sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM]

    :param self: self reference
    """
    self.__updateMetadataCache()
    for lfn in self.fileDict:
      lfnStatus = self.fileDict[lfn].get( 'Status' )
      if lfnStatus not in self.noSubmitStatus:
        cksmStr = ""
        # # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself
        if self.__cksmTest and self.__cksmType:
          checkSum = self.catalogMetadata.get( lfn, {} ).get( 'Checksum' )
          if checkSum:
            cksmStr = " %s:%s" % ( self.__cksmType, intAdlerToHex( hexAdlerToInt( checkSum ) ) )
        ftsFile = FTSFile()
        ftsFile.LFN = lfn
        ftsFile.SourceSURL = self.fileDict[lfn].get( 'Source' )
        ftsFile.TargetSURL = self.fileDict[lfn].get( 'Target' )
        ftsFile.SourceSE = self.sourceSE
        ftsFile.TargetSE = self.targetSE
        ftsFile.Status = self.fileDict[lfn].get( 'Status' )
        ftsFile.Checksum = cksmStr
        ftsFile.Size = self.catalogMetadata.get( lfn, {} ).get( 'Size' )
        self.ftsFiles.append( ftsFile )
        self.submittedFiles += 1
    return S_OK()
Beispiel #3
0
  def test01Ctor( self ):
    """ test ctor and (de-)serialization """

    ftsFile = FTSFile( self.fromDict )
    self.assertEqual( isinstance( ftsFile, FTSFile ), True )
    for k, v in self.fromDict.items():
      self.assertEqual( getattr( ftsFile, k ), v )

    json = ftsFile.toJSON()
    ftsFileJSON = FTSFile( json["Value"] )
    self.assertEqual( isinstance( ftsFileJSON, FTSFile ), True )
    for k, v in self.fromDict.items():
      self.assertEqual( getattr( ftsFileJSON, k ), v )
Beispiel #4
0
    def test01Ctor(self):
        """ test ctor and (de-)serialization """

        ftsFile = FTSFile(self.fromDict)
        self.assertEqual(isinstance(ftsFile, FTSFile), True)
        for k, v in self.fromDict.items():
            self.assertEqual(getattr(ftsFile, k), v)

        json = ftsFile.toJSON()
        ftsFileJSON = FTSFile(json["Value"])
        self.assertEqual(isinstance(ftsFileJSON, FTSFile), True)
        for k, v in self.fromDict.items():
            self.assertEqual(getattr(ftsFileJSON, k), v)
Beispiel #5
0
 def getFTSJobList(self, statusList=None, limit=500):
     """ select FTS jobs with statuses in :statusList: """
     statusList = statusList if statusList else list(FTSJob.INITSTATES +
                                                     FTSJob.TRANSSTATES)
     query = "SELECT * FROM `FTSJob` WHERE `Status` IN (%s) ORDER BY `LastUpdate` DESC LIMIT %s;" % (
         stringListToString(statusList), limit)
     trn = self._transaction([query])
     if not trn['OK']:
         self.log.error('Failed ftsJobSQL',
                        "getFTSJobList: %s" % trn['Message'])
         return trn
     ftsJobs = [FTSJob(ftsJobDict) for ftsJobDict in trn['Value'][query]]
     for ftsJob in ftsJobs:
         query = "SELECT * FROM `FTSFile` WHERE `FTSGUID` = '%s';" % ftsJob.FTSGUID
         trn = self._transaction(query)
         if not trn['OK']:
             self.log.error('Failed ftsFileSQL',
                            "getFTSJobList: %s" % trn['Message'])
             return trn
         ftsFiles = [
             FTSFile(ftsFileDict) for ftsFileDict in trn['Value'][query]
         ]
         for ftsFile in ftsFiles:
             ftsJob.addFile(ftsFile)
     return S_OK(ftsJobs)
Beispiel #6
0
    def getFTSJob(self, ftsJobID=None, readOnly=False):
        """ get FTSJob given FTSJobID """

        getJob = ["SELECT * FROM `FTSJob` WHERE `FTSJobID` = %s;" % ftsJobID]
        getJob = self._transaction(getJob)
        if not getJob["OK"]:
            self.log.error(getJob["Message"])
            return getJob
        getJob = getJob["Value"]
        if not getJob:
            return S_OK()
        ftsJob = FTSJob(getJob.values()[0][0])
        selectFiles = self._transaction([
            "SELECT * FROM `FTSFile` WHERE `FTSGUID` = '%s';" % ftsJob.FTSGUID
        ])
        if not selectFiles["OK"]:
            self.log.error(selectFiles["Message"])
            return selectFiles
        selectFiles = selectFiles["Value"]
        ftsFiles = [FTSFile(item) for item in selectFiles.values()[0]]
        for ftsFile in ftsFiles:
            ftsJob.addFile(ftsFile)
        if not readOnly:
            setAssigned = "UPDATE `FTSJob` SET `Status`='Assigned' WHERE `FTSJobID` = %s;" % ftsJobID
            setAssigned = self._query(setAssigned)
            if not setAssigned["OK"]:
                self.log.error(setAssigned["Message"])
                return setAssigned
        return S_OK(ftsJob)
Beispiel #7
0
 def export_putFTSFileList(self, ftsFilesJSONList):
     """ put FTS files list
 """
     ftsFiles = []
     for ftsFileJSON in ftsFilesJSONList:
         ftsFiles.append(FTSFile(ftsFileJSON))
     return self.ftsDB.putFTSFileList(ftsFiles)
Beispiel #8
0
    def getFTSJobsForRequest(self, requestID, statusList=None):
        """ get list of FTSJobs with status in :statusList: for request given its requestID

    TODO: should be more smart, i.e. one query to select all ftsfiles
    """

        statusList = statusList if statusList else list(FTSJob.INITSTATES +
                                                        FTSJob.TRANSSTATES)
        query = "SELECT * FROM `FTSJob` WHERE `RequestID` = %s AND `Status` in (%s)" % (
            requestID, stringListToString(statusList))
        ftsJobs = self._transaction([query])
        if not ftsJobs['OK']:
            self.log.error("Failed getFTSJobsForRequest",
                           "%s" % ftsJobs['Message'])
            return ftsJobs

        ftsJobs = ftsJobs['Value'][query] if query in ftsJobs['Value'] else []

        ftsJobs = [FTSJob(ftsJobDict) for ftsJobDict in ftsJobs]

        for ftsJob in ftsJobs:
            query = "SELECT * FROM `FTSFile` WHERE `FTSGUID` = '%s' AND `RequestID`=%s" % (
                ftsJob.FTSGUID, requestID)
            ftsFiles = self._transaction([query])
            if not ftsFiles['OK']:
                self.log.error("Failed getFTSJobsForRequest",
                               "%s" % ftsFiles['Message'])
                return ftsFiles
            ftsFiles = ftsFiles['Value'][query] if query in ftsFiles[
                'Value'] else []
            for ftsFileDict in ftsFiles:
                ftsJob.addFile(FTSFile(ftsFileDict))
        return S_OK(ftsJobs)
Beispiel #9
0
    def __init__(self, fromDict=None):
        """c'tor

    :param self: self reference
    :param dict fromDict: data dict
    """
        Record.__init__(self)
        now = datetime.datetime.utcnow().replace(microsecond=0)
        self.__data__["CreationTime"] = now
        self.__data__["SubmitTime"] = now
        self.__data__["LastUpdate"] = now
        self.__data__["Status"] = "Submitted"
        self.__data__["Completeness"] = 0
        self.__data__["FTSJobID"] = 0
        self.__files__ = TypedList(allowedTypes=FTSFile)

        self._log = gLogger.getSubLogger("FTSJob-%s" % self.FTSJobID, True)

        fromDict = fromDict if fromDict else {}
        for ftsFileDict in fromDict.get("FTSFiles", []):
            self += FTSFile(ftsFileDict)
        if "FTSFiles" in fromDict: del fromDict["FTSFiles"]
        for key, value in fromDict.items():
            if key not in self.__data__:
                raise AttributeError("Unknown FTSJob attribute '%s'" % key)
            if value:
                setattr(self, key, value)
Beispiel #10
0
    def getFTSFileList(self, statusList=None, limit=1000):
        """ get at most :limit: FTSFiles with status in :statusList:

    :param statusList: list with FTSFiles statuses
    :type statusList: python:list
    :param int limit: select query limit
    """
        statusList = statusList if statusList else ["Waiting"]
        reStatus = []
        inStatus = []
        for status in statusList:
            if "%" in status or ".*" in status or ".+" in status:
                reStatus.append(status)
            else:
                inStatus.append(status)
        reQuery = "`Status` REGEXP '%s'" % "|".join(
            reStatus) if reStatus else ""
        inQuery = "`Status` IN (%s)" % stringListToString(
            inStatus) if inStatus else ""
        whereClause = " OR ".join([q for q in (reQuery, inQuery) if q])
        if whereClause:
            whereClause = "WHERE %s" % whereClause
        query = "SELECT * FROM `FTSFile` %s ORDER BY `LastUpdate` DESC LIMIT %s;" % (
            whereClause, limit)
        trn = self._transaction([query])
        if not trn['OK']:
            self.log.error("Failed getFTSFileList", "%s" % trn['Message'])
            return trn
        return S_OK([FTSFile(fileDict) for fileDict in trn['Value'][query]])
Beispiel #11
0
 def _getFTSFileProperties(self, ftsFileID, columnNames=None):
     """ select :columnNames: from FTSJobFile table  """
     columnNames = columnNames if columnNames else FTSFile.tableDesc(
     )["Fields"].keys()
     columnNames = ",".join(
         ['`%s`' % str(columnName) for columnName in columnNames])
     return "SELECT %s FROM `FTSFile` WHERE `FTSFileID` = %s;" % (
         columnNames, int(ftsFileID))
Beispiel #12
0
 def getFTSFileList( self, statusList = None, limit = None ):
   """ get list of FTSFiles with status in statusList """
   statusList = statusList if statusList else [ "Waiting" ]
   limit = limit if limit else 1000
   getFTSFileList = self.ftsManager.getFTSFileList( statusList, limit )
   if not getFTSFileList['OK']:
     self.log.error( "Failed getFTSFileList", "%s" % getFTSFileList['Message'] )
     return getFTSFileList
   getFTSFileList = getFTSFileList['Value']
   return S_OK( [ FTSFile( ftsFile ) for ftsFile in getFTSFileList ] )
Beispiel #13
0
  def getAllFTSFilesForRequest( self, requestID ):
    """ read FTSFiles for a given :requestID:

    :param int requestID: ReqDB.Request.RequestID
    """
    ftsFiles = self.ftsManager.getAllFTSFilesForRequest( requestID )
    if not ftsFiles['OK']:
      self.log.error( "Failed getFTSFilesForRequest", "%s" % ftsFiles['Message'] )
      return ftsFiles
    return S_OK( [ FTSFile( ftsFileDict ) for ftsFileDict in ftsFiles['Value'] ] )
Beispiel #14
0
 def getAllFTSFilesForRequest(self, requestID):
     """ get FTSFiles with status in :statusList: for request given its :requestID: """
     requestID = int(requestID)
     query = "SELECT * FROM `FTSFile` WHERE `RequestID` = %s;" % (requestID)
     ftsFiles = self._transaction([query])
     if not ftsFiles['OK']:
         self.log.error("getFTSFilesForRequest: %s" % ftsFiles['Message'])
         return ftsFiles
     ftsFiles = ftsFiles['Value'].get(query, [])
     return S_OK([FTSFile(ftsFileDict) for ftsFileDict in ftsFiles])
Beispiel #15
0
 def getFTSFileList( self, statusList = None, limit = None ):
   """ get list of FTSFiles with status in statusList """
   statusList = statusList if statusList else [ "Waiting" ]
   limit = limit if limit else 1000
   getFTSFileList = self.ftsManager().getFTSFileList( statusList, limit )
   if not getFTSFileList["OK"]:
     self.log.error( "getFTSFileList: %s" % getFTSFileList["Message"] )
     return getFTSFileList
   getFTSFileList = getFTSFileList["Value"]
   return S_OK( [ FTSFile( ftsFile ) for ftsFile in getFTSFileList ] )
Beispiel #16
0
  def getFTSFilesForRequest( self, requestID, statusList = None ):
    """ read FTSFiles for a given :requestID:

    :param int requestID: ReqDB.Request.RequestID
    :param list statusList: List of statuses (default: Waiting)
    """
    ftsFiles = self.ftsManager.getFTSFilesForRequest( requestID, statusList )
    if not ftsFiles['OK']:
      self.log.error( "Failed getFTSFilesForRequest", "%s" % ftsFiles['Message'] )
      return ftsFiles
    return S_OK( [ FTSFile( ftsFileDict ) for ftsFileDict in ftsFiles['Value'] ] )
Beispiel #17
0
  def getFTSFilesForRequest( self, requestID, operationID = None ):
    """ read FTSFiles for a given :requestID:

    :param int requestID: ReqDB.Request.RequestID
    :param int operationID: ReqDB.Operation.OperationID
    """
    ftsFiles = self.ftsManager().getFTSFilesForRequest( requestID, operationID )
    if not ftsFiles["OK"]:
      self.log.error( "getFTSFilesForRequest: %s" % ftsFiles["Message"] )
      return ftsFiles
    return S_OK( [ FTSFile( ftsFileDict ) for ftsFileDict in ftsFiles["Value"] ] )
Beispiel #18
0
  def getFTSFile( self, ftsFileID = None ):
    """ get FTSFile

    :param int ftsFileID: FTSFileID
    """
    getFile = self.ftsManager.getFTSFile( ftsFileID )
    if not getFile['OK']:
      self.log.error( 'Failed to get FTS file', getFile['Message'] )
    # # de-serialize
    if getFile['Value']:
      ftsFile = FTSFile( getFile['Value'] )
    return S_OK( ftsFile )
Beispiel #19
0
 def getFTSFilesForRequest(self, requestID, statusList=None):
     """ get FTSFiles with status in :statusList: for request given its :requestID: """
     requestID = int(requestID)
     statusList = statusList if statusList else ["Waiting"]
     query = "SELECT * FROM `FTSFile` WHERE `RequestID` = %s AND `Status` IN (%s);" % (
         requestID, stringListToString(statusList))
     ftsFiles = self._transaction([query])
     if not ftsFiles['OK']:
         self.log.error("getFTSFilesForRequest: %s" % ftsFiles['Message'])
         return ftsFiles
     ftsFiles = ftsFiles['Value'].get(query, [])
     return S_OK([FTSFile(ftsFileDict) for ftsFileDict in ftsFiles])
Beispiel #20
0
 def export_putFTSFile(cls, ftsFileJSON):
     """ put FTSFile into FTSDB """
     try:
         ftsFile = FTSFile(ftsFileJSON)
         isValid = cls.ftsValidator().validate(ftsFile)
         if not isValid["OK"]:
             gLogger.error(isValid["Message"])
             return isValid
         return cls.__ftsDB.putFTSFile(ftsFile)
     except Exception, error:
         gLogger.exception(error)
         return S_ERROR(error)
Beispiel #21
0
 def getFTSFile(self, ftsFileID):
     """ read FTSFile from db given FTSFileID """
     select = "SELECT * FROM `FTSFile` WHERE `FTSFileID` = %s;" % ftsFileID
     select = self._transaction([select])
     if not select["OK"]:
         self.log.error(select["Message"])
         return select
     select = select["Value"]
     if not select.values()[0]:
         return S_OK()
     ftsFile = FTSFile(select.values()[0][0])
     return S_OK(ftsFile)
Beispiel #22
0
 def getFTSFile(self, ftsFileID):
     """ read FTSFile from db given FTSFileID """
     select = "SELECT * FROM `FTSFile` WHERE `FTSFileID` = %s;" % ftsFileID
     select = self._transaction([select])
     if not select['OK']:
         self.log.error('Failed FTS file selection', select['Message'])
         return select
     select = select['Value']
     if not select.values()[0]:
         return S_OK()
     ftsFile = FTSFile(select.values()[0][0])
     return S_OK(ftsFile)
Beispiel #23
0
    def setUp(self):
        """ test set up """

        self.fileA = FTSFile({
            "LFN": "/a",
            "ChecksumType": "ADLER32",
            "Checksum": "123456",
            "Size": 10,
            "SourceSE": "CERN-USER",
            "TargetSE": "PIC-USER",
            "SourceSURL": "sourceSURL",
            "TargetSURL": "targetSURL"
        })
        self.fileB = FTSFile({
            "LFN": "/b",
            "ChecksumType": "ADLER32",
            "Checksum": "654321",
            "Size": 10,
            "SourceSE": "CERN-USER",
            "TargetSE": "PIC-USER",
            "SourceSURL": "sourceSURL",
            "TargetSURL": "targetSURL"
        })
Beispiel #24
0
  def getFTSFile( self, ftsFileID = None ):
    """ get FTSFile

    :param int fileID: FileID
    :param int ftsFileID: FTSFileID
    """
    getFile = self.ftsManager().getFTSFile( ftsFileID )
    if not getFile["OK"]:
      self.log.error( getFile["Message"] )
    # # de-serialize
    if getFile["Value"]:
      getFile = FTSFile( getFile["Value"] )
      if not getFile["OK"]:
        self.log.error( getFile["Message"] )
    return getFile
Beispiel #25
0
    def export_putFTSJob(self, ftsJobJSON):
        """ put FTSJob (serialized in JSON into FTSDB """

        ftsFiles = []

        if "FTSFiles" in ftsJobJSON:
            ftsFiles = ftsJobJSON.get("FTSFiles", [])
            del ftsJobJSON["FTSFiles"]

        try:
            ftsJob = FTSJob(ftsJobJSON)
            for ftsFile in ftsFiles:
                ftsJob.addFile(FTSFile(ftsFile))
        except Exception, error:
            gLogger.exception(error)
            return S_ERROR(error)
Beispiel #26
0
    def __init__(self, fromDict=None):
        """c'tor

    :param self: self reference
    :param dict fromDict: data dict
    """
        self.__data__ = dict.fromkeys(self.tableDesc()["Fields"].keys(), None)

        now = datetime.datetime.utcnow().replace(microsecond=0)
        self.__data__["CreationTime"] = now
        self.__data__["SubmitTime"] = now
        self.__data__["LastUpdate"] = now
        self.__data__["Status"] = "Submitted"
        self.__data__["Completeness"] = 0
        self.__data__["FTSJobID"] = 0
        self._regTime = 0.
        self._regSuccess = 0
        self._regTotal = 0
        self.__files__ = TypedList(allowedTypes=FTSFile)

        self._fc = FileCatalog()
        self._fts3context = None

        self._states = tuple(
            set(self.INITSTATES + self.TRANSSTATES + self.FAILEDSTATES +
                self.FINALSTATES))

        fromDict = fromDict if fromDict else {}
        for ftsFileDict in fromDict.get("FTSFiles", []):
            self += FTSFile(ftsFileDict)
        if "FTSFiles" in fromDict:
            del fromDict["FTSFiles"]
        for key, value in fromDict.items():
            if key not in self.__data__:
                raise AttributeError("Unknown FTSJob attribute '%s'" % key)
            if value:
                setattr(self, key, value)
        self._log = gLogger.getSubLogger(
            "req_%s/FTSJob-%s" % (self.RequestID, self.FTSGUID), True)
Beispiel #27
0
    def getFTSJob(self, ftsJobID=None):
        """ get FTSJob given FTSJobID """

        getJob = ["SELECT * FROM `FTSJob` WHERE `FTSJobID` = %s;" % ftsJobID]
        getJob = self._transaction(getJob)
        if not getJob['OK']:
            self.log.error('Failed ftsFileSQL', getJob['Message'])
            return getJob
        getJob = getJob['Value']
        if not getJob:
            return S_OK()
        ftsJob = FTSJob(getJob.values()[0][0])
        selectFiles = self._transaction([
            "SELECT * FROM `FTSFile` WHERE `FTSGUID` = '%s';" % ftsJob.FTSGUID
        ])
        if not selectFiles['OK']:
            self.log.error('Failed ftsFileSQL', selectFiles['Message'])
            return selectFiles
        selectFiles = selectFiles['Value']
        ftsFiles = [FTSFile(item) for item in selectFiles.values()[0]]
        for ftsFile in ftsFiles:
            ftsJob.addFile(ftsFile)
        return S_OK(ftsJob)
Beispiel #28
0
    def monitorFTS2(self, command="glite-transfer-status", full=False):
        """ monitor fts job """
        if not self.FTSGUID:
            return S_ERROR("FTSGUID not set, FTS job not submitted?")

        monitorCommand = command.split() + \
                           ["--verbose",
                           "-s",
                           self.FTSServer,
                           self.FTSGUID ]

        if full:
            monitorCommand.append("-l")

        monitor = executeGridCommand("", monitorCommand)
        if not monitor["OK"]:
            return monitor
        returnCode, outputStr, errStr = monitor["Value"]

        # Returns a non zero status if error
        if returnCode != 0:
            if 'was not found' in outputStr and not errStr:
                errStr = 'Job was not found'
            return S_ERROR(errStr)

        outputStr = outputStr.replace("'", "").replace("<",
                                                       "").replace(">", "")

        # # set FTS job status
        regExp = re.compile("Status:\\s+(\\S+)")

        # with FTS3 this can be uppercase
        self.Status = re.search(regExp, outputStr).group(1)

        statusSummary = {}
        # This is capitalized, even in FTS3!
        for state in FTSFile.ALL_STATES:
            regExp = re.compile("\\s+%s:\\s+(\\d+)" % state)
            if regExp.search(outputStr):
                statusSummary[state] = int(
                    re.search(regExp, outputStr).group(1))

        total = sum(statusSummary.values())
        completed = sum(
            statusSummary.get(state, 0) for state in FTSFile.FINAL_STATES)
        self.Completeness = 100 * completed / total if total else 0

        if not full:
            return S_OK(statusSummary)

        # The order of informations is not the same for glite- and fts- !!!
        # In order: new fts-, old fts-, glite-
        realJob = len(self) != 0
        iExptr = None
        for iExptr, exptr in enumerate(
            ('[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Staging:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)',
             '[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)',
             '[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Retries:[ ]+(\\d+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)'
             )):
            regExp = re.compile(exptr, re.S)
            fileInfo = re.findall(regExp, outputStr)
            if fileInfo:
                break
        if not fileInfo:
            return S_ERROR("Error monitoring job (no regexp match)")
        for info in fileInfo:
            if iExptr == 0:
                # version >= 3.2.30
                sourceURL, targetURL, fileStatus, reason, duration, _retries, _staging = info
            elif iExptr == 1:
                # version FTS3 < 3.2.30
                sourceURL, targetURL, fileStatus, reason, duration, _retries = info
            elif iExptr == 2:
                # version FTS2
                sourceURL, targetURL, fileStatus, _retries, reason, duration = info
            else:
                return S_ERROR('Error monitoring job (implement match %d)' %
                               iExptr)
            candidateFile = None

            if not realJob:
                # This is used by the CLI monitoring of jobs in case no file was specified
                candidateFile = FTSFile()
                candidateFile.LFN = overlap(sourceURL, targetURL)
                candidateFile.SourceSURL = sourceURL
                candidateFile.Size = 0
                self += candidateFile
            else:
                for ftsFile in self:
                    if ftsFile.SourceSURL == sourceURL:
                        candidateFile = ftsFile
                        break
                if not candidateFile:
                    continue
            # Can be uppercase for FTS3
            if not candidateFile.TargetSURL:
                candidateFile.TargetSURL = targetURL
            candidateFile.Status = fileStatus
            candidateFile.Error = reason
            candidateFile._duration = duration

            if candidateFile.Status == "Failed":
                for missingSource in self.missingSourceErrors:
                    if missingSource.match(reason):
                        candidateFile.Error = "MissingSource"
            # If the staging info was present, record it
            if len(info) > 6:
                candidateFile._staging = info[6]
        # # register successful files
        if self.Status in FTSJob.FINALSTATES:
            return self.finalize()

        return S_OK()
Beispiel #29
0
    def setUp(self):
        """ test case set up """

        gLogger.setLevel('NOTICE')

        self.ftsSites = [
            FTSSite(
                ftsServer=
                'https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer',
                name='CERN.ch'),
            FTSSite(
                ftsServer=
                'https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer',
                name='PIC.es'),
            FTSSite(
                ftsServer=
                'https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer',
                name='RAL.uk'),
        ]

        self.ses = ['CERN-USER', 'RAL-USER']
        self.statuses = [
            'Submitted', 'Finished', 'FinishedDirty', 'Active', 'Ready'
        ]

        self.submitted = 0
        self.numberOfJobs = 10
        self.opIDs = []

        self.ftsJobs = []
        for i in xrange(self.numberOfJobs):

            opID = i % 3
            if opID not in self.opIDs:
                self.opIDs.append(opID)

            ftsJob = FTSJob()
            ftsJob.FTSGUID = str(uuid.uuid4())
            ftsJob.FTSServer = self.ftsSites[0].FTSServer
            ftsJob.Status = self.statuses[i % len(self.statuses)]
            ftsJob.OperationID = opID
            if ftsJob.Status in FTSJob.FINALSTATES:
                ftsJob.Completeness = 100
            if ftsJob.Status == 'Active':
                ftsJob.Completeness = 90
            ftsJob.SourceSE = self.ses[i % len(self.ses)]
            ftsJob.TargetSE = 'PIC-USER'
            ftsJob.RequestID = 12345

            ftsFile = FTSFile()
            ftsFile.FileID = i + 1
            ftsFile.OperationID = i + 1
            ftsFile.LFN = '/a/b/c/%d' % i
            ftsFile.Size = 1000000
            ftsFile.OperationID = opID
            ftsFile.SourceSE = ftsJob.SourceSE
            ftsFile.TargetSE = ftsJob.TargetSE
            ftsFile.SourceSURL = 'foo://source.bar.baz/%s' % ftsFile.LFN
            ftsFile.TargetSURL = 'foo://target.bar.baz/%s' % ftsFile.LFN
            ftsFile.Status = 'Waiting' if ftsJob.Status != 'FinishedDirty' else 'Failed'
            ftsFile.RequestID = 12345
            ftsFile.Checksum = 'addler'
            ftsFile.ChecksumType = 'adler32'

            ftsFile.FTSGUID = ftsJob.FTSGUID
            if ftsJob.Status == 'FinishedDirty':
                ftsJob.FailedFiles = 1
                ftsJob.FailedSize = ftsFile.Size

            ftsJob.addFile(ftsFile)
            self.ftsJobs.append(ftsJob)

        self.submitted = len(
            [i for i in self.ftsJobs if i.Status == 'Submitted'])

        self.ftsClient = FTSClient()
Beispiel #30
0
    def setUp(self):
        """ test case set up """
        # ## set some defaults
        gConfig.setOptionValue('DIRAC/Setup', 'Test')
        gConfig.setOptionValue('/DIRAC/Setups/Test/DataManagement', 'Test')
        gConfig.setOptionValue(
            '/Systems/DataManagement/Test/Databases/FTSDB/Host', 'localhost')
        gConfig.setOptionValue(
            '/Systems/DataManagement/Test/Databases/FTSDB/DBName', 'FTSDB')
        gConfig.setOptionValue(
            '/Systems/DataManagement/Test/Databases/FTSDB/User', 'Dirac')

        self.ftsSites = [
            FTSSite({
                "FTSServer":
                "https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer",
                "Name": "CERN.ch"
            }),
            FTSSite({
                "FTSServer":
                "https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer",
                "Name": "PIC.es"
            }),
            FTSSite({
                "FTSServer":
                "https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer",
                "Name": "RAL.uk"
            }),
            FTSSite({
                "FTSServer":
                "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer",
                "Name": "SARA.nl"
            }),
            FTSSite({
                "FTSServer":
                "https://fts.cr.cnaf.infn.it:8443/glite-data-transfer-fts/services/FileTransfer",
                "Name": "CNAF.it"
            }),
            FTSSite({
                "FTSServer":
                "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer",
                "Name": "NIKHEF.nl"
            }),
            FTSSite({
                "FTSServer":
                "https://fts-fzk.gridka.de:8443/glite-data-transfer-fts/services/FileTransfer",
                "Name": "GRIDKA.de"
            }),
            FTSSite({
                "FTSServer":
                "https://cclcgftsprod.in2p3.fr:8443/glite-data-transfer-fts/services/FileTransfer",
                "Name": "IN2P3.fr"
            })
        ]

        self.ftsFiles = []
        for i in range(100):
            ftsFile = FTSFile()
            ftsFile.FileID = i + 1
            ftsFile.OperationID = 9999
            ftsFile.LFN = "/a/b/c/%d" % i
            ftsFile.Size = 10
            ftsFile.SourceSE = "CERN-USER"
            ftsFile.TargetSE = "PIC-USER"
            ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN
            ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN
            ftsFile.Status = "Waiting"
            self.ftsFiles.append(ftsFile)

        ses = ["CERN-USER", "RAL-USER"]
        statuses = [
            "Submitted", "Finished", "FinishedDirty", "Active", "Ready"
        ]

        self.submitted = 0

        self.ftsJobs = []
        for i in range(7200):

            ftsJob = FTSJob()
            ftsJob.FTSGUID = str(uuid.uuid4())
            ftsJob.FTSServer = self.ftsSites[0].FTSServer
            ftsJob.Status = statuses[i % len(statuses)]

            if ftsJob.Status in FTSJob.FINALSTATES:
                ftsJob.Completeness = 100
            if ftsJob.Status == "Active":
                ftsJob.Completeness = 90

            ftsJob.SourceSE = ses[i % len(ses)]
            ftsJob.TargetSE = "PIC-USER"

            ftsFile = FTSFile()
            ftsFile.FileID = i + 1
            ftsFile.OperationID = i + 1
            ftsFile.LFN = "/a/b/c/%d" % i
            ftsFile.Size = 1000000
            ftsFile.SourceSE = ftsJob.SourceSE
            ftsFile.TargetSE = ftsJob.TargetSE
            ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN
            ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN
            ftsFile.Status = "Waiting" if ftsJob.Status != "FinishedDirty" else "Failed"

            ftsFile.FTSGUID = ftsJob.FTSGUID
            if ftsJob.Status == "FinishedDirty":
                ftsJob.FailedFiles = 1
                ftsJob.FailedSize = ftsFile.Size

            ftsJob.addFile(ftsFile)
            self.ftsJobs.append(ftsJob)

        self.submitted = len(
            [i for i in self.ftsJobs if i.Status == "Submitted"])
Beispiel #31
0
 def _getFTSFileProperties( self, ftsFileID, columnNames = None ):
   """ select :columnNames: from FTSJobFile table  """
   columnNames = columnNames if columnNames else FTSFile.tableDesc()["Fields"].keys()
   columnNames = ",".join( [ '`%s`' % str( columnName ) for columnName in columnNames ] )
   return "SELECT %s FROM `FTSFile` WHERE `FTSFileID` = %s;" % ( columnNames, int( ftsFileID ) )
Beispiel #32
0
    def export_ftsSchedule(self, requestID, operationID, fileJSONList):
        """ call FTS scheduler

    :param int requestID: ReqDB.Request.RequestID
    :param int operationID: ReqDB.Operation.OperationID
    :param list fileJSONList: [ (fileJSON, [sourceSE, ...], [ targetSE, ...] ) ]
    """
        # # this will be returned on success
        ret = {"Successful": [], "Failed": {}}

        requestID = int(requestID)
        operationID = int(operationID)

        fileIDs = []
        for fileJSON, sourceSEs, targetSEs in fileJSONList:
            fileID = int(fileJSON.get("FileID"))
            fileIDs.append(fileID)
        cleanUpFTSFiles = self.__ftsDB.cleanUpFTSFiles(requestID, fileIDs)
        if not cleanUpFTSFiles["OK"]:
            self.log.error("ftsSchedule: %s" % cleanUpFTSFiles["Message"])
            return S_ERROR(cleanUpFTSFiles["Message"])

        ftsFiles = []

        for fileJSON, sourceSEs, targetSEs in fileJSONList:

            lfn = fileJSON.get("LFN", "")
            size = int(fileJSON.get("Size", 0))
            fileID = int(fileJSON.get("FileID", 0))
            opID = int(fileJSON.get("OperationID", 0))

            gLogger.info(
                "ftsSchedule: LFN=%s FileID=%s OperationID=%s sources=%s targets=%s"
                % (lfn, fileID, opID, sourceSEs, targetSEs))

            replicaDict = self.replicaManager().getActiveReplicas(lfn)
            if not replicaDict["OK"]:
                gLogger.error("ftsSchedule: %s" % replicaDict["Message"])
                ret["Failed"][fileID] = replicaDict["Message"]
                continue
            replicaDict = replicaDict["Value"]

            if lfn in replicaDict["Failed"] and lfn not in replicaDict[
                    "Successful"]:
                ret["Failed"][fileID] = "no active replicas found"
                continue
            replicaDict = replicaDict["Successful"][lfn] if lfn in replicaDict[
                "Successful"] else {}
            # # use valid replicas only
            replicaDict = dict([(se, pfn) for se, pfn in replicaDict.items()
                                if se in sourceSEs])

            if not replicaDict:
                ret["Failed"][fileID] = "no active replicas found in sources"
                continue

            tree = self.ftsStrategy().replicationTree(sourceSEs, targetSEs,
                                                      size)
            if not tree["OK"]:
                gLogger.error("ftsSchedule: %s cannot be scheduled: %s" %
                              (lfn, tree["Message"]))
                ret["Failed"][fileID] = tree["Message"]
                continue
            tree = tree["Value"]

            gLogger.info("LFN=%s tree=%s" % (lfn, tree))

            for repDict in tree.values():
                gLogger.info(
                    "Strategy=%s Ancestor=%s SourceSE=%s TargetSE=%s" %
                    (repDict["Strategy"], repDict["Ancestor"],
                     repDict["SourceSE"], repDict["TargetSE"]))
                transferSURLs = self._getTransferURLs(lfn, repDict, sourceSEs,
                                                      replicaDict)
                if not transferSURLs["OK"]:
                    ret["Failed"][fileID] = transferSURLs["Message"]
                    continue

                sourceSURL, targetSURL, fileStatus = transferSURLs["Value"]
                if sourceSURL == targetSURL:
                    ret["Failed"][
                        fileID] = "sourceSURL equals to targetSURL for %s" % lfn
                    continue

                gLogger.info("sourceURL=%s targetURL=%s FTSFile.Status=%s" %
                             (sourceSURL, targetSURL, fileStatus))

                ftsFile = FTSFile()
                for key in ("LFN", "FileID", "OperationID", "Checksum",
                            "ChecksumType", "Size"):
                    setattr(ftsFile, key, fileJSON.get(key))
                ftsFile.RequestID = requestID
                ftsFile.OperationID = operationID
                ftsFile.SourceSURL = sourceSURL
                ftsFile.TargetSURL = targetSURL
                ftsFile.SourceSE = repDict["SourceSE"]
                ftsFile.TargetSE = repDict["TargetSE"]
                ftsFile.Status = fileStatus
                ftsFiles.append(ftsFile)

        if not ftsFiles:
            return S_ERROR("ftsSchedule: no FTSFiles to put")

        put = self.__ftsDB.putFTSFileList(ftsFiles)
        if not put["OK"]:
            gLogger.error("ftsSchedule: %s" % put["Message"])
            return put

        for fileJSON, sources, targets in fileJSONList:
            lfn = fileJSON.get("LFN", "")
            fileID = fileJSON.get("FileID", 0)
            if fileID not in ret["Failed"]:
                ret["Successful"].append(int(fileID))

        # # if we land here some files have been properly scheduled
        return S_OK(ret)
Beispiel #33
0
  def ftsSchedule( self, requestID, operationID, opFileList ):
    """ schedule lfn for FTS job

    :param int requestID: RequestDB.Request.RequestID
    :param int operationID: RequestDB.Operation.OperationID
    :param list opFileList: list of tuples ( File.toJSON()['Value'], sourcesList, targetList )
    """

    fileIDs = [int( fileJSON.get( 'FileID', 0 ) ) for fileJSON, _sourceSEs, _targetSEs in opFileList ]
    res = self.ftsManager.cleanUpFTSFiles( requestID, fileIDs )
    if not res['OK']:
      self.log.error( "ftsSchedule: %s" % res['Message'] )
      return S_ERROR( "ftsSchedule: %s" % res['Message'] )

    ftsFiles = []

    # # this will be returned on success
    result = { "Successful": [], "Failed": {} }

    for fileJSON, sourceSEs, targetSEs in opFileList:

      lfn = fileJSON.get( "LFN", "" )
      size = int( fileJSON.get( "Size", 0 ) )
      fileID = int( fileJSON.get( "FileID", 0 ) )
      opID = int( fileJSON.get( "OperationID", 0 ) )

      self.log.verbose( "ftsSchedule: LFN=%s FileID=%s OperationID=%s sources=%s targets=%s" % ( lfn, fileID, opID,
                                                                                                 sourceSEs,
                                                                                                 targetSEs ) )

      res = self.dataManager.getActiveReplicas( lfn )
      if not res['OK']:
        self.log.error( "ftsSchedule: %s" % res['Message'] )
        result["Failed"][fileID] = res['Message']
        continue
      replicaDict = res['Value']

      if lfn in replicaDict["Failed"] and lfn not in replicaDict["Successful"]:
        result["Failed"][fileID] = "no active replicas found"
        continue
      replicaDict = replicaDict["Successful"].get( lfn, {} )
      # # use valid replicas only
      validReplicasDict = dict( [ ( se, pfn ) for se, pfn in replicaDict.items() if se in sourceSEs ] )

      if not validReplicasDict:
        self.log.warn( "No active replicas found in sources" )
        result["Failed"][fileID] = "no active replicas found in sources"
        continue

      tree = self.ftsManager.getReplicationTree( sourceSEs, targetSEs, size )
      if not tree['OK']:
        self.log.error( "ftsSchedule: %s cannot be scheduled: %s" % ( lfn, tree['Message'] ) )
        result["Failed"][fileID] = tree['Message']
        continue
      tree = tree['Value']

      self.log.verbose( "LFN=%s tree=%s" % ( lfn, tree ) )

      for repDict in tree.values():
        self.log.verbose( "Strategy=%s Ancestor=%s SourceSE=%s TargetSE=%s" % ( repDict["Strategy"],
                                                                                repDict["Ancestor"],
                                                                                repDict["SourceSE"],
                                                                                repDict["TargetSE"] ) )
        transferSURLs = self._getTransferURLs( lfn, repDict, sourceSEs, validReplicasDict )
        if not transferSURLs['OK']:
          result["Failed"][fileID] = transferSURLs['Message']
          continue

        sourceSURL, targetSURL, fileStatus = transferSURLs['Value']
        if sourceSURL == targetSURL:
          result["Failed"][fileID] = "sourceSURL equals to targetSURL for %s" % lfn
          continue

        self.log.verbose( "sourceURL=%s targetURL=%s FTSFile.Status=%s" % ( sourceSURL, targetSURL, fileStatus ) )

        ftsFile = FTSFile()
        for key in ( "LFN", "FileID", "OperationID", "Checksum", "ChecksumType", "Size" ):
          if fileJSON.get( key ):
            setattr( ftsFile, key, fileJSON.get( key ) )
        ftsFile.RequestID = requestID
        ftsFile.OperationID = operationID
        ftsFile.SourceSURL = sourceSURL
        ftsFile.TargetSURL = targetSURL
        ftsFile.SourceSE = repDict["SourceSE"]
        ftsFile.TargetSE = repDict["TargetSE"]
        ftsFile.Status = fileStatus
        ftsFiles.append( ftsFile )

    if not ftsFiles:
      self.log.info( "ftsSchedule: no FTSFiles to put for request %d" % requestID )
      return S_OK( result )

    ftsFilesJSONList = [ftsFile.toJSON()['Value'] for ftsFile in ftsFiles]
    res = self.ftsManager.putFTSFileList( ftsFilesJSONList )
    if not res['OK']:
      self.log.error( "ftsSchedule: %s" % res['Message'] )
      return S_ERROR( "ftsSchedule: %s" % res['Message'] )

    result['Successful'] += [ fileID for fileID in fileIDs if fileID not in result['Failed']]

    # # if we land here some files have been properly scheduled
    return S_OK( result )
Beispiel #34
0
  def setUp( self ):
    """ test case set up """
    # ## set some defaults
    gConfig.setOptionValue( 'DIRAC/Setup', 'Test' )
    gConfig.setOptionValue( '/DIRAC/Setups/Test/DataManagement', 'Test' )
    gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/Host', 'localhost' )
    gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/DBName', 'FTSDB' )
    gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/User', 'Dirac' )

    self.ftsSites = [ FTSSite( { "FTSServer": "https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "CERN.ch" } ),
                      FTSSite( { "FTSServer": "https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "PIC.es" } ),
                      FTSSite( { "FTSServer": "https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "RAL.uk" } ),
                      FTSSite( { "FTSServer": "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "SARA.nl" } ),
                      FTSSite( { "FTSServer": "https://fts.cr.cnaf.infn.it:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "CNAF.it" } ),
                      FTSSite( { "FTSServer": "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "NIKHEF.nl" } ),
                      FTSSite( { "FTSServer": "https://fts-fzk.gridka.de:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "GRIDKA.de" } ),
                      FTSSite( { "FTSServer": "https://cclcgftsprod.in2p3.fr:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "IN2P3.fr" } ) ]


    self.ftsFiles = []
    for i in range ( 100 ):
      ftsFile = FTSFile()
      ftsFile.FileID = i + 1
      ftsFile.OperationID = 9999
      ftsFile.LFN = "/a/b/c/%d" % i
      ftsFile.Size = 10
      ftsFile.SourceSE = "CERN-USER"
      ftsFile.TargetSE = "PIC-USER"
      ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN
      ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN
      ftsFile.Status = "Waiting"
      self.ftsFiles.append( ftsFile )

    ses = [ "CERN-USER", "RAL-USER" ]
    statuses = [ "Submitted", "Finished", "FinishedDirty", "Active", "Ready" ]

    self.submitted = 0

    self.ftsJobs = []
    for i in range( 7200 ):

      ftsJob = FTSJob()
      ftsJob.FTSGUID = str( uuid.uuid4() )
      ftsJob.FTSServer = self.ftsSites[0].FTSServer
      ftsJob.Status = statuses[ i % len( statuses ) ]

      if ftsJob.Status in FTSJob.FINALSTATES:
        ftsJob.Completeness = 100
      if ftsJob.Status == "Active":
        ftsJob.Completeness = 90

      ftsJob.SourceSE = ses[ i % len( ses ) ]
      ftsJob.TargetSE = "PIC-USER"

      ftsFile = FTSFile()
      ftsFile.FileID = i + 1
      ftsFile.OperationID = i + 1
      ftsFile.LFN = "/a/b/c/%d" % i
      ftsFile.Size = 1000000
      ftsFile.SourceSE = ftsJob.SourceSE
      ftsFile.TargetSE = ftsJob.TargetSE
      ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN
      ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN
      ftsFile.Status = "Waiting" if ftsJob.Status != "FinishedDirty" else "Failed"

      ftsFile.FTSGUID = ftsJob.FTSGUID
      if ftsJob.Status == "FinishedDirty":
        ftsJob.FailedFiles = 1
        ftsJob.FailedSize = ftsFile.Size

      ftsJob.addFile( ftsFile )
      self.ftsJobs.append( ftsJob )

    self.submitted = len( [ i for i in self.ftsJobs if i.Status == "Submitted" ] )
  def export_ftsSchedule( self, requestID, operationID, fileJSONList ):
    """ call FTS scheduler

    :param int requestID: ReqDB.Request.RequestID
    :param int operationID: ReqDB.Operation.OperationID
    :param list fileJSONList: [ (fileJSON, [sourceSE, ...], [ targetSE, ...] ) ]
    """
    # # this will be returned on success
    ret = { "Successful": [], "Failed": {} }

    requestID = int( requestID )
    operationID = int( operationID )

    fileIDs = []
    for fileJSON, sourceSEs, targetSEs in fileJSONList:
      fileID = int( fileJSON.get( "FileID" ) )
      fileIDs.append( fileID )
    cleanUpFTSFiles = self.__ftsDB.cleanUpFTSFiles( requestID, fileIDs )
    if not cleanUpFTSFiles["OK"]:
      self.log.error( "ftsSchedule: %s" % cleanUpFTSFiles["Message"] )
      return S_ERROR( cleanUpFTSFiles["Message"] )

    ftsFiles = []

    for fileJSON, sourceSEs, targetSEs in fileJSONList:

      lfn = fileJSON.get( "LFN", "" )
      size = int( fileJSON.get( "Size", 0 ) )
      fileID = int( fileJSON.get( "FileID", 0 ) )
      opID = int( fileJSON.get( "OperationID", 0 ) )

      gLogger.info( "ftsSchedule: LFN=%s FileID=%s OperationID=%s sources=%s targets=%s" % ( lfn, fileID, opID,
                                                                                             sourceSEs, targetSEs ) )

      replicaDict = self.replicaManager().getActiveReplicas( lfn )
      if not replicaDict["OK"]:
        gLogger.error( "ftsSchedule: %s" % replicaDict["Message"] )
        ret["Failed"][fileID] = replicaDict["Message"]
        continue
      replicaDict = replicaDict["Value"]

      if lfn in replicaDict["Failed"] and lfn not in replicaDict["Successful"]:
        ret["Failed"][fileID] = "no active replicas found"
        continue
      replicaDict = replicaDict["Successful"][lfn] if lfn in replicaDict["Successful"] else {}
      # # use valid replicas only
      replicaDict = dict( [ ( se, pfn ) for se, pfn in replicaDict.items() if se in sourceSEs ] )

      if not replicaDict:
        ret["Failed"][fileID] = "no active replicas found in sources"
        continue

      tree = self.ftsStrategy().replicationTree( sourceSEs, targetSEs, size )
      if not tree["OK"]:
        gLogger.error( "ftsSchedule: %s cannot be scheduled: %s" % ( lfn, tree["Message"] ) )
        ret["Failed"][fileID] = tree["Message"]
        continue
      tree = tree["Value"]

      gLogger.info( "LFN=%s tree=%s" % ( lfn, tree ) )

      for repDict in tree.values():
        gLogger.info( "Strategy=%s Ancestor=%s SourceSE=%s TargetSE=%s" % ( repDict["Strategy"], repDict["Ancestor"],
                                                                            repDict["SourceSE"], repDict["TargetSE"] ) )
        transferSURLs = self._getTransferURLs( lfn, repDict, sourceSEs, replicaDict )
        if not transferSURLs["OK"]:
          ret["Failed"][fileID] = transferSURLs["Message"]
          continue

        sourceSURL, targetSURL, fileStatus = transferSURLs["Value"]
        if sourceSURL == targetSURL:
          ret["Failed"][fileID] = "sourceSURL equals to targetSURL for %s" % lfn
          continue

        gLogger.info( "sourceURL=%s targetURL=%s FTSFile.Status=%s" % ( sourceSURL, targetSURL, fileStatus ) )

        ftsFile = FTSFile()
        for key in ( "LFN", "FileID", "OperationID", "Checksum", "ChecksumType", "Size" ):
          setattr( ftsFile, key, fileJSON.get( key ) )
        ftsFile.RequestID = requestID
        ftsFile.OperationID = operationID
        ftsFile.SourceSURL = sourceSURL
        ftsFile.TargetSURL = targetSURL
        ftsFile.SourceSE = repDict["SourceSE"]
        ftsFile.TargetSE = repDict["TargetSE"]
        ftsFile.Status = fileStatus
        ftsFiles.append( ftsFile )

    if not ftsFiles:
      return S_ERROR( "ftsSchedule: no FTSFiles to put" )

    put = self.__ftsDB.putFTSFileList( ftsFiles )
    if not put["OK"]:
      gLogger.error( "ftsSchedule: %s" % put["Message"] )
      return put

    for fileJSON, sources, targets in fileJSONList:
      lfn = fileJSON.get( "LFN", "" )
      fileID = fileJSON.get( "FileID", 0 )
      if fileID not in ret["Failed"]:
        ret["Successful"].append( int( fileID ) )

    # # if we land here some files have been properly scheduled
    return S_OK( ret )
Beispiel #36
0
  def setUp( self ):
    """ test case set up """

    gLogger.setLevel( 'NOTICE' )

    self.ftsSites = [ FTSSite( ftsServer = 'https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer', name = 'CERN.ch' ),
                      FTSSite( ftsServer = 'https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer', name = 'PIC.es' ),
                      FTSSite( ftsServer = 'https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer', name = 'RAL.uk' ),
                    ]

    self.ses = [ 'CERN-USER', 'RAL-USER' ]
    self.statuses = [ 'Submitted', 'Finished', 'FinishedDirty', 'Active', 'Ready' ]

    self.submitted = 0
    self.numberOfJobs = 10
    self.opIDs = []

    self.ftsJobs = []
    for i in xrange( self.numberOfJobs ):

      opID = i % 3
      if opID not in self.opIDs:
        self.opIDs.append( opID )

      ftsJob = FTSJob()
      ftsJob.FTSGUID = str( uuid.uuid4() )
      ftsJob.FTSServer = self.ftsSites[0].FTSServer
      ftsJob.Status = self.statuses[ i % len( self.statuses ) ]
      ftsJob.OperationID = opID
      if ftsJob.Status in FTSJob.FINALSTATES:
        ftsJob.Completeness = 100
      if ftsJob.Status == 'Active':
        ftsJob.Completeness = 90
      ftsJob.SourceSE = self.ses[ i % len( self.ses ) ]
      ftsJob.TargetSE = 'PIC-USER'
      ftsJob.RequestID = 12345

      ftsFile = FTSFile()
      ftsFile.FileID = i + 1
      ftsFile.OperationID = i + 1
      ftsFile.LFN = '/a/b/c/%d' % i
      ftsFile.Size = 1000000
      ftsFile.OperationID = opID
      ftsFile.SourceSE = ftsJob.SourceSE
      ftsFile.TargetSE = ftsJob.TargetSE
      ftsFile.SourceSURL = 'foo://source.bar.baz/%s' % ftsFile.LFN
      ftsFile.TargetSURL = 'foo://target.bar.baz/%s' % ftsFile.LFN
      ftsFile.Status = 'Waiting' if ftsJob.Status != 'FinishedDirty' else 'Failed'
      ftsFile.RequestID = 12345
      ftsFile.Checksum = 'addler'
      ftsFile.ChecksumType = 'adler32'

      ftsFile.FTSGUID = ftsJob.FTSGUID
      if ftsJob.Status == 'FinishedDirty':
        ftsJob.FailedFiles = 1
        ftsJob.FailedSize = ftsFile.Size

      ftsJob.addFile( ftsFile )
      self.ftsJobs.append( ftsJob )

    self.submitted = len( [ i for i in self.ftsJobs if i.Status == 'Submitted' ] )

    self.ftsClient = FTSClient()
    self.ftsClient.replicaManager = mock.Mock()
    self.ftsClient.replicaManager.getActiveReplicas.return_value = {'OK': True,
                                                                    'Value': {'Successful': {'/a/b/c/1':{'CERN-USER':'******',
                                                                                                         'RAL-USER':'******'},
                                                                                             '/a/b/c/2':{'CERN-USER':'******',
                                                                                                         'RAL-USER':'******'},
                                                                                             '/a/b/c/3':{'CERN-USER':'******',
                                                                                                         'RAL-USER':'******'}
                                                                                             },
                                                                              'Failed': {'/a/b/c/4':'/aa/a/b/c/4d',
                                                                                         '/a/b/c/5':'/aa/a/b/c/5d'}
                                                                              }
                                                                    }
Beispiel #37
0
  def monitorFTS2( self, command = "glite-transfer-status", full = False ):
    """ monitor fts job """
    if not self.FTSGUID:
      return S_ERROR( "FTSGUID not set, FTS job not submitted?" )

    monitorCommand = command.split() + \
                       ["--verbose",
                       "-s",
                       self.FTSServer,
                       self.FTSGUID ]

    if full:
      monitorCommand.append( "-l" )

    monitor = executeGridCommand( "", monitorCommand )
    if not monitor["OK"]:
      return monitor
    returnCode, outputStr, errStr = monitor["Value"]

    # Returns a non zero status if error
    if returnCode != 0:
      if 'was not found' in outputStr and not errStr:
        errStr = 'Job was not found'
      return S_ERROR( errStr )

    outputStr = outputStr.replace( "'" , "" ).replace( "<", "" ).replace( ">", "" )

    # # set FTS job status
    regExp = re.compile( "Status:\\s+(\\S+)" )

    # with FTS3 this can be uppercase
    self.Status = re.search( regExp, outputStr ).group( 1 )

    statusSummary = {}
    # This is capitalized, even in FTS3!
    for state in FTSFile.ALL_STATES:
      regExp = re.compile( "\\s+%s:\\s+(\\d+)" % state )
      if regExp.search( outputStr ):
        statusSummary[state] = int( re.search( regExp, outputStr ).group( 1 ) )

    total = sum( statusSummary.values() )
    completed = sum( statusSummary.get( state, 0 ) for state in FTSFile.FINAL_STATES )
    self.Completeness = 100 * completed / total if total else 0

    if not full:
      return S_OK( statusSummary )

    # The order of informations is not the same for glite- and fts- !!!
    # In order: new fts-, old fts-, glite-
    realJob = len( self ) != 0
    iExptr = None
    for iExptr, exptr in enumerate( ( 
                   '[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Staging:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)',
                   '[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)',
                   '[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Retries:[ ]+(\\d+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)'
                   ) ):
      regExp = re.compile( exptr, re.S )
      fileInfo = re.findall( regExp, outputStr )
      if fileInfo:
        break
    if not fileInfo:
      return S_ERROR( "Error monitoring job (no regexp match)" )
    for info in fileInfo:
      if iExptr == 0:
        # version >= 3.2.30
        sourceURL, targetURL, fileStatus, reason, duration, _retries, _staging = info
      elif iExptr == 1:
        # version FTS3 < 3.2.30
        sourceURL, targetURL, fileStatus, reason, duration, _retries = info
      elif iExptr == 2:
        # version FTS2
        sourceURL, targetURL, fileStatus, _retries, reason, duration = info
      else:
        return S_ERROR( 'Error monitoring job (implement match %d)' % iExptr )
      candidateFile = None

      if not realJob:
        # This is used by the CLI monitoring of jobs in case no file was specified
        candidateFile = FTSFile()
        candidateFile.LFN = overlap( sourceURL, targetURL )
        candidateFile.SourceSURL = sourceURL
        candidateFile.Size = 0
        self +=candidateFile
      else:
        for ftsFile in self:
          if ftsFile.SourceSURL == sourceURL:
            candidateFile = ftsFile
            break
        if not candidateFile:
          continue
      # Can be uppercase for FTS3
      if not candidateFile.TargetSURL:
        candidateFile.TargetSURL = targetURL
      candidateFile.Status = fileStatus
      candidateFile.Error = reason
      candidateFile._duration = duration

      if candidateFile.Status == "Failed":
        for missingSource in self.missingSourceErrors:
          if missingSource.match( reason ):
            candidateFile.Error = "MissingSource"
      # If the staging info was present, record it
      if len( info ) > 6:
        candidateFile._staging = info[6]
    # # register successful files
    if self.Status in FTSJob.FINALSTATES:
      return self.finalize()

    return S_OK()
Beispiel #38
0
  def ftsSchedule( self, requestID, operationID, opFileList ):
    """ schedule lfn for FTS job

    :param int requestID: RequestDB.Request.RequestID
    :param int operationID: RequestDB.Operation.OperationID
    :param list opFileList: list of tuples ( File.toJSON()['Value'], sourcesList, targetList )
    """

    # Check whether there are duplicates
    fList = []
    for fileJSON, sourceSEs, targetSEs in opFileList:
      fTuple = ( json.loads( fileJSON ), sourceSEs, targetSEs )
      if fTuple not in fList:
        fList.append( fTuple )
      else:
        self.log.warn( 'File list for FTS scheduling has duplicates, fix it:\n', fTuple )
    fileIDs = [int( fileJSON.get( 'FileID', 0 ) ) for fileJSON, _sourceSEs, _targetSEs in fList ]
    res = self.ftsManager.cleanUpFTSFiles( requestID, fileIDs )
    if not res['OK']:
      self.log.error( "Failed ftsSchedule", "%s" % res['Message'] )
      return S_ERROR( "ftsSchedule: %s" % res['Message'] )
    ftsFiles = []

    # # this will be returned on success
    result = { "Successful": [], "Failed": {} }

    for fileJSON, sourceSEs, targetSEs in fList:

      lfn = fileJSON.get( "LFN", "" )
      size = int( fileJSON.get( "Size", 0 ) )
      fileID = int( fileJSON.get( "FileID", 0 ) )
      opID = int( fileJSON.get( "OperationID", 0 ) )

      self.log.verbose( "ftsSchedule: LFN=%s FileID=%s OperationID=%s sources=%s targets=%s" % ( lfn, fileID, opID,
                                                                                                 sourceSEs,
                                                                                                 targetSEs ) )

      res = self.dataManager.getActiveReplicas( lfn )
      if not res['OK']:
        self.log.error( "Failed ftsSchedule", "%s" % res['Message'] )
        result["Failed"][fileID] = res['Message']
        continue
      replicaDict = res['Value']

      if lfn in replicaDict["Failed"] and lfn not in replicaDict["Successful"]:
        result["Failed"][fileID] = "no active replicas found"
        continue
      replicaDict = replicaDict["Successful"].get( lfn, {} )
      # # use valid replicas only
      validReplicasDict = dict( [ ( se, pfn ) for se, pfn in replicaDict.items() if se in sourceSEs ] )

      if not validReplicasDict:
        self.log.warn( "No active replicas found in sources" )
        result["Failed"][fileID] = "no active replicas found in sources"
        continue

      tree = self.ftsManager.getReplicationTree( sourceSEs, targetSEs, size )
      if not tree['OK']:
        self.log.error( "Failed ftsSchedule", "%s cannot be scheduled: %s" % ( lfn, tree['Message'] ) )
        result["Failed"][fileID] = tree['Message']
        continue
      tree = tree['Value']

      self.log.verbose( "LFN=%s tree=%s" % ( lfn, tree ) )

      treeBranches = []
      printed = False
      for repDict in tree.values():
        if repDict in treeBranches:
          if not printed:
            self.log.warn( 'Duplicate tree branch', str( tree ) )
            printed = True
        else:
          treeBranches.append( repDict )

      for repDict in treeBranches:
        self.log.verbose( "Strategy=%s Ancestor=%s SourceSE=%s TargetSE=%s" % ( repDict["Strategy"],
                                                                                repDict["Ancestor"],
                                                                                repDict["SourceSE"],
                                                                                repDict["TargetSE"] ) )
        transferSURLs = self._getTransferURLs( lfn, repDict, sourceSEs, validReplicasDict )
        if not transferSURLs['OK']:
          result["Failed"][fileID] = transferSURLs['Message']
          continue

        sourceSURL, targetSURL, fileStatus = transferSURLs['Value']
        if sourceSURL == targetSURL:
          result["Failed"][fileID] = "sourceSURL equals to targetSURL for %s" % lfn
          continue

        self.log.verbose( "sourceURL=%s targetURL=%s FTSFile.Status=%s" % ( sourceSURL, targetSURL, fileStatus ) )

        ftsFile = FTSFile()
        for key in ( "LFN", "FileID", "OperationID", "Checksum", "ChecksumType", "Size" ):
          if fileJSON.get( key ):
            setattr( ftsFile, key, fileJSON.get( key ) )
        ftsFile.RequestID = requestID
        ftsFile.OperationID = operationID
        ftsFile.SourceSURL = sourceSURL
        ftsFile.TargetSURL = targetSURL
        ftsFile.SourceSE = repDict["SourceSE"]
        ftsFile.TargetSE = repDict["TargetSE"]
        ftsFile.Status = fileStatus
        ftsFiles.append( ftsFile )

    if not ftsFiles:
      self.log.info( "ftsSchedule: no FTSFiles to put for request %d" % requestID )
      return S_OK( result )

    ftsFilesJSONList = [ftsFile.toJSON()['Value'] for ftsFile in ftsFiles]
    res = self.ftsManager.putFTSFileList( ftsFilesJSONList )
    if not res['OK']:
      self.log.error( "Failed ftsSchedule", "%s" % res['Message'] )
      return S_ERROR( "ftsSchedule: %s" % res['Message'] )

    result['Successful'] += [ fileID for fileID in fileIDs if fileID not in result['Failed']]

    # # if we land here some files have been properly scheduled
    return S_OK( result )