def finalize(self): """ register successfully transferred files """ if self.Status not in FTSJob.FINALSTATES: return S_OK() targetSE = StorageElement(self.TargetSE) toRegister = [ ftsFile for ftsFile in self if ftsFile.Status == "Finished" ] toRegisterDict = {} for ftsFile in toRegister: pfn = targetSE.getPfnForProtocol(ftsFile.TargetSURL, "SRM2", withPort=False) if not pfn["OK"]: continue pfn = pfn["Value"] toRegisterDict[ftsFile.LFN] = {"PFN": pfn, "SE": self.TargetSE} if toRegisterDict: register = self.replicaManager().addCatalogReplica(toRegisterDict) if not register["OK"]: for ftsFile in toRegister: ftsFile.Error = "AddCatalogReplicaFailed" print ftsFile.Error return register register = register["Value"] failedFiles = register["Failed"] if "Failed" in register else {} for ftsFile in toRegister: if ftsFile.LFN in failedFiles: ftsFile.Error = "AddCatalogReplicaFailed" print ftsFile.Error return S_OK()
def finalize( self ): """ register successfully transferred files """ if self.Status not in FTSJob.FINALSTATES: return S_OK() targetSE = StorageElement( self.TargetSE ) toRegister = [ ftsFile for ftsFile in self if ftsFile.Status == "Finished" ] toRegisterDict = {} for ftsFile in toRegister: pfn = targetSE.getPfnForProtocol( ftsFile.TargetSURL, "SRM2", withPort = False ) if not pfn["OK"]: continue pfn = pfn["Value"] toRegisterDict[ ftsFile.LFN ] = { "PFN": pfn, "SE": self.TargetSE } if toRegisterDict: register = self.replicaManager().addCatalogReplica( toRegisterDict ) if not register["OK"]: for ftsFile in toRegister: ftsFile.Error = "AddCatalogReplicaFailed" print ftsFile.Error return register register = register["Value"] failedFiles = register["Failed"] if "Failed" in register else {} for ftsFile in toRegister: if ftsFile.LFN in failedFiles: ftsFile.Error = "AddCatalogReplicaFailed" print ftsFile.Error return S_OK()
def finalize( self ): """ register successfully transferred files """ if self.Status not in FTSJob.FINALSTATES: return S_OK() startTime = time.time() targetSE = StorageElement( self.TargetSE ) toRegister = [ ftsFile for ftsFile in self if ftsFile.Status == "Finished" ] toRegisterDict = {} for ftsFile in toRegister: pfn = returnSingleResult( targetSE.getPfnForProtocol( ftsFile.TargetSURL, protocol = "SRM2", withPort = False ) ) if not pfn["OK"]: continue pfn = pfn["Value"] toRegisterDict[ ftsFile.LFN ] = { "PFN": pfn, "SE": self.TargetSE } if toRegisterDict: self._regTotal += len( toRegisterDict ) register = self._fc.addReplica( toRegisterDict ) self._regTime += time.time() - startTime if not register["OK"]: for ftsFile in toRegister: ftsFile.Error = "AddCatalogReplicaFailed" return register register = register["Value"] self._regSuccess += len( register.get( 'Successful', {} ) ) failedFiles = register.get( "Failed", {} ) for ftsFile in toRegister: if ftsFile.LFN in failedFiles: ftsFile.Error = "AddCatalogReplicaFailed" return S_OK()
def finalize(self): """ register successfully transferred files """ if self.Status not in FTSJob.FINALSTATES: return S_OK() startTime = time.time() targetSE = StorageElement(self.TargetSE) toRegister = [ ftsFile for ftsFile in self if ftsFile.Status == "Finished" ] toRegisterDict = {} for ftsFile in toRegister: pfn = targetSE.getPfnForProtocol(ftsFile.TargetSURL, "SRM2", withPort=False) if not pfn["OK"]: continue pfn = pfn["Value"] toRegisterDict[ftsFile.LFN] = {"PFN": pfn, "SE": self.TargetSE} if toRegisterDict: self._regTotal += len(toRegisterDict) register = self.replicaManager().addCatalogReplica(toRegisterDict) self._regTime += time.time() - startTime if not register["OK"]: # FIXME: shouldn't be a print! for ftsFile in toRegister: ftsFile.Error = "AddCatalogReplicaFailed" print ftsFile.Error return register register = register["Value"] self._regSuccess += len(register.get('Successful', {})) failedFiles = register.get("Failed", {}) # FIXME for ftsFile in toRegister: if ftsFile.LFN in failedFiles: ftsFile.Error = "AddCatalogReplicaFailed" print ftsFile.Error return S_OK()
def execute( self ): IntegrityDB = RPCClient( 'DataManagement/DataIntegrity' ) res = self.RequestDBClient.getRequest( 'integrity' ) if not res['OK']: gLogger.info( "SEvsLFCAgent.execute: Failed to get request from database." ) return S_OK() elif not res['Value']: gLogger.info( "SEvsLFCAgent.execute: No requests to be executed found." ) return S_OK() requestString = res['Value']['requestString'] requestName = res['Value']['requestName'] sourceServer = res['Value']['Server'] gLogger.info( "SEvsLFCAgent.execute: Obtained request %s" % requestName ) oRequest = RequestContainer( request = requestString ) ################################################ # Find the number of sub-requests from the request res = oRequest.getNumSubRequests( 'integrity' ) if not res['OK']: errStr = "SEvsLFCAgent.execute: Failed to obtain number of integrity subrequests." gLogger.error( errStr, res['Message'] ) return S_OK() gLogger.info( "SEvsLFCAgent.execute: Found %s sub requests." % res['Value'] ) ################################################ # For all the sub-requests in the request for ind in range( res['Value'] ): gLogger.info( "SEvsLFCAgent.execute: Processing sub-request %s." % ind ) subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'integrity' )['Value'] if subRequestAttributes['Status'] == 'Waiting': subRequestFiles = oRequest.getSubRequestFiles( ind, 'integrity' )['Value'] operation = subRequestAttributes['Operation'] ################################################ # If the sub-request is a lfcvsse operation if operation == 'SEvsLFC': gLogger.info( "SEvsLFCAgent.execute: Attempting to execute %s sub-request." % operation ) storageElementName = subRequestAttributes['StorageElement'] for subRequestFile in subRequestFiles: if subRequestFile['Status'] == 'Waiting': lfn = subRequestFile['LFN'] storageElement = StorageElement( storageElementName ) res = storageElement.isValid() if not res['OK']: errStr = "SEvsLFCAgent.execute: Failed to instantiate destination StorageElement." gLogger.error( errStr, storageElement ) else: res = storageElement.getPfnForLfn( lfn ) if not res['OK']: gLogger.info( 'shit bugger do something.' ) else: oNamespaceBrowser = NamespaceBrowser( res['Value'] ) # Loop over all the directories and sub-directories while ( oNamespaceBrowser.isActive() ): currentDir = oNamespaceBrowser.getActiveDir() gLogger.info( "SEvsLFCAgent.execute: Attempting to list the contents of %s." % currentDir ) res = storageElement.listDirectory( currentDir ) if not res['Value']['Successful'].has_key( currentDir ): gLogger.error( "SEvsLFCAgent.execute: Failed to list the directory contents.", "%s %s" % ( currentDir, res['Value']['Successful']['Failed'][currentDir] ) ) subDirs = [currentDir] else: subDirs = [] files = {} for surl, surlDict in res['Value']['Successful'][currentDir]['Files'].items(): pfnRes = storageElement.getPfnForProtocol( surl, 'SRM2', withPort = False ) surl = pfnRes['Value'] files[surl] = surlDict for surl, surlDict in res['Value']['Successful'][currentDir]['SubDirs'].items(): pfnRes = storageElement.getPfnForProtocol( surl, 'SRM2', withPort = False ) surl = pfnRes['Value'] subDirs.append( surl ) #subDirs = res['Value']['Successful'][currentDir]['SubDirs'] gLogger.info( "SEvsLFCAgent.execute: Successfully obtained %s sub-directories." % len( subDirs ) ) #files = res['Value']['Successful'][currentDir]['Files'] gLogger.info( "SEvsLFCAgent.execute: Successfully obtained %s files." % len( files ) ) selectedLfns = [] lfnPfnDict = {} pfnSize = {} for pfn, pfnDict in files.items(): res = storageElement.getPfnPath( pfn ) if not res['OK']: gLogger.error( "SEvsLFCAgent.execute: Failed to get determine LFN from pfn.", "%s %s" % ( pfn, res['Message'] ) ) fileMetadata = {'Prognosis':'NonConventionPfn', 'LFN':'', 'PFN':pfn, 'StorageElement':storageElementName, 'Size':pfnDict['Size']} res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata ) if res['OK']: gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) else: lfn = res['Value'] selectedLfns.append( lfn ) lfnPfnDict[lfn] = pfn pfnSize[pfn] = pfnDict['Size'] res = self.ReplicaManager.getCatalogFileMetadata( selectedLfns ) if not res['OK']: subDirs = [currentDir] else: for lfn in res['Value']['Failed'].keys(): gLogger.error( "SEvsLFCAgent.execute: Failed to get metadata.", "%s %s" % ( lfn, res['Value']['Failed'][lfn] ) ) pfn = lfnPfnDict[lfn] fileMetadata = {'Prognosis':'SEPfnNoLfn', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName, 'Size':pfnSize[pfn]} res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata ) if res['OK']: gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) for lfn, lfnDict in res['Value']['Successful'].items(): pfn = lfnPfnDict[lfn] storageSize = pfnSize[pfn] catalogSize = lfnDict['Size'] if int( catalogSize ) == int( storageSize ): gLogger.info( "SEvsLFCAgent.execute: Catalog and storage sizes match.", "%s %s" % ( pfn, storageElementName ) ) gLogger.info( "Change the status in the LFC" ) elif int( storageSize ) == 0: gLogger.error( "SEvsLFCAgent.execute: Physical file size is 0.", "%s %s" % ( pfn, storageElementName ) ) fileMetadata = {'Prognosis':'ZeroSizePfn', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName} res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata ) if res['OK']: gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) else: gLogger.error( "SEvsLFCAgent.execute: Catalog and storage size mis-match.", "%s %s" % ( pfn, storageElementName ) ) fileMetadata = {'Prognosis':'PfnSizeMismatch', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName} res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata ) if res['OK']: gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) res = self.ReplicaManager.getCatalogReplicas( selectedLfns ) if not res['OK']: subDirs = [currentDir] else: for lfn in res['Value']['Failed'].keys(): gLogger.error( "SEvsLFCAgent.execute: Failed to get replica information.", "%s %s" % ( lfn, res['Value']['Failed'][lfn] ) ) pfn = lfnPfnDict[lfn] fileMetadata = {'Prognosis':'PfnNoReplica', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName, 'Size':pfnSize[pfn]} res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata ) if res['OK']: gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) for lfn, repDict in res['Value']['Successful'].items(): pfn = lfnPfnDict[lfn] registeredPfns = repDict.values() if not pfn in registeredPfns: gLogger.error( "SEvsLFCAgent.execute: SE PFN not registered.", "%s %s" % ( lfn, pfn ) ) fileMetadata = {'Prognosis':'PfnNoReplica', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName} res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata ) if res['OK']: gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) else: gLogger.info( "SEvsLFCAgent.execute: SE Pfn verified.", pfn ) oNamespaceBrowser.updateDirs( subDirs ) oRequest.setSubRequestFileAttributeValue( ind, 'integrity', lfn, 'Status', 'Done' ) ################################################ # If the sub-request is none of the above types else: gLogger.info( "SEvsLFCAgent.execute: Operation not supported.", operation ) ################################################ # Determine whether there are any active files if oRequest.isSubRequestEmpty( ind, 'integrity' )['Value']: oRequest.setSubRequestStatus( ind, 'integrity', 'Done' ) ################################################ # If the sub-request is already in terminal state else: gLogger.info( "SEvsLFCAgent.execute: Sub-request %s is status '%s' and not to be executed." % ( ind, subRequestAttributes['Status'] ) ) ################################################ # Generate the new request string after operation requestString = oRequest.toXML()['Value'] res = self.RequestDBClient.updateRequest( requestName, requestString, sourceServer ) return S_OK()
class FTSRequest( object ): """ .. class:: FTSRequest Helper class for FTS job submission and monitoring. """ # # default checksum type __defaultCksmType = "ADLER32" # # flag to disablr/enable checksum test, default: disabled __cksmTest = False def __init__( self ): """c'tor :param self: self reference """ self.log = gLogger.getSubLogger( self.__class__.__name__, True ) # # final states tuple self.finalStates = ( 'Canceled', 'Failed', 'Hold', 'Finished', 'FinishedDirty' ) # # failed states tuple self.failedStates = ( 'Canceled', 'Failed', 'Hold', 'FinishedDirty' ) # # successful states tuple self.successfulStates = ( 'Finished', 'Done' ) # # all file states tuple self.fileStates = ( 'Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed', 'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting' ) self.statusSummary = {} # # request status self.requestStatus = 'Unknown' # # dict for FTS job files self.fileDict = {} # # dict for replicas information self.catalogReplicas = {} # # dict for metadata information self.catalogMetadata = {} # # dict for files that failed to register self.failedRegistrations = {} # # placehoder for FileCatalog reference self.oCatalog = None # # submit timestamp self.submitTime = '' # # placeholder FTS job GUID self.ftsGUID = '' # # placeholder for FTS server URL self.ftsServer = '' # # flag marking FTS job completness self.isTerminal = False # # completness percentage self.percentageComplete = 0.0 # # source SE name self.sourceSE = '' # # flag marking source SE validity self.sourceValid = False # # source space token self.sourceToken = '' # # target SE name self.targetSE = '' # # flag marking target SE validity self.targetValid = False # # target space token self.targetToken = '' # # placeholder for target StorageElement self.oTargetSE = None # # placeholder for source StorageElement self.oSourceSE = None # # checksum type, set it to default self.__cksmType = self.__defaultCksmType # # disable checksum test by default self.__cksmTest = False # # statuses that prevent submitting to FTS self.noSubmitStatus = ( 'Failed', 'Done', 'Staging' ) # # were sources resolved? self.sourceResolved = False # # Number of file transfers actually submitted self.submittedFiles = 0 self.transferTime = 0 self.submitCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/SubmitCommand', 'glite-transfer-submit' ) self.monitorCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/MonitorCommand', 'glite-transfer-status' ) self.ftsJob = None self.ftsFiles = [] #################################################################### # # Methods for setting/getting/checking the SEs # def setSourceSE( self, se ): """ set SE for source :param self: self reference :param str se: source SE name """ if se == self.targetSE: return S_ERROR( "SourceSE is TargetSE" ) self.sourceSE = se self.oSourceSE = StorageElement( self.sourceSE ) return self.__checkSourceSE() def __checkSourceSE( self ): """ check source SE availability :param self: self reference """ if not self.sourceSE: return S_ERROR( "SourceSE not set" ) res = self.oSourceSE.isValid( 'Read' ) if not res['OK']: return S_ERROR( "SourceSE not available for reading" ) res = self.__getSESpaceToken( self.oSourceSE ) if not res['OK']: self.log.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message'] ) return S_ERROR( "SourceSE does not support FTS transfers" ) if self.__cksmTest: res = self.oSourceSE.getChecksumType() if not res["OK"]: self.log.error( "Unable to get checksum type for SourceSE %s: %s" % ( self.sourceSE, res["Message"] ) ) cksmType = res["Value"] if cksmType in ( "NONE", "NULL" ): self.log.warn( "Checksum type set to %s at SourceSE %s, disabling checksum test" % ( cksmType, self.sourceSE ) ) self.__cksmTest = False elif cksmType != self.__cksmType: self.log.warn( "Checksum type mismatch, disabling checksum test" ) self.__cksmTest = False self.sourceToken = res['Value'] self.sourceValid = True return S_OK() def setTargetSE( self, se ): """ set target SE :param self: self reference :param str se: target SE name """ if se == self.sourceSE: return S_ERROR( "TargetSE is SourceSE" ) self.targetSE = se self.oTargetSE = StorageElement( self.targetSE ) return self.__checkTargetSE() def setTargetToken( self, token ): """ target space token setter :param self: self reference :param str token: target space token """ self.targetToken = token return S_OK() def __checkTargetSE( self ): """ check target SE availability :param self: self reference """ if not self.targetSE: return S_ERROR( "TargetSE not set" ) res = self.oTargetSE.isValid( 'Write' ) if not res['OK']: return S_ERROR( "TargetSE not available for writing" ) res = self.__getSESpaceToken( self.oTargetSE ) if not res['OK']: self.log.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message'] ) return S_ERROR( "TargetSE does not support FTS transfers" ) # # check checksum types if self.__cksmTest: res = self.oTargetSE.getChecksumType() if not res["OK"]: self.log.error( "Unable to get checksum type for TargetSE %s: %s" % ( self.targetSE, res["Message"] ) ) cksmType = res["Value"] if cksmType in ( "NONE", "NULL" ): self.log.warn( "Checksum type set to %s at TargetSE %s, disabling checksum test" % ( cksmType, self.targetSE ) ) self.__cksmTest = False elif cksmType != self.__cksmType: self.log.warn( "Checksum type mismatch, disabling checksum test" ) self.__cksmTest = False self.targetToken = res['Value'] self.targetValid = True return S_OK() @staticmethod def __getSESpaceToken( oSE ): """ get space token from StorageElement instance :param self: self reference :param StorageElement oSE: StorageElement instance """ res = oSE.getStorageParameters( "SRM2" ) if not res['OK']: return res return S_OK( res['Value'].get( 'SpaceToken' ) ) #################################################################### # # Methods for setting/getting FTS request parameters # def setFTSGUID( self, guid ): """ FTS job GUID setter :param self: self reference :param str guid: string containg GUID """ if not checkGuid( guid ): return S_ERROR( "Incorrect GUID format" ) self.ftsGUID = guid return S_OK() def setFTSServer( self, server ): """ FTS server setter :param self: self reference :param str server: FTS server URL """ self.ftsServer = server return S_OK() def isRequestTerminal( self ): """ check if FTS job has terminated :param self: self reference """ if self.requestStatus in self.finalStates: self.isTerminal = True return S_OK( self.isTerminal ) def setCksmTest( self, cksmTest = False ): """ set cksm test :param self: self reference :param bool cksmTest: flag to enable/disable checksum test """ self.__cksmTest = bool( cksmTest ) return S_OK( self.__cksmTest ) #################################################################### # # Methods for setting/getting/checking files and their metadata # def setLFN( self, lfn ): """ add LFN :lfn: to :fileDict: :param self: self reference :param str lfn: LFN to add to """ self.fileDict.setdefault( lfn, {'Status':'Waiting'} ) return S_OK() def setSourceSURL( self, lfn, surl ): """ source SURL setter :param self: self reference :param str lfn: LFN :param str surl: source SURL """ target = self.fileDict[lfn].get( 'Target' ) if target == surl: return S_ERROR( "Source and target the same" ) return self.__setFileParameter( lfn, 'Source', surl ) def getSourceSURL( self, lfn ): """ get source SURL for LFN :lfn: :param self: self reference :param str lfn: LFN """ return self.__getFileParameter( lfn, 'Source' ) def setTargetSURL( self, lfn, surl ): """ set target SURL for LFN :lfn: :param self: self reference :param str lfn: LFN :param str surl: target SURL """ source = self.fileDict[lfn].get( 'Source' ) if source == surl: return S_ERROR( "Source and target the same" ) return self.__setFileParameter( lfn, 'Target', surl ) def getFailReason( self, lfn ): """ get fail reason for file :lfn: :param self: self reference :param str lfn: LFN """ return self.__getFileParameter( lfn, 'Reason' ) def getRetries( self, lfn ): """ get number of attepmts made to transfer file :lfn: :param self: self reference :param str lfn: LFN """ return self.__getFileParameter( lfn, 'Retries' ) def getTransferTime( self, lfn ): """ get duration of transfer for file :lfn: :param self: self reference :param str lfn: LFN """ return self.__getFileParameter( lfn, 'Duration' ) def getFailed( self ): """ get list of wrongly transferred LFNs :param self: self reference """ return S_OK( [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( 'Status', '' ) in self.failedStates ] ) def getStaging( self ): """ get files set for prestaging """ return S_OK( [lfn for lfn in self.fileDict if self.fileDict[lfn].get( 'Status', '' ) == 'Staging'] ) def getDone( self ): """ get list of succesfully transferred LFNs :param self: self reference """ return S_OK( [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( 'Status', '' ) in self.successfulStates ] ) def __setFileParameter( self, lfn, paramName, paramValue ): """ set :paramName: to :paramValue: for :lfn: file :param self: self reference :param str lfn: LFN :param str paramName: parameter name :param mixed paramValue: a new parameter value """ self.setLFN( lfn ) self.fileDict[lfn][paramName] = paramValue return S_OK() def __getFileParameter( self, lfn, paramName ): """ get value of :paramName: for file :lfn: :param self: self reference :param str lfn: LFN :param str paramName: parameter name """ if lfn not in self.fileDict: return S_ERROR( "Supplied file not set" ) if paramName not in self.fileDict[lfn]: return S_ERROR( "%s not set for file" % paramName ) return S_OK( self.fileDict[lfn][paramName] ) #################################################################### # # Methods for submission # def submit( self, monitor = False, printOutput = True ): """ submit FTS job :param self: self reference :param bool monitor: flag to monitor progress of FTS job :param bool printOutput: flag to print output of execution to stdout """ res = self.__prepareForSubmission() if not res['OK']: return res res = self.__submitFTSTransfer() if not res['OK']: return res resDict = { 'ftsGUID' : self.ftsGUID, 'ftsServer' : self.ftsServer, 'submittedFiles' : self.submittedFiles } if monitor or printOutput: gLogger.always( "Submitted %s@%s" % ( self.ftsGUID, self.ftsServer ) ) if monitor: self.monitor( untilTerminal = True, printOutput = printOutput, full = False ) return S_OK( resDict ) def __prepareForSubmission( self ): """ check validity of job before submission :param self: self reference """ if not self.fileDict: return S_ERROR( "No files set" ) if not self.sourceValid: return S_ERROR( "SourceSE not valid" ) if not self.targetValid: return S_ERROR( "TargetSE not valid" ) if not self.ftsServer: res = self.__resolveFTSServer() if not res['OK']: return S_ERROR( "FTSServer not valid" ) self.resolveSource() self.resolveTarget() res = self.__filesToSubmit() if not res['OK']: return S_ERROR( "No files to submit" ) return S_OK() def __getCatalogObject( self ): """ CatalogInterface instance facade :param self: self reference """ try: if not self.oCatalog: self.oCatalog = FileCatalog() return S_OK() except: return S_ERROR() def __updateReplicaCache( self, lfns = None, overwrite = False ): """ update replica cache for list of :lfns: :param self: self reference :param mixed lfns: list of LFNs :param bool overwrite: flag to trigger cache clearing and updating """ if not lfns: lfns = self.fileDict.keys() toUpdate = [ lfn for lfn in lfns if ( lfn not in self.catalogReplicas ) or overwrite ] if not toUpdate: return S_OK() res = self.__getCatalogObject() if not res['OK']: return res res = self.oCatalog.getReplicas( toUpdate ) if not res['OK']: return S_ERROR( "Failed to update replica cache: %s" % res['Message'] ) for lfn, error in res['Value']['Failed'].items(): self.__setFileParameter( lfn, 'Reason', error ) self.__setFileParameter( lfn, 'Status', 'Failed' ) for lfn, replicas in res['Value']['Successful'].items(): self.catalogReplicas[lfn] = replicas return S_OK() def __updateMetadataCache( self, lfns = None ): """ update metadata cache for list of LFNs :param self: self reference :param list lnfs: list of LFNs """ if not lfns: lfns = self.fileDict.keys() toUpdate = [ lfn for lfn in lfns if lfn not in self.catalogMetadata ] if not toUpdate: return S_OK() res = self.__getCatalogObject() if not res['OK']: return res res = self.oCatalog.getFileMetadata( toUpdate ) if not res['OK']: return S_ERROR( "Failed to get source catalog metadata: %s" % res['Message'] ) for lfn, error in res['Value']['Failed'].items(): self.__setFileParameter( lfn, 'Reason', error ) self.__setFileParameter( lfn, 'Status', 'Failed' ) for lfn, metadata in res['Value']['Successful'].items(): self.catalogMetadata[lfn] = metadata return S_OK() def resolveSource( self ): """ resolve source SE eligible for submission :param self: self reference """ # Avoid resolving sources twice if self.sourceResolved: return S_OK() # Only resolve files that need a transfer toResolve = [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( "Status", "" ) != "Failed" ] if not toResolve: return S_OK() res = self.__updateMetadataCache( toResolve ) if not res['OK']: return res res = self.__updateReplicaCache( toResolve ) if not res['OK']: return res # Define the source URLs for lfn in toResolve: replicas = self.catalogReplicas.get( lfn, {} ) if self.sourceSE not in replicas: gLogger.warn( "resolveSource: skipping %s - not replicas at SourceSE %s" % ( lfn, self.sourceSE ) ) self.__setFileParameter( lfn, 'Reason', "No replica at SourceSE" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue # Fix first the PFN pfn = self.oSourceSE.getPfnForLfn( lfn ).get( 'Value', {} ).get( 'Successful', {} ).get( lfn, replicas[self.sourceSE] ) res = returnSingleResult( self.oSourceSE.getPfnForProtocol( pfn, protocol = 'SRM2', withPort = True ) ) if not res['OK']: gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) ) self.__setFileParameter( lfn, 'Reason', res['Message'] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue res = self.setSourceSURL( lfn, res['Value'] ) if not res['OK']: gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) ) self.__setFileParameter( lfn, 'Reason', res['Message'] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue toResolve = {} for lfn in self.fileDict: if "Source" in self.fileDict[lfn]: toResolve[self.fileDict[lfn]['Source']] = lfn if not toResolve: return S_ERROR( "No eligible Source files" ) # Get metadata of the sources, to check for existance, availability and caching res = self.oSourceSE.getFileMetadata( toResolve.keys() ) if not res['OK']: return S_ERROR( "Failed to check source file metadata" ) for pfn, error in res['Value']['Failed'].items(): lfn = toResolve[pfn] if re.search( 'File does not exist', error ): gLogger.warn( "resolveSource: skipping %s - source file does not exists" % lfn ) self.__setFileParameter( lfn, 'Reason', "Source file does not exist" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) else: gLogger.warn( "resolveSource: skipping %s - failed to get source metadata" % lfn ) self.__setFileParameter( lfn, 'Reason', "Failed to get Source metadata" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) toStage = [] nbStagedFiles = 0 for pfn, metadata in res['Value']['Successful'].items(): lfn = toResolve[pfn] lfnStatus = self.fileDict.get( lfn, {} ).get( 'Status' ) if metadata['Unavailable']: gLogger.warn( "resolveSource: skipping %s - source file unavailable" % lfn ) self.__setFileParameter( lfn, 'Reason', "Source file Unavailable" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif metadata['Lost']: gLogger.warn( "resolveSource: skipping %s - source file lost" % lfn ) self.__setFileParameter( lfn, 'Reason', "Source file Lost" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif not metadata['Cached']: if lfnStatus != 'Staging': toStage.append( pfn ) elif metadata['Size'] != self.catalogMetadata[lfn]['Size']: gLogger.warn( "resolveSource: skipping %s - source file size mismatch" % lfn ) self.__setFileParameter( lfn, 'Reason', "Source size mismatch" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \ not compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ): gLogger.warn( "resolveSource: skipping %s - source file checksum mismatch" % lfn ) self.__setFileParameter( lfn, 'Reason', "Source checksum mismatch" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif lfnStatus == 'Staging': # file that was staging is now cached self.__setFileParameter( lfn, 'Status', 'Waiting' ) nbStagedFiles += 1 # Some files were being staged if nbStagedFiles: self.log.info( 'resolveSource: %d files have been staged' % nbStagedFiles ) # Launching staging of files not in cache if toStage: gLogger.warn( "resolveSource: %s source files not cached, prestaging..." % len( toStage ) ) stage = self.oSourceSE.prestageFile( toStage ) if not stage["OK"]: gLogger.error( "resolveSource: error is prestaging - %s" % stage["Message"] ) for pfn in toStage: lfn = toResolve[pfn] self.__setFileParameter( lfn, 'Reason', stage["Message"] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) else: for pfn in toStage: lfn = toResolve[pfn] if pfn in stage['Value']['Successful']: self.__setFileParameter( lfn, 'Status', 'Staging' ) elif pfn in stage['Value']['Failed']: self.__setFileParameter( lfn, 'Reason', stage['Value']['Failed'][pfn] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) self.sourceResolved = True return S_OK() def resolveTarget( self ): """ find target SE eligible for submission :param self: self reference """ toResolve = [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( 'Status' ) not in self.noSubmitStatus ] if not toResolve: return S_OK() res = self.__updateReplicaCache( toResolve ) if not res['OK']: return res for lfn in toResolve: res = self.oTargetSE.getPfnForLfn( lfn ) if not res['OK'] or lfn not in res['Value']['Successful']: gLogger.warn( "resolveTarget: skipping %s - failed to create target pfn" % lfn ) self.__setFileParameter( lfn, 'Reason', "Failed to create Target" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue pfn = res['Value']['Successful'][lfn] res = self.oTargetSE.getPfnForProtocol( pfn, protocol = 'SRM2', withPort = True ) if not res['OK'] or pfn not in res['Value']['Successful']: reason = res.get( 'Message', res.get( 'Value', {} ).get( 'Failed', {} ).get( pfn ) ) gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, reason ) ) self.__setFileParameter( lfn, 'Reason', reason ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue pfn = res['Value']['Successful'][pfn] res = self.setTargetSURL( lfn, pfn ) if not res['OK']: gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) ) self.__setFileParameter( lfn, 'Reason', res['Message'] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue toResolve = {} for lfn in self.fileDict: if "Target" in self.fileDict[lfn]: toResolve[self.fileDict[lfn]['Target']] = lfn if not toResolve: return S_ERROR( "No eligible Target files" ) res = self.oTargetSE.exists( toResolve.keys() ) if not res['OK']: return S_ERROR( "Failed to check target existence" ) for pfn, error in res['Value']['Failed'].items(): lfn = toResolve[pfn] self.__setFileParameter( lfn, 'Reason', error ) self.__setFileParameter( lfn, 'Status', 'Failed' ) toRemove = [] for pfn, exists in res['Value']['Successful'].items(): if exists: lfn = toResolve[pfn] res = self.getSourceSURL( lfn ) if not res['OK']: gLogger.warn( "resolveTarget: skipping %s - target exists" % lfn ) self.__setFileParameter( lfn, 'Reason', "Target exists" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif res['Value'] == pfn: gLogger.warn( "resolveTarget: skipping %s - source and target pfns are the same" % lfn ) self.__setFileParameter( lfn, 'Reason', "Source and Target the same" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) else: toRemove.append( pfn ) if toRemove: self.oTargetSE.removeFile( toRemove ) return S_OK() def __filesToSubmit( self ): """ check if there is at least one file to submit :return: S_OK if at least one file is present, S_ERROR otherwise """ for lfn in self.fileDict: lfnStatus = self.fileDict[lfn].get( 'Status' ) source = self.fileDict[lfn].get( 'Source' ) target = self.fileDict[lfn].get( 'Target' ) if lfnStatus not in self.noSubmitStatus and source and target: return S_OK() return S_ERROR() def __createFTSFiles( self ): """ create LFNs file for glite-transfer-submit command This file consists one line for each fiel to be transferred: sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM] :param self: self reference """ self.__updateMetadataCache() for lfn in self.fileDict: lfnStatus = self.fileDict[lfn].get( 'Status' ) if lfnStatus not in self.noSubmitStatus: cksmStr = "" # # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself if self.__cksmTest and self.__cksmType: checkSum = self.catalogMetadata.get( lfn, {} ).get( 'Checksum' ) if checkSum: cksmStr = " %s:%s" % ( self.__cksmType, intAdlerToHex( hexAdlerToInt( checkSum ) ) ) ftsFile = FTSFile() ftsFile.LFN = lfn ftsFile.SourceSURL = self.fileDict[lfn].get( 'Source' ) ftsFile.TargetSURL = self.fileDict[lfn].get( 'Target' ) ftsFile.SourceSE = self.sourceSE ftsFile.TargetSE = self.targetSE ftsFile.Status = self.fileDict[lfn].get( 'Status' ) ftsFile.Checksum = cksmStr ftsFile.Size = self.catalogMetadata.get( lfn, {} ).get( 'Size' ) self.ftsFiles.append( ftsFile ) self.submittedFiles += 1 return S_OK() def __createFTSJob( self, guid = None ): self.__createFTSFiles() ftsJob = FTSJob() ftsJob.RequestID = 0 ftsJob.OperationID = 0 ftsJob.SourceSE = self.sourceSE ftsJob.TargetSE = self.targetSE ftsJob.SourceToken = self.sourceToken ftsJob.TargetToken = self.targetToken ftsJob.FTSServer = self.ftsServer if guid: ftsJob.FTSGUID = guid for ftsFile in self.ftsFiles: ftsFile.Attempt += 1 ftsFile.Error = "" ftsJob.addFile( ftsFile ) self.ftsJob = ftsJob def __submitFTSTransfer( self ): """ create and execute glite-transfer-submit CLI command :param self: self reference """ log = gLogger.getSubLogger( 'Submit' ) self.__createFTSJob() submit = self.ftsJob.submitFTS2( command = self.submitCommand ) if not submit["OK"]: log.error( "unable to submit FTSJob: %s" % submit["Message"] ) return submit log.info( "FTSJob '%s'@'%s' has been submitted" % ( self.ftsJob.FTSGUID, self.ftsJob.FTSServer ) ) # # update statuses for job files for ftsFile in self.ftsJob: ftsFile.FTSGUID = self.ftsJob.FTSGUID ftsFile.Status = "Submitted" ftsFile.Attempt += 1 log.info( "FTSJob '%s'@'%s' has been submitted" % ( self.ftsJob.FTSGUID, self.ftsJob.FTSServer ) ) self.ftsGUID = self.ftsJob.FTSGUID return S_OK() def __resolveFTSServer( self ): """ resolve FTS server to use, it should be the closest one from target SE :param self: self reference """ from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTSServersForSites if not self.targetSE: return S_ERROR( "Target SE not set" ) res = getSitesForSE( self.targetSE ) if not res['OK'] or not res['Value']: return S_ERROR( "Could not determine target site" ) targetSites = res['Value'] targetSite = '' for targetSite in targetSites: targetFTS = getFTSServersForSites( [targetSite] ) if targetFTS['OK']: ftsTarget = targetFTS['Value'][targetSite] if ftsTarget: self.ftsServer = ftsTarget return S_OK( self.ftsServer ) else: return targetFTS return S_ERROR( 'No FTS server found for %s' % targetSite ) #################################################################### # # Methods for monitoring # def summary( self, untilTerminal = False, printOutput = False ): """ summary of FTS job :param self: self reference :param bool untilTerminal: flag to monitor FTS job to its final state :param bool printOutput: flag to print out monitoring information to the stdout """ res = self.__isSummaryValid() if not res['OK']: return res while not self.isTerminal: res = self.__parseOutput( full = True ) if not res['OK']: return res if untilTerminal: self.__print() self.isRequestTerminal() if res['Value'] or ( not untilTerminal ): break time.sleep( 1 ) if untilTerminal: print "" if printOutput and ( not untilTerminal ): return self.dumpSummary( printOutput = printOutput ) return S_OK() def monitor( self, untilTerminal = False, printOutput = False, full = True ): """ monitor FTS job :param self: self reference :param bool untilTerminal: flag to monitor FTS job to its final state :param bool printOutput: flag to print out monitoring information to the stdout """ if not self.ftsJob: self.resolveSource() self.__createFTSJob( self.ftsGUID ) res = self.__isSummaryValid() if not res['OK']: return res if untilTerminal: res = self.summary( untilTerminal = untilTerminal, printOutput = printOutput ) if not res['OK']: return res res = self.__parseOutput( full = full ) if not res['OK']: return res if untilTerminal: self.finalize() if printOutput: self.dump() return res def dumpSummary( self, printOutput = False ): """ get FTS job summary as str :param self: self reference :param bool printOutput: print summary to stdout """ outStr = '' for status in sorted( self.statusSummary ): if self.statusSummary[status]: outStr = '%s\t%-10s : %-10s\n' % ( outStr, status, str( self.statusSummary[status] ) ) outStr = outStr.rstrip( '\n' ) if printOutput: print outStr return S_OK( outStr ) def __print( self ): """ print progress bar of FTS job completeness to stdout :param self: self reference """ width = 100 bits = int( ( width * self.percentageComplete ) / 100 ) outStr = "|%s>%s| %.1f%s %s %s" % ( "="*bits, " "*( width - bits ), self.percentageComplete, "%", self.requestStatus, " "*10 ) sys.stdout.write( "%s\r" % ( outStr ) ) sys.stdout.flush() def dump( self ): """ print FTS job parameters and files to stdout :param self: self reference """ print "%-10s : %-10s" % ( "Status", self.requestStatus ) print "%-10s : %-10s" % ( "Source", self.sourceSE ) print "%-10s : %-10s" % ( "Target", self.targetSE ) print "%-10s : %-128s" % ( "Server", self.ftsServer ) print "%-10s : %-128s" % ( "GUID", self.ftsGUID ) for lfn in sorted( self.fileDict ): print "\n %-15s : %-128s" % ( 'LFN', lfn ) for key in ['Source', 'Target', 'Status', 'Reason', 'Duration']: print " %-15s : %-128s" % ( key, str( self.fileDict[lfn].get( key ) ) ) return S_OK() def __isSummaryValid( self ): """ check validity of FTS job summary report :param self: self reference """ if not self.ftsServer: return S_ERROR( "FTSServer not set" ) if not self.ftsGUID: return S_ERROR( "FTSGUID not set" ) return S_OK() def __parseOutput( self, full = False ): """ execute glite-transfer-status command and parse its output :param self: self reference :param bool full: glite-transfer-status verbosity level, when set, collect information of files as well """ monitor = self.ftsJob.monitorFTS2( command = self.monitorCommand, full = full ) if not monitor['OK']: return monitor self.percentageComplete = self.ftsJob.Completeness self.requestStatus = self.ftsJob.Status self.submitTime = self.ftsJob.SubmitTime statusSummary = monitor['Value'] if statusSummary: for state in statusSummary: self.statusSummary[state] = statusSummary[state] self.transferTime = 0 for ftsFile in self.ftsJob: lfn = ftsFile.LFN self.__setFileParameter( lfn, 'Status', ftsFile.Status ) self.__setFileParameter( lfn, 'Reason', ftsFile.Error ) self.__setFileParameter( lfn, 'Duration', ftsFile._duration ) targetURL = self.__getFileParameter( lfn, 'Target' ) if not targetURL['OK']: self.__setFileParameter( lfn, 'Target', ftsFile.TargetSURL ) self.transferTime += int( ftsFile._duration ) return S_OK() #################################################################### # # Methods for finalization # def finalize( self ): """ finalize FTS job :param self: self reference """ self.__updateMetadataCache() transEndTime = dateTime() regStartTime = time.time() res = self.getTransferStatistics() transDict = res['Value'] res = self.__registerSuccessful( transDict['transLFNs'] ) regSuc, regTotal = res['Value'] regTime = time.time() - regStartTime if self.sourceSE and self.targetSE: self.__sendAccounting( regSuc, regTotal, regTime, transEndTime, transDict ) return S_OK() def getTransferStatistics( self ): """ collect information of Transfers that can be used by Accounting :param self: self reference """ transDict = { 'transTotal': len( self.fileDict ), 'transLFNs': [], 'transOK': 0, 'transSize': 0 } for lfn in self.fileDict: if self.fileDict[lfn].get( 'Status' ) in self.successfulStates: if self.fileDict[lfn].get( 'Duration', 0 ): transDict['transLFNs'].append( lfn ) transDict['transOK'] += 1 if lfn in self.catalogMetadata: transDict['transSize'] += self.catalogMetadata[lfn].get( 'Size', 0 ) return S_OK( transDict ) def getFailedRegistrations( self ): """ get failed registrations dict :param self: self reference """ return S_OK( self.failedRegistrations ) def __registerSuccessful( self, transLFNs ): """ register successfully transferred files to the catalogs, fill failedRegistrations dict for files that failed to register :param self: self reference :param list transLFNs: LFNs in FTS job """ self.failedRegistrations = {} toRegister = {} for lfn in transLFNs: res = returnSingleResult( self.oTargetSE.getPfnForProtocol( self.fileDict[lfn].get( 'Target' ), protocol = 'SRM2', withPort = False ) ) if not res['OK']: self.__setFileParameter( lfn, 'Reason', res['Message'] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) else: toRegister[lfn] = { 'PFN' : res['Value'], 'SE' : self.targetSE } if not toRegister: return S_OK( ( 0, 0 ) ) res = self.__getCatalogObject() if not res['OK']: for lfn in toRegister: self.failedRegistrations = toRegister self.log.error( 'Failed to get Catalog Object', res['Message'] ) return S_OK( ( 0, len( toRegister ) ) ) res = self.oCatalog.addReplica( toRegister ) if not res['OK']: self.failedRegistrations = toRegister self.log.error( 'Failed to get Catalog Object', res['Message'] ) return S_OK( ( 0, len( toRegister ) ) ) for lfn, error in res['Value']['Failed'].items(): self.failedRegistrations[lfn] = toRegister[lfn] self.log.error( 'Registration of Replica failed', '%s : %s' % ( lfn, str( error ) ) ) return S_OK( ( len( res['Value']['Successful'] ), len( toRegister ) ) ) def __sendAccounting( self, regSuc, regTotal, regTime, transEndTime, transDict ): """ send accounting record :param self: self reference :param regSuc: number of files successfully registered :param regTotal: number of files attepted to register :param regTime: time stamp at the end of registration :param transEndTime: time stamp at the end of FTS job :param dict transDict: dict holding couters for files being transerred, their sizes and successfull transfers """ oAccounting = DataOperation() oAccounting.setEndTime( transEndTime ) oAccounting.setStartTime( self.submitTime ) accountingDict = {} accountingDict['OperationType'] = 'replicateAndRegister' result = getProxyInfo() if not result['OK']: userName = '******' else: userName = result['Value'].get( 'username', 'unknown' ) accountingDict['User'] = userName accountingDict['Protocol'] = 'FTS' if 'fts3' not in self.ftsServer else 'FTS3' accountingDict['RegistrationTime'] = regTime accountingDict['RegistrationOK'] = regSuc accountingDict['RegistrationTotal'] = regTotal accountingDict['TransferOK'] = transDict['transOK'] accountingDict['TransferTotal'] = transDict['transTotal'] accountingDict['TransferSize'] = transDict['transSize'] accountingDict['FinalStatus'] = self.requestStatus accountingDict['Source'] = self.sourceSE accountingDict['Destination'] = self.targetSE accountingDict['TransferTime'] = self.transferTime oAccounting.setValuesFromDict( accountingDict ) self.log.verbose( "Attempting to commit accounting message..." ) oAccounting.commit() self.log.verbose( "...committed." ) return S_OK()
class FTSRequest( object ): """ .. class:: FTSRequest Helper class for FTS job submission and monitoring. """ # # default checksum type __defaultCksmType = "ADLER32" # # flag to disablr/enable checksum test, default: disabled __cksmTest = False def __init__( self ): """c'tor :param self: self reference """ self.log = gLogger.getSubLogger( self.__class__.__name__, True ) # # final states tuple self.finalStates = ( 'Canceled', 'Failed', 'Hold', 'Finished', 'FinishedDirty' ) # # failed states tuple self.failedStates = ( 'Canceled', 'Failed', 'Hold', 'FinishedDirty' ) # # successful states tuple self.successfulStates = ( 'Finished', 'Done' ) # # all file states tuple self.fileStates = ( 'Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed', 'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting' ) self.newlyCompletedFiles = [] self.newlyFailedFiles = [] self.statusSummary = {} # # request status self.requestStatus = 'Unknown' # # dict for FTS job files self.fileDict = {} # # dict for replicas information self.catalogReplicas = {} # # dict for metadata information self.catalogMetadata = {} # # dict for files that failed to register self.failedRegistrations = {} # # placehoder for CatalogInterface reference self.oCatalog = None # # submit timestamp self.submitTime = '' # # placeholder FTS job GUID self.ftsGUID = '' # # placeholder for FTS server URL self.ftsServer = '' # # not used self.priority = 3 # # flag marking FTS job completness self.isTerminal = False # # completness percentage self.percentageComplete = 0.0 # # source SE name self.sourceSE = '' # # flag marking source SE validity self.sourceValid = False # # source space token self.sourceToken = '' # # target SE name self.targetSE = '' # # flag marking target SE validity self.targetValid = False # # target space token self.targetToken = '' # # whatever self.dumpStr = '' # # placeholder for surl file self.surlFile = None # # placeholder for target StorageElement self.oTargetSE = None # # placeholder for source StorageElement self.oSourceSE = None # # checksum type, set it to default self.__cksmType = self.__defaultCksmType # # disable checksum test by default self.__cksmTest = False # # replica manager handler self.replicaManager = ReplicaManager() # # statuses that prevent submitting to FTS self.noSubmitStatus = ( 'Failed', 'Done', 'Staging' ) # # were sources resolved? self.sourceResolved = False # # Number of file transfers actually submitted self.submittedFiles = 0 #################################################################### # # Methods for setting/getting/checking the SEs # def setSourceSE( self, se ): """ set SE for source :param self: self reference :param str se: source SE name """ if se == self.targetSE: return S_ERROR( "SourceSE is TargetSE" ) self.sourceSE = se self.oSourceSE = StorageElement( self.sourceSE ) return self.__checkSourceSE() def getSourceSE( self ): """ source SE getter :param self: self reference """ if not self.sourceSE: return S_ERROR( "Source SE not defined" ) return S_OK( self.sourceSE ) def setSourceToken( self, token ): """ set source space token :param self: self reference :param str token: source space token """ self.sourceToken = token return S_OK() def getSourceToken( self ): """ source space token getter :param self: self reference """ if not self.sourceToken: return S_ERROR( "Source token not defined" ) return S_OK( self.sourceToken ) def __checkSourceSE( self ): """ check source SE availability :param self: self reference """ if not self.sourceSE: return S_ERROR( "SourceSE not set" ) res = self.oSourceSE.isValid( 'Read' ) if not res['OK']: return S_ERROR( "SourceSE not available for reading" ) res = self.__getSESpaceToken( self.oSourceSE ) if not res['OK']: self.log.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message'] ) return S_ERROR( "SourceSE does not support FTS transfers" ) if self.__cksmTest: res = self.oSourceSE.getChecksumType() if not res["OK"]: self.log.error( "Unable to get checksum type for SourceSE %s: %s" % ( self.sourceSE, res["Message"] ) ) cksmType = res["Value"] if cksmType in ( "NONE", "NULL" ): self.log.warn( "Checksum type set to %s at SourceSE %s, disabling checksum test" % ( cksmType, self.sourceSE ) ) self.__cksmTest = False elif cksmType != self.__cksmType: self.log.warn( "Checksum type mismatch, disabling checksum test" ) self.__cksmTest = False self.sourceToken = res['Value'] self.sourceValid = True return S_OK() def setTargetSE( self, se ): """ set target SE :param self: self reference :param str se: target SE name """ if se == self.sourceSE: return S_ERROR( "TargetSE is SourceSE" ) self.targetSE = se self.oTargetSE = StorageElement( self.targetSE ) return self.__checkTargetSE() def getTargetSE( self ): """ target SE getter :param self: self reference """ if not self.targetSE: return S_ERROR( "Target SE not defined" ) return S_OK( self.targetSE ) def setTargetToken( self, token ): """ target space token setter :param self: self reference :param str token: target space token """ self.targetToken = token return S_OK() def getTargetToken( self ): """ target space token getter :param self: self reference """ if not self.targetToken: return S_ERROR( "Target token not defined" ) return S_OK( self.targetToken ) def __checkTargetSE( self ): """ check target SE availability :param self: self reference """ if not self.targetSE: return S_ERROR( "TargetSE not set" ) res = self.oTargetSE.isValid( 'Write' ) if not res['OK']: return S_ERROR( "TargetSE not available for writing" ) res = self.__getSESpaceToken( self.oTargetSE ) if not res['OK']: self.log.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message'] ) return S_ERROR( "TargetSE does not support FTS transfers" ) # # check checksum types if self.__cksmTest: res = self.oTargetSE.getChecksumType() if not res["OK"]: self.log.error( "Unable to get checksum type for TargetSE %s: %s" % ( self.targetSE, res["Message"] ) ) cksmType = res["Value"] if cksmType in ( "NONE", "NULL" ): self.log.warn( "Checksum type set to %s at TargetSE %s, disabling checksum test" % ( cksmType, self.targetSE ) ) self.__cksmTest = False elif cksmType != self.__cksmType: self.log.warn( "Checksum type mismatch, disabling checksum test" ) self.__cksmTest = False self.targetToken = res['Value'] self.targetValid = True return S_OK() @staticmethod def __getSESpaceToken( oSE ): """ get space token from StorageElement instance :param self: self reference :param StorageElement oSE: StorageElement instance """ res = oSE.getStorageParameters( "SRM2" ) if not res['OK']: return res return S_OK( res['Value'].get( 'SpaceToken' ) ) #################################################################### # # Methods for setting/getting FTS request parameters # def setFTSGUID( self, guid ): """ FTS job GUID setter :param self: self reference :param str guid: string containg GUID """ if not checkGuid( guid ): return S_ERROR( "Incorrect GUID format" ) self.ftsGUID = guid return S_OK() def getFTSGUID( self ): """ FTS job GUID getter :param self: self refenece """ if not self.ftsGUID: return S_ERROR( "FTSGUID not set" ) return S_OK( self.ftsGUID ) def setFTSServer( self, server ): """ FTS server setter :param self: self reference :param str server: FTS server URL """ self.ftsServer = server return S_OK() def getFTSServer( self ): """ FTS server getter :param self: self reference """ if not self.ftsServer: return S_ERROR( "FTSServer not set" ) return S_OK( self.ftsServer ) def setPriority( self, priority ): """ set priority for FTS job :param self: self reference :param int priority: a new priority """ if not type( priority ) in ( IntType, LongType ): return S_ERROR( "Priority must be integer" ) if priority < 0: priority = 0 elif priority > 5: priority = 5 self.priority = priority return S_OK( self.priority ) def getPriority( self ): """ FTS job priority getter :param self: self reference """ return S_OK( self.priority ) def getPercentageComplete( self ): """ get completness percentage :param self: self reference """ completedFiles = 0 totalFiles = 0 for state in self.statusSummary: if state in self.successfulStates: completedFiles += self.statusSummary[state] totalFiles += self.statusSummary[state] self.percentageComplete = ( float( completedFiles ) * 100.0 ) / float( totalFiles ) return S_OK( self.percentageComplete ) def isRequestTerminal( self ): """ check if FTS job has terminated :param self: self reference """ if self.requestStatus in self.finalStates: self.isTerminal = True return S_OK( self.isTerminal ) def getStatus( self ): """ get FTS job status :param self: self reference """ return S_OK( self.requestStatus ) def setCksmType( self, cksm = None ): """ set checksum type to use :param self: self reference :param mixed cksm: checksum type, should be one of 'Adler32', 'md5', 'sha1', None """ if str( cksm ).upper() not in ( "ADLER32", "MD5", "SHA1", "NONE" ): return S_ERROR( "Not supported checksum type: %s" % str( cksm ) ) if not cksm: self.__cksmType = None return S_OK( False ) self.__cksmType = str( cksm ).upper() return S_OK( True ) def getCksmType( self ): """ get checksum type :param self: self reference """ return S_OK( self.__cksmType ) def setCksmTest( self, cksmTest = False ): """ set cksm test :param self: self reference :param bool cksmTest: flag to enable/disable checksum test """ self.__cksmTest = bool( cksmTest ) return S_OK( self.__cksmTest ) def getCksmTest( self ): """ get cksm test flag :param self: self reference """ return S_OK( self.__cksmTest ) #################################################################### # # Methods for setting/getting/checking files and their metadata # def setLFN( self, lfn ): """ add LFN :lfn: to :fileDict: :param self: self reference :param str lfn: LFN to add to """ self.fileDict.setdefault( lfn, {'Status':'Waiting'} ) return S_OK() def setStatus( self, lfn, status ): """ set status of a file """ return( self.__setFileParameter( lfn, 'Status', status ) ) def setSourceSURL( self, lfn, surl ): """ source SURL setter :param self: self reference :param str lfn: LFN :param str surl: source SURL """ target = self.fileDict[lfn].get( 'Target' ) if target == surl: return S_ERROR( "Source and target the same" ) return( self.__setFileParameter( lfn, 'Source', surl ) ) def getSourceSURL( self, lfn ): """ get source SURL for LFN :lfn: :param self: self reference :param str lfn: LFN """ return self.__getFileParameter( lfn, 'Source' ) def setTargetSURL( self, lfn, surl ): """ set target SURL for LFN :lfn: :param self: self reference :param str lfn: LFN :param str surl: target SURL """ source = self.fileDict[lfn].get( 'Source' ) if source == surl: return S_ERROR( "Source and target the same" ) return( self.__setFileParameter( lfn, 'Target', surl ) ) def getTargetSURL( self, lfn ): """ target SURL getter :param self: self reference :param str lfn: LFN """ return self.__getFileParameter( lfn, 'Target' ) def getFailReason( self, lfn ): """ get fail reason for file :lfn: :param self: self reference :param str lfn: LFN """ return self.__getFileParameter( lfn, 'Reason' ) def getRetries( self, lfn ): """ get number of attepmts made to transfer file :lfn: :param self: self reference :param str lfn: LFN """ return self.__getFileParameter( lfn, 'Retries' ) def getTransferTime( self, lfn ): """ get duration of transfer for file :lfn: :param self: self reference :param str lfn: LFN """ return self.__getFileParameter( lfn, 'Duration' ) def getFailed( self ): """ get list of wrongly transferred LFNs :param self: self reference """ return S_OK( [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( 'Status', '' ) in self.failedStates ] ) def getStaging( self ): """ get files set for prestaging """ return S_OK( [lfn for lfn in self.fileDict if self.fileDict[lfn].get( 'Status', '' ) == 'Staging'] ) def getDone( self ): """ get list of succesfully transferred LFNs :param self: self reference """ return S_OK( [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( 'Status', '' ) in self.successfulStates ] ) def __setFileParameter( self, lfn, paramName, paramValue ): """ set :paramName: to :paramValue: for :lfn: file :param self: self reference :param str lfn: LFN :param str paramName: parameter name :param mixed paramValue: a new parameter value """ self.setLFN( lfn ) self.fileDict[lfn][paramName] = paramValue return S_OK() def __getFileParameter( self, lfn, paramName ): """ get value of :paramName: for file :lfn: :param self: self reference :param str lfn: LFN :param str paramName: parameter name """ if lfn not in self.fileDict: return S_ERROR( "Supplied file not set" ) if paramName not in self.fileDict[lfn]: return S_ERROR( "%s not set for file" % paramName ) return S_OK( self.fileDict[lfn][paramName] ) #################################################################### # # Methods for submission # def submit( self, monitor = False, printOutput = True ): """ submit FTS job :param self: self reference :param bool monitor: flag to monitor progress of FTS job :param bool printOutput: flag to print output of execution to stdout """ res = self.__isSubmissionValid() if not res['OK']: return res res = self.__createSURLPairFile() if not res['OK']: return res res = self.__submitFTSTransfer() if not res['OK']: return res resDict = { 'ftsGUID' : self.ftsGUID, 'ftsServer' : self.ftsServer, 'submittedFiles' : self.submittedFiles } # print "Submitted %s @ %s" % ( self.ftsGUID, self.ftsServer ) if monitor: self.monitor( untilTerminal = True, printOutput = printOutput ) return S_OK( resDict ) def __isSubmissionValid( self ): """ check validity of job before submission :param self: self reference """ if not self.fileDict: return S_ERROR( "No files set" ) if not self.sourceValid: return S_ERROR( "SourceSE not valid" ) if not self.targetValid: return S_ERROR( "TargetSE not valid" ) if not self.ftsServer: res = self.__resolveFTSServer() if not res['OK']: return S_ERROR( "FTSServer not valid" ) self.resolveSource() self.resolveTarget() res = self.__filesToSubmit() if not res['OK']: return S_ERROR( "No files to submit" ) return S_OK() def __getCatalogObject( self ): """ CatalogInterface instance facade :param self: self reference """ try: if not self.oCatalog: self.oCatalog = CatalogInterface() return S_OK() except: return S_ERROR() def __updateReplicaCache( self, lfns = None, overwrite = False ): """ update replica cache for list of :lfns: :param self: self reference :param mixed lfns: list of LFNs :param bool overwrite: flag to trigger cache clearing and updating """ if not lfns: lfns = self.fileDict.keys() toUpdate = [ lfn for lfn in lfns if ( lfn not in self.catalogReplicas ) or overwrite ] if not toUpdate: return S_OK() res = self.__getCatalogObject() if not res['OK']: return res res = self.oCatalog.getCatalogReplicas( toUpdate ) if not res['OK']: return S_ERROR( "Failed to update replica cache: %s" % res['Message'] ) for lfn, error in res['Value']['Failed'].items(): self.__setFileParameter( lfn, 'Reason', error ) self.__setFileParameter( lfn, 'Status', 'Failed' ) for lfn, replicas in res['Value']['Successful'].items(): self.catalogReplicas[lfn] = replicas return S_OK() def __updateMetadataCache( self, lfns = None, overwrite = False ): """ update metadata cache for list of LFNs :param self: self reference :param list lnfs: list of LFNs :param bool overwrite: flag to trigger cache clearing and updating """ if not lfns: lfns = self.fileDict.keys() toUpdate = [ lfn for lfn in lfns if ( lfn not in self.catalogMetadata ) or overwrite ] if not toUpdate: return S_OK() res = self.__getCatalogObject() if not res['OK']: return res res = self.oCatalog.getCatalogFileMetadata( toUpdate ) if not res['OK']: return S_ERROR( "Failed to get source catalog metadata: %s" % res['Message'] ) for lfn, error in res['Value']['Failed'].items(): self.__setFileParameter( lfn, 'Reason', error ) self.__setFileParameter( lfn, 'Status', 'Failed' ) for lfn, metadata in res['Value']['Successful'].items(): self.catalogMetadata[lfn] = metadata return S_OK() def resolveSource( self ): """ resolve source SE eligible for submission :param self: self reference """ # Avoid resolving sources twice if self.sourceResolved: return S_OK() # Only resolve files that need a transfer toResolve = [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( "Status", "" ) != "Failed" ] if not toResolve: return S_OK() res = self.__updateMetadataCache( toResolve ) if not res['OK']: return res res = self.__updateReplicaCache( toResolve ) if not res['OK']: return res # Define the source URLs for lfn in toResolve: replicas = self.catalogReplicas.get( lfn, {} ) if self.sourceSE not in replicas: gLogger.warn( "resolveSource: skipping %s - not replicas at SourceSE %s" % ( lfn, self.sourceSE ) ) self.__setFileParameter( lfn, 'Reason', "No replica at SourceSE" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue res = self.oSourceSE.getPfnForProtocol( replicas[self.sourceSE], 'SRM2', withPort = True ) if not res['OK']: gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) ) self.__setFileParameter( lfn, 'Reason', res['Message'] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue res = self.setSourceSURL( lfn, res['Value'] ) if not res['OK']: gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) ) self.__setFileParameter( lfn, 'Reason', res['Message'] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue toResolve = {} for lfn in self.fileDict: if "Source" in self.fileDict[lfn]: toResolve[self.fileDict[lfn]['Source']] = lfn if not toResolve: return S_ERROR( "No eligible Source files" ) # Get metadata of the sources, to check for existance, availability and caching res = self.oSourceSE.getFileMetadata( toResolve.keys() ) if not res['OK']: return S_ERROR( "Failed to check source file metadata" ) for pfn, error in res['Value']['Failed'].items(): lfn = toResolve[pfn] if re.search( 'File does not exist', error ): gLogger.warn( "resolveSource: skipping %s - source file does not exists" % lfn ) self.__setFileParameter( lfn, 'Reason', "Source file does not exist" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) else: gLogger.warn( "resolveSource: skipping %s - failed to get source metadata" % lfn ) self.__setFileParameter( lfn, 'Reason', "Failed to get Source metadata" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) toStage = [] nbStagedFiles = 0 for pfn, metadata in res['Value']['Successful'].items(): lfn = toResolve[pfn] lfnStatus = self.fileDict.get( lfn, {} ).get( 'Status' ) if metadata['Unavailable']: gLogger.warn( "resolveSource: skipping %s - source file unavailable" % lfn ) self.__setFileParameter( lfn, 'Reason', "Source file Unavailable" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif metadata['Lost']: gLogger.warn( "resolveSource: skipping %s - source file lost" % lfn ) self.__setFileParameter( lfn, 'Reason', "Source file Lost" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif not metadata['Cached']: if lfnStatus != 'Staging': toStage.append( pfn ) elif metadata['Size'] != self.catalogMetadata[lfn]['Size']: gLogger.warn( "resolveSource: skipping %s - source file size mismatch" % lfn ) self.__setFileParameter( lfn, 'Reason', "Source size mismatch" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \ not ( compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ) ): gLogger.warn( "resolveSource: skipping %s - source file checksum mismatch" % lfn ) self.__setFileParameter( lfn, 'Reason', "Source checksum mismatch" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif lfnStatus == 'Staging': # file that was staging is now cached self.__setFileParameter( lfn, 'Status', 'Waiting' ) nbStagedFiles += 1 # Some files were being staged if nbStagedFiles: self.log.info( 'resolveSource: %d files have been staged' % nbStagedFiles ) # Launching staging of files not in cache if toStage: gLogger.warn( "resolveSource: %s source files not cached, prestaging..." % len( toStage ) ) stage = self.replicaManager.prestageStorageFile( toStage, self.sourceSE ) if not stage["OK"]: gLogger.error( "resolveSource: error is prestaging - %s" % stage["Message"] ) for pfn in toStage: lfn = toResolve[pfn] self.__setFileParameter( lfn, 'Reason', stage["Message"] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) else: for pfn in toStage: lfn = toResolve[pfn] if pfn in stage['Value']['Successful']: self.__setFileParameter( lfn, 'Status', 'Staging' ) elif pfn in stage['Value']['Failed']: self.__setFileParameter( lfn, 'Reason', stage['Value']['Failed'][pfn] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) self.sourceResolved = True return S_OK() def resolveTarget( self ): """ find target SE eligible for submission :param self: self reference """ toResolve = [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( 'Status' ) not in self.noSubmitStatus ] if not toResolve: return S_OK() res = self.__updateReplicaCache( toResolve ) if not res['OK']: return res for lfn in toResolve: res = self.oTargetSE.getPfnForLfn( lfn ) if not res['OK']: gLogger.warn( "resolveTarget: skipping %s - failed to create target pfn" % lfn ) self.__setFileParameter( lfn, 'Reason', "Failed to create Target" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue res = self.oTargetSE.getPfnForProtocol( res['Value'], 'SRM2', withPort = True ) if not res['OK']: gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) ) self.__setFileParameter( lfn, 'Reason', res['Message'] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue res = self.setTargetSURL( lfn, res['Value'] ) if not res['OK']: gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) ) self.__setFileParameter( lfn, 'Reason', res['Message'] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue toResolve = {} for lfn in self.fileDict: if "Target" in self.fileDict[lfn]: toResolve[self.fileDict[lfn]['Target']] = lfn if not toResolve: return S_ERROR( "No eligible Target files" ) res = self.oTargetSE.exists( toResolve.keys() ) if not res['OK']: return S_ERROR( "Failed to check target existence" ) for pfn, error in res['Value']['Failed'].items(): lfn = toResolve[pfn] self.__setFileParameter( lfn, 'Reason', error ) self.__setFileParameter( lfn, 'Status', 'Failed' ) toRemove = [] for pfn, exists in res['Value']['Successful'].items(): if exists: lfn = toResolve[pfn] res = self.getSourceSURL( lfn ) if not res['OK']: gLogger.warn( "resolveTarget: skipping %s - target exists" % lfn ) self.__setFileParameter( lfn, 'Reason', "Target exists" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif res['Value'] == pfn: gLogger.warn( "resolveTarget: skipping %s - source and target pfns are the same" % lfn ) self.__setFileParameter( lfn, 'Reason', "Source and Target the same" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) else: toRemove.append( pfn ) if toRemove: self.oTargetSE.removeFile( toRemove ) return S_OK() def __filesToSubmit( self ): """ check if there is at least one file to submit :return: S_OK if at least one file is present, S_ERROR otherwise """ for lfn in self.fileDict: lfnStatus = self.fileDict[lfn].get( 'Status' ) source = self.fileDict[lfn].get( 'Source' ) target = self.fileDict[lfn].get( 'Target' ) if lfnStatus not in self.noSubmitStatus and source and target: return S_OK() return S_ERROR() def __createSURLPairFile( self ): """ create LFNs file for glite-transfer-submit command This file consists one line for each fiel to be transferred: sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM] :param self: self reference """ fd, fileName = tempfile.mkstemp() surlFile = os.fdopen( fd, 'w' ) for lfn in self.fileDict: lfnStatus = self.fileDict[lfn].get( 'Status' ) source = self.fileDict[lfn].get( 'Source' ) target = self.fileDict[lfn].get( 'Target' ) if lfnStatus not in self.noSubmitStatus and source and target: cksmStr = "" # # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself if self.__cksmTest and self.__cksmType: checkSum = self.catalogMetadata.get( lfn, {} ).get( 'Checksum' ) if checkSum: cksmStr = " %s:%s" % ( self.__cksmType, intAdlerToHex( hexAdlerToInt( checkSum ) ) ) surlFile.write( "%s %s%s\n" % ( source, target, cksmStr ) ) self.submittedFiles += 1 surlFile.close() self.surlFile = fileName return S_OK() def __submitFTSTransfer( self ): """ create and execute glite-transfer-submit CLI command :param self: self reference """ comm = [ 'glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile, '-o' ] if self.targetToken: comm += [ '-t', self.targetToken ] if self.sourceToken: comm += [ '-S', self.sourceToken ] if self.__cksmTest: comm.append( "--compare-checksums" ) gLogger.verbose( 'Executing %s' % ' '.join( comm ) ) res = executeGridCommand( '', comm ) os.remove( self.surlFile ) if not res['OK']: return res returnCode, output, errStr = res['Value'] if not returnCode == 0: return S_ERROR( errStr ) guid = output.replace( '\n', '' ) if not checkGuid( guid ): return S_ERROR( 'Wrong GUID format returned' ) self.ftsGUID = guid # if self.priority != 3: # comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)] # executeGridCommand('',comm) return res def __getFTSServer( self, site ): try: configPath = '/Resources/FTSEndpoints/%s' % site endpointURL = gConfig.getValue( configPath ) if not endpointURL: errStr = "FTSRequest.__getFTSServer: Failed to find FTS endpoint, check CS entry for '%s'." % site return S_ERROR( errStr ) return S_OK( endpointURL ) except Exception, x: return S_ERROR( 'FTSRequest.__getFTSServer: Failed to obtain endpoint details from CS' )
class FTSRequest: def __init__( self ): self.gridEnv = '/afs/cern.ch/project/gd/LCG-share/3.2.8-0/etc/profile.d/grid-env' self.finalStates = ['Canceled', 'Failed', 'Hold', 'Finished', 'FinishedDirty'] self.failedStates = ['Canceled', 'Failed', 'Hold', 'FinishedDirty'] self.successfulStates = ['Finished', 'Done'] self.fileStates = ['Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed', 'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting'] self.newlyCompletedFiles = [] self.newlyFailedFiles = [] self.statusSummary = {} self.requestStatus = 'Unknown' self.fileDict = {} self.catalogReplicas = {} self.catalogMetadata = {} self.oCatalog = None self.submitTime = '' self.ftsGUID = '' self.ftsServer = '' self.priority = 3 self.isTerminal = False self.percentageComplete = 0.0 self.sourceSE = '' self.sourceValid = False self.sourceToken = '' self.targetSE = '' self.targetValid = False self.targetToken = '' self.dumpStr = '' #################################################################### # # Methods for setting/getting/checking the SEs # def setSourceSE( self, se ): if se == self.targetSE: return S_ERROR( "SourceSE is TargetSE" ) self.sourceSE = se self.oSourceSE = StorageElement( self.sourceSE ) return self.__checkSourceSE() def getSourceSE( self ): if not self.sourceSE: return S_ERROR( "Source SE not defined" ) return S_OK( self.sourceSE ) def setSourceToken( self, token ): self.sourceToken = token return S_OK() def getSourceToken( self ): if not self.sourceToken: return S_ERROR( "Source token not defined" ) return S_OK( self.sourceToken ) def __checkSourceSE( self ): if not self.sourceSE: return S_ERROR( "SourceSE not set" ) res = self.oSourceSE.isValid( 'Read' ) if not res['OK']: return S_ERROR( "SourceSE not available for reading" ) res = self.__getSESpaceToken( self.oSourceSE ) if not res['OK']: gLogger.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message'] ) return S_ERROR( "SourceSE does not support FTS transfers" ) self.sourceToken = res['Value'] self.sourceValid = True return S_OK() def setTargetSE( self, se ): if se == self.sourceSE: return S_ERROR( "TargetSE is SourceSE" ) self.targetSE = se self.oTargetSE = StorageElement( self.targetSE ) return self.__checkTargetSE() def getTargetSE( self ): if not self.targetSE: return S_ERROR( "Target SE not defined" ) return S_OK( self.targetSE ) def setTargetToken( self, token ): self.targetToken = token return S_OK() def getTargetToken( self ): if not self.targetToken: return S_ERROR( "Target token not defined" ) return S_OK( self.targetToken ) def __checkTargetSE( self ): if not self.targetSE: return S_ERROR( "TargetSE not set" ) res = self.oTargetSE.isValid( 'Write' ) if not res['OK']: return S_ERROR( "TargetSE not available for writing" ) res = self.__getSESpaceToken( self.oTargetSE ) if not res['OK']: gLogger.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message'] ) return S_ERROR( "SourceSE does not support FTS transfers" ) self.targetToken = res['Value'] self.targetValid = True return S_OK() def __getSESpaceToken( self, oSE ): res = oSE.getStorageParameters( "SRM2" ) if not res['OK']: return res return S_OK( res['Value'].get( 'SpaceToken' ) ) #################################################################### # # Methods for setting/getting FTS request parameters # def setFTSGUID( self, guid ): if not checkGuid( guid ): return S_ERROR( "Incorrect GUID format" ) self.ftsGUID = guid return S_OK() def getFTSGUID( self ): if not self.ftsGUID: return S_ERROR( "FTSGUID not set" ) return S_OK( self.ftsGUID ) def setFTSServer( self, server ): self.ftsServer = server return S_OK() def getFTSServer( self ): if not self.ftsServer: return S_ERROR( "FTSServer not set" ) return S_OK( self.ftsServer ) def setPriority( self, priority ): if not type( priority ) in [types.IntType, types.LongType]: return S_ERROR( "Priority must be integer" ) if priority < 0: priority = 0 elif priority > 5: priority = 5 self.priority = priority return S_OK( self.priority ) def getPriority( self ): return S_OK( self.priority ) def getPercentageComplete( self ): completedFiles = 0 totalFiles = 0 for state in ( self.statusSummary.keys() ): if state in self.successfulStates: completedFiles += self.statusSummary[state] totalFiles += self.statusSummary[state] self.percentageComplete = ( float( completedFiles ) * 100.0 ) / float( totalFiles ) return S_OK( self.percentageComplete ) def isRequestTerminal( self ): if self.requestStatus in self.finalStates: self.isTerminal = True return S_OK( self.isTerminal ) def getStatus( self ): return S_OK( self.requestStatus ) #################################################################### # # Methods for setting/getting/checking files and their metadata # def setLFN( self, lfn ): if not self.fileDict.has_key( lfn ): self.fileDict[lfn] = {} return S_OK() def setSourceSURL( self, lfn, surl ): target = self.fileDict[lfn].get( 'Target' ) if target == surl: return S_ERROR( "Source and target the same" ) self.__setFileParameter( lfn, 'Source', surl ) return S_OK() def getSourceSURL( self, lfn ): return self.__getFileParameter( lfn, 'Source' ) def setTargetSURL( self, lfn, surl ): source = self.fileDict[lfn].get( 'Source' ) if source == surl: return S_ERROR( "Source and target the same" ) self.__setFileParameter( lfn, 'Target', surl ) return S_OK() def getTargetSURL( self, lfn ): return self.__getFileParameter( lfn, 'Target' ) def getFailReason( self, lfn ): return self.__getFileParameter( lfn, 'Reason' ) def getRetries( self, lfn ): return self.__getFileParameter( lfn, 'Retries' ) def getTransferTime( self, lfn ): return self.__getFileParameter( lfn, 'Duration' ) def getFailed( self ): failed = [] for lfn in self.fileDict.keys(): status = self.fileDict[lfn].get( 'Status', '' ) if status in self.failedStates: failed.append( lfn ) return S_OK( failed ) def getDone( self ): done = [] for lfn in self.fileDict.keys(): status = self.fileDict[lfn].get( 'Status', '' ) if status in self.successfulStates: done.append( lfn ) return S_OK( done ) def __setFileParameter( self, lfn, paramName, paramValue ): self.setLFN( lfn ) self.fileDict[lfn][paramName] = paramValue return S_OK() def __getFileParameter( self, lfn, paramName ): if not self.fileDict.has_key( lfn ): return S_ERROR( "Supplied file not set" ) if not self.fileDict[lfn].has_key( paramName ): return S_ERROR( "%s not set for file" % paramName ) return S_OK( self.fileDict[lfn][paramName] ) #################################################################### # # Methods for submission # def submit( self, monitor = False, printOutput = True ): res = self.__isSubmissionValid() if not res['OK']: return res res = self.__createSURLPairFile() if not res['OK']: return res res = self.__submitFTSTransfer() if not res['OK']: return res resDict = {'ftsGUID':self.ftsGUID, 'ftsServer':self.ftsServer} print "Submitted %s @ %s" % ( self.ftsGUID, self.ftsServer ) if monitor: self.monitor( untilTerminal = True, printOutput = printOutput ) return S_OK( resDict ) def __isSubmissionValid( self ): if not self.fileDict: return S_ERROR( "No files set" ) if not self.sourceValid: return S_ERROR( "SourceSE not valid" ) if not self.targetValid: return S_ERROR( "TargetSE not valid" ) if not self.ftsServer: res = self.__resolveFTSServer() if not res['OK']: return S_ERROR( "FTSServer not valid" ) self.__resolveSource() self.__resolveTarget() res = self.__filesToSubmit() if not res['OK']: return S_ERROR( "No files to submit" ) return S_OK() def __getCatalogObject( self ): try: if not self.oCatalog: self.oCatalog = CatalogInterface() return S_OK() except: return S_ERROR() def __updateReplicaCache( self, lfns = [], overwrite = False ): if not lfns: lfns = self.fileDict.keys() toUpdate = [] for lfn in lfns: if ( not lfn in self.catalogReplicas.keys() ) or overwrite: toUpdate.append( lfn ) if not toUpdate: return S_OK() res = self.__getCatalogObject() if not res['OK']: return res res = self.oCatalog.getCatalogReplicas( toUpdate ) if not res['OK']: return S_ERROR( "Failed to update replica cache", res['Message'] ) for lfn, error in res['Value']['Failed'].items(): self.__setFileParameter( lfn, 'Reason', error ) self.__setFileParameter( lfn, 'Status', 'Failed' ) for lfn, replicas in res['Value']['Successful'].items(): self.catalogReplicas[lfn] = replicas return S_OK() def __updateMetadataCache( self, lfns = [], overwrite = False ): if not lfns: lfns = self.fileDict.keys() toUpdate = [] for lfn in lfns: if ( not lfn in self.catalogMetadata.keys() ) or overwrite: toUpdate.append( lfn ) if not toUpdate: return S_OK() res = self.__getCatalogObject() if not res['OK']: return res res = self.oCatalog.getCatalogFileMetadata( toUpdate ) if not res['OK']: return S_ERROR( "Failed to get source catalog metadata", res['Message'] ) for lfn, error in res['Value']['Failed'].items(): self.__setFileParameter( lfn, 'Reason', error ) self.__setFileParameter( lfn, 'Status', 'Failed' ) for lfn, metadata in res['Value']['Successful'].items(): self.catalogMetadata[lfn] = metadata return S_OK() def __resolveSource( self ): toResolve = [] for lfn in self.fileDict.keys(): if ( not self.fileDict[lfn].has_key( 'Source' ) ) and ( self.fileDict[lfn].get( 'Status' ) != 'Failed' ): toResolve.append( lfn ) if not toResolve: return S_OK() res = self.__updateMetadataCache( toResolve ) if not res['OK']: return res res = self.__updateReplicaCache( toResolve ) if not res['OK']: return res for lfn in toResolve: if self.fileDict[lfn].get( 'Status' ) == 'Failed': continue replicas = self.catalogReplicas.get( lfn, {} ) if not replicas.has_key( self.sourceSE ): self.__setFileParameter( lfn, 'Reason', "No replica at SourceSE" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue res = self.oSourceSE.getPfnForProtocol( replicas[self.sourceSE], 'SRM2', withPort = True ) if not res['OK']: self.__setFileParameter( lfn, 'Reason', res['Message'] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue res = self.setSourceSURL( lfn, res['Value'] ) if not res['OK']: self.__setFileParameter( lfn, 'Reason', res['Message'] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue toResolve = {} for lfn in self.fileDict.keys(): if self.fileDict[lfn].has_key( 'Source' ): toResolve[self.fileDict[lfn]['Source']] = lfn if not toResolve: return S_ERROR( "No eligible Source files" ) res = self.oSourceSE.getFileMetadata( toResolve.keys() ) if not res['OK']: return S_ERROR( "Failed to check source file metadata" ) for pfn, error in res['Value']['Failed'].items(): lfn = toResolve[pfn] if re.search( 'File does not exist', error ): self.__setFileParameter( lfn, 'Reason', "Source file does not exist" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) else: self.__setFileParameter( lfn, 'Reason', "Failed to get Source metadata" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) for pfn, metadata in res['Value']['Successful'].items(): lfn = toResolve[pfn] if metadata['Unavailable']: self.__setFileParameter( lfn, 'Reason', "Source file Unavailable" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif metadata['Lost']: self.__setFileParameter( lfn, 'Reason', "Source file Lost" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif not metadata['Cached']: self.__setFileParameter( lfn, 'Reason', "Source file not Cached" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif metadata['Size'] != self.catalogMetadata[lfn]['Size']: self.__setFileParameter( lfn, 'Reason', "Source size mismatch" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and not ( compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ) ): self.__setFileParameter( lfn, 'Reason', "Source checksum mismatch" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) return S_OK() def __resolveTarget( self ): toResolve = [] for lfn in self.fileDict.keys(): if not self.fileDict[lfn].has_key( 'Target' ) and ( self.fileDict[lfn].get( 'Status' ) != 'Failed' ): toResolve.append( lfn ) if not toResolve: return S_OK() res = self.__updateReplicaCache( toResolve ) if not res['OK']: return res atTarget = [] for lfn in sortList( toResolve ): if self.fileDict[lfn].get( 'Status' ) == 'Failed': continue replicas = self.catalogReplicas.get( lfn, {} ) if replicas.has_key( self.targetSE ): self.__setFileParameter( lfn, 'Reason', "File already at Target" ) self.__setFileParameter( lfn, 'Status', 'Done' ) atTarget.append( lfn ) for lfn in toResolve: if ( self.fileDict[lfn].get( 'Status' ) == 'Failed' ) or ( lfn in atTarget ): continue res = self.oTargetSE.getPfnForLfn( lfn ) if not res['OK']: self.__setFileParameter( lfn, 'Reason', "Failed to create Target" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue res = self.oTargetSE.getPfnForProtocol( res['Value'], 'SRM2', withPort = True ) if not res['OK']: self.__setFileParameter( lfn, 'Reason', res['Message'] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue res = self.setTargetSURL( lfn, res['Value'] ) if not res['OK']: self.__setFileParameter( lfn, 'Reason', res['Message'] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue toResolve = {} for lfn in self.fileDict.keys(): if self.fileDict[lfn].has_key( 'Target' ): toResolve[self.fileDict[lfn]['Target']] = lfn if not toResolve: return S_ERROR( "No eligible Target files" ) res = self.oTargetSE.exists( toResolve.keys() ) if not res['OK']: return S_ERROR( "Failed to check target existence" ) for pfn, error in res['Value']['Failed'].items(): lfn = toResolve[pfn] self.__setFileParameter( lfn, 'Reason', error ) self.__setFileParameter( lfn, 'Status', 'Failed' ) toRemove = [] for pfn, exists in res['Value']['Successful'].items(): if exists: lfn = toResolve[pfn] res = self.getSourceSURL( lfn ) if not res['OK']: self.__setFileParameter( lfn, 'Reason', "Target exists" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif res['Value'] == pfn: self.__setFileParameter( lfn, 'Reason', "Source and Target the same" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) else: toRemove.append( pfn ) if toRemove: self.oTargetSE.removeFile( toRemove ) return S_OK() def __filesToSubmit( self ): for lfn in self.fileDict.keys(): lfnStatus = self.fileDict[lfn].get( 'Status' ) source = self.fileDict[lfn].get( 'Source' ) target = self.fileDict[lfn].get( 'Target' ) if ( lfnStatus != 'Failed' ) and ( lfnStatus != 'Done' ) and source and target: return S_OK() return S_ERROR() def __createSURLPairFile( self ): fd, fileName = tempfile.mkstemp() surlFile = os.fdopen( fd, 'w' ) for lfn in self.fileDict.keys(): lfnStatus = self.fileDict[lfn].get( 'Status' ) source = self.fileDict[lfn].get( 'Source' ) target = self.fileDict[lfn].get( 'Target' ) if ( lfnStatus != 'Failed' ) and ( lfnStatus != 'Done' ) and source and target: surlString = '%s %s\n' % ( source, target ) surlFile.write( surlString ) surlFile.close() self.surlFile = fileName return S_OK() def __submitFTSTransfer( self ): comm = ['glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile, '-o'] if self.targetToken: comm.append( '-t' ) comm.append( self.targetToken ) if self.sourceToken: comm.append( '-S' ) comm.append( self.sourceToken ) res = executeGridCommand( '', comm, self.gridEnv ) os.remove( self.surlFile ) if not res['OK']: return res returnCode, output, errStr = res['Value'] if not returnCode == 0: return S_ERROR( errStr ) guid = output.replace( '\n', '' ) if not checkGuid( guid ): return S_ERROR( 'Wrong GUID format returned' ) self.ftsGUID = guid #if self.priority != 3: # comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)] # executeGridCommand('',comm,self.gridEnv) return res def __resolveFTSServer( self ): if not self.sourceSE: return S_ERROR( "Source SE not set" ) if not self.targetSE: return S_ERROR( "Target SE not set" ) res = getSitesForSE( self.sourceSE, 'LCG' ) if not res['OK'] or not res['Value']: return S_ERROR( "Could not determine source site" ) sourceSite = res['Value'][0] res = getSitesForSE( self.targetSE, 'LCG' ) if not res['OK'] or not res['Value']: return S_ERROR( "Could not determine target site" ) targetSite = res['Value'][0] if ( sourceSite == 'LCG.CERN.ch' ) or ( targetSite == 'LCG.CERN.ch' ): ep = 'LCG.CERN.ch' else: # Target site FTS server should be used ep = targetSite try: configPath = '/Resources/FTSEndpoints/%s' % ep endpointURL = gConfig.getValue( configPath ) if not endpointURL: errStr = "FTSRequest.__resolveFTSEndpoint: Failed to find FTS endpoint, check CS entry for '%s'." % ep return S_ERROR( errStr ) self.ftsServer = endpointURL return S_OK( endpointURL ) except Exception, x: return S_ERROR( 'FTSRequest.__resolveFTSEndpoint: Failed to obtain endpoint details from CS' )
class FTSRequest(object): """ .. class:: FTSRequest Helper class for FTS job submission and monitoring. """ ## default checksum type __defaultCksmType = "ADLER32" ## flag to disablr/enable checksum test, default: disabled __cksmTest = False def __init__(self): """c'tor :param self: self reference """ self.log = gLogger.getSubLogger(self.__class__.__name__, True) ## final states tuple self.finalStates = ('Canceled', 'Failed', 'Hold', 'Finished', 'FinishedDirty') ## failed states tuple self.failedStates = ('Canceled', 'Failed', 'Hold', 'FinishedDirty') ## successful states tuple self.successfulStates = ('Finished', 'Done') ## all file states tuple self.fileStates = ('Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed', 'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting') self.newlyCompletedFiles = [] self.newlyFailedFiles = [] self.statusSummary = {} ## request status self.requestStatus = 'Unknown' ## dict for FTS job files self.fileDict = {} ## dict for replicas information self.catalogReplicas = {} ## dict for metadata information self.catalogMetadata = {} ## dict for files that failed to register self.failedRegistrations = {} ## placehoder for CatalogInterface reference self.oCatalog = None ## submit timestamp self.submitTime = '' ## placeholder FTS job GUID self.ftsGUID = '' ## placeholder for FTS server URL self.ftsServer = '' ## not used self.priority = 3 ## flag marking FTS job completness self.isTerminal = False ## completness percentage self.percentageComplete = 0.0 ## source SE name self.sourceSE = '' ## flag marking source SE validity self.sourceValid = False ## source space token self.sourceToken = '' ## target SE name self.targetSE = '' ## flag marking target SE validity self.targetValid = False ## target space token self.targetToken = '' ## whatever self.dumpStr = '' ## placeholder for surl file self.surlFile = None ## placeholder for target StorageElement self.oTargetSE = None ## placeholder for source StorageElement self.oSourceSE = None ## checksum type, set it to default self.__cksmType = self.__defaultCksmType ## disable checksum test by default self.__cksmTest = False ## replica manager handler self.replicaManager = ReplicaManager() #################################################################### # # Methods for setting/getting/checking the SEs # def setSourceSE(self, se): """ set SE for source :param self: self reference :param str se: source SE name """ if se == self.targetSE: return S_ERROR("SourceSE is TargetSE") self.sourceSE = se self.oSourceSE = StorageElement(self.sourceSE) return self.__checkSourceSE() def getSourceSE(self): """ source SE getter :param self: self reference """ if not self.sourceSE: return S_ERROR("Source SE not defined") return S_OK(self.sourceSE) def setSourceToken(self, token): """ set source space token :param self: self reference :param str token: source space token """ self.sourceToken = token return S_OK() def getSourceToken(self): """ source space token getter :param self: self reference """ if not self.sourceToken: return S_ERROR("Source token not defined") return S_OK(self.sourceToken) def __checkSourceSE(self): """ check source SE availability :param self: self reference """ if not self.sourceSE: return S_ERROR("SourceSE not set") res = self.oSourceSE.isValid('Read') if not res['OK']: return S_ERROR("SourceSE not available for reading") res = self.__getSESpaceToken(self.oSourceSE) if not res['OK']: self.log.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message']) return S_ERROR("SourceSE does not support FTS transfers") if self.__cksmTest: res = self.oSourceSE.getChecksumType() if not res["OK"]: self.log.error( "Unable to get checksum type for SourceSE %s: %s" % (self.sourceSE, res["Message"])) cksmType = res["Value"] if cksmType in ("NONE", "NULL"): self.log.warn( "Checksum type set to %s at SourceSE %s, disabling checksum test" % (cksmType, self.sourceSE)) self.__cksmTest = False elif cksmType != self.__cksmType: self.log.warn( "Checksum type mismatch, disabling checksum test") self.__cksmTest = False self.sourceToken = res['Value'] self.sourceValid = True return S_OK() def setTargetSE(self, se): """ set target SE :param self: self reference :param str se: target SE name """ if se == self.sourceSE: return S_ERROR("TargetSE is SourceSE") self.targetSE = se self.oTargetSE = StorageElement(self.targetSE) return self.__checkTargetSE() def getTargetSE(self): """ target SE getter :param self: self reference """ if not self.targetSE: return S_ERROR("Target SE not defined") return S_OK(self.targetSE) def setTargetToken(self, token): """ target space token setter :param self: self reference :param str token: target space token """ self.targetToken = token return S_OK() def getTargetToken(self): """ target space token getter :param self: self reference """ if not self.targetToken: return S_ERROR("Target token not defined") return S_OK(self.targetToken) def __checkTargetSE(self): """ check target SE availability :param self: self reference """ if not self.targetSE: return S_ERROR("TargetSE not set") res = self.oTargetSE.isValid('Write') if not res['OK']: return S_ERROR("TargetSE not available for writing") res = self.__getSESpaceToken(self.oTargetSE) if not res['OK']: self.log.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message']) return S_ERROR("TargetSE does not support FTS transfers") ## check checksum types if self.__cksmTest: res = self.oTargetSE.getChecksumType() if not res["OK"]: self.log.error( "Unable to get checksum type for TargetSE %s: %s" % (self.targetSE, res["Message"])) cksmType = res["Value"] if cksmType in ("NONE", "NULL"): self.log.warn( "Checksum type set to %s at TargetSE %s, disabling checksum test" % (cksmType, self.targetSE)) self.__cksmTest = False elif cksmType != self.__cksmType: self.log.warn( "Checksum type mismatch, disabling checksum test") self.__cksmTest = False self.targetToken = res['Value'] self.targetValid = True return S_OK() @staticmethod def __getSESpaceToken(oSE): """ get space token from StorageElement instance :param self: self reference :param StorageElement oSE: StorageElement instance """ res = oSE.getStorageParameters("SRM2") if not res['OK']: return res return S_OK(res['Value'].get('SpaceToken')) #################################################################### # # Methods for setting/getting FTS request parameters # def setFTSGUID(self, guid): """ FTS job GUID setter :param self: self reference :param str guid: string containg GUID """ if not checkGuid(guid): return S_ERROR("Incorrect GUID format") self.ftsGUID = guid return S_OK() def getFTSGUID(self): """ FTS job GUID getter :param self: self refenece """ if not self.ftsGUID: return S_ERROR("FTSGUID not set") return S_OK(self.ftsGUID) def setFTSServer(self, server): """ FTS server setter :param self: self reference :param str server: FTS server URL """ self.ftsServer = server return S_OK() def getFTSServer(self): """ FTS server getter :param self: self reference """ if not self.ftsServer: return S_ERROR("FTSServer not set") return S_OK(self.ftsServer) def setPriority(self, priority): """ set priority for FTS job :param self: self reference :param int priority: a new priority """ if not type(priority) in (IntType, LongType): return S_ERROR("Priority must be integer") if priority < 0: priority = 0 elif priority > 5: priority = 5 self.priority = priority return S_OK(self.priority) def getPriority(self): """ FTS job priority getter :param self: self reference """ return S_OK(self.priority) def getPercentageComplete(self): """ get completness percentage :param self: self reference """ completedFiles = 0 totalFiles = 0 for state in (self.statusSummary.keys()): if state in self.successfulStates: completedFiles += self.statusSummary[state] totalFiles += self.statusSummary[state] self.percentageComplete = (float(completedFiles) * 100.0) / float(totalFiles) return S_OK(self.percentageComplete) def isRequestTerminal(self): """ check if FTS job has terminated :param self: self reference """ if self.requestStatus in self.finalStates: self.isTerminal = True return S_OK(self.isTerminal) def getStatus(self): """ get FTS job status :param self: self reference """ return S_OK(self.requestStatus) def setCksmType(self, cksm=None): """ set checksum type to use :param self: self reference :param mixed cksm: checksum type, should be one of 'Adler32', 'md5', 'sha1', None """ if str(cksm).upper() not in ("ADLER32", "MD5", "SHA1", "NONE"): return S_ERROR("Not supported checksum type: %s" % str(cksm)) if not cksm: self.__cksmType = None return S_OK(False) self.__cksmType = str(cksm).upper() return S_OK(True) def getCksmType(self): """ get checksum type :param self: self reference """ return S_OK(self.__cksmType) def setCksmTest(self, cksmTest=False): """ set cksm test :param self: self reference :param bool cksmTest: flag to enable/disable checksum test """ self.__cksmTest = bool(cksmTest) return S_OK(self.__cksmTest) def getCksmTest(self): """ get cksm test flag :param self: self reference """ return S_OK(self.__cksmTest) #################################################################### # # Methods for setting/getting/checking files and their metadata # def setLFN(self, lfn): """ add LFN :lfn: to :fileDict: :param self: self reference :param str lfn: LFN to add to """ if lfn not in self.fileDict: self.fileDict[lfn] = {} return S_OK() def setSourceSURL(self, lfn, surl): """ source SURL setter :param self: self reference :param str lfn: LFN :param str surl: source SURL """ target = self.fileDict[lfn].get('Target') if target == surl: return S_ERROR("Source and target the same") self.__setFileParameter(lfn, 'Source', surl) return S_OK() def getSourceSURL(self, lfn): """ get source SURL for LFN :lfn: :param self: self reference :param str lfn: LFN """ return self.__getFileParameter(lfn, 'Source') def setTargetSURL(self, lfn, surl): """ set target SURL for LFN :lfn: :param self: self reference :param str lfn: LFN :param str surl: target SURL """ source = self.fileDict[lfn].get('Source') if source == surl: return S_ERROR("Source and target the same") self.__setFileParameter(lfn, 'Target', surl) return S_OK() def getTargetSURL(self, lfn): """ target SURL getter :param self: self reference :param str lfn: LFN """ return self.__getFileParameter(lfn, 'Target') def getFailReason(self, lfn): """ get fail reason for file :lfn: :param self: self reference :param str lfn: LFN """ return self.__getFileParameter(lfn, 'Reason') def getRetries(self, lfn): """ get number of attepmts made to transfer file :lfn: :param self: self reference :param str lfn: LFN """ return self.__getFileParameter(lfn, 'Retries') def getTransferTime(self, lfn): """ get duration of transfer for file :lfn: :param self: self reference :param str lfn: LFN """ return self.__getFileParameter(lfn, 'Duration') def getFailed(self): """ get list of wrongly transferred LFNs :param self: self reference """ return S_OK([ lfn for lfn in self.fileDict if self.fileDict[lfn].get('Status', '') in self.failedStates ]) def getDone(self): """ get list of succesfully transferred LFNs :param self: self reference """ return S_OK([ lfn for lfn in self.fileDict if self.fileDict[lfn].get('Status', '') in self.successfulStates ]) def __setFileParameter(self, lfn, paramName, paramValue): """ set :paramName: to :paramValue: for :lfn: file :param self: self reference :param str lfn: LFN :param str paramName: parameter name :param mixed paramValue: a new parameter value """ self.setLFN(lfn) self.fileDict[lfn][paramName] = paramValue return S_OK() def __getFileParameter(self, lfn, paramName): """ get value of :paramName: for file :lfn: :param self: self reference :param str lfn: LFN :param str paramName: parameter name """ if lfn not in self.fileDict: return S_ERROR("Supplied file not set") if paramName not in self.fileDict[lfn]: return S_ERROR("%s not set for file" % paramName) return S_OK(self.fileDict[lfn][paramName]) #################################################################### # # Methods for submission # def submit(self, monitor=False, printOutput=True): """ submit FTS job :param self: self reference :param bool monitor: flag to monitor progress of FTS job :param bool printOutput: flag to print output of execution to stdout """ res = self.__isSubmissionValid() if not res['OK']: return res res = self.__createSURLPairFile() if not res['OK']: return res res = self.__submitFTSTransfer() if not res['OK']: return res resDict = {'ftsGUID': self.ftsGUID, 'ftsServer': self.ftsServer} print "Submitted %s @ %s" % (self.ftsGUID, self.ftsServer) if monitor: self.monitor(untilTerminal=True, printOutput=printOutput) return S_OK(resDict) def __isSubmissionValid(self): """ check validity of job before submission :param self: self reference """ if not self.fileDict: return S_ERROR("No files set") if not self.sourceValid: return S_ERROR("SourceSE not valid") if not self.targetValid: return S_ERROR("TargetSE not valid") if not self.ftsServer: res = self.__resolveFTSServer() if not res['OK']: return S_ERROR("FTSServer not valid") self.resolveSource() self.resolveTarget() res = self.__filesToSubmit() if not res['OK']: return S_ERROR("No files to submit") return S_OK() def __getCatalogObject(self): """ CatalogInterface instance facade :param self: self reference """ try: if not self.oCatalog: self.oCatalog = CatalogInterface() return S_OK() except: return S_ERROR() def __updateReplicaCache(self, lfns=None, overwrite=False): """ update replica cache for list of :lfns: :param self: self reference :param mixed lfns: list of LFNs :param bool overwrite: flag to trigger cache clearing and updating """ if not lfns: lfns = self.fileDict.keys() toUpdate = [ lfn for lfn in lfns if (lfn not in self.catalogReplicas) or overwrite ] if not toUpdate: return S_OK() res = self.__getCatalogObject() if not res['OK']: return res res = self.oCatalog.getCatalogReplicas(toUpdate) if not res['OK']: return S_ERROR("Failed to update replica cache: %s" % res['Message']) for lfn, error in res['Value']['Failed'].items(): self.__setFileParameter(lfn, 'Reason', error) self.__setFileParameter(lfn, 'Status', 'Failed') for lfn, replicas in res['Value']['Successful'].items(): self.catalogReplicas[lfn] = replicas return S_OK() def __updateMetadataCache(self, lfns=None, overwrite=False): """ update metadata cache for list of LFNs :param self: self reference :param list lnfs: list of LFNs :param bool overwrite: flag to trigger cache clearing and updating """ if not lfns: lfns = self.fileDict.keys() toUpdate = [ lfn for lfn in lfns if (lfn not in self.catalogMetadata) or overwrite ] if not toUpdate: return S_OK() res = self.__getCatalogObject() if not res['OK']: return res res = self.oCatalog.getCatalogFileMetadata(toUpdate) if not res['OK']: return S_ERROR("Failed to get source catalog metadata: %s" % res['Message']) for lfn, error in res['Value']['Failed'].items(): self.__setFileParameter(lfn, 'Reason', error) self.__setFileParameter(lfn, 'Status', 'Failed') for lfn, metadata in res['Value']['Successful'].items(): self.catalogMetadata[lfn] = metadata return S_OK() def resolveSource(self): """ resolve source SE eligible for submission :param self: self reference """ toResolve = [lfn for lfn in self.fileDict] if not toResolve: return S_OK() res = self.__updateMetadataCache(toResolve) if not res['OK']: return res res = self.__updateReplicaCache(toResolve) if not res['OK']: return res for lfn in toResolve: if self.fileDict[lfn].get("Status", "") == "Failed": continue replicas = self.catalogReplicas.get(lfn, {}) if self.sourceSE not in replicas: gLogger.warn( "resolveSource: skipping %s - not replicas at SourceSE %s" % (lfn, self.sourceSE)) self.__setFileParameter(lfn, 'Reason', "No replica at SourceSE") self.__setFileParameter(lfn, 'Status', 'Failed') continue res = self.oSourceSE.getPfnForProtocol(replicas[self.sourceSE], 'SRM2', withPort=True) if not res['OK']: gLogger.warn("resolveSource: skipping %s - %s" % (lfn, res["Message"])) self.__setFileParameter(lfn, 'Reason', res['Message']) self.__setFileParameter(lfn, 'Status', 'Failed') continue res = self.setSourceSURL(lfn, res['Value']) if not res['OK']: gLogger.warn("resolveSource: skipping %s - %s" % (lfn, res["Message"])) self.__setFileParameter(lfn, 'Reason', res['Message']) self.__setFileParameter(lfn, 'Status', 'Failed') continue toResolve = {} for lfn in self.fileDict: if "Source" in self.fileDict[lfn]: toResolve[self.fileDict[lfn]['Source']] = lfn if not toResolve: return S_ERROR("No eligible Source files") res = self.oSourceSE.getFileMetadata(toResolve.keys()) if not res['OK']: return S_ERROR("Failed to check source file metadata") for pfn, error in res['Value']['Failed'].items(): lfn = toResolve[pfn] if re.search('File does not exist', error): gLogger.warn( "resolveSource: skipping %s - source file does not exists" % lfn) self.__setFileParameter(lfn, 'Reason', "Source file does not exist") self.__setFileParameter(lfn, 'Status', 'Failed') else: gLogger.warn( "resolveSource: skipping %s - failed to get source metadata" % lfn) self.__setFileParameter(lfn, 'Reason', "Failed to get Source metadata") self.__setFileParameter(lfn, 'Status', 'Failed') for pfn, metadata in res['Value']['Successful'].items(): lfn = toResolve[pfn] if metadata['Unavailable']: gLogger.warn( "resolveSource: skipping %s - source file unavailable" % lfn) self.__setFileParameter(lfn, 'Reason', "Source file Unavailable") self.__setFileParameter(lfn, 'Status', 'Failed') elif metadata['Lost']: gLogger.warn("resolveSource: skipping %s - source file lost" % lfn) self.__setFileParameter(lfn, 'Reason', "Source file Lost") self.__setFileParameter(lfn, 'Status', 'Failed') elif not metadata['Cached']: gLogger.warn( "resolveSource: source file %s not cached, prestaging..." % lfn) stage = self.replicaManager.prestageStorageFile( pfn, self.sourceSE, singleFile=True) if not stage["OK"]: gLogger.warn("resolveSource: skipping %s - %s" % (lfn, stage["Message"])) self.__setFileParameter(lfn, 'Reason', stage["Message"]) self.__setFileParameter(lfn, 'Status', 'Failed') elif metadata['Size'] != self.catalogMetadata[lfn]['Size']: gLogger.warn( "resolveSource: skipping %s - source file size mismatch" % lfn) self.__setFileParameter(lfn, 'Reason', "Source size mismatch") self.__setFileParameter(lfn, 'Status', 'Failed') elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \ not ( compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ) ): gLogger.warn( "resolveSource: skipping %s - source file checksum mismatch" % lfn) self.__setFileParameter(lfn, 'Reason', "Source checksum mismatch") self.__setFileParameter(lfn, 'Status', 'Failed') return S_OK() def resolveTarget(self): """ find target SE eligible for submission :param self: self reference """ toResolve = [lfn for lfn in self.fileDict] if not toResolve: return S_OK() res = self.__updateReplicaCache(toResolve) if not res['OK']: return res for lfn in toResolve: if (self.fileDict[lfn].get('Status') == 'Failed'): continue res = self.oTargetSE.getPfnForLfn(lfn) if not res['OK']: gLogger.warn( "resolveTarget: skipping %s - failed to create target pfn" % lfn) self.__setFileParameter(lfn, 'Reason', "Failed to create Target") self.__setFileParameter(lfn, 'Status', 'Failed') continue res = self.oTargetSE.getPfnForProtocol(res['Value'], 'SRM2', withPort=True) if not res['OK']: gLogger.warn("resolveTarget: skipping %s - %s" % (lfn, res["Message"])) self.__setFileParameter(lfn, 'Reason', res['Message']) self.__setFileParameter(lfn, 'Status', 'Failed') continue res = self.setTargetSURL(lfn, res['Value']) if not res['OK']: gLogger.warn("resolveTarget: skipping %s - %s" % (lfn, res["Message"])) self.__setFileParameter(lfn, 'Reason', res['Message']) self.__setFileParameter(lfn, 'Status', 'Failed') continue toResolve = {} for lfn in self.fileDict: if "Target" in self.fileDict[lfn]: toResolve[self.fileDict[lfn]['Target']] = lfn if not toResolve: return S_ERROR("No eligible Target files") res = self.oTargetSE.exists(toResolve.keys()) if not res['OK']: return S_ERROR("Failed to check target existence") for pfn, error in res['Value']['Failed'].items(): lfn = toResolve[pfn] self.__setFileParameter(lfn, 'Reason', error) self.__setFileParameter(lfn, 'Status', 'Failed') toRemove = [] for pfn, exists in res['Value']['Successful'].items(): if exists: lfn = toResolve[pfn] res = self.getSourceSURL(lfn) if not res['OK']: gLogger.warn("resolveTarget: skipping %s - target exists" % lfn) self.__setFileParameter(lfn, 'Reason', "Target exists") self.__setFileParameter(lfn, 'Status', 'Failed') elif res['Value'] == pfn: gLogger.warn( "resolveTarget: skipping %s - source and target pfns are teh same" % lfn) self.__setFileParameter(lfn, 'Reason', "Source and Target the same") self.__setFileParameter(lfn, 'Status', 'Failed') else: toRemove.append(pfn) if toRemove: self.oTargetSE.removeFile(toRemove) return S_OK() def __filesToSubmit(self): """ check if there is at least one file to submit :return: S_OK if at least one file is present, S_ERROR otherwise """ for lfn in self.fileDict: lfnStatus = self.fileDict[lfn].get('Status') source = self.fileDict[lfn].get('Source') target = self.fileDict[lfn].get('Target') if (lfnStatus != 'Failed') and (lfnStatus != 'Done') and source and target: return S_OK() return S_ERROR() def __createSURLPairFile(self): """ create LFNs file for glite-transfer-submit command This file consists one line for each fiel to be transferred: sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM] :param self: self reference """ fd, fileName = tempfile.mkstemp() surlFile = os.fdopen(fd, 'w') for lfn in self.fileDict: lfnStatus = self.fileDict[lfn].get('Status') source = self.fileDict[lfn].get('Source') target = self.fileDict[lfn].get('Target') if (lfnStatus not in ('Failed', 'Done')) and source and target: cksmStr = "" ## add chsmType:cksm only if cksmType is specified, else let FTS decide by itself if self.__cksmTest and self.__cksmType: if lfn in self.catalogMetadata and "Checksum" in self.catalogMetadata[ lfn]: cksmStr = " %s:%s" % (self.__cksmType, self. catalogMetadata[lfn]["Checksum"]) surlFile.write("%s %s%s\n" % (source, target, cksmStr)) surlFile.close() self.surlFile = fileName return S_OK() def __submitFTSTransfer(self): """ create and execute glite-transfer-submit CLI command :param self: self reference """ comm = [ 'glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile, '-o' ] if self.targetToken: comm.append('-t') comm.append(self.targetToken) if self.sourceToken: comm.append('-S') comm.append(self.sourceToken) if self.__cksmTest: comm.append("--compare-checksums") res = executeGridCommand('', comm) os.remove(self.surlFile) if not res['OK']: return res returnCode, output, errStr = res['Value'] if not returnCode == 0: return S_ERROR(errStr) guid = output.replace('\n', '') if not checkGuid(guid): return S_ERROR('Wrong GUID format returned') self.ftsGUID = guid #if self.priority != 3: # comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)] # executeGridCommand('',comm) return res def __resolveFTSServer(self): """ resolve FTS server to use, it should be the closest one from target SE :param self: self reference """ if not self.sourceSE: return S_ERROR("Source SE not set") if not self.targetSE: return S_ERROR("Target SE not set") res = getSitesForSE(self.sourceSE, 'LCG') if not res['OK'] or not res['Value']: return S_ERROR("Could not determine source site") sourceSite = res['Value'][0] res = getSitesForSE(self.targetSE, 'LCG') if not res['OK'] or not res['Value']: return S_ERROR("Could not determine target site") targetSite = res['Value'][0] if (sourceSite == 'LCG.CERN.ch') or (targetSite == 'LCG.CERN.ch'): ep = 'LCG.CERN.ch' else: # Target site FTS server should be used ep = targetSite try: configPath = '/Resources/FTSEndpoints/%s' % ep endpointURL = gConfig.getValue(configPath) if not endpointURL: errStr = "FTSRequest.__resolveFTSServer: Failed to find FTS endpoint, check CS entry for '%s'." % ep return S_ERROR(errStr) self.ftsServer = endpointURL return S_OK(endpointURL) except Exception, x: return S_ERROR( 'FTSRequest.__resolveFTSServer: Failed to obtain endpoint details from CS' )
class FTSRequest(object): """ .. class:: FTSRequest Helper class for FTS job submission and monitoring. """ # # default checksum type __defaultCksmType = "ADLER32" # # flag to disablr/enable checksum test, default: disabled __cksmTest = False def __init__(self): """c'tor :param self: self reference """ self.log = gLogger.getSubLogger(self.__class__.__name__, True) # # final states tuple self.finalStates = ('Canceled', 'Failed', 'Hold', 'Finished', 'FinishedDirty') # # failed states tuple self.failedStates = ('Canceled', 'Failed', 'Hold', 'FinishedDirty') # # successful states tuple self.successfulStates = ('Finished', 'Done') # # all file states tuple self.fileStates = ('Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed', 'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting') self.statusSummary = {} # # request status self.requestStatus = 'Unknown' # # dict for FTS job files self.fileDict = {} # # dict for replicas information self.catalogReplicas = {} # # dict for metadata information self.catalogMetadata = {} # # dict for files that failed to register self.failedRegistrations = {} # # placehoder for FileCatalog reference self.oCatalog = None # # submit timestamp self.submitTime = '' # # placeholder FTS job GUID self.ftsGUID = '' # # placeholder for FTS server URL self.ftsServer = '' # # flag marking FTS job completness self.isTerminal = False # # completness percentage self.percentageComplete = 0.0 # # source SE name self.sourceSE = '' # # flag marking source SE validity self.sourceValid = False # # source space token self.sourceToken = '' # # target SE name self.targetSE = '' # # flag marking target SE validity self.targetValid = False # # target space token self.targetToken = '' # # placeholder for target StorageElement self.oTargetSE = None # # placeholder for source StorageElement self.oSourceSE = None # # checksum type, set it to default self.__cksmType = self.__defaultCksmType # # disable checksum test by default self.__cksmTest = False # # statuses that prevent submitting to FTS self.noSubmitStatus = ('Failed', 'Done', 'Staging') # # were sources resolved? self.sourceResolved = False # # Number of file transfers actually submitted self.submittedFiles = 0 self.transferTime = 0 self.submitCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/SubmitCommand', 'glite-transfer-submit') self.monitorCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/MonitorCommand', 'glite-transfer-status') self.ftsJob = None self.ftsFiles = [] #################################################################### # # Methods for setting/getting/checking the SEs # def setSourceSE(self, se): """ set SE for source :param self: self reference :param str se: source SE name """ if se == self.targetSE: return S_ERROR("SourceSE is TargetSE") self.sourceSE = se self.oSourceSE = StorageElement(self.sourceSE) return self.__checkSourceSE() def __checkSourceSE(self): """ check source SE availability :param self: self reference """ if not self.sourceSE: return S_ERROR("SourceSE not set") res = self.oSourceSE.isValid('Read') if not res['OK']: return S_ERROR("SourceSE not available for reading") res = self.__getSESpaceToken(self.oSourceSE) if not res['OK']: self.log.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message']) return S_ERROR("SourceSE does not support FTS transfers") if self.__cksmTest: res = self.oSourceSE.getChecksumType() if not res["OK"]: self.log.error( "Unable to get checksum type for SourceSE %s: %s" % (self.sourceSE, res["Message"])) cksmType = res["Value"] if cksmType in ("NONE", "NULL"): self.log.warn( "Checksum type set to %s at SourceSE %s, disabling checksum test" % (cksmType, self.sourceSE)) self.__cksmTest = False elif cksmType != self.__cksmType: self.log.warn( "Checksum type mismatch, disabling checksum test") self.__cksmTest = False self.sourceToken = res['Value'] self.sourceValid = True return S_OK() def setTargetSE(self, se): """ set target SE :param self: self reference :param str se: target SE name """ if se == self.sourceSE: return S_ERROR("TargetSE is SourceSE") self.targetSE = se self.oTargetSE = StorageElement(self.targetSE) return self.__checkTargetSE() def setTargetToken(self, token): """ target space token setter :param self: self reference :param str token: target space token """ self.targetToken = token return S_OK() def __checkTargetSE(self): """ check target SE availability :param self: self reference """ if not self.targetSE: return S_ERROR("TargetSE not set") res = self.oTargetSE.isValid('Write') if not res['OK']: return S_ERROR("TargetSE not available for writing") res = self.__getSESpaceToken(self.oTargetSE) if not res['OK']: self.log.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message']) return S_ERROR("TargetSE does not support FTS transfers") # # check checksum types if self.__cksmTest: res = self.oTargetSE.getChecksumType() if not res["OK"]: self.log.error( "Unable to get checksum type for TargetSE %s: %s" % (self.targetSE, res["Message"])) cksmType = res["Value"] if cksmType in ("NONE", "NULL"): self.log.warn( "Checksum type set to %s at TargetSE %s, disabling checksum test" % (cksmType, self.targetSE)) self.__cksmTest = False elif cksmType != self.__cksmType: self.log.warn( "Checksum type mismatch, disabling checksum test") self.__cksmTest = False self.targetToken = res['Value'] self.targetValid = True return S_OK() @staticmethod def __getSESpaceToken(oSE): """ get space token from StorageElement instance :param self: self reference :param StorageElement oSE: StorageElement instance """ res = oSE.getStorageParameters("SRM2") if not res['OK']: return res return S_OK(res['Value'].get('SpaceToken')) #################################################################### # # Methods for setting/getting FTS request parameters # def setFTSGUID(self, guid): """ FTS job GUID setter :param self: self reference :param str guid: string containg GUID """ if not checkGuid(guid): return S_ERROR("Incorrect GUID format") self.ftsGUID = guid return S_OK() def setFTSServer(self, server): """ FTS server setter :param self: self reference :param str server: FTS server URL """ self.ftsServer = server return S_OK() def isRequestTerminal(self): """ check if FTS job has terminated :param self: self reference """ if self.requestStatus in self.finalStates: self.isTerminal = True return S_OK(self.isTerminal) def setCksmTest(self, cksmTest=False): """ set cksm test :param self: self reference :param bool cksmTest: flag to enable/disable checksum test """ self.__cksmTest = bool(cksmTest) return S_OK(self.__cksmTest) #################################################################### # # Methods for setting/getting/checking files and their metadata # def setLFN(self, lfn): """ add LFN :lfn: to :fileDict: :param self: self reference :param str lfn: LFN to add to """ self.fileDict.setdefault(lfn, {'Status': 'Waiting'}) return S_OK() def setSourceSURL(self, lfn, surl): """ source SURL setter :param self: self reference :param str lfn: LFN :param str surl: source SURL """ target = self.fileDict[lfn].get('Target') if target == surl: return S_ERROR("Source and target the same") return self.__setFileParameter(lfn, 'Source', surl) def getSourceSURL(self, lfn): """ get source SURL for LFN :lfn: :param self: self reference :param str lfn: LFN """ return self.__getFileParameter(lfn, 'Source') def setTargetSURL(self, lfn, surl): """ set target SURL for LFN :lfn: :param self: self reference :param str lfn: LFN :param str surl: target SURL """ source = self.fileDict[lfn].get('Source') if source == surl: return S_ERROR("Source and target the same") return self.__setFileParameter(lfn, 'Target', surl) def getFailReason(self, lfn): """ get fail reason for file :lfn: :param self: self reference :param str lfn: LFN """ return self.__getFileParameter(lfn, 'Reason') def getRetries(self, lfn): """ get number of attepmts made to transfer file :lfn: :param self: self reference :param str lfn: LFN """ return self.__getFileParameter(lfn, 'Retries') def getTransferTime(self, lfn): """ get duration of transfer for file :lfn: :param self: self reference :param str lfn: LFN """ return self.__getFileParameter(lfn, 'Duration') def getFailed(self): """ get list of wrongly transferred LFNs :param self: self reference """ return S_OK([ lfn for lfn in self.fileDict if self.fileDict[lfn].get('Status', '') in self.failedStates ]) def getStaging(self): """ get files set for prestaging """ return S_OK([ lfn for lfn in self.fileDict if self.fileDict[lfn].get('Status', '') == 'Staging' ]) def getDone(self): """ get list of succesfully transferred LFNs :param self: self reference """ return S_OK([ lfn for lfn in self.fileDict if self.fileDict[lfn].get('Status', '') in self.successfulStates ]) def __setFileParameter(self, lfn, paramName, paramValue): """ set :paramName: to :paramValue: for :lfn: file :param self: self reference :param str lfn: LFN :param str paramName: parameter name :param mixed paramValue: a new parameter value """ self.setLFN(lfn) self.fileDict[lfn][paramName] = paramValue return S_OK() def __getFileParameter(self, lfn, paramName): """ get value of :paramName: for file :lfn: :param self: self reference :param str lfn: LFN :param str paramName: parameter name """ if lfn not in self.fileDict: return S_ERROR("Supplied file not set") if paramName not in self.fileDict[lfn]: return S_ERROR("%s not set for file" % paramName) return S_OK(self.fileDict[lfn][paramName]) #################################################################### # # Methods for submission # def submit(self, monitor=False, printOutput=True): """ submit FTS job :param self: self reference :param bool monitor: flag to monitor progress of FTS job :param bool printOutput: flag to print output of execution to stdout """ res = self.__prepareForSubmission() if not res['OK']: return res res = self.__submitFTSTransfer() if not res['OK']: return res resDict = { 'ftsGUID': self.ftsGUID, 'ftsServer': self.ftsServer, 'submittedFiles': self.submittedFiles } if monitor or printOutput: gLogger.always("Submitted %s@%s" % (self.ftsGUID, self.ftsServer)) if monitor: self.monitor(untilTerminal=True, printOutput=printOutput, full=False) return S_OK(resDict) def __prepareForSubmission(self): """ check validity of job before submission :param self: self reference """ if not self.fileDict: return S_ERROR("No files set") if not self.sourceValid: return S_ERROR("SourceSE not valid") if not self.targetValid: return S_ERROR("TargetSE not valid") if not self.ftsServer: res = self.__resolveFTSServer() if not res['OK']: return S_ERROR("FTSServer not valid") self.resolveSource() self.resolveTarget() res = self.__filesToSubmit() if not res['OK']: return S_ERROR("No files to submit") return S_OK() def __getCatalogObject(self): """ CatalogInterface instance facade :param self: self reference """ try: if not self.oCatalog: self.oCatalog = FileCatalog() return S_OK() except: return S_ERROR() def __updateReplicaCache(self, lfns=None, overwrite=False): """ update replica cache for list of :lfns: :param self: self reference :param mixed lfns: list of LFNs :param bool overwrite: flag to trigger cache clearing and updating """ if not lfns: lfns = self.fileDict.keys() toUpdate = [ lfn for lfn in lfns if (lfn not in self.catalogReplicas) or overwrite ] if not toUpdate: return S_OK() res = self.__getCatalogObject() if not res['OK']: return res res = self.oCatalog.getReplicas(toUpdate) if not res['OK']: return S_ERROR("Failed to update replica cache: %s" % res['Message']) for lfn, error in res['Value']['Failed'].items(): self.__setFileParameter(lfn, 'Reason', error) self.__setFileParameter(lfn, 'Status', 'Failed') for lfn, replicas in res['Value']['Successful'].items(): self.catalogReplicas[lfn] = replicas return S_OK() def __updateMetadataCache(self, lfns=None): """ update metadata cache for list of LFNs :param self: self reference :param list lnfs: list of LFNs """ if not lfns: lfns = self.fileDict.keys() toUpdate = [lfn for lfn in lfns if lfn not in self.catalogMetadata] if not toUpdate: return S_OK() res = self.__getCatalogObject() if not res['OK']: return res res = self.oCatalog.getFileMetadata(toUpdate) if not res['OK']: return S_ERROR("Failed to get source catalog metadata: %s" % res['Message']) for lfn, error in res['Value']['Failed'].items(): self.__setFileParameter(lfn, 'Reason', error) self.__setFileParameter(lfn, 'Status', 'Failed') for lfn, metadata in res['Value']['Successful'].items(): self.catalogMetadata[lfn] = metadata return S_OK() def resolveSource(self): """ resolve source SE eligible for submission :param self: self reference """ # Avoid resolving sources twice if self.sourceResolved: return S_OK() # Only resolve files that need a transfer toResolve = [ lfn for lfn in self.fileDict if self.fileDict[lfn].get("Status", "") != "Failed" ] if not toResolve: return S_OK() res = self.__updateMetadataCache(toResolve) if not res['OK']: return res res = self.__updateReplicaCache(toResolve) if not res['OK']: return res # Define the source URLs for lfn in toResolve: replicas = self.catalogReplicas.get(lfn, {}) if self.sourceSE not in replicas: gLogger.warn( "resolveSource: skipping %s - not replicas at SourceSE %s" % (lfn, self.sourceSE)) self.__setFileParameter(lfn, 'Reason', "No replica at SourceSE") self.__setFileParameter(lfn, 'Status', 'Failed') continue # Fix first the PFN pfn = self.oSourceSE.getPfnForLfn(lfn).get('Value', {}).get( 'Successful', {}).get(lfn, replicas[self.sourceSE]) res = returnSingleResult( self.oSourceSE.getPfnForProtocol(pfn, protocol='SRM2', withPort=True)) if not res['OK']: gLogger.warn("resolveSource: skipping %s - %s" % (lfn, res["Message"])) self.__setFileParameter(lfn, 'Reason', res['Message']) self.__setFileParameter(lfn, 'Status', 'Failed') continue res = self.setSourceSURL(lfn, res['Value']) if not res['OK']: gLogger.warn("resolveSource: skipping %s - %s" % (lfn, res["Message"])) self.__setFileParameter(lfn, 'Reason', res['Message']) self.__setFileParameter(lfn, 'Status', 'Failed') continue toResolve = {} for lfn in self.fileDict: if "Source" in self.fileDict[lfn]: toResolve[self.fileDict[lfn]['Source']] = lfn if not toResolve: return S_ERROR("No eligible Source files") # Get metadata of the sources, to check for existance, availability and caching res = self.oSourceSE.getFileMetadata(toResolve.keys()) if not res['OK']: return S_ERROR("Failed to check source file metadata") for pfn, error in res['Value']['Failed'].items(): lfn = toResolve[pfn] if re.search('File does not exist', error): gLogger.warn( "resolveSource: skipping %s - source file does not exists" % lfn) self.__setFileParameter(lfn, 'Reason', "Source file does not exist") self.__setFileParameter(lfn, 'Status', 'Failed') else: gLogger.warn( "resolveSource: skipping %s - failed to get source metadata" % lfn) self.__setFileParameter(lfn, 'Reason', "Failed to get Source metadata") self.__setFileParameter(lfn, 'Status', 'Failed') toStage = [] nbStagedFiles = 0 for pfn, metadata in res['Value']['Successful'].items(): lfn = toResolve[pfn] lfnStatus = self.fileDict.get(lfn, {}).get('Status') if metadata['Unavailable']: gLogger.warn( "resolveSource: skipping %s - source file unavailable" % lfn) self.__setFileParameter(lfn, 'Reason', "Source file Unavailable") self.__setFileParameter(lfn, 'Status', 'Failed') elif metadata['Lost']: gLogger.warn("resolveSource: skipping %s - source file lost" % lfn) self.__setFileParameter(lfn, 'Reason', "Source file Lost") self.__setFileParameter(lfn, 'Status', 'Failed') elif not metadata['Cached']: if lfnStatus != 'Staging': toStage.append(pfn) elif metadata['Size'] != self.catalogMetadata[lfn]['Size']: gLogger.warn( "resolveSource: skipping %s - source file size mismatch" % lfn) self.__setFileParameter(lfn, 'Reason', "Source size mismatch") self.__setFileParameter(lfn, 'Status', 'Failed') elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \ not compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ): gLogger.warn( "resolveSource: skipping %s - source file checksum mismatch" % lfn) self.__setFileParameter(lfn, 'Reason', "Source checksum mismatch") self.__setFileParameter(lfn, 'Status', 'Failed') elif lfnStatus == 'Staging': # file that was staging is now cached self.__setFileParameter(lfn, 'Status', 'Waiting') nbStagedFiles += 1 # Some files were being staged if nbStagedFiles: self.log.info('resolveSource: %d files have been staged' % nbStagedFiles) # Launching staging of files not in cache if toStage: gLogger.warn( "resolveSource: %s source files not cached, prestaging..." % len(toStage)) stage = self.oSourceSE.prestageFile(toStage) if not stage["OK"]: gLogger.error("resolveSource: error is prestaging - %s" % stage["Message"]) for pfn in toStage: lfn = toResolve[pfn] self.__setFileParameter(lfn, 'Reason', stage["Message"]) self.__setFileParameter(lfn, 'Status', 'Failed') else: for pfn in toStage: lfn = toResolve[pfn] if pfn in stage['Value']['Successful']: self.__setFileParameter(lfn, 'Status', 'Staging') elif pfn in stage['Value']['Failed']: self.__setFileParameter(lfn, 'Reason', stage['Value']['Failed'][pfn]) self.__setFileParameter(lfn, 'Status', 'Failed') self.sourceResolved = True return S_OK() def resolveTarget(self): """ find target SE eligible for submission :param self: self reference """ toResolve = [ lfn for lfn in self.fileDict if self.fileDict[lfn].get('Status') not in self.noSubmitStatus ] if not toResolve: return S_OK() res = self.__updateReplicaCache(toResolve) if not res['OK']: return res for lfn in toResolve: res = self.oTargetSE.getPfnForLfn(lfn) if not res['OK'] or lfn not in res['Value']['Successful']: gLogger.warn( "resolveTarget: skipping %s - failed to create target pfn" % lfn) self.__setFileParameter(lfn, 'Reason', "Failed to create Target") self.__setFileParameter(lfn, 'Status', 'Failed') continue pfn = res['Value']['Successful'][lfn] res = self.oTargetSE.getPfnForProtocol(pfn, protocol='SRM2', withPort=True) if not res['OK'] or pfn not in res['Value']['Successful']: reason = res.get( 'Message', res.get('Value', {}).get('Failed', {}).get(pfn)) gLogger.warn("resolveTarget: skipping %s - %s" % (lfn, reason)) self.__setFileParameter(lfn, 'Reason', reason) self.__setFileParameter(lfn, 'Status', 'Failed') continue pfn = res['Value']['Successful'][pfn] res = self.setTargetSURL(lfn, pfn) if not res['OK']: gLogger.warn("resolveTarget: skipping %s - %s" % (lfn, res["Message"])) self.__setFileParameter(lfn, 'Reason', res['Message']) self.__setFileParameter(lfn, 'Status', 'Failed') continue toResolve = {} for lfn in self.fileDict: if "Target" in self.fileDict[lfn]: toResolve[self.fileDict[lfn]['Target']] = lfn if not toResolve: return S_ERROR("No eligible Target files") res = self.oTargetSE.exists(toResolve.keys()) if not res['OK']: return S_ERROR("Failed to check target existence") for pfn, error in res['Value']['Failed'].items(): lfn = toResolve[pfn] self.__setFileParameter(lfn, 'Reason', error) self.__setFileParameter(lfn, 'Status', 'Failed') toRemove = [] for pfn, exists in res['Value']['Successful'].items(): if exists: lfn = toResolve[pfn] res = self.getSourceSURL(lfn) if not res['OK']: gLogger.warn("resolveTarget: skipping %s - target exists" % lfn) self.__setFileParameter(lfn, 'Reason', "Target exists") self.__setFileParameter(lfn, 'Status', 'Failed') elif res['Value'] == pfn: gLogger.warn( "resolveTarget: skipping %s - source and target pfns are the same" % lfn) self.__setFileParameter(lfn, 'Reason', "Source and Target the same") self.__setFileParameter(lfn, 'Status', 'Failed') else: toRemove.append(pfn) if toRemove: self.oTargetSE.removeFile(toRemove) return S_OK() def __filesToSubmit(self): """ check if there is at least one file to submit :return: S_OK if at least one file is present, S_ERROR otherwise """ for lfn in self.fileDict: lfnStatus = self.fileDict[lfn].get('Status') source = self.fileDict[lfn].get('Source') target = self.fileDict[lfn].get('Target') if lfnStatus not in self.noSubmitStatus and source and target: return S_OK() return S_ERROR() def __createFTSFiles(self): """ create LFNs file for glite-transfer-submit command This file consists one line for each fiel to be transferred: sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM] :param self: self reference """ self.__updateMetadataCache() for lfn in self.fileDict: lfnStatus = self.fileDict[lfn].get('Status') if lfnStatus not in self.noSubmitStatus: cksmStr = "" # # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself if self.__cksmTest and self.__cksmType: checkSum = self.catalogMetadata.get(lfn, {}).get('Checksum') if checkSum: cksmStr = " %s:%s" % (self.__cksmType, intAdlerToHex( hexAdlerToInt(checkSum))) ftsFile = FTSFile() ftsFile.LFN = lfn ftsFile.SourceSURL = self.fileDict[lfn].get('Source') ftsFile.TargetSURL = self.fileDict[lfn].get('Target') ftsFile.SourceSE = self.sourceSE ftsFile.TargetSE = self.targetSE ftsFile.Status = self.fileDict[lfn].get('Status') ftsFile.Checksum = cksmStr ftsFile.Size = self.catalogMetadata.get(lfn, {}).get('Size') self.ftsFiles.append(ftsFile) self.submittedFiles += 1 return S_OK() def __createFTSJob(self, guid=None): self.__createFTSFiles() ftsJob = FTSJob() ftsJob.RequestID = 0 ftsJob.OperationID = 0 ftsJob.SourceSE = self.sourceSE ftsJob.TargetSE = self.targetSE ftsJob.SourceToken = self.sourceToken ftsJob.TargetToken = self.targetToken ftsJob.FTSServer = self.ftsServer if guid: ftsJob.FTSGUID = guid for ftsFile in self.ftsFiles: ftsFile.Attempt += 1 ftsFile.Error = "" ftsJob.addFile(ftsFile) self.ftsJob = ftsJob def __submitFTSTransfer(self): """ create and execute glite-transfer-submit CLI command :param self: self reference """ log = gLogger.getSubLogger('Submit') self.__createFTSJob() submit = self.ftsJob.submitFTS2(command=self.submitCommand) if not submit["OK"]: log.error("unable to submit FTSJob: %s" % submit["Message"]) return submit log.info("FTSJob '%s'@'%s' has been submitted" % (self.ftsJob.FTSGUID, self.ftsJob.FTSServer)) # # update statuses for job files for ftsFile in self.ftsJob: ftsFile.FTSGUID = self.ftsJob.FTSGUID ftsFile.Status = "Submitted" ftsFile.Attempt += 1 log.info("FTSJob '%s'@'%s' has been submitted" % (self.ftsJob.FTSGUID, self.ftsJob.FTSServer)) self.ftsGUID = self.ftsJob.FTSGUID return S_OK() def __resolveFTSServer(self): """ resolve FTS server to use, it should be the closest one from target SE :param self: self reference """ from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTSServersForSites if not self.targetSE: return S_ERROR("Target SE not set") res = getSitesForSE(self.targetSE) if not res['OK'] or not res['Value']: return S_ERROR("Could not determine target site") targetSites = res['Value'] targetSite = '' for targetSite in targetSites: targetFTS = getFTSServersForSites([targetSite]) if targetFTS['OK']: ftsTarget = targetFTS['Value'][targetSite] if ftsTarget: self.ftsServer = ftsTarget return S_OK(self.ftsServer) else: return targetFTS return S_ERROR('No FTS server found for %s' % targetSite) #################################################################### # # Methods for monitoring # def summary(self, untilTerminal=False, printOutput=False): """ summary of FTS job :param self: self reference :param bool untilTerminal: flag to monitor FTS job to its final state :param bool printOutput: flag to print out monitoring information to the stdout """ res = self.__isSummaryValid() if not res['OK']: return res while not self.isTerminal: res = self.__parseOutput(full=True) if not res['OK']: return res if untilTerminal: self.__print() self.isRequestTerminal() if res['Value'] or (not untilTerminal): break time.sleep(1) if untilTerminal: print "" if printOutput and (not untilTerminal): return self.dumpSummary(printOutput=printOutput) return S_OK() def monitor(self, untilTerminal=False, printOutput=False, full=True): """ monitor FTS job :param self: self reference :param bool untilTerminal: flag to monitor FTS job to its final state :param bool printOutput: flag to print out monitoring information to the stdout """ if not self.ftsJob: self.resolveSource() self.__createFTSJob(self.ftsGUID) res = self.__isSummaryValid() if not res['OK']: return res if untilTerminal: res = self.summary(untilTerminal=untilTerminal, printOutput=printOutput) if not res['OK']: return res res = self.__parseOutput(full=full) if not res['OK']: return res if untilTerminal: self.finalize() if printOutput: self.dump() return res def dumpSummary(self, printOutput=False): """ get FTS job summary as str :param self: self reference :param bool printOutput: print summary to stdout """ outStr = '' for status in sorted(self.statusSummary): if self.statusSummary[status]: outStr = '%s\t%-10s : %-10s\n' % ( outStr, status, str(self.statusSummary[status])) outStr = outStr.rstrip('\n') if printOutput: print outStr return S_OK(outStr) def __print(self): """ print progress bar of FTS job completeness to stdout :param self: self reference """ width = 100 bits = int((width * self.percentageComplete) / 100) outStr = "|%s>%s| %.1f%s %s %s" % ("=" * bits, " " * (width - bits), self.percentageComplete, "%", self.requestStatus, " " * 10) sys.stdout.write("%s\r" % (outStr)) sys.stdout.flush() def dump(self): """ print FTS job parameters and files to stdout :param self: self reference """ print "%-10s : %-10s" % ("Status", self.requestStatus) print "%-10s : %-10s" % ("Source", self.sourceSE) print "%-10s : %-10s" % ("Target", self.targetSE) print "%-10s : %-128s" % ("Server", self.ftsServer) print "%-10s : %-128s" % ("GUID", self.ftsGUID) for lfn in sorted(self.fileDict): print "\n %-15s : %-128s" % ('LFN', lfn) for key in ['Source', 'Target', 'Status', 'Reason', 'Duration']: print " %-15s : %-128s" % (key, str(self.fileDict[lfn].get(key))) return S_OK() def __isSummaryValid(self): """ check validity of FTS job summary report :param self: self reference """ if not self.ftsServer: return S_ERROR("FTSServer not set") if not self.ftsGUID: return S_ERROR("FTSGUID not set") return S_OK() def __parseOutput(self, full=False): """ execute glite-transfer-status command and parse its output :param self: self reference :param bool full: glite-transfer-status verbosity level, when set, collect information of files as well """ monitor = self.ftsJob.monitorFTS2(command=self.monitorCommand, full=full) if not monitor['OK']: return monitor self.percentageComplete = self.ftsJob.Completeness self.requestStatus = self.ftsJob.Status self.submitTime = self.ftsJob.SubmitTime statusSummary = monitor['Value'] if statusSummary: for state in statusSummary: self.statusSummary[state] = statusSummary[state] self.transferTime = 0 for ftsFile in self.ftsJob: lfn = ftsFile.LFN self.__setFileParameter(lfn, 'Status', ftsFile.Status) self.__setFileParameter(lfn, 'Reason', ftsFile.Error) self.__setFileParameter(lfn, 'Duration', ftsFile._duration) targetURL = self.__getFileParameter(lfn, 'Target') if not targetURL['OK']: self.__setFileParameter(lfn, 'Target', ftsFile.TargetSURL) self.transferTime += int(ftsFile._duration) return S_OK() #################################################################### # # Methods for finalization # def finalize(self): """ finalize FTS job :param self: self reference """ self.__updateMetadataCache() transEndTime = dateTime() regStartTime = time.time() res = self.getTransferStatistics() transDict = res['Value'] res = self.__registerSuccessful(transDict['transLFNs']) regSuc, regTotal = res['Value'] regTime = time.time() - regStartTime if self.sourceSE and self.targetSE: self.__sendAccounting(regSuc, regTotal, regTime, transEndTime, transDict) return S_OK() def getTransferStatistics(self): """ collect information of Transfers that can be used by Accounting :param self: self reference """ transDict = { 'transTotal': len(self.fileDict), 'transLFNs': [], 'transOK': 0, 'transSize': 0 } for lfn in self.fileDict: if self.fileDict[lfn].get('Status') in self.successfulStates: if self.fileDict[lfn].get('Duration', 0): transDict['transLFNs'].append(lfn) transDict['transOK'] += 1 if lfn in self.catalogMetadata: transDict['transSize'] += self.catalogMetadata[ lfn].get('Size', 0) return S_OK(transDict) def getFailedRegistrations(self): """ get failed registrations dict :param self: self reference """ return S_OK(self.failedRegistrations) def __registerSuccessful(self, transLFNs): """ register successfully transferred files to the catalogs, fill failedRegistrations dict for files that failed to register :param self: self reference :param list transLFNs: LFNs in FTS job """ self.failedRegistrations = {} toRegister = {} for lfn in transLFNs: res = returnSingleResult( self.oTargetSE.getPfnForProtocol( self.fileDict[lfn].get('Target'), protocol='SRM2', withPort=False)) if not res['OK']: self.__setFileParameter(lfn, 'Reason', res['Message']) self.__setFileParameter(lfn, 'Status', 'Failed') else: toRegister[lfn] = {'PFN': res['Value'], 'SE': self.targetSE} if not toRegister: return S_OK((0, 0)) res = self.__getCatalogObject() if not res['OK']: for lfn in toRegister: self.failedRegistrations = toRegister self.log.error('Failed to get Catalog Object', res['Message']) return S_OK((0, len(toRegister))) res = self.oCatalog.addReplica(toRegister) if not res['OK']: self.failedRegistrations = toRegister self.log.error('Failed to get Catalog Object', res['Message']) return S_OK((0, len(toRegister))) for lfn, error in res['Value']['Failed'].items(): self.failedRegistrations[lfn] = toRegister[lfn] self.log.error('Registration of Replica failed', '%s : %s' % (lfn, str(error))) return S_OK((len(res['Value']['Successful']), len(toRegister))) def __sendAccounting(self, regSuc, regTotal, regTime, transEndTime, transDict): """ send accounting record :param self: self reference :param regSuc: number of files successfully registered :param regTotal: number of files attepted to register :param regTime: time stamp at the end of registration :param transEndTime: time stamp at the end of FTS job :param dict transDict: dict holding couters for files being transerred, their sizes and successfull transfers """ oAccounting = DataOperation() oAccounting.setEndTime(transEndTime) oAccounting.setStartTime(self.submitTime) accountingDict = {} accountingDict['OperationType'] = 'replicateAndRegister' result = getProxyInfo() if not result['OK']: userName = '******' else: userName = result['Value'].get('username', 'unknown') accountingDict['User'] = userName accountingDict[ 'Protocol'] = 'FTS' if 'fts3' not in self.ftsServer else 'FTS3' accountingDict['RegistrationTime'] = regTime accountingDict['RegistrationOK'] = regSuc accountingDict['RegistrationTotal'] = regTotal accountingDict['TransferOK'] = transDict['transOK'] accountingDict['TransferTotal'] = transDict['transTotal'] accountingDict['TransferSize'] = transDict['transSize'] accountingDict['FinalStatus'] = self.requestStatus accountingDict['Source'] = self.sourceSE accountingDict['Destination'] = self.targetSE accountingDict['TransferTime'] = self.transferTime oAccounting.setValuesFromDict(accountingDict) self.log.verbose("Attempting to commit accounting message...") oAccounting.commit() self.log.verbose("...committed.") return S_OK()
def resolvePFNNotRegistered( self, problematicDict ): """ This takes the problematic dictionary returned by the integrity DB and resolved the PFNNotRegistered prognosis """ lfn = problematicDict['LFN'] pfn = problematicDict['PFN'] seName = problematicDict['SE'] fileID = problematicDict['FileID'] se = StorageElement( seName ) res = Utils.executeSingleFileOrDirWrapper( self.fc.exists( lfn ) ) if not res['OK']: return self.__returnProblematicError( fileID, res ) if not res['Value']: # The file does not exist in the catalog res = Utils.executeSingleFileOrDirWrapper( se.removeFile( pfn ) ) if not res['OK']: return self.__returnProblematicError( fileID, res ) return self.__updateCompletedFiles( 'PFNNotRegistered', fileID ) res = Utils.executeSingleFileOrDirWrapper( se.getFileMetadata( pfn ) ) if ( not res['OK'] ) and ( re.search( 'File does not exist', res['Message'] ) ): gLogger.info( "PFNNotRegistered replica (%d) found to be missing." % fileID ) return self.__updateCompletedFiles( 'PFNNotRegistered', fileID ) elif not res['OK']: return self.__returnProblematicError( fileID, res ) storageMetadata = res['Value'] if storageMetadata['Lost']: gLogger.info( "PFNNotRegistered replica (%d) found to be Lost. Updating prognosis" % fileID ) return self.changeProblematicPrognosis( fileID, 'PFNLost' ) if storageMetadata['Unavailable']: gLogger.info( "PFNNotRegistered replica (%d) found to be Unavailable. Updating retry count" % fileID ) return self.incrementProblematicRetry( fileID ) # HACK until we can obtain the space token descriptions through GFAL site = seName.split( '_' )[0].split( '-' )[0] if not storageMetadata['Cached']: if lfn.endswith( '.raw' ): seName = '%s-RAW' % site else: seName = '%s-RDST' % site elif storageMetadata['Migrated']: if lfn.startswith( '/lhcb/data' ): seName = '%s_M-DST' % site else: seName = '%s_MC_M-DST' % site else: if lfn.startswith( '/lhcb/data' ): seName = '%s-DST' % site else: seName = '%s_MC-DST' % site problematicDict['SE'] = seName res = se.getPfnForProtocol( pfn, withPort = False ) if not res['OK']: return self.__returnProblematicError( fileID, res ) for pfn, error in res['Value']['Failed'].items(): gLogger.error( 'Failed to obtain registered PFN for physical file', '%s %s' % ( pfn, error ) ) return S_ERROR( 'Failed to obtain registered PFNs from physical file' ) problematicDict['PFN'] = res['Value']['Successful'][pfn] res = Utils.executeSingleFileOrDirWrapper( self.fc.addReplica( {lfn:problematicDict} ) ) if not res['OK']: return self.__returnProblematicError( fileID, res ) res = Utils.executeSingleFileOrDirWrapper( self.fc.getFileMetadata( lfn ) ) if not res['OK']: return self.__returnProblematicError( fileID, res ) if res['Value']['Size'] != storageMetadata['Size']: gLogger.info( "PFNNotRegistered replica (%d) found with catalog size mismatch. Updating prognosis" % fileID ) return self.changeProblematicPrognosis( fileID, 'CatalogPFNSizeMismatch' ) return self.__updateCompletedFiles( 'PFNNotRegistered', fileID )
def execute(self): IntegrityDB = RPCClient('DataManagement/DataIntegrity') res = self.RequestDBClient.getRequest('integrity') if not res['OK']: gLogger.info( "SEvsLFCAgent.execute: Failed to get request from database.") return S_OK() elif not res['Value']: gLogger.info( "SEvsLFCAgent.execute: No requests to be executed found.") return S_OK() requestString = res['Value']['requestString'] requestName = res['Value']['requestName'] sourceServer = res['Value']['Server'] gLogger.info("SEvsLFCAgent.execute: Obtained request %s" % requestName) oRequest = RequestContainer(request=requestString) ################################################ # Find the number of sub-requests from the request res = oRequest.getNumSubRequests('integrity') if not res['OK']: errStr = "SEvsLFCAgent.execute: Failed to obtain number of integrity subrequests." gLogger.error(errStr, res['Message']) return S_OK() gLogger.info("SEvsLFCAgent.execute: Found %s sub requests." % res['Value']) ################################################ # For all the sub-requests in the request for ind in range(res['Value']): gLogger.info("SEvsLFCAgent.execute: Processing sub-request %s." % ind) subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'integrity')['Value'] if subRequestAttributes['Status'] == 'Waiting': subRequestFiles = oRequest.getSubRequestFiles( ind, 'integrity')['Value'] operation = subRequestAttributes['Operation'] ################################################ # If the sub-request is a lfcvsse operation if operation == 'SEvsLFC': gLogger.info( "SEvsLFCAgent.execute: Attempting to execute %s sub-request." % operation) storageElementName = subRequestAttributes['StorageElement'] for subRequestFile in subRequestFiles: if subRequestFile['Status'] == 'Waiting': lfn = subRequestFile['LFN'] storageElement = StorageElement(storageElementName) res = storageElement.isValid() if not res['OK']: errStr = "SEvsLFCAgent.execute: Failed to instantiate destination StorageElement." gLogger.error(errStr, storageElement) else: res = storageElement.getPfnForLfn(lfn) if not res['OK']: gLogger.info('shit bugger do something.') else: oNamespaceBrowser = NamespaceBrowser( res['Value']) # Loop over all the directories and sub-directories while (oNamespaceBrowser.isActive()): currentDir = oNamespaceBrowser.getActiveDir( ) gLogger.info( "SEvsLFCAgent.execute: Attempting to list the contents of %s." % currentDir) res = storageElement.listDirectory( currentDir) if not res['Value'][ 'Successful'].has_key( currentDir): gLogger.error( "SEvsLFCAgent.execute: Failed to list the directory contents.", "%s %s" % (currentDir, res['Value']['Successful'] ['Failed'][currentDir])) subDirs = [currentDir] else: subDirs = [] files = {} for surl, surlDict in res['Value'][ 'Successful'][currentDir][ 'Files'].items(): pfnRes = storageElement.getPfnForProtocol( surl, 'SRM2', withPort=False) surl = pfnRes['Value'] files[surl] = surlDict for surl, surlDict in res['Value'][ 'Successful'][currentDir][ 'SubDirs'].items(): pfnRes = storageElement.getPfnForProtocol( surl, 'SRM2', withPort=False) surl = pfnRes['Value'] subDirs.append(surl) #subDirs = res['Value']['Successful'][currentDir]['SubDirs'] gLogger.info( "SEvsLFCAgent.execute: Successfully obtained %s sub-directories." % len(subDirs)) #files = res['Value']['Successful'][currentDir]['Files'] gLogger.info( "SEvsLFCAgent.execute: Successfully obtained %s files." % len(files)) selectedLfns = [] lfnPfnDict = {} pfnSize = {} for pfn, pfnDict in files.items(): res = storageElement.getPfnPath( pfn) if not res['OK']: gLogger.error( "SEvsLFCAgent.execute: Failed to get determine LFN from pfn.", "%s %s" % (pfn, res['Message'])) fileMetadata = { 'Prognosis': 'NonConventionPfn', 'LFN': '', 'PFN': pfn, 'StorageElement': storageElementName, 'Size': pfnDict['Size'] } res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata) if res['OK']: gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) else: lfn = res['Value'] selectedLfns.append(lfn) lfnPfnDict[lfn] = pfn pfnSize[pfn] = pfnDict[ 'Size'] res = self.ReplicaManager.getCatalogFileMetadata( selectedLfns) if not res['OK']: subDirs = [currentDir] else: for lfn in res['Value'][ 'Failed'].keys(): gLogger.error( "SEvsLFCAgent.execute: Failed to get metadata.", "%s %s" % (lfn, res['Value'] ['Failed'][lfn])) pfn = lfnPfnDict[lfn] fileMetadata = { 'Prognosis': 'SEPfnNoLfn', 'LFN': lfn, 'PFN': pfn, 'StorageElement': storageElementName, 'Size': pfnSize[pfn] } res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata) if res['OK']: gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) for lfn, lfnDict in res[ 'Value'][ 'Successful'].items( ): pfn = lfnPfnDict[lfn] storageSize = pfnSize[pfn] catalogSize = lfnDict[ 'Size'] if int(catalogSize) == int( storageSize): gLogger.info( "SEvsLFCAgent.execute: Catalog and storage sizes match.", "%s %s" % (pfn, storageElementName )) gLogger.info( "Change the status in the LFC" ) elif int(storageSize) == 0: gLogger.error( "SEvsLFCAgent.execute: Physical file size is 0.", "%s %s" % (pfn, storageElementName )) fileMetadata = { 'Prognosis': 'ZeroSizePfn', 'LFN': lfn, 'PFN': pfn, 'StorageElement': storageElementName } res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata) if res['OK']: gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) else: gLogger.error( "SEvsLFCAgent.execute: Catalog and storage size mis-match.", "%s %s" % (pfn, storageElementName )) fileMetadata = { 'Prognosis': 'PfnSizeMismatch', 'LFN': lfn, 'PFN': pfn, 'StorageElement': storageElementName } res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata) if res['OK']: gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) res = self.ReplicaManager.getCatalogReplicas( selectedLfns) if not res['OK']: subDirs = [currentDir] else: for lfn in res['Value'][ 'Failed'].keys(): gLogger.error( "SEvsLFCAgent.execute: Failed to get replica information.", "%s %s" % (lfn, res['Value'] ['Failed'][lfn])) pfn = lfnPfnDict[lfn] fileMetadata = { 'Prognosis': 'PfnNoReplica', 'LFN': lfn, 'PFN': pfn, 'StorageElement': storageElementName, 'Size': pfnSize[pfn] } res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata) if res['OK']: gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) for lfn, repDict in res[ 'Value'][ 'Successful'].items( ): pfn = lfnPfnDict[lfn] registeredPfns = repDict.values( ) if not pfn in registeredPfns: gLogger.error( "SEvsLFCAgent.execute: SE PFN not registered.", "%s %s" % (lfn, pfn)) fileMetadata = { 'Prognosis': 'PfnNoReplica', 'LFN': lfn, 'PFN': pfn, 'StorageElement': storageElementName } res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata) if res['OK']: gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." ) gLogger.error( "Change the status in the LFC,ProcDB...." ) else: gLogger.error( "Shit, f**k, bugger. Add the failover." ) else: gLogger.info( "SEvsLFCAgent.execute: SE Pfn verified.", pfn) oNamespaceBrowser.updateDirs(subDirs) oRequest.setSubRequestFileAttributeValue( ind, 'integrity', lfn, 'Status', 'Done') ################################################ # If the sub-request is none of the above types else: gLogger.info( "SEvsLFCAgent.execute: Operation not supported.", operation) ################################################ # Determine whether there are any active files if oRequest.isSubRequestEmpty(ind, 'integrity')['Value']: oRequest.setSubRequestStatus(ind, 'integrity', 'Done') ################################################ # If the sub-request is already in terminal state else: gLogger.info( "SEvsLFCAgent.execute: Sub-request %s is status '%s' and not to be executed." % (ind, subRequestAttributes['Status'])) ################################################ # Generate the new request string after operation requestString = oRequest.toXML()['Value'] res = self.RequestDBClient.updateRequest(requestName, requestString, sourceServer) return S_OK()
def resolvePFNNotRegistered(self, problematicDict): """ This takes the problematic dictionary returned by the integrity DB and resolved the PFNNotRegistered prognosis """ lfn = problematicDict['LFN'] pfn = problematicDict['PFN'] seName = problematicDict['SE'] fileID = problematicDict['FileID'] se = StorageElement(seName) res = returnSingleResult(self.fc.exists(lfn)) if not res['OK']: return self.__returnProblematicError(fileID, res) if not res['Value']: # The file does not exist in the catalog res = returnSingleResult(se.removeFile(pfn)) if not res['OK']: return self.__returnProblematicError(fileID, res) return self.__updateCompletedFiles('PFNNotRegistered', fileID) res = returnSingleResult(se.getFileMetadata(pfn)) if (not res['OK']) and (re.search('File does not exist', res['Message'])): gLogger.info("PFNNotRegistered replica (%d) found to be missing." % fileID) return self.__updateCompletedFiles('PFNNotRegistered', fileID) elif not res['OK']: return self.__returnProblematicError(fileID, res) storageMetadata = res['Value'] if storageMetadata['Lost']: gLogger.info( "PFNNotRegistered replica (%d) found to be Lost. Updating prognosis" % fileID) return self.changeProblematicPrognosis(fileID, 'PFNLost') if storageMetadata['Unavailable']: gLogger.info( "PFNNotRegistered replica (%d) found to be Unavailable. Updating retry count" % fileID) return self.incrementProblematicRetry(fileID) # HACK until we can obtain the space token descriptions through GFAL site = seName.split('_')[0].split('-')[0] if not storageMetadata['Cached']: if lfn.endswith('.raw'): seName = '%s-RAW' % site else: seName = '%s-RDST' % site elif storageMetadata['Migrated']: if lfn.startswith('/lhcb/data'): seName = '%s_M-DST' % site else: seName = '%s_MC_M-DST' % site else: if lfn.startswith('/lhcb/data'): seName = '%s-DST' % site else: seName = '%s_MC-DST' % site problematicDict['SE'] = seName res = se.getPfnForProtocol(pfn, withPort=False) if not res['OK']: return self.__returnProblematicError(fileID, res) for pfn, error in res['Value']['Failed'].items(): gLogger.error('Failed to obtain registered PFN for physical file', '%s %s' % (pfn, error)) return S_ERROR( 'Failed to obtain registered PFNs from physical file') problematicDict['PFN'] = res['Value']['Successful'][pfn] res = returnSingleResult(self.fc.addReplica({lfn: problematicDict})) if not res['OK']: return self.__returnProblematicError(fileID, res) res = returnSingleResult(self.fc.getFileMetadata(lfn)) if not res['OK']: return self.__returnProblematicError(fileID, res) if res['Value']['Size'] != storageMetadata['Size']: gLogger.info( "PFNNotRegistered replica (%d) found with catalog size mismatch. Updating prognosis" % fileID) return self.changeProblematicPrognosis(fileID, 'CatalogPFNSizeMismatch') return self.__updateCompletedFiles('PFNNotRegistered', fileID)
class FTSRequest: def __init__(self): self.gridEnv = '/afs/cern.ch/project/gd/LCG-share/3.2.8-0/etc/profile.d/grid-env' self.finalStates = [ 'Canceled', 'Failed', 'Hold', 'Finished', 'FinishedDirty' ] self.failedStates = ['Canceled', 'Failed', 'Hold', 'FinishedDirty'] self.successfulStates = ['Finished', 'Done'] self.fileStates = [ 'Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed', 'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting' ] self.newlyCompletedFiles = [] self.newlyFailedFiles = [] self.statusSummary = {} self.requestStatus = 'Unknown' self.fileDict = {} self.catalogReplicas = {} self.catalogMetadata = {} self.oCatalog = None self.submitTime = '' self.ftsGUID = '' self.ftsServer = '' self.priority = 3 self.isTerminal = False self.percentageComplete = 0.0 self.sourceSE = '' self.sourceValid = False self.sourceToken = '' self.targetSE = '' self.targetValid = False self.targetToken = '' self.dumpStr = '' #################################################################### # # Methods for setting/getting/checking the SEs # def setSourceSE(self, se): if se == self.targetSE: return S_ERROR("SourceSE is TargetSE") self.sourceSE = se self.oSourceSE = StorageElement(self.sourceSE) return self.__checkSourceSE() def getSourceSE(self): if not self.sourceSE: return S_ERROR("Source SE not defined") return S_OK(self.sourceSE) def setSourceToken(self, token): self.sourceToken = token return S_OK() def getSourceToken(self): if not self.sourceToken: return S_ERROR("Source token not defined") return S_OK(self.sourceToken) def __checkSourceSE(self): if not self.sourceSE: return S_ERROR("SourceSE not set") res = self.oSourceSE.isValid('Read') if not res['OK']: return S_ERROR("SourceSE not available for reading") res = self.__getSESpaceToken(self.oSourceSE) if not res['OK']: gLogger.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message']) return S_ERROR("SourceSE does not support FTS transfers") self.sourceToken = res['Value'] self.sourceValid = True return S_OK() def setTargetSE(self, se): if se == self.sourceSE: return S_ERROR("TargetSE is SourceSE") self.targetSE = se self.oTargetSE = StorageElement(self.targetSE) return self.__checkTargetSE() def getTargetSE(self): if not self.targetSE: return S_ERROR("Target SE not defined") return S_OK(self.targetSE) def setTargetToken(self, token): self.targetToken = token return S_OK() def getTargetToken(self): if not self.targetToken: return S_ERROR("Target token not defined") return S_OK(self.targetToken) def __checkTargetSE(self): if not self.targetSE: return S_ERROR("TargetSE not set") res = self.oTargetSE.isValid('Write') if not res['OK']: return S_ERROR("TargetSE not available for writing") res = self.__getSESpaceToken(self.oTargetSE) if not res['OK']: gLogger.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message']) return S_ERROR("SourceSE does not support FTS transfers") self.targetToken = res['Value'] self.targetValid = True return S_OK() def __getSESpaceToken(self, oSE): res = oSE.getStorageParameters("SRM2") if not res['OK']: return res return S_OK(res['Value'].get('SpaceToken')) #################################################################### # # Methods for setting/getting FTS request parameters # def setFTSGUID(self, guid): if not checkGuid(guid): return S_ERROR("Incorrect GUID format") self.ftsGUID = guid return S_OK() def getFTSGUID(self): if not self.ftsGUID: return S_ERROR("FTSGUID not set") return S_OK(self.ftsGUID) def setFTSServer(self, server): self.ftsServer = server return S_OK() def getFTSServer(self): if not self.ftsServer: return S_ERROR("FTSServer not set") return S_OK(self.ftsServer) def setPriority(self, priority): if not type(priority) in [types.IntType, types.LongType]: return S_ERROR("Priority must be integer") if priority < 0: priority = 0 elif priority > 5: priority = 5 self.priority = priority return S_OK(self.priority) def getPriority(self): return S_OK(self.priority) def getPercentageComplete(self): completedFiles = 0 totalFiles = 0 for state in (self.statusSummary.keys()): if state in self.successfulStates: completedFiles += self.statusSummary[state] totalFiles += self.statusSummary[state] self.percentageComplete = (float(completedFiles) * 100.0) / float(totalFiles) return S_OK(self.percentageComplete) def isRequestTerminal(self): if self.requestStatus in self.finalStates: self.isTerminal = True return S_OK(self.isTerminal) def getStatus(self): return S_OK(self.requestStatus) #################################################################### # # Methods for setting/getting/checking files and their metadata # def setLFN(self, lfn): if not self.fileDict.has_key(lfn): self.fileDict[lfn] = {} return S_OK() def setSourceSURL(self, lfn, surl): target = self.fileDict[lfn].get('Target') if target == surl: return S_ERROR("Source and target the same") self.__setFileParameter(lfn, 'Source', surl) return S_OK() def getSourceSURL(self, lfn): return self.__getFileParameter(lfn, 'Source') def setTargetSURL(self, lfn, surl): source = self.fileDict[lfn].get('Source') if source == surl: return S_ERROR("Source and target the same") self.__setFileParameter(lfn, 'Target', surl) return S_OK() def getTargetSURL(self, lfn): return self.__getFileParameter(lfn, 'Target') def getFailReason(self, lfn): return self.__getFileParameter(lfn, 'Reason') def getRetries(self, lfn): return self.__getFileParameter(lfn, 'Retries') def getTransferTime(self, lfn): return self.__getFileParameter(lfn, 'Duration') def getFailed(self): failed = [] for lfn in self.fileDict.keys(): status = self.fileDict[lfn].get('Status', '') if status in self.failedStates: failed.append(lfn) return S_OK(failed) def getDone(self): done = [] for lfn in self.fileDict.keys(): status = self.fileDict[lfn].get('Status', '') if status in self.successfulStates: done.append(lfn) return S_OK(done) def __setFileParameter(self, lfn, paramName, paramValue): self.setLFN(lfn) self.fileDict[lfn][paramName] = paramValue return S_OK() def __getFileParameter(self, lfn, paramName): if not self.fileDict.has_key(lfn): return S_ERROR("Supplied file not set") if not self.fileDict[lfn].has_key(paramName): return S_ERROR("%s not set for file" % paramName) return S_OK(self.fileDict[lfn][paramName]) #################################################################### # # Methods for submission # def submit(self, monitor=False, printOutput=True): res = self.__isSubmissionValid() if not res['OK']: return res res = self.__createSURLPairFile() if not res['OK']: return res res = self.__submitFTSTransfer() if not res['OK']: return res resDict = {'ftsGUID': self.ftsGUID, 'ftsServer': self.ftsServer} print "Submitted %s @ %s" % (self.ftsGUID, self.ftsServer) if monitor: self.monitor(untilTerminal=True, printOutput=printOutput) return S_OK(resDict) def __isSubmissionValid(self): if not self.fileDict: return S_ERROR("No files set") if not self.sourceValid: return S_ERROR("SourceSE not valid") if not self.targetValid: return S_ERROR("TargetSE not valid") if not self.ftsServer: res = self.__resolveFTSServer() if not res['OK']: return S_ERROR("FTSServer not valid") self.__resolveSource() self.__resolveTarget() res = self.__filesToSubmit() if not res['OK']: return S_ERROR("No files to submit") return S_OK() def __getCatalogObject(self): try: if not self.oCatalog: self.oCatalog = CatalogInterface() return S_OK() except: return S_ERROR() def __updateReplicaCache(self, lfns=[], overwrite=False): if not lfns: lfns = self.fileDict.keys() toUpdate = [] for lfn in lfns: if (not lfn in self.catalogReplicas.keys()) or overwrite: toUpdate.append(lfn) if not toUpdate: return S_OK() res = self.__getCatalogObject() if not res['OK']: return res res = self.oCatalog.getCatalogReplicas(toUpdate) if not res['OK']: return S_ERROR("Failed to update replica cache", res['Message']) for lfn, error in res['Value']['Failed'].items(): self.__setFileParameter(lfn, 'Reason', error) self.__setFileParameter(lfn, 'Status', 'Failed') for lfn, replicas in res['Value']['Successful'].items(): self.catalogReplicas[lfn] = replicas return S_OK() def __updateMetadataCache(self, lfns=[], overwrite=False): if not lfns: lfns = self.fileDict.keys() toUpdate = [] for lfn in lfns: if (not lfn in self.catalogMetadata.keys()) or overwrite: toUpdate.append(lfn) if not toUpdate: return S_OK() res = self.__getCatalogObject() if not res['OK']: return res res = self.oCatalog.getCatalogFileMetadata(toUpdate) if not res['OK']: return S_ERROR("Failed to get source catalog metadata", res['Message']) for lfn, error in res['Value']['Failed'].items(): self.__setFileParameter(lfn, 'Reason', error) self.__setFileParameter(lfn, 'Status', 'Failed') for lfn, metadata in res['Value']['Successful'].items(): self.catalogMetadata[lfn] = metadata return S_OK() def __resolveSource(self): toResolve = [] for lfn in self.fileDict.keys(): if (not self.fileDict[lfn].has_key('Source')) and ( self.fileDict[lfn].get('Status') != 'Failed'): toResolve.append(lfn) if not toResolve: return S_OK() res = self.__updateMetadataCache(toResolve) if not res['OK']: return res res = self.__updateReplicaCache(toResolve) if not res['OK']: return res for lfn in toResolve: if self.fileDict[lfn].get('Status') == 'Failed': continue replicas = self.catalogReplicas.get(lfn, {}) if not replicas.has_key(self.sourceSE): self.__setFileParameter(lfn, 'Reason', "No replica at SourceSE") self.__setFileParameter(lfn, 'Status', 'Failed') continue res = self.oSourceSE.getPfnForProtocol(replicas[self.sourceSE], 'SRM2', withPort=True) if not res['OK']: self.__setFileParameter(lfn, 'Reason', res['Message']) self.__setFileParameter(lfn, 'Status', 'Failed') continue res = self.setSourceSURL(lfn, res['Value']) if not res['OK']: self.__setFileParameter(lfn, 'Reason', res['Message']) self.__setFileParameter(lfn, 'Status', 'Failed') continue toResolve = {} for lfn in self.fileDict.keys(): if self.fileDict[lfn].has_key('Source'): toResolve[self.fileDict[lfn]['Source']] = lfn if not toResolve: return S_ERROR("No eligible Source files") res = self.oSourceSE.getFileMetadata(toResolve.keys()) if not res['OK']: return S_ERROR("Failed to check source file metadata") for pfn, error in res['Value']['Failed'].items(): lfn = toResolve[pfn] if re.search('File does not exist', error): self.__setFileParameter(lfn, 'Reason', "Source file does not exist") self.__setFileParameter(lfn, 'Status', 'Failed') else: self.__setFileParameter(lfn, 'Reason', "Failed to get Source metadata") self.__setFileParameter(lfn, 'Status', 'Failed') for pfn, metadata in res['Value']['Successful'].items(): lfn = toResolve[pfn] if metadata['Unavailable']: self.__setFileParameter(lfn, 'Reason', "Source file Unavailable") self.__setFileParameter(lfn, 'Status', 'Failed') elif metadata['Lost']: self.__setFileParameter(lfn, 'Reason', "Source file Lost") self.__setFileParameter(lfn, 'Status', 'Failed') elif not metadata['Cached']: self.__setFileParameter(lfn, 'Reason', "Source file not Cached") self.__setFileParameter(lfn, 'Status', 'Failed') elif metadata['Size'] != self.catalogMetadata[lfn]['Size']: self.__setFileParameter(lfn, 'Reason', "Source size mismatch") self.__setFileParameter(lfn, 'Status', 'Failed') elif self.catalogMetadata[lfn]['Checksum'] and metadata[ 'Checksum'] and not (compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'])): self.__setFileParameter(lfn, 'Reason', "Source checksum mismatch") self.__setFileParameter(lfn, 'Status', 'Failed') return S_OK() def __resolveTarget(self): toResolve = [] for lfn in self.fileDict.keys(): if not self.fileDict[lfn].has_key('Target') and ( self.fileDict[lfn].get('Status') != 'Failed'): toResolve.append(lfn) if not toResolve: return S_OK() res = self.__updateReplicaCache(toResolve) if not res['OK']: return res atTarget = [] for lfn in sortList(toResolve): if self.fileDict[lfn].get('Status') == 'Failed': continue replicas = self.catalogReplicas.get(lfn, {}) if replicas.has_key(self.targetSE): self.__setFileParameter(lfn, 'Reason', "File already at Target") self.__setFileParameter(lfn, 'Status', 'Done') atTarget.append(lfn) for lfn in toResolve: if (self.fileDict[lfn].get('Status') == 'Failed') or (lfn in atTarget): continue res = self.oTargetSE.getPfnForLfn(lfn) if not res['OK']: self.__setFileParameter(lfn, 'Reason', "Failed to create Target") self.__setFileParameter(lfn, 'Status', 'Failed') continue res = self.oTargetSE.getPfnForProtocol(res['Value'], 'SRM2', withPort=True) if not res['OK']: self.__setFileParameter(lfn, 'Reason', res['Message']) self.__setFileParameter(lfn, 'Status', 'Failed') continue res = self.setTargetSURL(lfn, res['Value']) if not res['OK']: self.__setFileParameter(lfn, 'Reason', res['Message']) self.__setFileParameter(lfn, 'Status', 'Failed') continue toResolve = {} for lfn in self.fileDict.keys(): if self.fileDict[lfn].has_key('Target'): toResolve[self.fileDict[lfn]['Target']] = lfn if not toResolve: return S_ERROR("No eligible Target files") res = self.oTargetSE.exists(toResolve.keys()) if not res['OK']: return S_ERROR("Failed to check target existence") for pfn, error in res['Value']['Failed'].items(): lfn = toResolve[pfn] self.__setFileParameter(lfn, 'Reason', error) self.__setFileParameter(lfn, 'Status', 'Failed') toRemove = [] for pfn, exists in res['Value']['Successful'].items(): if exists: lfn = toResolve[pfn] res = self.getSourceSURL(lfn) if not res['OK']: self.__setFileParameter(lfn, 'Reason', "Target exists") self.__setFileParameter(lfn, 'Status', 'Failed') elif res['Value'] == pfn: self.__setFileParameter(lfn, 'Reason', "Source and Target the same") self.__setFileParameter(lfn, 'Status', 'Failed') else: toRemove.append(pfn) if toRemove: self.oTargetSE.removeFile(toRemove) return S_OK() def __filesToSubmit(self): for lfn in self.fileDict.keys(): lfnStatus = self.fileDict[lfn].get('Status') source = self.fileDict[lfn].get('Source') target = self.fileDict[lfn].get('Target') if (lfnStatus != 'Failed') and (lfnStatus != 'Done') and source and target: return S_OK() return S_ERROR() def __createSURLPairFile(self): fd, fileName = tempfile.mkstemp() surlFile = os.fdopen(fd, 'w') for lfn in self.fileDict.keys(): lfnStatus = self.fileDict[lfn].get('Status') source = self.fileDict[lfn].get('Source') target = self.fileDict[lfn].get('Target') if (lfnStatus != 'Failed') and (lfnStatus != 'Done') and source and target: surlString = '%s %s\n' % (source, target) surlFile.write(surlString) surlFile.close() self.surlFile = fileName return S_OK() def __submitFTSTransfer(self): comm = [ 'glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile, '-o' ] if self.targetToken: comm.append('-t') comm.append(self.targetToken) if self.sourceToken: comm.append('-S') comm.append(self.sourceToken) res = executeGridCommand('', comm, self.gridEnv) os.remove(self.surlFile) if not res['OK']: return res returnCode, output, errStr = res['Value'] if not returnCode == 0: return S_ERROR(errStr) guid = output.replace('\n', '') if not checkGuid(guid): return S_ERROR('Wrong GUID format returned') self.ftsGUID = guid #if self.priority != 3: # comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)] # executeGridCommand('',comm,self.gridEnv) return res def __resolveFTSServer(self): if not self.sourceSE: return S_ERROR("Source SE not set") if not self.targetSE: return S_ERROR("Target SE not set") res = getSitesForSE(self.sourceSE, 'LCG') if not res['OK'] or not res['Value']: return S_ERROR("Could not determine source site") sourceSite = res['Value'][0] res = getSitesForSE(self.targetSE, 'LCG') if not res['OK'] or not res['Value']: return S_ERROR("Could not determine target site") targetSite = res['Value'][0] if (sourceSite == 'LCG.CERN.ch') or (targetSite == 'LCG.CERN.ch'): ep = 'LCG.CERN.ch' else: # Target site FTS server should be used ep = targetSite try: configPath = '/Resources/FTSEndpoints/%s' % ep endpointURL = gConfig.getValue(configPath) if not endpointURL: errStr = "FTSRequest.__resolveFTSEndpoint: Failed to find FTS endpoint, check CS entry for '%s'." % ep return S_ERROR(errStr) self.ftsServer = endpointURL return S_OK(endpointURL) except Exception, x: return S_ERROR( 'FTSRequest.__resolveFTSEndpoint: Failed to obtain endpoint details from CS' )