def __getCatalogObject(self): try: if not self.oCatalog: self.oCatalog = CatalogInterface() return S_OK() except: return S_ERROR()
def __getCatalogObject(self): """ CatalogInterface instance facade :param self: self reference """ try: if not self.oCatalog: self.oCatalog = CatalogInterface() return S_OK() except: return S_ERROR()
def __getCatalogObject( self ): try: if not self.oCatalog: self.oCatalog = CatalogInterface() return S_OK() except: return S_ERROR()
def __getCatalogObject( self ): """ CatalogInterface instance facade :param self: self reference """ try: if not self.oCatalog: self.oCatalog = CatalogInterface() return S_OK() except: return S_ERROR()
class FTSRequest( object ): """ .. class:: FTSRequest Helper class for FTS job submission and monitoring. """ # # default checksum type __defaultCksmType = "ADLER32" # # flag to disablr/enable checksum test, default: disabled __cksmTest = False def __init__( self ): """c'tor :param self: self reference """ self.log = gLogger.getSubLogger( self.__class__.__name__, True ) # # final states tuple self.finalStates = ( 'Canceled', 'Failed', 'Hold', 'Finished', 'FinishedDirty' ) # # failed states tuple self.failedStates = ( 'Canceled', 'Failed', 'Hold', 'FinishedDirty' ) # # successful states tuple self.successfulStates = ( 'Finished', 'Done' ) # # all file states tuple self.fileStates = ( 'Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed', 'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting' ) self.newlyCompletedFiles = [] self.newlyFailedFiles = [] self.statusSummary = {} # # request status self.requestStatus = 'Unknown' # # dict for FTS job files self.fileDict = {} # # dict for replicas information self.catalogReplicas = {} # # dict for metadata information self.catalogMetadata = {} # # dict for files that failed to register self.failedRegistrations = {} # # placehoder for CatalogInterface reference self.oCatalog = None # # submit timestamp self.submitTime = '' # # placeholder FTS job GUID self.ftsGUID = '' # # placeholder for FTS server URL self.ftsServer = '' # # not used self.priority = 3 # # flag marking FTS job completness self.isTerminal = False # # completness percentage self.percentageComplete = 0.0 # # source SE name self.sourceSE = '' # # flag marking source SE validity self.sourceValid = False # # source space token self.sourceToken = '' # # target SE name self.targetSE = '' # # flag marking target SE validity self.targetValid = False # # target space token self.targetToken = '' # # whatever self.dumpStr = '' # # placeholder for surl file self.surlFile = None # # placeholder for target StorageElement self.oTargetSE = None # # placeholder for source StorageElement self.oSourceSE = None # # checksum type, set it to default self.__cksmType = self.__defaultCksmType # # disable checksum test by default self.__cksmTest = False # # replica manager handler self.replicaManager = ReplicaManager() # # statuses that prevent submitting to FTS self.noSubmitStatus = ( 'Failed', 'Done', 'Staging' ) # # were sources resolved? self.sourceResolved = False # # Number of file transfers actually submitted self.submittedFiles = 0 #################################################################### # # Methods for setting/getting/checking the SEs # def setSourceSE( self, se ): """ set SE for source :param self: self reference :param str se: source SE name """ if se == self.targetSE: return S_ERROR( "SourceSE is TargetSE" ) self.sourceSE = se self.oSourceSE = StorageElement( self.sourceSE ) return self.__checkSourceSE() def getSourceSE( self ): """ source SE getter :param self: self reference """ if not self.sourceSE: return S_ERROR( "Source SE not defined" ) return S_OK( self.sourceSE ) def setSourceToken( self, token ): """ set source space token :param self: self reference :param str token: source space token """ self.sourceToken = token return S_OK() def getSourceToken( self ): """ source space token getter :param self: self reference """ if not self.sourceToken: return S_ERROR( "Source token not defined" ) return S_OK( self.sourceToken ) def __checkSourceSE( self ): """ check source SE availability :param self: self reference """ if not self.sourceSE: return S_ERROR( "SourceSE not set" ) res = self.oSourceSE.isValid( 'Read' ) if not res['OK']: return S_ERROR( "SourceSE not available for reading" ) res = self.__getSESpaceToken( self.oSourceSE ) if not res['OK']: self.log.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message'] ) return S_ERROR( "SourceSE does not support FTS transfers" ) if self.__cksmTest: res = self.oSourceSE.getChecksumType() if not res["OK"]: self.log.error( "Unable to get checksum type for SourceSE %s: %s" % ( self.sourceSE, res["Message"] ) ) cksmType = res["Value"] if cksmType in ( "NONE", "NULL" ): self.log.warn( "Checksum type set to %s at SourceSE %s, disabling checksum test" % ( cksmType, self.sourceSE ) ) self.__cksmTest = False elif cksmType != self.__cksmType: self.log.warn( "Checksum type mismatch, disabling checksum test" ) self.__cksmTest = False self.sourceToken = res['Value'] self.sourceValid = True return S_OK() def setTargetSE( self, se ): """ set target SE :param self: self reference :param str se: target SE name """ if se == self.sourceSE: return S_ERROR( "TargetSE is SourceSE" ) self.targetSE = se self.oTargetSE = StorageElement( self.targetSE ) return self.__checkTargetSE() def getTargetSE( self ): """ target SE getter :param self: self reference """ if not self.targetSE: return S_ERROR( "Target SE not defined" ) return S_OK( self.targetSE ) def setTargetToken( self, token ): """ target space token setter :param self: self reference :param str token: target space token """ self.targetToken = token return S_OK() def getTargetToken( self ): """ target space token getter :param self: self reference """ if not self.targetToken: return S_ERROR( "Target token not defined" ) return S_OK( self.targetToken ) def __checkTargetSE( self ): """ check target SE availability :param self: self reference """ if not self.targetSE: return S_ERROR( "TargetSE not set" ) res = self.oTargetSE.isValid( 'Write' ) if not res['OK']: return S_ERROR( "TargetSE not available for writing" ) res = self.__getSESpaceToken( self.oTargetSE ) if not res['OK']: self.log.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message'] ) return S_ERROR( "TargetSE does not support FTS transfers" ) # # check checksum types if self.__cksmTest: res = self.oTargetSE.getChecksumType() if not res["OK"]: self.log.error( "Unable to get checksum type for TargetSE %s: %s" % ( self.targetSE, res["Message"] ) ) cksmType = res["Value"] if cksmType in ( "NONE", "NULL" ): self.log.warn( "Checksum type set to %s at TargetSE %s, disabling checksum test" % ( cksmType, self.targetSE ) ) self.__cksmTest = False elif cksmType != self.__cksmType: self.log.warn( "Checksum type mismatch, disabling checksum test" ) self.__cksmTest = False self.targetToken = res['Value'] self.targetValid = True return S_OK() @staticmethod def __getSESpaceToken( oSE ): """ get space token from StorageElement instance :param self: self reference :param StorageElement oSE: StorageElement instance """ res = oSE.getStorageParameters( "SRM2" ) if not res['OK']: return res return S_OK( res['Value'].get( 'SpaceToken' ) ) #################################################################### # # Methods for setting/getting FTS request parameters # def setFTSGUID( self, guid ): """ FTS job GUID setter :param self: self reference :param str guid: string containg GUID """ if not checkGuid( guid ): return S_ERROR( "Incorrect GUID format" ) self.ftsGUID = guid return S_OK() def getFTSGUID( self ): """ FTS job GUID getter :param self: self refenece """ if not self.ftsGUID: return S_ERROR( "FTSGUID not set" ) return S_OK( self.ftsGUID ) def setFTSServer( self, server ): """ FTS server setter :param self: self reference :param str server: FTS server URL """ self.ftsServer = server return S_OK() def getFTSServer( self ): """ FTS server getter :param self: self reference """ if not self.ftsServer: return S_ERROR( "FTSServer not set" ) return S_OK( self.ftsServer ) def setPriority( self, priority ): """ set priority for FTS job :param self: self reference :param int priority: a new priority """ if not type( priority ) in ( IntType, LongType ): return S_ERROR( "Priority must be integer" ) if priority < 0: priority = 0 elif priority > 5: priority = 5 self.priority = priority return S_OK( self.priority ) def getPriority( self ): """ FTS job priority getter :param self: self reference """ return S_OK( self.priority ) def getPercentageComplete( self ): """ get completness percentage :param self: self reference """ completedFiles = 0 totalFiles = 0 for state in self.statusSummary: if state in self.successfulStates: completedFiles += self.statusSummary[state] totalFiles += self.statusSummary[state] self.percentageComplete = ( float( completedFiles ) * 100.0 ) / float( totalFiles ) return S_OK( self.percentageComplete ) def isRequestTerminal( self ): """ check if FTS job has terminated :param self: self reference """ if self.requestStatus in self.finalStates: self.isTerminal = True return S_OK( self.isTerminal ) def getStatus( self ): """ get FTS job status :param self: self reference """ return S_OK( self.requestStatus ) def setCksmType( self, cksm = None ): """ set checksum type to use :param self: self reference :param mixed cksm: checksum type, should be one of 'Adler32', 'md5', 'sha1', None """ if str( cksm ).upper() not in ( "ADLER32", "MD5", "SHA1", "NONE" ): return S_ERROR( "Not supported checksum type: %s" % str( cksm ) ) if not cksm: self.__cksmType = None return S_OK( False ) self.__cksmType = str( cksm ).upper() return S_OK( True ) def getCksmType( self ): """ get checksum type :param self: self reference """ return S_OK( self.__cksmType ) def setCksmTest( self, cksmTest = False ): """ set cksm test :param self: self reference :param bool cksmTest: flag to enable/disable checksum test """ self.__cksmTest = bool( cksmTest ) return S_OK( self.__cksmTest ) def getCksmTest( self ): """ get cksm test flag :param self: self reference """ return S_OK( self.__cksmTest ) #################################################################### # # Methods for setting/getting/checking files and their metadata # def setLFN( self, lfn ): """ add LFN :lfn: to :fileDict: :param self: self reference :param str lfn: LFN to add to """ self.fileDict.setdefault( lfn, {'Status':'Waiting'} ) return S_OK() def setStatus( self, lfn, status ): """ set status of a file """ return( self.__setFileParameter( lfn, 'Status', status ) ) def setSourceSURL( self, lfn, surl ): """ source SURL setter :param self: self reference :param str lfn: LFN :param str surl: source SURL """ target = self.fileDict[lfn].get( 'Target' ) if target == surl: return S_ERROR( "Source and target the same" ) return( self.__setFileParameter( lfn, 'Source', surl ) ) def getSourceSURL( self, lfn ): """ get source SURL for LFN :lfn: :param self: self reference :param str lfn: LFN """ return self.__getFileParameter( lfn, 'Source' ) def setTargetSURL( self, lfn, surl ): """ set target SURL for LFN :lfn: :param self: self reference :param str lfn: LFN :param str surl: target SURL """ source = self.fileDict[lfn].get( 'Source' ) if source == surl: return S_ERROR( "Source and target the same" ) return( self.__setFileParameter( lfn, 'Target', surl ) ) def getTargetSURL( self, lfn ): """ target SURL getter :param self: self reference :param str lfn: LFN """ return self.__getFileParameter( lfn, 'Target' ) def getFailReason( self, lfn ): """ get fail reason for file :lfn: :param self: self reference :param str lfn: LFN """ return self.__getFileParameter( lfn, 'Reason' ) def getRetries( self, lfn ): """ get number of attepmts made to transfer file :lfn: :param self: self reference :param str lfn: LFN """ return self.__getFileParameter( lfn, 'Retries' ) def getTransferTime( self, lfn ): """ get duration of transfer for file :lfn: :param self: self reference :param str lfn: LFN """ return self.__getFileParameter( lfn, 'Duration' ) def getFailed( self ): """ get list of wrongly transferred LFNs :param self: self reference """ return S_OK( [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( 'Status', '' ) in self.failedStates ] ) def getStaging( self ): """ get files set for prestaging """ return S_OK( [lfn for lfn in self.fileDict if self.fileDict[lfn].get( 'Status', '' ) == 'Staging'] ) def getDone( self ): """ get list of succesfully transferred LFNs :param self: self reference """ return S_OK( [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( 'Status', '' ) in self.successfulStates ] ) def __setFileParameter( self, lfn, paramName, paramValue ): """ set :paramName: to :paramValue: for :lfn: file :param self: self reference :param str lfn: LFN :param str paramName: parameter name :param mixed paramValue: a new parameter value """ self.setLFN( lfn ) self.fileDict[lfn][paramName] = paramValue return S_OK() def __getFileParameter( self, lfn, paramName ): """ get value of :paramName: for file :lfn: :param self: self reference :param str lfn: LFN :param str paramName: parameter name """ if lfn not in self.fileDict: return S_ERROR( "Supplied file not set" ) if paramName not in self.fileDict[lfn]: return S_ERROR( "%s not set for file" % paramName ) return S_OK( self.fileDict[lfn][paramName] ) #################################################################### # # Methods for submission # def submit( self, monitor = False, printOutput = True ): """ submit FTS job :param self: self reference :param bool monitor: flag to monitor progress of FTS job :param bool printOutput: flag to print output of execution to stdout """ res = self.__isSubmissionValid() if not res['OK']: return res res = self.__createSURLPairFile() if not res['OK']: return res res = self.__submitFTSTransfer() if not res['OK']: return res resDict = { 'ftsGUID' : self.ftsGUID, 'ftsServer' : self.ftsServer, 'submittedFiles' : self.submittedFiles } # print "Submitted %s @ %s" % ( self.ftsGUID, self.ftsServer ) if monitor: self.monitor( untilTerminal = True, printOutput = printOutput ) return S_OK( resDict ) def __isSubmissionValid( self ): """ check validity of job before submission :param self: self reference """ if not self.fileDict: return S_ERROR( "No files set" ) if not self.sourceValid: return S_ERROR( "SourceSE not valid" ) if not self.targetValid: return S_ERROR( "TargetSE not valid" ) if not self.ftsServer: res = self.__resolveFTSServer() if not res['OK']: return S_ERROR( "FTSServer not valid" ) self.resolveSource() self.resolveTarget() res = self.__filesToSubmit() if not res['OK']: return S_ERROR( "No files to submit" ) return S_OK() def __getCatalogObject( self ): """ CatalogInterface instance facade :param self: self reference """ try: if not self.oCatalog: self.oCatalog = CatalogInterface() return S_OK() except: return S_ERROR() def __updateReplicaCache( self, lfns = None, overwrite = False ): """ update replica cache for list of :lfns: :param self: self reference :param mixed lfns: list of LFNs :param bool overwrite: flag to trigger cache clearing and updating """ if not lfns: lfns = self.fileDict.keys() toUpdate = [ lfn for lfn in lfns if ( lfn not in self.catalogReplicas ) or overwrite ] if not toUpdate: return S_OK() res = self.__getCatalogObject() if not res['OK']: return res res = self.oCatalog.getCatalogReplicas( toUpdate ) if not res['OK']: return S_ERROR( "Failed to update replica cache: %s" % res['Message'] ) for lfn, error in res['Value']['Failed'].items(): self.__setFileParameter( lfn, 'Reason', error ) self.__setFileParameter( lfn, 'Status', 'Failed' ) for lfn, replicas in res['Value']['Successful'].items(): self.catalogReplicas[lfn] = replicas return S_OK() def __updateMetadataCache( self, lfns = None, overwrite = False ): """ update metadata cache for list of LFNs :param self: self reference :param list lnfs: list of LFNs :param bool overwrite: flag to trigger cache clearing and updating """ if not lfns: lfns = self.fileDict.keys() toUpdate = [ lfn for lfn in lfns if ( lfn not in self.catalogMetadata ) or overwrite ] if not toUpdate: return S_OK() res = self.__getCatalogObject() if not res['OK']: return res res = self.oCatalog.getCatalogFileMetadata( toUpdate ) if not res['OK']: return S_ERROR( "Failed to get source catalog metadata: %s" % res['Message'] ) for lfn, error in res['Value']['Failed'].items(): self.__setFileParameter( lfn, 'Reason', error ) self.__setFileParameter( lfn, 'Status', 'Failed' ) for lfn, metadata in res['Value']['Successful'].items(): self.catalogMetadata[lfn] = metadata return S_OK() def resolveSource( self ): """ resolve source SE eligible for submission :param self: self reference """ # Avoid resolving sources twice if self.sourceResolved: return S_OK() # Only resolve files that need a transfer toResolve = [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( "Status", "" ) != "Failed" ] if not toResolve: return S_OK() res = self.__updateMetadataCache( toResolve ) if not res['OK']: return res res = self.__updateReplicaCache( toResolve ) if not res['OK']: return res # Define the source URLs for lfn in toResolve: replicas = self.catalogReplicas.get( lfn, {} ) if self.sourceSE not in replicas: gLogger.warn( "resolveSource: skipping %s - not replicas at SourceSE %s" % ( lfn, self.sourceSE ) ) self.__setFileParameter( lfn, 'Reason', "No replica at SourceSE" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue res = self.oSourceSE.getPfnForProtocol( replicas[self.sourceSE], 'SRM2', withPort = True ) if not res['OK']: gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) ) self.__setFileParameter( lfn, 'Reason', res['Message'] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue res = self.setSourceSURL( lfn, res['Value'] ) if not res['OK']: gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) ) self.__setFileParameter( lfn, 'Reason', res['Message'] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue toResolve = {} for lfn in self.fileDict: if "Source" in self.fileDict[lfn]: toResolve[self.fileDict[lfn]['Source']] = lfn if not toResolve: return S_ERROR( "No eligible Source files" ) # Get metadata of the sources, to check for existance, availability and caching res = self.oSourceSE.getFileMetadata( toResolve.keys() ) if not res['OK']: return S_ERROR( "Failed to check source file metadata" ) for pfn, error in res['Value']['Failed'].items(): lfn = toResolve[pfn] if re.search( 'File does not exist', error ): gLogger.warn( "resolveSource: skipping %s - source file does not exists" % lfn ) self.__setFileParameter( lfn, 'Reason', "Source file does not exist" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) else: gLogger.warn( "resolveSource: skipping %s - failed to get source metadata" % lfn ) self.__setFileParameter( lfn, 'Reason', "Failed to get Source metadata" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) toStage = [] nbStagedFiles = 0 for pfn, metadata in res['Value']['Successful'].items(): lfn = toResolve[pfn] lfnStatus = self.fileDict.get( lfn, {} ).get( 'Status' ) if metadata['Unavailable']: gLogger.warn( "resolveSource: skipping %s - source file unavailable" % lfn ) self.__setFileParameter( lfn, 'Reason', "Source file Unavailable" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif metadata['Lost']: gLogger.warn( "resolveSource: skipping %s - source file lost" % lfn ) self.__setFileParameter( lfn, 'Reason', "Source file Lost" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif not metadata['Cached']: if lfnStatus != 'Staging': toStage.append( pfn ) elif metadata['Size'] != self.catalogMetadata[lfn]['Size']: gLogger.warn( "resolveSource: skipping %s - source file size mismatch" % lfn ) self.__setFileParameter( lfn, 'Reason', "Source size mismatch" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \ not ( compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ) ): gLogger.warn( "resolveSource: skipping %s - source file checksum mismatch" % lfn ) self.__setFileParameter( lfn, 'Reason', "Source checksum mismatch" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif lfnStatus == 'Staging': # file that was staging is now cached self.__setFileParameter( lfn, 'Status', 'Waiting' ) nbStagedFiles += 1 # Some files were being staged if nbStagedFiles: self.log.info( 'resolveSource: %d files have been staged' % nbStagedFiles ) # Launching staging of files not in cache if toStage: gLogger.warn( "resolveSource: %s source files not cached, prestaging..." % len( toStage ) ) stage = self.replicaManager.prestageStorageFile( toStage, self.sourceSE ) if not stage["OK"]: gLogger.error( "resolveSource: error is prestaging - %s" % stage["Message"] ) for pfn in toStage: lfn = toResolve[pfn] self.__setFileParameter( lfn, 'Reason', stage["Message"] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) else: for pfn in toStage: lfn = toResolve[pfn] if pfn in stage['Value']['Successful']: self.__setFileParameter( lfn, 'Status', 'Staging' ) elif pfn in stage['Value']['Failed']: self.__setFileParameter( lfn, 'Reason', stage['Value']['Failed'][pfn] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) self.sourceResolved = True return S_OK() def resolveTarget( self ): """ find target SE eligible for submission :param self: self reference """ toResolve = [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( 'Status' ) not in self.noSubmitStatus ] if not toResolve: return S_OK() res = self.__updateReplicaCache( toResolve ) if not res['OK']: return res for lfn in toResolve: res = self.oTargetSE.getPfnForLfn( lfn ) if not res['OK']: gLogger.warn( "resolveTarget: skipping %s - failed to create target pfn" % lfn ) self.__setFileParameter( lfn, 'Reason', "Failed to create Target" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue res = self.oTargetSE.getPfnForProtocol( res['Value'], 'SRM2', withPort = True ) if not res['OK']: gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) ) self.__setFileParameter( lfn, 'Reason', res['Message'] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue res = self.setTargetSURL( lfn, res['Value'] ) if not res['OK']: gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) ) self.__setFileParameter( lfn, 'Reason', res['Message'] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue toResolve = {} for lfn in self.fileDict: if "Target" in self.fileDict[lfn]: toResolve[self.fileDict[lfn]['Target']] = lfn if not toResolve: return S_ERROR( "No eligible Target files" ) res = self.oTargetSE.exists( toResolve.keys() ) if not res['OK']: return S_ERROR( "Failed to check target existence" ) for pfn, error in res['Value']['Failed'].items(): lfn = toResolve[pfn] self.__setFileParameter( lfn, 'Reason', error ) self.__setFileParameter( lfn, 'Status', 'Failed' ) toRemove = [] for pfn, exists in res['Value']['Successful'].items(): if exists: lfn = toResolve[pfn] res = self.getSourceSURL( lfn ) if not res['OK']: gLogger.warn( "resolveTarget: skipping %s - target exists" % lfn ) self.__setFileParameter( lfn, 'Reason', "Target exists" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif res['Value'] == pfn: gLogger.warn( "resolveTarget: skipping %s - source and target pfns are the same" % lfn ) self.__setFileParameter( lfn, 'Reason', "Source and Target the same" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) else: toRemove.append( pfn ) if toRemove: self.oTargetSE.removeFile( toRemove ) return S_OK() def __filesToSubmit( self ): """ check if there is at least one file to submit :return: S_OK if at least one file is present, S_ERROR otherwise """ for lfn in self.fileDict: lfnStatus = self.fileDict[lfn].get( 'Status' ) source = self.fileDict[lfn].get( 'Source' ) target = self.fileDict[lfn].get( 'Target' ) if lfnStatus not in self.noSubmitStatus and source and target: return S_OK() return S_ERROR() def __createSURLPairFile( self ): """ create LFNs file for glite-transfer-submit command This file consists one line for each fiel to be transferred: sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM] :param self: self reference """ fd, fileName = tempfile.mkstemp() surlFile = os.fdopen( fd, 'w' ) for lfn in self.fileDict: lfnStatus = self.fileDict[lfn].get( 'Status' ) source = self.fileDict[lfn].get( 'Source' ) target = self.fileDict[lfn].get( 'Target' ) if lfnStatus not in self.noSubmitStatus and source and target: cksmStr = "" # # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself if self.__cksmTest and self.__cksmType: checkSum = self.catalogMetadata.get( lfn, {} ).get( 'Checksum' ) if checkSum: cksmStr = " %s:%s" % ( self.__cksmType, intAdlerToHex( hexAdlerToInt( checkSum ) ) ) surlFile.write( "%s %s%s\n" % ( source, target, cksmStr ) ) self.submittedFiles += 1 surlFile.close() self.surlFile = fileName return S_OK() def __submitFTSTransfer( self ): """ create and execute glite-transfer-submit CLI command :param self: self reference """ comm = [ 'glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile, '-o' ] if self.targetToken: comm += [ '-t', self.targetToken ] if self.sourceToken: comm += [ '-S', self.sourceToken ] if self.__cksmTest: comm.append( "--compare-checksums" ) gLogger.verbose( 'Executing %s' % ' '.join( comm ) ) res = executeGridCommand( '', comm ) os.remove( self.surlFile ) if not res['OK']: return res returnCode, output, errStr = res['Value'] if not returnCode == 0: return S_ERROR( errStr ) guid = output.replace( '\n', '' ) if not checkGuid( guid ): return S_ERROR( 'Wrong GUID format returned' ) self.ftsGUID = guid # if self.priority != 3: # comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)] # executeGridCommand('',comm) return res def __getFTSServer( self, site ): try: configPath = '/Resources/FTSEndpoints/%s' % site endpointURL = gConfig.getValue( configPath ) if not endpointURL: errStr = "FTSRequest.__getFTSServer: Failed to find FTS endpoint, check CS entry for '%s'." % site return S_ERROR( errStr ) return S_OK( endpointURL ) except Exception, x: return S_ERROR( 'FTSRequest.__getFTSServer: Failed to obtain endpoint details from CS' )
class FTSRequest: def __init__( self ): self.gridEnv = '/afs/cern.ch/project/gd/LCG-share/3.2.8-0/etc/profile.d/grid-env' self.finalStates = ['Canceled', 'Failed', 'Hold', 'Finished', 'FinishedDirty'] self.failedStates = ['Canceled', 'Failed', 'Hold', 'FinishedDirty'] self.successfulStates = ['Finished', 'Done'] self.fileStates = ['Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed', 'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting'] self.newlyCompletedFiles = [] self.newlyFailedFiles = [] self.statusSummary = {} self.requestStatus = 'Unknown' self.fileDict = {} self.catalogReplicas = {} self.catalogMetadata = {} self.oCatalog = None self.submitTime = '' self.ftsGUID = '' self.ftsServer = '' self.priority = 3 self.isTerminal = False self.percentageComplete = 0.0 self.sourceSE = '' self.sourceValid = False self.sourceToken = '' self.targetSE = '' self.targetValid = False self.targetToken = '' self.dumpStr = '' #################################################################### # # Methods for setting/getting/checking the SEs # def setSourceSE( self, se ): if se == self.targetSE: return S_ERROR( "SourceSE is TargetSE" ) self.sourceSE = se self.oSourceSE = StorageElement( self.sourceSE ) return self.__checkSourceSE() def getSourceSE( self ): if not self.sourceSE: return S_ERROR( "Source SE not defined" ) return S_OK( self.sourceSE ) def setSourceToken( self, token ): self.sourceToken = token return S_OK() def getSourceToken( self ): if not self.sourceToken: return S_ERROR( "Source token not defined" ) return S_OK( self.sourceToken ) def __checkSourceSE( self ): if not self.sourceSE: return S_ERROR( "SourceSE not set" ) res = self.oSourceSE.isValid( 'Read' ) if not res['OK']: return S_ERROR( "SourceSE not available for reading" ) res = self.__getSESpaceToken( self.oSourceSE ) if not res['OK']: gLogger.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message'] ) return S_ERROR( "SourceSE does not support FTS transfers" ) self.sourceToken = res['Value'] self.sourceValid = True return S_OK() def setTargetSE( self, se ): if se == self.sourceSE: return S_ERROR( "TargetSE is SourceSE" ) self.targetSE = se self.oTargetSE = StorageElement( self.targetSE ) return self.__checkTargetSE() def getTargetSE( self ): if not self.targetSE: return S_ERROR( "Target SE not defined" ) return S_OK( self.targetSE ) def setTargetToken( self, token ): self.targetToken = token return S_OK() def getTargetToken( self ): if not self.targetToken: return S_ERROR( "Target token not defined" ) return S_OK( self.targetToken ) def __checkTargetSE( self ): if not self.targetSE: return S_ERROR( "TargetSE not set" ) res = self.oTargetSE.isValid( 'Write' ) if not res['OK']: return S_ERROR( "TargetSE not available for writing" ) res = self.__getSESpaceToken( self.oTargetSE ) if not res['OK']: gLogger.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message'] ) return S_ERROR( "SourceSE does not support FTS transfers" ) self.targetToken = res['Value'] self.targetValid = True return S_OK() def __getSESpaceToken( self, oSE ): res = oSE.getStorageParameters( "SRM2" ) if not res['OK']: return res return S_OK( res['Value'].get( 'SpaceToken' ) ) #################################################################### # # Methods for setting/getting FTS request parameters # def setFTSGUID( self, guid ): if not checkGuid( guid ): return S_ERROR( "Incorrect GUID format" ) self.ftsGUID = guid return S_OK() def getFTSGUID( self ): if not self.ftsGUID: return S_ERROR( "FTSGUID not set" ) return S_OK( self.ftsGUID ) def setFTSServer( self, server ): self.ftsServer = server return S_OK() def getFTSServer( self ): if not self.ftsServer: return S_ERROR( "FTSServer not set" ) return S_OK( self.ftsServer ) def setPriority( self, priority ): if not type( priority ) in [types.IntType, types.LongType]: return S_ERROR( "Priority must be integer" ) if priority < 0: priority = 0 elif priority > 5: priority = 5 self.priority = priority return S_OK( self.priority ) def getPriority( self ): return S_OK( self.priority ) def getPercentageComplete( self ): completedFiles = 0 totalFiles = 0 for state in ( self.statusSummary.keys() ): if state in self.successfulStates: completedFiles += self.statusSummary[state] totalFiles += self.statusSummary[state] self.percentageComplete = ( float( completedFiles ) * 100.0 ) / float( totalFiles ) return S_OK( self.percentageComplete ) def isRequestTerminal( self ): if self.requestStatus in self.finalStates: self.isTerminal = True return S_OK( self.isTerminal ) def getStatus( self ): return S_OK( self.requestStatus ) #################################################################### # # Methods for setting/getting/checking files and their metadata # def setLFN( self, lfn ): if not self.fileDict.has_key( lfn ): self.fileDict[lfn] = {} return S_OK() def setSourceSURL( self, lfn, surl ): target = self.fileDict[lfn].get( 'Target' ) if target == surl: return S_ERROR( "Source and target the same" ) self.__setFileParameter( lfn, 'Source', surl ) return S_OK() def getSourceSURL( self, lfn ): return self.__getFileParameter( lfn, 'Source' ) def setTargetSURL( self, lfn, surl ): source = self.fileDict[lfn].get( 'Source' ) if source == surl: return S_ERROR( "Source and target the same" ) self.__setFileParameter( lfn, 'Target', surl ) return S_OK() def getTargetSURL( self, lfn ): return self.__getFileParameter( lfn, 'Target' ) def getFailReason( self, lfn ): return self.__getFileParameter( lfn, 'Reason' ) def getRetries( self, lfn ): return self.__getFileParameter( lfn, 'Retries' ) def getTransferTime( self, lfn ): return self.__getFileParameter( lfn, 'Duration' ) def getFailed( self ): failed = [] for lfn in self.fileDict.keys(): status = self.fileDict[lfn].get( 'Status', '' ) if status in self.failedStates: failed.append( lfn ) return S_OK( failed ) def getDone( self ): done = [] for lfn in self.fileDict.keys(): status = self.fileDict[lfn].get( 'Status', '' ) if status in self.successfulStates: done.append( lfn ) return S_OK( done ) def __setFileParameter( self, lfn, paramName, paramValue ): self.setLFN( lfn ) self.fileDict[lfn][paramName] = paramValue return S_OK() def __getFileParameter( self, lfn, paramName ): if not self.fileDict.has_key( lfn ): return S_ERROR( "Supplied file not set" ) if not self.fileDict[lfn].has_key( paramName ): return S_ERROR( "%s not set for file" % paramName ) return S_OK( self.fileDict[lfn][paramName] ) #################################################################### # # Methods for submission # def submit( self, monitor = False, printOutput = True ): res = self.__isSubmissionValid() if not res['OK']: return res res = self.__createSURLPairFile() if not res['OK']: return res res = self.__submitFTSTransfer() if not res['OK']: return res resDict = {'ftsGUID':self.ftsGUID, 'ftsServer':self.ftsServer} print "Submitted %s @ %s" % ( self.ftsGUID, self.ftsServer ) if monitor: self.monitor( untilTerminal = True, printOutput = printOutput ) return S_OK( resDict ) def __isSubmissionValid( self ): if not self.fileDict: return S_ERROR( "No files set" ) if not self.sourceValid: return S_ERROR( "SourceSE not valid" ) if not self.targetValid: return S_ERROR( "TargetSE not valid" ) if not self.ftsServer: res = self.__resolveFTSServer() if not res['OK']: return S_ERROR( "FTSServer not valid" ) self.__resolveSource() self.__resolveTarget() res = self.__filesToSubmit() if not res['OK']: return S_ERROR( "No files to submit" ) return S_OK() def __getCatalogObject( self ): try: if not self.oCatalog: self.oCatalog = CatalogInterface() return S_OK() except: return S_ERROR() def __updateReplicaCache( self, lfns = [], overwrite = False ): if not lfns: lfns = self.fileDict.keys() toUpdate = [] for lfn in lfns: if ( not lfn in self.catalogReplicas.keys() ) or overwrite: toUpdate.append( lfn ) if not toUpdate: return S_OK() res = self.__getCatalogObject() if not res['OK']: return res res = self.oCatalog.getCatalogReplicas( toUpdate ) if not res['OK']: return S_ERROR( "Failed to update replica cache", res['Message'] ) for lfn, error in res['Value']['Failed'].items(): self.__setFileParameter( lfn, 'Reason', error ) self.__setFileParameter( lfn, 'Status', 'Failed' ) for lfn, replicas in res['Value']['Successful'].items(): self.catalogReplicas[lfn] = replicas return S_OK() def __updateMetadataCache( self, lfns = [], overwrite = False ): if not lfns: lfns = self.fileDict.keys() toUpdate = [] for lfn in lfns: if ( not lfn in self.catalogMetadata.keys() ) or overwrite: toUpdate.append( lfn ) if not toUpdate: return S_OK() res = self.__getCatalogObject() if not res['OK']: return res res = self.oCatalog.getCatalogFileMetadata( toUpdate ) if not res['OK']: return S_ERROR( "Failed to get source catalog metadata", res['Message'] ) for lfn, error in res['Value']['Failed'].items(): self.__setFileParameter( lfn, 'Reason', error ) self.__setFileParameter( lfn, 'Status', 'Failed' ) for lfn, metadata in res['Value']['Successful'].items(): self.catalogMetadata[lfn] = metadata return S_OK() def __resolveSource( self ): toResolve = [] for lfn in self.fileDict.keys(): if ( not self.fileDict[lfn].has_key( 'Source' ) ) and ( self.fileDict[lfn].get( 'Status' ) != 'Failed' ): toResolve.append( lfn ) if not toResolve: return S_OK() res = self.__updateMetadataCache( toResolve ) if not res['OK']: return res res = self.__updateReplicaCache( toResolve ) if not res['OK']: return res for lfn in toResolve: if self.fileDict[lfn].get( 'Status' ) == 'Failed': continue replicas = self.catalogReplicas.get( lfn, {} ) if not replicas.has_key( self.sourceSE ): self.__setFileParameter( lfn, 'Reason', "No replica at SourceSE" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue res = self.oSourceSE.getPfnForProtocol( replicas[self.sourceSE], 'SRM2', withPort = True ) if not res['OK']: self.__setFileParameter( lfn, 'Reason', res['Message'] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue res = self.setSourceSURL( lfn, res['Value'] ) if not res['OK']: self.__setFileParameter( lfn, 'Reason', res['Message'] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue toResolve = {} for lfn in self.fileDict.keys(): if self.fileDict[lfn].has_key( 'Source' ): toResolve[self.fileDict[lfn]['Source']] = lfn if not toResolve: return S_ERROR( "No eligible Source files" ) res = self.oSourceSE.getFileMetadata( toResolve.keys() ) if not res['OK']: return S_ERROR( "Failed to check source file metadata" ) for pfn, error in res['Value']['Failed'].items(): lfn = toResolve[pfn] if re.search( 'File does not exist', error ): self.__setFileParameter( lfn, 'Reason', "Source file does not exist" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) else: self.__setFileParameter( lfn, 'Reason', "Failed to get Source metadata" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) for pfn, metadata in res['Value']['Successful'].items(): lfn = toResolve[pfn] if metadata['Unavailable']: self.__setFileParameter( lfn, 'Reason', "Source file Unavailable" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif metadata['Lost']: self.__setFileParameter( lfn, 'Reason', "Source file Lost" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif not metadata['Cached']: self.__setFileParameter( lfn, 'Reason', "Source file not Cached" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif metadata['Size'] != self.catalogMetadata[lfn]['Size']: self.__setFileParameter( lfn, 'Reason', "Source size mismatch" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and not ( compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ) ): self.__setFileParameter( lfn, 'Reason', "Source checksum mismatch" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) return S_OK() def __resolveTarget( self ): toResolve = [] for lfn in self.fileDict.keys(): if not self.fileDict[lfn].has_key( 'Target' ) and ( self.fileDict[lfn].get( 'Status' ) != 'Failed' ): toResolve.append( lfn ) if not toResolve: return S_OK() res = self.__updateReplicaCache( toResolve ) if not res['OK']: return res atTarget = [] for lfn in sortList( toResolve ): if self.fileDict[lfn].get( 'Status' ) == 'Failed': continue replicas = self.catalogReplicas.get( lfn, {} ) if replicas.has_key( self.targetSE ): self.__setFileParameter( lfn, 'Reason', "File already at Target" ) self.__setFileParameter( lfn, 'Status', 'Done' ) atTarget.append( lfn ) for lfn in toResolve: if ( self.fileDict[lfn].get( 'Status' ) == 'Failed' ) or ( lfn in atTarget ): continue res = self.oTargetSE.getPfnForLfn( lfn ) if not res['OK']: self.__setFileParameter( lfn, 'Reason', "Failed to create Target" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue res = self.oTargetSE.getPfnForProtocol( res['Value'], 'SRM2', withPort = True ) if not res['OK']: self.__setFileParameter( lfn, 'Reason', res['Message'] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue res = self.setTargetSURL( lfn, res['Value'] ) if not res['OK']: self.__setFileParameter( lfn, 'Reason', res['Message'] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue toResolve = {} for lfn in self.fileDict.keys(): if self.fileDict[lfn].has_key( 'Target' ): toResolve[self.fileDict[lfn]['Target']] = lfn if not toResolve: return S_ERROR( "No eligible Target files" ) res = self.oTargetSE.exists( toResolve.keys() ) if not res['OK']: return S_ERROR( "Failed to check target existence" ) for pfn, error in res['Value']['Failed'].items(): lfn = toResolve[pfn] self.__setFileParameter( lfn, 'Reason', error ) self.__setFileParameter( lfn, 'Status', 'Failed' ) toRemove = [] for pfn, exists in res['Value']['Successful'].items(): if exists: lfn = toResolve[pfn] res = self.getSourceSURL( lfn ) if not res['OK']: self.__setFileParameter( lfn, 'Reason', "Target exists" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif res['Value'] == pfn: self.__setFileParameter( lfn, 'Reason', "Source and Target the same" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) else: toRemove.append( pfn ) if toRemove: self.oTargetSE.removeFile( toRemove ) return S_OK() def __filesToSubmit( self ): for lfn in self.fileDict.keys(): lfnStatus = self.fileDict[lfn].get( 'Status' ) source = self.fileDict[lfn].get( 'Source' ) target = self.fileDict[lfn].get( 'Target' ) if ( lfnStatus != 'Failed' ) and ( lfnStatus != 'Done' ) and source and target: return S_OK() return S_ERROR() def __createSURLPairFile( self ): fd, fileName = tempfile.mkstemp() surlFile = os.fdopen( fd, 'w' ) for lfn in self.fileDict.keys(): lfnStatus = self.fileDict[lfn].get( 'Status' ) source = self.fileDict[lfn].get( 'Source' ) target = self.fileDict[lfn].get( 'Target' ) if ( lfnStatus != 'Failed' ) and ( lfnStatus != 'Done' ) and source and target: surlString = '%s %s\n' % ( source, target ) surlFile.write( surlString ) surlFile.close() self.surlFile = fileName return S_OK() def __submitFTSTransfer( self ): comm = ['glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile, '-o'] if self.targetToken: comm.append( '-t' ) comm.append( self.targetToken ) if self.sourceToken: comm.append( '-S' ) comm.append( self.sourceToken ) res = executeGridCommand( '', comm, self.gridEnv ) os.remove( self.surlFile ) if not res['OK']: return res returnCode, output, errStr = res['Value'] if not returnCode == 0: return S_ERROR( errStr ) guid = output.replace( '\n', '' ) if not checkGuid( guid ): return S_ERROR( 'Wrong GUID format returned' ) self.ftsGUID = guid #if self.priority != 3: # comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)] # executeGridCommand('',comm,self.gridEnv) return res def __resolveFTSServer( self ): if not self.sourceSE: return S_ERROR( "Source SE not set" ) if not self.targetSE: return S_ERROR( "Target SE not set" ) res = getSitesForSE( self.sourceSE, 'LCG' ) if not res['OK'] or not res['Value']: return S_ERROR( "Could not determine source site" ) sourceSite = res['Value'][0] res = getSitesForSE( self.targetSE, 'LCG' ) if not res['OK'] or not res['Value']: return S_ERROR( "Could not determine target site" ) targetSite = res['Value'][0] if ( sourceSite == 'LCG.CERN.ch' ) or ( targetSite == 'LCG.CERN.ch' ): ep = 'LCG.CERN.ch' else: # Target site FTS server should be used ep = targetSite try: configPath = '/Resources/FTSEndpoints/%s' % ep endpointURL = gConfig.getValue( configPath ) if not endpointURL: errStr = "FTSRequest.__resolveFTSEndpoint: Failed to find FTS endpoint, check CS entry for '%s'." % ep return S_ERROR( errStr ) self.ftsServer = endpointURL return S_OK( endpointURL ) except Exception, x: return S_ERROR( 'FTSRequest.__resolveFTSEndpoint: Failed to obtain endpoint details from CS' )
######################################################################## __RCSID__ = "02424f4 (2012-03-21 17:26:24 +0100) Andrei Tsaregorodtsev <*****@*****.**>" from DIRAC.Core.Base import Script Script.setUsageMessage(""" Change status of replica of a given file or a list of files at a given Storage Element Usage: %s <lfn | fileContainingLfns> <SE> <status> """ % Script.scriptName) Script.parseCommandLine() from DIRAC.DataManagementSystem.Client.ReplicaManager import CatalogInterface catalog = CatalogInterface() import sys,os if not len(sys.argv) == 4: Script.showHelp() DIRAC.exit( -1 ) else: inputFileName = sys.argv[1] se = sys.argv[2] newStatus = sys.argv[3] if os.path.exists(inputFileName): inputFile = open(inputFileName,'r') string = inputFile.read() lfns = string.splitlines() inputFile.close()
from DIRAC import exit as DIRACExit from DIRAC.Core.Base import Script Script.setUsageMessage(""" Change status of replica of a given file or a list of files at a given Storage Element Usage: %s <lfn | fileContainingLfns> <SE> <status> """ % Script.scriptName) Script.parseCommandLine() from DIRAC.DataManagementSystem.Client.ReplicaManager import CatalogInterface catalog = CatalogInterface() import sys, os if not len(sys.argv) == 4: Script.showHelp() DIRACExit(-1) else: inputFileName = sys.argv[1] se = sys.argv[2] newStatus = sys.argv[3] if os.path.exists(inputFileName): inputFile = open(inputFileName, 'r') string = inputFile.read() lfns = string.splitlines() inputFile.close()
class FTSRequest(object): """ .. class:: FTSRequest Helper class for FTS job submission and monitoring. """ ## default checksum type __defaultCksmType = "ADLER32" ## flag to disablr/enable checksum test, default: disabled __cksmTest = False def __init__(self): """c'tor :param self: self reference """ self.log = gLogger.getSubLogger(self.__class__.__name__, True) ## final states tuple self.finalStates = ('Canceled', 'Failed', 'Hold', 'Finished', 'FinishedDirty') ## failed states tuple self.failedStates = ('Canceled', 'Failed', 'Hold', 'FinishedDirty') ## successful states tuple self.successfulStates = ('Finished', 'Done') ## all file states tuple self.fileStates = ('Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed', 'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting') self.newlyCompletedFiles = [] self.newlyFailedFiles = [] self.statusSummary = {} ## request status self.requestStatus = 'Unknown' ## dict for FTS job files self.fileDict = {} ## dict for replicas information self.catalogReplicas = {} ## dict for metadata information self.catalogMetadata = {} ## dict for files that failed to register self.failedRegistrations = {} ## placehoder for CatalogInterface reference self.oCatalog = None ## submit timestamp self.submitTime = '' ## placeholder FTS job GUID self.ftsGUID = '' ## placeholder for FTS server URL self.ftsServer = '' ## not used self.priority = 3 ## flag marking FTS job completness self.isTerminal = False ## completness percentage self.percentageComplete = 0.0 ## source SE name self.sourceSE = '' ## flag marking source SE validity self.sourceValid = False ## source space token self.sourceToken = '' ## target SE name self.targetSE = '' ## flag marking target SE validity self.targetValid = False ## target space token self.targetToken = '' ## whatever self.dumpStr = '' ## placeholder for surl file self.surlFile = None ## placeholder for target StorageElement self.oTargetSE = None ## placeholder for source StorageElement self.oSourceSE = None ## checksum type, set it to default self.__cksmType = self.__defaultCksmType ## disable checksum test by default self.__cksmTest = False ## replica manager handler self.replicaManager = ReplicaManager() #################################################################### # # Methods for setting/getting/checking the SEs # def setSourceSE(self, se): """ set SE for source :param self: self reference :param str se: source SE name """ if se == self.targetSE: return S_ERROR("SourceSE is TargetSE") self.sourceSE = se self.oSourceSE = StorageElement(self.sourceSE) return self.__checkSourceSE() def getSourceSE(self): """ source SE getter :param self: self reference """ if not self.sourceSE: return S_ERROR("Source SE not defined") return S_OK(self.sourceSE) def setSourceToken(self, token): """ set source space token :param self: self reference :param str token: source space token """ self.sourceToken = token return S_OK() def getSourceToken(self): """ source space token getter :param self: self reference """ if not self.sourceToken: return S_ERROR("Source token not defined") return S_OK(self.sourceToken) def __checkSourceSE(self): """ check source SE availability :param self: self reference """ if not self.sourceSE: return S_ERROR("SourceSE not set") res = self.oSourceSE.isValid('Read') if not res['OK']: return S_ERROR("SourceSE not available for reading") res = self.__getSESpaceToken(self.oSourceSE) if not res['OK']: self.log.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message']) return S_ERROR("SourceSE does not support FTS transfers") if self.__cksmTest: res = self.oSourceSE.getChecksumType() if not res["OK"]: self.log.error( "Unable to get checksum type for SourceSE %s: %s" % (self.sourceSE, res["Message"])) cksmType = res["Value"] if cksmType in ("NONE", "NULL"): self.log.warn( "Checksum type set to %s at SourceSE %s, disabling checksum test" % (cksmType, self.sourceSE)) self.__cksmTest = False elif cksmType != self.__cksmType: self.log.warn( "Checksum type mismatch, disabling checksum test") self.__cksmTest = False self.sourceToken = res['Value'] self.sourceValid = True return S_OK() def setTargetSE(self, se): """ set target SE :param self: self reference :param str se: target SE name """ if se == self.sourceSE: return S_ERROR("TargetSE is SourceSE") self.targetSE = se self.oTargetSE = StorageElement(self.targetSE) return self.__checkTargetSE() def getTargetSE(self): """ target SE getter :param self: self reference """ if not self.targetSE: return S_ERROR("Target SE not defined") return S_OK(self.targetSE) def setTargetToken(self, token): """ target space token setter :param self: self reference :param str token: target space token """ self.targetToken = token return S_OK() def getTargetToken(self): """ target space token getter :param self: self reference """ if not self.targetToken: return S_ERROR("Target token not defined") return S_OK(self.targetToken) def __checkTargetSE(self): """ check target SE availability :param self: self reference """ if not self.targetSE: return S_ERROR("TargetSE not set") res = self.oTargetSE.isValid('Write') if not res['OK']: return S_ERROR("TargetSE not available for writing") res = self.__getSESpaceToken(self.oTargetSE) if not res['OK']: self.log.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message']) return S_ERROR("TargetSE does not support FTS transfers") ## check checksum types if self.__cksmTest: res = self.oTargetSE.getChecksumType() if not res["OK"]: self.log.error( "Unable to get checksum type for TargetSE %s: %s" % (self.targetSE, res["Message"])) cksmType = res["Value"] if cksmType in ("NONE", "NULL"): self.log.warn( "Checksum type set to %s at TargetSE %s, disabling checksum test" % (cksmType, self.targetSE)) self.__cksmTest = False elif cksmType != self.__cksmType: self.log.warn( "Checksum type mismatch, disabling checksum test") self.__cksmTest = False self.targetToken = res['Value'] self.targetValid = True return S_OK() @staticmethod def __getSESpaceToken(oSE): """ get space token from StorageElement instance :param self: self reference :param StorageElement oSE: StorageElement instance """ res = oSE.getStorageParameters("SRM2") if not res['OK']: return res return S_OK(res['Value'].get('SpaceToken')) #################################################################### # # Methods for setting/getting FTS request parameters # def setFTSGUID(self, guid): """ FTS job GUID setter :param self: self reference :param str guid: string containg GUID """ if not checkGuid(guid): return S_ERROR("Incorrect GUID format") self.ftsGUID = guid return S_OK() def getFTSGUID(self): """ FTS job GUID getter :param self: self refenece """ if not self.ftsGUID: return S_ERROR("FTSGUID not set") return S_OK(self.ftsGUID) def setFTSServer(self, server): """ FTS server setter :param self: self reference :param str server: FTS server URL """ self.ftsServer = server return S_OK() def getFTSServer(self): """ FTS server getter :param self: self reference """ if not self.ftsServer: return S_ERROR("FTSServer not set") return S_OK(self.ftsServer) def setPriority(self, priority): """ set priority for FTS job :param self: self reference :param int priority: a new priority """ if not type(priority) in (IntType, LongType): return S_ERROR("Priority must be integer") if priority < 0: priority = 0 elif priority > 5: priority = 5 self.priority = priority return S_OK(self.priority) def getPriority(self): """ FTS job priority getter :param self: self reference """ return S_OK(self.priority) def getPercentageComplete(self): """ get completness percentage :param self: self reference """ completedFiles = 0 totalFiles = 0 for state in (self.statusSummary.keys()): if state in self.successfulStates: completedFiles += self.statusSummary[state] totalFiles += self.statusSummary[state] self.percentageComplete = (float(completedFiles) * 100.0) / float(totalFiles) return S_OK(self.percentageComplete) def isRequestTerminal(self): """ check if FTS job has terminated :param self: self reference """ if self.requestStatus in self.finalStates: self.isTerminal = True return S_OK(self.isTerminal) def getStatus(self): """ get FTS job status :param self: self reference """ return S_OK(self.requestStatus) def setCksmType(self, cksm=None): """ set checksum type to use :param self: self reference :param mixed cksm: checksum type, should be one of 'Adler32', 'md5', 'sha1', None """ if str(cksm).upper() not in ("ADLER32", "MD5", "SHA1", "NONE"): return S_ERROR("Not supported checksum type: %s" % str(cksm)) if not cksm: self.__cksmType = None return S_OK(False) self.__cksmType = str(cksm).upper() return S_OK(True) def getCksmType(self): """ get checksum type :param self: self reference """ return S_OK(self.__cksmType) def setCksmTest(self, cksmTest=False): """ set cksm test :param self: self reference :param bool cksmTest: flag to enable/disable checksum test """ self.__cksmTest = bool(cksmTest) return S_OK(self.__cksmTest) def getCksmTest(self): """ get cksm test flag :param self: self reference """ return S_OK(self.__cksmTest) #################################################################### # # Methods for setting/getting/checking files and their metadata # def setLFN(self, lfn): """ add LFN :lfn: to :fileDict: :param self: self reference :param str lfn: LFN to add to """ if lfn not in self.fileDict: self.fileDict[lfn] = {} return S_OK() def setSourceSURL(self, lfn, surl): """ source SURL setter :param self: self reference :param str lfn: LFN :param str surl: source SURL """ target = self.fileDict[lfn].get('Target') if target == surl: return S_ERROR("Source and target the same") self.__setFileParameter(lfn, 'Source', surl) return S_OK() def getSourceSURL(self, lfn): """ get source SURL for LFN :lfn: :param self: self reference :param str lfn: LFN """ return self.__getFileParameter(lfn, 'Source') def setTargetSURL(self, lfn, surl): """ set target SURL for LFN :lfn: :param self: self reference :param str lfn: LFN :param str surl: target SURL """ source = self.fileDict[lfn].get('Source') if source == surl: return S_ERROR("Source and target the same") self.__setFileParameter(lfn, 'Target', surl) return S_OK() def getTargetSURL(self, lfn): """ target SURL getter :param self: self reference :param str lfn: LFN """ return self.__getFileParameter(lfn, 'Target') def getFailReason(self, lfn): """ get fail reason for file :lfn: :param self: self reference :param str lfn: LFN """ return self.__getFileParameter(lfn, 'Reason') def getRetries(self, lfn): """ get number of attepmts made to transfer file :lfn: :param self: self reference :param str lfn: LFN """ return self.__getFileParameter(lfn, 'Retries') def getTransferTime(self, lfn): """ get duration of transfer for file :lfn: :param self: self reference :param str lfn: LFN """ return self.__getFileParameter(lfn, 'Duration') def getFailed(self): """ get list of wrongly transferred LFNs :param self: self reference """ return S_OK([ lfn for lfn in self.fileDict if self.fileDict[lfn].get('Status', '') in self.failedStates ]) def getDone(self): """ get list of succesfully transferred LFNs :param self: self reference """ return S_OK([ lfn for lfn in self.fileDict if self.fileDict[lfn].get('Status', '') in self.successfulStates ]) def __setFileParameter(self, lfn, paramName, paramValue): """ set :paramName: to :paramValue: for :lfn: file :param self: self reference :param str lfn: LFN :param str paramName: parameter name :param mixed paramValue: a new parameter value """ self.setLFN(lfn) self.fileDict[lfn][paramName] = paramValue return S_OK() def __getFileParameter(self, lfn, paramName): """ get value of :paramName: for file :lfn: :param self: self reference :param str lfn: LFN :param str paramName: parameter name """ if lfn not in self.fileDict: return S_ERROR("Supplied file not set") if paramName not in self.fileDict[lfn]: return S_ERROR("%s not set for file" % paramName) return S_OK(self.fileDict[lfn][paramName]) #################################################################### # # Methods for submission # def submit(self, monitor=False, printOutput=True): """ submit FTS job :param self: self reference :param bool monitor: flag to monitor progress of FTS job :param bool printOutput: flag to print output of execution to stdout """ res = self.__isSubmissionValid() if not res['OK']: return res res = self.__createSURLPairFile() if not res['OK']: return res res = self.__submitFTSTransfer() if not res['OK']: return res resDict = {'ftsGUID': self.ftsGUID, 'ftsServer': self.ftsServer} print "Submitted %s @ %s" % (self.ftsGUID, self.ftsServer) if monitor: self.monitor(untilTerminal=True, printOutput=printOutput) return S_OK(resDict) def __isSubmissionValid(self): """ check validity of job before submission :param self: self reference """ if not self.fileDict: return S_ERROR("No files set") if not self.sourceValid: return S_ERROR("SourceSE not valid") if not self.targetValid: return S_ERROR("TargetSE not valid") if not self.ftsServer: res = self.__resolveFTSServer() if not res['OK']: return S_ERROR("FTSServer not valid") self.resolveSource() self.resolveTarget() res = self.__filesToSubmit() if not res['OK']: return S_ERROR("No files to submit") return S_OK() def __getCatalogObject(self): """ CatalogInterface instance facade :param self: self reference """ try: if not self.oCatalog: self.oCatalog = CatalogInterface() return S_OK() except: return S_ERROR() def __updateReplicaCache(self, lfns=None, overwrite=False): """ update replica cache for list of :lfns: :param self: self reference :param mixed lfns: list of LFNs :param bool overwrite: flag to trigger cache clearing and updating """ if not lfns: lfns = self.fileDict.keys() toUpdate = [ lfn for lfn in lfns if (lfn not in self.catalogReplicas) or overwrite ] if not toUpdate: return S_OK() res = self.__getCatalogObject() if not res['OK']: return res res = self.oCatalog.getCatalogReplicas(toUpdate) if not res['OK']: return S_ERROR("Failed to update replica cache: %s" % res['Message']) for lfn, error in res['Value']['Failed'].items(): self.__setFileParameter(lfn, 'Reason', error) self.__setFileParameter(lfn, 'Status', 'Failed') for lfn, replicas in res['Value']['Successful'].items(): self.catalogReplicas[lfn] = replicas return S_OK() def __updateMetadataCache(self, lfns=None, overwrite=False): """ update metadata cache for list of LFNs :param self: self reference :param list lnfs: list of LFNs :param bool overwrite: flag to trigger cache clearing and updating """ if not lfns: lfns = self.fileDict.keys() toUpdate = [ lfn for lfn in lfns if (lfn not in self.catalogMetadata) or overwrite ] if not toUpdate: return S_OK() res = self.__getCatalogObject() if not res['OK']: return res res = self.oCatalog.getCatalogFileMetadata(toUpdate) if not res['OK']: return S_ERROR("Failed to get source catalog metadata: %s" % res['Message']) for lfn, error in res['Value']['Failed'].items(): self.__setFileParameter(lfn, 'Reason', error) self.__setFileParameter(lfn, 'Status', 'Failed') for lfn, metadata in res['Value']['Successful'].items(): self.catalogMetadata[lfn] = metadata return S_OK() def resolveSource(self): """ resolve source SE eligible for submission :param self: self reference """ toResolve = [lfn for lfn in self.fileDict] if not toResolve: return S_OK() res = self.__updateMetadataCache(toResolve) if not res['OK']: return res res = self.__updateReplicaCache(toResolve) if not res['OK']: return res for lfn in toResolve: if self.fileDict[lfn].get("Status", "") == "Failed": continue replicas = self.catalogReplicas.get(lfn, {}) if self.sourceSE not in replicas: gLogger.warn( "resolveSource: skipping %s - not replicas at SourceSE %s" % (lfn, self.sourceSE)) self.__setFileParameter(lfn, 'Reason', "No replica at SourceSE") self.__setFileParameter(lfn, 'Status', 'Failed') continue res = self.oSourceSE.getPfnForProtocol(replicas[self.sourceSE], 'SRM2', withPort=True) if not res['OK']: gLogger.warn("resolveSource: skipping %s - %s" % (lfn, res["Message"])) self.__setFileParameter(lfn, 'Reason', res['Message']) self.__setFileParameter(lfn, 'Status', 'Failed') continue res = self.setSourceSURL(lfn, res['Value']) if not res['OK']: gLogger.warn("resolveSource: skipping %s - %s" % (lfn, res["Message"])) self.__setFileParameter(lfn, 'Reason', res['Message']) self.__setFileParameter(lfn, 'Status', 'Failed') continue toResolve = {} for lfn in self.fileDict: if "Source" in self.fileDict[lfn]: toResolve[self.fileDict[lfn]['Source']] = lfn if not toResolve: return S_ERROR("No eligible Source files") res = self.oSourceSE.getFileMetadata(toResolve.keys()) if not res['OK']: return S_ERROR("Failed to check source file metadata") for pfn, error in res['Value']['Failed'].items(): lfn = toResolve[pfn] if re.search('File does not exist', error): gLogger.warn( "resolveSource: skipping %s - source file does not exists" % lfn) self.__setFileParameter(lfn, 'Reason', "Source file does not exist") self.__setFileParameter(lfn, 'Status', 'Failed') else: gLogger.warn( "resolveSource: skipping %s - failed to get source metadata" % lfn) self.__setFileParameter(lfn, 'Reason', "Failed to get Source metadata") self.__setFileParameter(lfn, 'Status', 'Failed') for pfn, metadata in res['Value']['Successful'].items(): lfn = toResolve[pfn] if metadata['Unavailable']: gLogger.warn( "resolveSource: skipping %s - source file unavailable" % lfn) self.__setFileParameter(lfn, 'Reason', "Source file Unavailable") self.__setFileParameter(lfn, 'Status', 'Failed') elif metadata['Lost']: gLogger.warn("resolveSource: skipping %s - source file lost" % lfn) self.__setFileParameter(lfn, 'Reason', "Source file Lost") self.__setFileParameter(lfn, 'Status', 'Failed') elif not metadata['Cached']: gLogger.warn( "resolveSource: source file %s not cached, prestaging..." % lfn) stage = self.replicaManager.prestageStorageFile( pfn, self.sourceSE, singleFile=True) if not stage["OK"]: gLogger.warn("resolveSource: skipping %s - %s" % (lfn, stage["Message"])) self.__setFileParameter(lfn, 'Reason', stage["Message"]) self.__setFileParameter(lfn, 'Status', 'Failed') elif metadata['Size'] != self.catalogMetadata[lfn]['Size']: gLogger.warn( "resolveSource: skipping %s - source file size mismatch" % lfn) self.__setFileParameter(lfn, 'Reason', "Source size mismatch") self.__setFileParameter(lfn, 'Status', 'Failed') elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \ not ( compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ) ): gLogger.warn( "resolveSource: skipping %s - source file checksum mismatch" % lfn) self.__setFileParameter(lfn, 'Reason', "Source checksum mismatch") self.__setFileParameter(lfn, 'Status', 'Failed') return S_OK() def resolveTarget(self): """ find target SE eligible for submission :param self: self reference """ toResolve = [lfn for lfn in self.fileDict] if not toResolve: return S_OK() res = self.__updateReplicaCache(toResolve) if not res['OK']: return res for lfn in toResolve: if (self.fileDict[lfn].get('Status') == 'Failed'): continue res = self.oTargetSE.getPfnForLfn(lfn) if not res['OK']: gLogger.warn( "resolveTarget: skipping %s - failed to create target pfn" % lfn) self.__setFileParameter(lfn, 'Reason', "Failed to create Target") self.__setFileParameter(lfn, 'Status', 'Failed') continue res = self.oTargetSE.getPfnForProtocol(res['Value'], 'SRM2', withPort=True) if not res['OK']: gLogger.warn("resolveTarget: skipping %s - %s" % (lfn, res["Message"])) self.__setFileParameter(lfn, 'Reason', res['Message']) self.__setFileParameter(lfn, 'Status', 'Failed') continue res = self.setTargetSURL(lfn, res['Value']) if not res['OK']: gLogger.warn("resolveTarget: skipping %s - %s" % (lfn, res["Message"])) self.__setFileParameter(lfn, 'Reason', res['Message']) self.__setFileParameter(lfn, 'Status', 'Failed') continue toResolve = {} for lfn in self.fileDict: if "Target" in self.fileDict[lfn]: toResolve[self.fileDict[lfn]['Target']] = lfn if not toResolve: return S_ERROR("No eligible Target files") res = self.oTargetSE.exists(toResolve.keys()) if not res['OK']: return S_ERROR("Failed to check target existence") for pfn, error in res['Value']['Failed'].items(): lfn = toResolve[pfn] self.__setFileParameter(lfn, 'Reason', error) self.__setFileParameter(lfn, 'Status', 'Failed') toRemove = [] for pfn, exists in res['Value']['Successful'].items(): if exists: lfn = toResolve[pfn] res = self.getSourceSURL(lfn) if not res['OK']: gLogger.warn("resolveTarget: skipping %s - target exists" % lfn) self.__setFileParameter(lfn, 'Reason', "Target exists") self.__setFileParameter(lfn, 'Status', 'Failed') elif res['Value'] == pfn: gLogger.warn( "resolveTarget: skipping %s - source and target pfns are teh same" % lfn) self.__setFileParameter(lfn, 'Reason', "Source and Target the same") self.__setFileParameter(lfn, 'Status', 'Failed') else: toRemove.append(pfn) if toRemove: self.oTargetSE.removeFile(toRemove) return S_OK() def __filesToSubmit(self): """ check if there is at least one file to submit :return: S_OK if at least one file is present, S_ERROR otherwise """ for lfn in self.fileDict: lfnStatus = self.fileDict[lfn].get('Status') source = self.fileDict[lfn].get('Source') target = self.fileDict[lfn].get('Target') if (lfnStatus != 'Failed') and (lfnStatus != 'Done') and source and target: return S_OK() return S_ERROR() def __createSURLPairFile(self): """ create LFNs file for glite-transfer-submit command This file consists one line for each fiel to be transferred: sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM] :param self: self reference """ fd, fileName = tempfile.mkstemp() surlFile = os.fdopen(fd, 'w') for lfn in self.fileDict: lfnStatus = self.fileDict[lfn].get('Status') source = self.fileDict[lfn].get('Source') target = self.fileDict[lfn].get('Target') if (lfnStatus not in ('Failed', 'Done')) and source and target: cksmStr = "" ## add chsmType:cksm only if cksmType is specified, else let FTS decide by itself if self.__cksmTest and self.__cksmType: if lfn in self.catalogMetadata and "Checksum" in self.catalogMetadata[ lfn]: cksmStr = " %s:%s" % (self.__cksmType, self. catalogMetadata[lfn]["Checksum"]) surlFile.write("%s %s%s\n" % (source, target, cksmStr)) surlFile.close() self.surlFile = fileName return S_OK() def __submitFTSTransfer(self): """ create and execute glite-transfer-submit CLI command :param self: self reference """ comm = [ 'glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile, '-o' ] if self.targetToken: comm.append('-t') comm.append(self.targetToken) if self.sourceToken: comm.append('-S') comm.append(self.sourceToken) if self.__cksmTest: comm.append("--compare-checksums") res = executeGridCommand('', comm) os.remove(self.surlFile) if not res['OK']: return res returnCode, output, errStr = res['Value'] if not returnCode == 0: return S_ERROR(errStr) guid = output.replace('\n', '') if not checkGuid(guid): return S_ERROR('Wrong GUID format returned') self.ftsGUID = guid #if self.priority != 3: # comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)] # executeGridCommand('',comm) return res def __resolveFTSServer(self): """ resolve FTS server to use, it should be the closest one from target SE :param self: self reference """ if not self.sourceSE: return S_ERROR("Source SE not set") if not self.targetSE: return S_ERROR("Target SE not set") res = getSitesForSE(self.sourceSE, 'LCG') if not res['OK'] or not res['Value']: return S_ERROR("Could not determine source site") sourceSite = res['Value'][0] res = getSitesForSE(self.targetSE, 'LCG') if not res['OK'] or not res['Value']: return S_ERROR("Could not determine target site") targetSite = res['Value'][0] if (sourceSite == 'LCG.CERN.ch') or (targetSite == 'LCG.CERN.ch'): ep = 'LCG.CERN.ch' else: # Target site FTS server should be used ep = targetSite try: configPath = '/Resources/FTSEndpoints/%s' % ep endpointURL = gConfig.getValue(configPath) if not endpointURL: errStr = "FTSRequest.__resolveFTSServer: Failed to find FTS endpoint, check CS entry for '%s'." % ep return S_ERROR(errStr) self.ftsServer = endpointURL return S_OK(endpointURL) except Exception, x: return S_ERROR( 'FTSRequest.__resolveFTSServer: Failed to obtain endpoint details from CS' )
class FTSRequest: def __init__(self): self.gridEnv = '/afs/cern.ch/project/gd/LCG-share/3.2.8-0/etc/profile.d/grid-env' self.finalStates = [ 'Canceled', 'Failed', 'Hold', 'Finished', 'FinishedDirty' ] self.failedStates = ['Canceled', 'Failed', 'Hold', 'FinishedDirty'] self.successfulStates = ['Finished', 'Done'] self.fileStates = [ 'Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed', 'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting' ] self.newlyCompletedFiles = [] self.newlyFailedFiles = [] self.statusSummary = {} self.requestStatus = 'Unknown' self.fileDict = {} self.catalogReplicas = {} self.catalogMetadata = {} self.oCatalog = None self.submitTime = '' self.ftsGUID = '' self.ftsServer = '' self.priority = 3 self.isTerminal = False self.percentageComplete = 0.0 self.sourceSE = '' self.sourceValid = False self.sourceToken = '' self.targetSE = '' self.targetValid = False self.targetToken = '' self.dumpStr = '' #################################################################### # # Methods for setting/getting/checking the SEs # def setSourceSE(self, se): if se == self.targetSE: return S_ERROR("SourceSE is TargetSE") self.sourceSE = se self.oSourceSE = StorageElement(self.sourceSE) return self.__checkSourceSE() def getSourceSE(self): if not self.sourceSE: return S_ERROR("Source SE not defined") return S_OK(self.sourceSE) def setSourceToken(self, token): self.sourceToken = token return S_OK() def getSourceToken(self): if not self.sourceToken: return S_ERROR("Source token not defined") return S_OK(self.sourceToken) def __checkSourceSE(self): if not self.sourceSE: return S_ERROR("SourceSE not set") res = self.oSourceSE.isValid('Read') if not res['OK']: return S_ERROR("SourceSE not available for reading") res = self.__getSESpaceToken(self.oSourceSE) if not res['OK']: gLogger.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message']) return S_ERROR("SourceSE does not support FTS transfers") self.sourceToken = res['Value'] self.sourceValid = True return S_OK() def setTargetSE(self, se): if se == self.sourceSE: return S_ERROR("TargetSE is SourceSE") self.targetSE = se self.oTargetSE = StorageElement(self.targetSE) return self.__checkTargetSE() def getTargetSE(self): if not self.targetSE: return S_ERROR("Target SE not defined") return S_OK(self.targetSE) def setTargetToken(self, token): self.targetToken = token return S_OK() def getTargetToken(self): if not self.targetToken: return S_ERROR("Target token not defined") return S_OK(self.targetToken) def __checkTargetSE(self): if not self.targetSE: return S_ERROR("TargetSE not set") res = self.oTargetSE.isValid('Write') if not res['OK']: return S_ERROR("TargetSE not available for writing") res = self.__getSESpaceToken(self.oTargetSE) if not res['OK']: gLogger.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message']) return S_ERROR("SourceSE does not support FTS transfers") self.targetToken = res['Value'] self.targetValid = True return S_OK() def __getSESpaceToken(self, oSE): res = oSE.getStorageParameters("SRM2") if not res['OK']: return res return S_OK(res['Value'].get('SpaceToken')) #################################################################### # # Methods for setting/getting FTS request parameters # def setFTSGUID(self, guid): if not checkGuid(guid): return S_ERROR("Incorrect GUID format") self.ftsGUID = guid return S_OK() def getFTSGUID(self): if not self.ftsGUID: return S_ERROR("FTSGUID not set") return S_OK(self.ftsGUID) def setFTSServer(self, server): self.ftsServer = server return S_OK() def getFTSServer(self): if not self.ftsServer: return S_ERROR("FTSServer not set") return S_OK(self.ftsServer) def setPriority(self, priority): if not type(priority) in [types.IntType, types.LongType]: return S_ERROR("Priority must be integer") if priority < 0: priority = 0 elif priority > 5: priority = 5 self.priority = priority return S_OK(self.priority) def getPriority(self): return S_OK(self.priority) def getPercentageComplete(self): completedFiles = 0 totalFiles = 0 for state in (self.statusSummary.keys()): if state in self.successfulStates: completedFiles += self.statusSummary[state] totalFiles += self.statusSummary[state] self.percentageComplete = (float(completedFiles) * 100.0) / float(totalFiles) return S_OK(self.percentageComplete) def isRequestTerminal(self): if self.requestStatus in self.finalStates: self.isTerminal = True return S_OK(self.isTerminal) def getStatus(self): return S_OK(self.requestStatus) #################################################################### # # Methods for setting/getting/checking files and their metadata # def setLFN(self, lfn): if not self.fileDict.has_key(lfn): self.fileDict[lfn] = {} return S_OK() def setSourceSURL(self, lfn, surl): target = self.fileDict[lfn].get('Target') if target == surl: return S_ERROR("Source and target the same") self.__setFileParameter(lfn, 'Source', surl) return S_OK() def getSourceSURL(self, lfn): return self.__getFileParameter(lfn, 'Source') def setTargetSURL(self, lfn, surl): source = self.fileDict[lfn].get('Source') if source == surl: return S_ERROR("Source and target the same") self.__setFileParameter(lfn, 'Target', surl) return S_OK() def getTargetSURL(self, lfn): return self.__getFileParameter(lfn, 'Target') def getFailReason(self, lfn): return self.__getFileParameter(lfn, 'Reason') def getRetries(self, lfn): return self.__getFileParameter(lfn, 'Retries') def getTransferTime(self, lfn): return self.__getFileParameter(lfn, 'Duration') def getFailed(self): failed = [] for lfn in self.fileDict.keys(): status = self.fileDict[lfn].get('Status', '') if status in self.failedStates: failed.append(lfn) return S_OK(failed) def getDone(self): done = [] for lfn in self.fileDict.keys(): status = self.fileDict[lfn].get('Status', '') if status in self.successfulStates: done.append(lfn) return S_OK(done) def __setFileParameter(self, lfn, paramName, paramValue): self.setLFN(lfn) self.fileDict[lfn][paramName] = paramValue return S_OK() def __getFileParameter(self, lfn, paramName): if not self.fileDict.has_key(lfn): return S_ERROR("Supplied file not set") if not self.fileDict[lfn].has_key(paramName): return S_ERROR("%s not set for file" % paramName) return S_OK(self.fileDict[lfn][paramName]) #################################################################### # # Methods for submission # def submit(self, monitor=False, printOutput=True): res = self.__isSubmissionValid() if not res['OK']: return res res = self.__createSURLPairFile() if not res['OK']: return res res = self.__submitFTSTransfer() if not res['OK']: return res resDict = {'ftsGUID': self.ftsGUID, 'ftsServer': self.ftsServer} print "Submitted %s @ %s" % (self.ftsGUID, self.ftsServer) if monitor: self.monitor(untilTerminal=True, printOutput=printOutput) return S_OK(resDict) def __isSubmissionValid(self): if not self.fileDict: return S_ERROR("No files set") if not self.sourceValid: return S_ERROR("SourceSE not valid") if not self.targetValid: return S_ERROR("TargetSE not valid") if not self.ftsServer: res = self.__resolveFTSServer() if not res['OK']: return S_ERROR("FTSServer not valid") self.__resolveSource() self.__resolveTarget() res = self.__filesToSubmit() if not res['OK']: return S_ERROR("No files to submit") return S_OK() def __getCatalogObject(self): try: if not self.oCatalog: self.oCatalog = CatalogInterface() return S_OK() except: return S_ERROR() def __updateReplicaCache(self, lfns=[], overwrite=False): if not lfns: lfns = self.fileDict.keys() toUpdate = [] for lfn in lfns: if (not lfn in self.catalogReplicas.keys()) or overwrite: toUpdate.append(lfn) if not toUpdate: return S_OK() res = self.__getCatalogObject() if not res['OK']: return res res = self.oCatalog.getCatalogReplicas(toUpdate) if not res['OK']: return S_ERROR("Failed to update replica cache", res['Message']) for lfn, error in res['Value']['Failed'].items(): self.__setFileParameter(lfn, 'Reason', error) self.__setFileParameter(lfn, 'Status', 'Failed') for lfn, replicas in res['Value']['Successful'].items(): self.catalogReplicas[lfn] = replicas return S_OK() def __updateMetadataCache(self, lfns=[], overwrite=False): if not lfns: lfns = self.fileDict.keys() toUpdate = [] for lfn in lfns: if (not lfn in self.catalogMetadata.keys()) or overwrite: toUpdate.append(lfn) if not toUpdate: return S_OK() res = self.__getCatalogObject() if not res['OK']: return res res = self.oCatalog.getCatalogFileMetadata(toUpdate) if not res['OK']: return S_ERROR("Failed to get source catalog metadata", res['Message']) for lfn, error in res['Value']['Failed'].items(): self.__setFileParameter(lfn, 'Reason', error) self.__setFileParameter(lfn, 'Status', 'Failed') for lfn, metadata in res['Value']['Successful'].items(): self.catalogMetadata[lfn] = metadata return S_OK() def __resolveSource(self): toResolve = [] for lfn in self.fileDict.keys(): if (not self.fileDict[lfn].has_key('Source')) and ( self.fileDict[lfn].get('Status') != 'Failed'): toResolve.append(lfn) if not toResolve: return S_OK() res = self.__updateMetadataCache(toResolve) if not res['OK']: return res res = self.__updateReplicaCache(toResolve) if not res['OK']: return res for lfn in toResolve: if self.fileDict[lfn].get('Status') == 'Failed': continue replicas = self.catalogReplicas.get(lfn, {}) if not replicas.has_key(self.sourceSE): self.__setFileParameter(lfn, 'Reason', "No replica at SourceSE") self.__setFileParameter(lfn, 'Status', 'Failed') continue res = self.oSourceSE.getPfnForProtocol(replicas[self.sourceSE], 'SRM2', withPort=True) if not res['OK']: self.__setFileParameter(lfn, 'Reason', res['Message']) self.__setFileParameter(lfn, 'Status', 'Failed') continue res = self.setSourceSURL(lfn, res['Value']) if not res['OK']: self.__setFileParameter(lfn, 'Reason', res['Message']) self.__setFileParameter(lfn, 'Status', 'Failed') continue toResolve = {} for lfn in self.fileDict.keys(): if self.fileDict[lfn].has_key('Source'): toResolve[self.fileDict[lfn]['Source']] = lfn if not toResolve: return S_ERROR("No eligible Source files") res = self.oSourceSE.getFileMetadata(toResolve.keys()) if not res['OK']: return S_ERROR("Failed to check source file metadata") for pfn, error in res['Value']['Failed'].items(): lfn = toResolve[pfn] if re.search('File does not exist', error): self.__setFileParameter(lfn, 'Reason', "Source file does not exist") self.__setFileParameter(lfn, 'Status', 'Failed') else: self.__setFileParameter(lfn, 'Reason', "Failed to get Source metadata") self.__setFileParameter(lfn, 'Status', 'Failed') for pfn, metadata in res['Value']['Successful'].items(): lfn = toResolve[pfn] if metadata['Unavailable']: self.__setFileParameter(lfn, 'Reason', "Source file Unavailable") self.__setFileParameter(lfn, 'Status', 'Failed') elif metadata['Lost']: self.__setFileParameter(lfn, 'Reason', "Source file Lost") self.__setFileParameter(lfn, 'Status', 'Failed') elif not metadata['Cached']: self.__setFileParameter(lfn, 'Reason', "Source file not Cached") self.__setFileParameter(lfn, 'Status', 'Failed') elif metadata['Size'] != self.catalogMetadata[lfn]['Size']: self.__setFileParameter(lfn, 'Reason', "Source size mismatch") self.__setFileParameter(lfn, 'Status', 'Failed') elif self.catalogMetadata[lfn]['Checksum'] and metadata[ 'Checksum'] and not (compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'])): self.__setFileParameter(lfn, 'Reason', "Source checksum mismatch") self.__setFileParameter(lfn, 'Status', 'Failed') return S_OK() def __resolveTarget(self): toResolve = [] for lfn in self.fileDict.keys(): if not self.fileDict[lfn].has_key('Target') and ( self.fileDict[lfn].get('Status') != 'Failed'): toResolve.append(lfn) if not toResolve: return S_OK() res = self.__updateReplicaCache(toResolve) if not res['OK']: return res atTarget = [] for lfn in sortList(toResolve): if self.fileDict[lfn].get('Status') == 'Failed': continue replicas = self.catalogReplicas.get(lfn, {}) if replicas.has_key(self.targetSE): self.__setFileParameter(lfn, 'Reason', "File already at Target") self.__setFileParameter(lfn, 'Status', 'Done') atTarget.append(lfn) for lfn in toResolve: if (self.fileDict[lfn].get('Status') == 'Failed') or (lfn in atTarget): continue res = self.oTargetSE.getPfnForLfn(lfn) if not res['OK']: self.__setFileParameter(lfn, 'Reason', "Failed to create Target") self.__setFileParameter(lfn, 'Status', 'Failed') continue res = self.oTargetSE.getPfnForProtocol(res['Value'], 'SRM2', withPort=True) if not res['OK']: self.__setFileParameter(lfn, 'Reason', res['Message']) self.__setFileParameter(lfn, 'Status', 'Failed') continue res = self.setTargetSURL(lfn, res['Value']) if not res['OK']: self.__setFileParameter(lfn, 'Reason', res['Message']) self.__setFileParameter(lfn, 'Status', 'Failed') continue toResolve = {} for lfn in self.fileDict.keys(): if self.fileDict[lfn].has_key('Target'): toResolve[self.fileDict[lfn]['Target']] = lfn if not toResolve: return S_ERROR("No eligible Target files") res = self.oTargetSE.exists(toResolve.keys()) if not res['OK']: return S_ERROR("Failed to check target existence") for pfn, error in res['Value']['Failed'].items(): lfn = toResolve[pfn] self.__setFileParameter(lfn, 'Reason', error) self.__setFileParameter(lfn, 'Status', 'Failed') toRemove = [] for pfn, exists in res['Value']['Successful'].items(): if exists: lfn = toResolve[pfn] res = self.getSourceSURL(lfn) if not res['OK']: self.__setFileParameter(lfn, 'Reason', "Target exists") self.__setFileParameter(lfn, 'Status', 'Failed') elif res['Value'] == pfn: self.__setFileParameter(lfn, 'Reason', "Source and Target the same") self.__setFileParameter(lfn, 'Status', 'Failed') else: toRemove.append(pfn) if toRemove: self.oTargetSE.removeFile(toRemove) return S_OK() def __filesToSubmit(self): for lfn in self.fileDict.keys(): lfnStatus = self.fileDict[lfn].get('Status') source = self.fileDict[lfn].get('Source') target = self.fileDict[lfn].get('Target') if (lfnStatus != 'Failed') and (lfnStatus != 'Done') and source and target: return S_OK() return S_ERROR() def __createSURLPairFile(self): fd, fileName = tempfile.mkstemp() surlFile = os.fdopen(fd, 'w') for lfn in self.fileDict.keys(): lfnStatus = self.fileDict[lfn].get('Status') source = self.fileDict[lfn].get('Source') target = self.fileDict[lfn].get('Target') if (lfnStatus != 'Failed') and (lfnStatus != 'Done') and source and target: surlString = '%s %s\n' % (source, target) surlFile.write(surlString) surlFile.close() self.surlFile = fileName return S_OK() def __submitFTSTransfer(self): comm = [ 'glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile, '-o' ] if self.targetToken: comm.append('-t') comm.append(self.targetToken) if self.sourceToken: comm.append('-S') comm.append(self.sourceToken) res = executeGridCommand('', comm, self.gridEnv) os.remove(self.surlFile) if not res['OK']: return res returnCode, output, errStr = res['Value'] if not returnCode == 0: return S_ERROR(errStr) guid = output.replace('\n', '') if not checkGuid(guid): return S_ERROR('Wrong GUID format returned') self.ftsGUID = guid #if self.priority != 3: # comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)] # executeGridCommand('',comm,self.gridEnv) return res def __resolveFTSServer(self): if not self.sourceSE: return S_ERROR("Source SE not set") if not self.targetSE: return S_ERROR("Target SE not set") res = getSitesForSE(self.sourceSE, 'LCG') if not res['OK'] or not res['Value']: return S_ERROR("Could not determine source site") sourceSite = res['Value'][0] res = getSitesForSE(self.targetSE, 'LCG') if not res['OK'] or not res['Value']: return S_ERROR("Could not determine target site") targetSite = res['Value'][0] if (sourceSite == 'LCG.CERN.ch') or (targetSite == 'LCG.CERN.ch'): ep = 'LCG.CERN.ch' else: # Target site FTS server should be used ep = targetSite try: configPath = '/Resources/FTSEndpoints/%s' % ep endpointURL = gConfig.getValue(configPath) if not endpointURL: errStr = "FTSRequest.__resolveFTSEndpoint: Failed to find FTS endpoint, check CS entry for '%s'." % ep return S_ERROR(errStr) self.ftsServer = endpointURL return S_OK(endpointURL) except Exception, x: return S_ERROR( 'FTSRequest.__resolveFTSEndpoint: Failed to obtain endpoint details from CS' )