class FTSRequest( object ):
"""
.. class:: FTSRequest
Helper class for FTS job submission and monitoring.
"""
# # default checksum type
__defaultCksmType = "ADLER32"
# # flag to disablr/enable checksum test, default: disabled
__cksmTest = False
def __init__( self ):
"""c'tor
:param self: self reference
"""
self.log = gLogger.getSubLogger( self.__class__.__name__, True )
# # final states tuple
self.finalStates = ( 'Canceled', 'Failed', 'Hold',
'Finished', 'FinishedDirty' )
# # failed states tuple
self.failedStates = ( 'Canceled', 'Failed',
'Hold', 'FinishedDirty' )
# # successful states tuple
self.successfulStates = ( 'Finished', 'Done' )
# # all file states tuple
self.fileStates = ( 'Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed',
'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting' )
self.newlyCompletedFiles = []
self.newlyFailedFiles = []
self.statusSummary = {}
# # request status
self.requestStatus = 'Unknown'
# # dict for FTS job files
self.fileDict = {}
# # dict for replicas information
self.catalogReplicas = {}
# # dict for metadata information
self.catalogMetadata = {}
# # dict for files that failed to register
self.failedRegistrations = {}
# # placehoder for CatalogInterface reference
self.oCatalog = None
# # submit timestamp
self.submitTime = ''
# # placeholder FTS job GUID
self.ftsGUID = ''
# # placeholder for FTS server URL
self.ftsServer = ''
# # not used
self.priority = 3
# # flag marking FTS job completness
self.isTerminal = False
# # completness percentage
self.percentageComplete = 0.0
# # source SE name
self.sourceSE = ''
# # flag marking source SE validity
self.sourceValid = False
# # source space token
self.sourceToken = ''
# # target SE name
self.targetSE = ''
# # flag marking target SE validity
self.targetValid = False
# # target space token
self.targetToken = ''
# # whatever
self.dumpStr = ''
# # placeholder for surl file
self.surlFile = None
# # placeholder for target StorageElement
self.oTargetSE = None
# # placeholder for source StorageElement
self.oSourceSE = None
# # checksum type, set it to default
self.__cksmType = self.__defaultCksmType
# # disable checksum test by default
self.__cksmTest = False
# # replica manager handler
self.replicaManager = ReplicaManager()
# # statuses that prevent submitting to FTS
self.noSubmitStatus = ( 'Failed', 'Done', 'Staging' )
# # were sources resolved?
self.sourceResolved = False
# # Number of file transfers actually submitted
self.submittedFiles = 0
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE( self, se ):
""" set SE for source
:param self: self reference
:param str se: source SE name
"""
if se == self.targetSE:
return S_ERROR( "SourceSE is TargetSE" )
self.sourceSE = se
self.oSourceSE = StorageElement( self.sourceSE )
return self.__checkSourceSE()
def getSourceSE( self ):
""" source SE getter
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR( "Source SE not defined" )
return S_OK( self.sourceSE )
def setSourceToken( self, token ):
""" set source space token
:param self: self reference
:param str token: source space token
"""
self.sourceToken = token
return S_OK()
def getSourceToken( self ):
""" source space token getter
:param self: self reference
"""
if not self.sourceToken:
return S_ERROR( "Source token not defined" )
return S_OK( self.sourceToken )
def __checkSourceSE( self ):
""" check source SE availability
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR( "SourceSE not set" )
res = self.oSourceSE.isValid( 'Read' )
if not res['OK']:
return S_ERROR( "SourceSE not available for reading" )
res = self.__getSESpaceToken( self.oSourceSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message'] )
return S_ERROR( "SourceSE does not support FTS transfers" )
if self.__cksmTest:
res = self.oSourceSE.getChecksumType()
if not res["OK"]:
self.log.error( "Unable to get checksum type for SourceSE %s: %s" % ( self.sourceSE,
res["Message"] ) )
cksmType = res["Value"]
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at SourceSE %s, disabling checksum test" % ( cksmType,
self.sourceSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE( self, se ):
""" set target SE
:param self: self reference
:param str se: target SE name
"""
if se == self.sourceSE:
return S_ERROR( "TargetSE is SourceSE" )
self.targetSE = se
self.oTargetSE = StorageElement( self.targetSE )
return self.__checkTargetSE()
def getTargetSE( self ):
""" target SE getter
:param self: self reference
"""
if not self.targetSE:
return S_ERROR( "Target SE not defined" )
return S_OK( self.targetSE )
def setTargetToken( self, token ):
""" target space token setter
:param self: self reference
:param str token: target space token
"""
self.targetToken = token
return S_OK()
def getTargetToken( self ):
""" target space token getter
:param self: self reference
"""
if not self.targetToken:
return S_ERROR( "Target token not defined" )
return S_OK( self.targetToken )
def __checkTargetSE( self ):
""" check target SE availability
:param self: self reference
"""
if not self.targetSE:
return S_ERROR( "TargetSE not set" )
res = self.oTargetSE.isValid( 'Write' )
if not res['OK']:
return S_ERROR( "TargetSE not available for writing" )
res = self.__getSESpaceToken( self.oTargetSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message'] )
return S_ERROR( "TargetSE does not support FTS transfers" )
# # check checksum types
if self.__cksmTest:
res = self.oTargetSE.getChecksumType()
if not res["OK"]:
self.log.error( "Unable to get checksum type for TargetSE %s: %s" % ( self.targetSE,
res["Message"] ) )
cksmType = res["Value"]
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at TargetSE %s, disabling checksum test" % ( cksmType,
self.targetSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
@staticmethod
def __getSESpaceToken( oSE ):
""" get space token from StorageElement instance
:param self: self reference
:param StorageElement oSE: StorageElement instance
"""
res = oSE.getStorageParameters( "SRM2" )
if not res['OK']:
return res
return S_OK( res['Value'].get( 'SpaceToken' ) )
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID( self, guid ):
""" FTS job GUID setter
:param self: self reference
:param str guid: string containg GUID
"""
if not checkGuid( guid ):
return S_ERROR( "Incorrect GUID format" )
self.ftsGUID = guid
return S_OK()
def getFTSGUID( self ):
""" FTS job GUID getter
:param self: self refenece
"""
if not self.ftsGUID:
return S_ERROR( "FTSGUID not set" )
return S_OK( self.ftsGUID )
def setFTSServer( self, server ):
""" FTS server setter
:param self: self reference
:param str server: FTS server URL
"""
self.ftsServer = server
return S_OK()
def getFTSServer( self ):
""" FTS server getter
:param self: self reference
"""
if not self.ftsServer:
return S_ERROR( "FTSServer not set" )
return S_OK( self.ftsServer )
def setPriority( self, priority ):
""" set priority for FTS job
:param self: self reference
:param int priority: a new priority
"""
if not type( priority ) in ( IntType, LongType ):
return S_ERROR( "Priority must be integer" )
if priority < 0:
priority = 0
elif priority > 5:
priority = 5
self.priority = priority
return S_OK( self.priority )
def getPriority( self ):
""" FTS job priority getter
:param self: self reference
"""
return S_OK( self.priority )
def getPercentageComplete( self ):
""" get completness percentage
:param self: self reference
"""
completedFiles = 0
totalFiles = 0
for state in self.statusSummary:
if state in self.successfulStates:
completedFiles += self.statusSummary[state]
totalFiles += self.statusSummary[state]
self.percentageComplete = ( float( completedFiles ) * 100.0 ) / float( totalFiles )
return S_OK( self.percentageComplete )
def isRequestTerminal( self ):
""" check if FTS job has terminated
:param self: self reference
"""
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK( self.isTerminal )
def getStatus( self ):
""" get FTS job status
:param self: self reference
"""
return S_OK( self.requestStatus )
def setCksmType( self, cksm = None ):
""" set checksum type to use
:param self: self reference
:param mixed cksm: checksum type, should be one of 'Adler32', 'md5', 'sha1', None
"""
if str( cksm ).upper() not in ( "ADLER32", "MD5", "SHA1", "NONE" ):
return S_ERROR( "Not supported checksum type: %s" % str( cksm ) )
if not cksm:
self.__cksmType = None
return S_OK( False )
self.__cksmType = str( cksm ).upper()
return S_OK( True )
def getCksmType( self ):
""" get checksum type
:param self: self reference
"""
return S_OK( self.__cksmType )
def setCksmTest( self, cksmTest = False ):
""" set cksm test
:param self: self reference
:param bool cksmTest: flag to enable/disable checksum test
"""
self.__cksmTest = bool( cksmTest )
return S_OK( self.__cksmTest )
def getCksmTest( self ):
""" get cksm test flag
:param self: self reference
"""
return S_OK( self.__cksmTest )
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN( self, lfn ):
""" add LFN :lfn: to :fileDict:
:param self: self reference
:param str lfn: LFN to add to
"""
self.fileDict.setdefault( lfn, {'Status':'Waiting'} )
return S_OK()
def setStatus( self, lfn, status ):
""" set status of a file """
return( self.__setFileParameter( lfn, 'Status', status ) )
def setSourceSURL( self, lfn, surl ):
""" source SURL setter
:param self: self reference
:param str lfn: LFN
:param str surl: source SURL
"""
target = self.fileDict[lfn].get( 'Target' )
if target == surl:
return S_ERROR( "Source and target the same" )
return( self.__setFileParameter( lfn, 'Source', surl ) )
def getSourceSURL( self, lfn ):
""" get source SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Source' )
def setTargetSURL( self, lfn, surl ):
""" set target SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
:param str surl: target SURL
"""
source = self.fileDict[lfn].get( 'Source' )
if source == surl:
return S_ERROR( "Source and target the same" )
return( self.__setFileParameter( lfn, 'Target', surl ) )
def getTargetSURL( self, lfn ):
""" target SURL getter
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Target' )
def getFailReason( self, lfn ):
""" get fail reason for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Reason' )
def getRetries( self, lfn ):
""" get number of attepmts made to transfer file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Retries' )
def getTransferTime( self, lfn ):
""" get duration of transfer for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Duration' )
def getFailed( self ):
""" get list of wrongly transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.failedStates ] )
def getStaging( self ):
""" get files set for prestaging """
return S_OK( [lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) == 'Staging'] )
def getDone( self ):
""" get list of succesfully transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.successfulStates ] )
def __setFileParameter( self, lfn, paramName, paramValue ):
""" set :paramName: to :paramValue: for :lfn: file
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
:param mixed paramValue: a new parameter value
"""
self.setLFN( lfn )
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter( self, lfn, paramName ):
""" get value of :paramName: for file :lfn:
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
"""
if lfn not in self.fileDict:
return S_ERROR( "Supplied file not set" )
if paramName not in self.fileDict[lfn]:
return S_ERROR( "%s not set for file" % paramName )
return S_OK( self.fileDict[lfn][paramName] )
####################################################################
#
# Methods for submission
#
def submit( self, monitor = False, printOutput = True ):
""" submit FTS job
:param self: self reference
:param bool monitor: flag to monitor progress of FTS job
:param bool printOutput: flag to print output of execution to stdout
"""
res = self.__isSubmissionValid()
if not res['OK']:
return res
res = self.__createSURLPairFile()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = { 'ftsGUID' : self.ftsGUID, 'ftsServer' : self.ftsServer, 'submittedFiles' : self.submittedFiles }
# print "Submitted %s @ %s" % ( self.ftsGUID, self.ftsServer )
if monitor:
self.monitor( untilTerminal = True, printOutput = printOutput )
return S_OK( resDict )
def __isSubmissionValid( self ):
""" check validity of job before submission
:param self: self reference
"""
if not self.fileDict:
return S_ERROR( "No files set" )
if not self.sourceValid:
return S_ERROR( "SourceSE not valid" )
if not self.targetValid:
return S_ERROR( "TargetSE not valid" )
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR( "FTSServer not valid" )
self.resolveSource()
self.resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR( "No files to submit" )
return S_OK()
def __getCatalogObject( self ):
""" CatalogInterface instance facade
:param self: self reference
"""
try:
if not self.oCatalog:
self.oCatalog = CatalogInterface()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache( self, lfns = None, overwrite = False ):
""" update replica cache for list of :lfns:
:param self: self reference
:param mixed lfns: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if ( lfn not in self.catalogReplicas ) or overwrite ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogReplicas( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to update replica cache: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache( self, lfns = None, overwrite = False ):
""" update metadata cache for list of LFNs
:param self: self reference
:param list lnfs: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if ( lfn not in self.catalogMetadata ) or overwrite ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogFileMetadata( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to get source catalog metadata: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def resolveSource( self ):
""" resolve source SE eligible for submission
:param self: self reference
"""
# Avoid resolving sources twice
if self.sourceResolved:
return S_OK()
# Only resolve files that need a transfer
toResolve = [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( "Status", "" ) != "Failed" ]
if not toResolve:
return S_OK()
res = self.__updateMetadataCache( toResolve )
if not res['OK']:
return res
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
# Define the source URLs
for lfn in toResolve:
replicas = self.catalogReplicas.get( lfn, {} )
if self.sourceSE not in replicas:
gLogger.warn( "resolveSource: skipping %s - not replicas at SourceSE %s" % ( lfn, self.sourceSE ) )
self.__setFileParameter( lfn, 'Reason', "No replica at SourceSE" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.oSourceSE.getPfnForProtocol( replicas[self.sourceSE], 'SRM2', withPort = True )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setSourceSURL( lfn, res['Value'] )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict:
if "Source" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Source']] = lfn
if not toResolve:
return S_ERROR( "No eligible Source files" )
# Get metadata of the sources, to check for existance, availability and caching
res = self.oSourceSE.getFileMetadata( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check source file metadata" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
if re.search( 'File does not exist', error ):
gLogger.warn( "resolveSource: skipping %s - source file does not exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file does not exist" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
gLogger.warn( "resolveSource: skipping %s - failed to get source metadata" % lfn )
self.__setFileParameter( lfn, 'Reason', "Failed to get Source metadata" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toStage = []
nbStagedFiles = 0
for pfn, metadata in res['Value']['Successful'].items():
lfn = toResolve[pfn]
lfnStatus = self.fileDict.get( lfn, {} ).get( 'Status' )
if metadata['Unavailable']:
gLogger.warn( "resolveSource: skipping %s - source file unavailable" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Unavailable" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif metadata['Lost']:
gLogger.warn( "resolveSource: skipping %s - source file lost" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Lost" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif not metadata['Cached']:
if lfnStatus != 'Staging':
toStage.append( pfn )
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
gLogger.warn( "resolveSource: skipping %s - source file size mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source size mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \
not ( compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ) ):
gLogger.warn( "resolveSource: skipping %s - source file checksum mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source checksum mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif lfnStatus == 'Staging':
# file that was staging is now cached
self.__setFileParameter( lfn, 'Status', 'Waiting' )
nbStagedFiles += 1
# Some files were being staged
if nbStagedFiles:
self.log.info( 'resolveSource: %d files have been staged' % nbStagedFiles )
# Launching staging of files not in cache
if toStage:
gLogger.warn( "resolveSource: %s source files not cached, prestaging..." % len( toStage ) )
stage = self.replicaManager.prestageStorageFile( toStage, self.sourceSE )
if not stage["OK"]:
gLogger.error( "resolveSource: error is prestaging - %s" % stage["Message"] )
for pfn in toStage:
lfn = toResolve[pfn]
self.__setFileParameter( lfn, 'Reason', stage["Message"] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
for pfn in toStage:
lfn = toResolve[pfn]
if pfn in stage['Value']['Successful']:
self.__setFileParameter( lfn, 'Status', 'Staging' )
elif pfn in stage['Value']['Failed']:
self.__setFileParameter( lfn, 'Reason', stage['Value']['Failed'][pfn] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
self.sourceResolved = True
return S_OK()
def resolveTarget( self ):
""" find target SE eligible for submission
:param self: self reference
"""
toResolve = [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status' ) not in self.noSubmitStatus ]
if not toResolve:
return S_OK()
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
for lfn in toResolve:
res = self.oTargetSE.getPfnForLfn( lfn )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - failed to create target pfn" % lfn )
self.__setFileParameter( lfn, 'Reason', "Failed to create Target" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.oTargetSE.getPfnForProtocol( res['Value'], 'SRM2', withPort = True )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setTargetSURL( lfn, res['Value'] )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict:
if "Target" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Target']] = lfn
if not toResolve:
return S_ERROR( "No eligible Target files" )
res = self.oTargetSE.exists( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check target existence" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toRemove = []
for pfn, exists in res['Value']['Successful'].items():
if exists:
lfn = toResolve[pfn]
res = self.getSourceSURL( lfn )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - target exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Target exists" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif res['Value'] == pfn:
gLogger.warn( "resolveTarget: skipping %s - source and target pfns are the same" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source and Target the same" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRemove.append( pfn )
if toRemove:
self.oTargetSE.removeFile( toRemove )
return S_OK()
def __filesToSubmit( self ):
"""
check if there is at least one file to submit
:return: S_OK if at least one file is present, S_ERROR otherwise
"""
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if lfnStatus not in self.noSubmitStatus and source and target:
return S_OK()
return S_ERROR()
def __createSURLPairFile( self ):
""" create LFNs file for glite-transfer-submit command
This file consists one line for each fiel to be transferred:
sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM]
:param self: self reference
"""
fd, fileName = tempfile.mkstemp()
surlFile = os.fdopen( fd, 'w' )
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if lfnStatus not in self.noSubmitStatus and source and target:
cksmStr = ""
# # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself
if self.__cksmTest and self.__cksmType:
checkSum = self.catalogMetadata.get( lfn, {} ).get( 'Checksum' )
if checkSum:
cksmStr = " %s:%s" % ( self.__cksmType, intAdlerToHex( hexAdlerToInt( checkSum ) ) )
surlFile.write( "%s %s%s\n" % ( source, target, cksmStr ) )
self.submittedFiles += 1
surlFile.close()
self.surlFile = fileName
return S_OK()
def __submitFTSTransfer( self ):
""" create and execute glite-transfer-submit CLI command
:param self: self reference
"""
comm = [ 'glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile, '-o' ]
if self.targetToken:
comm += [ '-t', self.targetToken ]
if self.sourceToken:
comm += [ '-S', self.sourceToken ]
if self.__cksmTest:
comm.append( "--compare-checksums" )
gLogger.verbose( 'Executing %s' % ' '.join( comm ) )
res = executeGridCommand( '', comm )
os.remove( self.surlFile )
if not res['OK']:
return res
returnCode, output, errStr = res['Value']
if not returnCode == 0:
return S_ERROR( errStr )
guid = output.replace( '\n', '' )
if not checkGuid( guid ):
return S_ERROR( 'Wrong GUID format returned' )
self.ftsGUID = guid
# if self.priority != 3:
# comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)]
# executeGridCommand('',comm)
return res
def __getFTSServer( self, site ):
try:
configPath = '/Resources/FTSEndpoints/%s' % site
endpointURL = gConfig.getValue( configPath )
if not endpointURL:
errStr = "FTSRequest.__getFTSServer: Failed to find FTS endpoint, check CS entry for '%s'." % site
return S_ERROR( errStr )
return S_OK( endpointURL )
except Exception, x:
return S_ERROR( 'FTSRequest.__getFTSServer: Failed to obtain endpoint details from CS' )
class FTSRequest(object):
"""
.. class:: FTSRequest
Helper class for FTS job submission and monitoring.
"""
## default checksum type
__defaultCksmType = "ADLER32"
## flag to disablr/enable checksum test, default: disabled
__cksmTest = False
def __init__(self):
"""c'tor
:param self: self reference
"""
self.log = gLogger.getSubLogger(self.__class__.__name__, True)
## final states tuple
self.finalStates = ('Canceled', 'Failed', 'Hold', 'Finished',
'FinishedDirty')
## failed states tuple
self.failedStates = ('Canceled', 'Failed', 'Hold', 'FinishedDirty')
## successful states tuple
self.successfulStates = ('Finished', 'Done')
## all file states tuple
self.fileStates = ('Done', 'Active', 'Pending', 'Ready', 'Canceled',
'Failed', 'Finishing', 'Finished', 'Submitted',
'Hold', 'Waiting')
self.newlyCompletedFiles = []
self.newlyFailedFiles = []
self.statusSummary = {}
## request status
self.requestStatus = 'Unknown'
## dict for FTS job files
self.fileDict = {}
## dict for replicas information
self.catalogReplicas = {}
## dict for metadata information
self.catalogMetadata = {}
## dict for files that failed to register
self.failedRegistrations = {}
## placehoder for CatalogInterface reference
self.oCatalog = None
## submit timestamp
self.submitTime = ''
## placeholder FTS job GUID
self.ftsGUID = ''
## placeholder for FTS server URL
self.ftsServer = ''
## not used
self.priority = 3
## flag marking FTS job completness
self.isTerminal = False
## completness percentage
self.percentageComplete = 0.0
## source SE name
self.sourceSE = ''
## flag marking source SE validity
self.sourceValid = False
## source space token
self.sourceToken = ''
## target SE name
self.targetSE = ''
## flag marking target SE validity
self.targetValid = False
## target space token
self.targetToken = ''
## whatever
self.dumpStr = ''
## placeholder for surl file
self.surlFile = None
## placeholder for target StorageElement
self.oTargetSE = None
## placeholder for source StorageElement
self.oSourceSE = None
## checksum type, set it to default
self.__cksmType = self.__defaultCksmType
## disable checksum test by default
self.__cksmTest = False
## replica manager handler
self.replicaManager = ReplicaManager()
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE(self, se):
""" set SE for source
:param self: self reference
:param str se: source SE name
"""
if se == self.targetSE:
return S_ERROR("SourceSE is TargetSE")
self.sourceSE = se
self.oSourceSE = StorageElement(self.sourceSE)
return self.__checkSourceSE()
def getSourceSE(self):
""" source SE getter
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR("Source SE not defined")
return S_OK(self.sourceSE)
def setSourceToken(self, token):
""" set source space token
:param self: self reference
:param str token: source space token
"""
self.sourceToken = token
return S_OK()
def getSourceToken(self):
""" source space token getter
:param self: self reference
"""
if not self.sourceToken:
return S_ERROR("Source token not defined")
return S_OK(self.sourceToken)
def __checkSourceSE(self):
""" check source SE availability
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR("SourceSE not set")
res = self.oSourceSE.isValid('Read')
if not res['OK']:
return S_ERROR("SourceSE not available for reading")
res = self.__getSESpaceToken(self.oSourceSE)
if not res['OK']:
self.log.error(
"FTSRequest failed to get SRM Space Token for SourceSE",
res['Message'])
return S_ERROR("SourceSE does not support FTS transfers")
if self.__cksmTest:
res = self.oSourceSE.getChecksumType()
if not res["OK"]:
self.log.error(
"Unable to get checksum type for SourceSE %s: %s" %
(self.sourceSE, res["Message"]))
cksmType = res["Value"]
if cksmType in ("NONE", "NULL"):
self.log.warn(
"Checksum type set to %s at SourceSE %s, disabling checksum test"
% (cksmType, self.sourceSE))
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn(
"Checksum type mismatch, disabling checksum test")
self.__cksmTest = False
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE(self, se):
""" set target SE
:param self: self reference
:param str se: target SE name
"""
if se == self.sourceSE:
return S_ERROR("TargetSE is SourceSE")
self.targetSE = se
self.oTargetSE = StorageElement(self.targetSE)
return self.__checkTargetSE()
def getTargetSE(self):
""" target SE getter
:param self: self reference
"""
if not self.targetSE:
return S_ERROR("Target SE not defined")
return S_OK(self.targetSE)
def setTargetToken(self, token):
""" target space token setter
:param self: self reference
:param str token: target space token
"""
self.targetToken = token
return S_OK()
def getTargetToken(self):
""" target space token getter
:param self: self reference
"""
if not self.targetToken:
return S_ERROR("Target token not defined")
return S_OK(self.targetToken)
def __checkTargetSE(self):
""" check target SE availability
:param self: self reference
"""
if not self.targetSE:
return S_ERROR("TargetSE not set")
res = self.oTargetSE.isValid('Write')
if not res['OK']:
return S_ERROR("TargetSE not available for writing")
res = self.__getSESpaceToken(self.oTargetSE)
if not res['OK']:
self.log.error(
"FTSRequest failed to get SRM Space Token for TargetSE",
res['Message'])
return S_ERROR("TargetSE does not support FTS transfers")
## check checksum types
if self.__cksmTest:
res = self.oTargetSE.getChecksumType()
if not res["OK"]:
self.log.error(
"Unable to get checksum type for TargetSE %s: %s" %
(self.targetSE, res["Message"]))
cksmType = res["Value"]
if cksmType in ("NONE", "NULL"):
self.log.warn(
"Checksum type set to %s at TargetSE %s, disabling checksum test"
% (cksmType, self.targetSE))
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn(
"Checksum type mismatch, disabling checksum test")
self.__cksmTest = False
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
@staticmethod
def __getSESpaceToken(oSE):
""" get space token from StorageElement instance
:param self: self reference
:param StorageElement oSE: StorageElement instance
"""
res = oSE.getStorageParameters("SRM2")
if not res['OK']:
return res
return S_OK(res['Value'].get('SpaceToken'))
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID(self, guid):
""" FTS job GUID setter
:param self: self reference
:param str guid: string containg GUID
"""
if not checkGuid(guid):
return S_ERROR("Incorrect GUID format")
self.ftsGUID = guid
return S_OK()
def getFTSGUID(self):
""" FTS job GUID getter
:param self: self refenece
"""
if not self.ftsGUID:
return S_ERROR("FTSGUID not set")
return S_OK(self.ftsGUID)
def setFTSServer(self, server):
""" FTS server setter
:param self: self reference
:param str server: FTS server URL
"""
self.ftsServer = server
return S_OK()
def getFTSServer(self):
""" FTS server getter
:param self: self reference
"""
if not self.ftsServer:
return S_ERROR("FTSServer not set")
return S_OK(self.ftsServer)
def setPriority(self, priority):
""" set priority for FTS job
:param self: self reference
:param int priority: a new priority
"""
if not type(priority) in (IntType, LongType):
return S_ERROR("Priority must be integer")
if priority < 0:
priority = 0
elif priority > 5:
priority = 5
self.priority = priority
return S_OK(self.priority)
def getPriority(self):
""" FTS job priority getter
:param self: self reference
"""
return S_OK(self.priority)
def getPercentageComplete(self):
""" get completness percentage
:param self: self reference
"""
completedFiles = 0
totalFiles = 0
for state in (self.statusSummary.keys()):
if state in self.successfulStates:
completedFiles += self.statusSummary[state]
totalFiles += self.statusSummary[state]
self.percentageComplete = (float(completedFiles) *
100.0) / float(totalFiles)
return S_OK(self.percentageComplete)
def isRequestTerminal(self):
""" check if FTS job has terminated
:param self: self reference
"""
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK(self.isTerminal)
def getStatus(self):
""" get FTS job status
:param self: self reference
"""
return S_OK(self.requestStatus)
def setCksmType(self, cksm=None):
""" set checksum type to use
:param self: self reference
:param mixed cksm: checksum type, should be one of 'Adler32', 'md5', 'sha1', None
"""
if str(cksm).upper() not in ("ADLER32", "MD5", "SHA1", "NONE"):
return S_ERROR("Not supported checksum type: %s" % str(cksm))
if not cksm:
self.__cksmType = None
return S_OK(False)
self.__cksmType = str(cksm).upper()
return S_OK(True)
def getCksmType(self):
""" get checksum type
:param self: self reference
"""
return S_OK(self.__cksmType)
def setCksmTest(self, cksmTest=False):
""" set cksm test
:param self: self reference
:param bool cksmTest: flag to enable/disable checksum test
"""
self.__cksmTest = bool(cksmTest)
return S_OK(self.__cksmTest)
def getCksmTest(self):
""" get cksm test flag
:param self: self reference
"""
return S_OK(self.__cksmTest)
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN(self, lfn):
""" add LFN :lfn: to :fileDict:
:param self: self reference
:param str lfn: LFN to add to
"""
if lfn not in self.fileDict:
self.fileDict[lfn] = {}
return S_OK()
def setSourceSURL(self, lfn, surl):
""" source SURL setter
:param self: self reference
:param str lfn: LFN
:param str surl: source SURL
"""
target = self.fileDict[lfn].get('Target')
if target == surl:
return S_ERROR("Source and target the same")
self.__setFileParameter(lfn, 'Source', surl)
return S_OK()
def getSourceSURL(self, lfn):
""" get source SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter(lfn, 'Source')
def setTargetSURL(self, lfn, surl):
""" set target SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
:param str surl: target SURL
"""
source = self.fileDict[lfn].get('Source')
if source == surl:
return S_ERROR("Source and target the same")
self.__setFileParameter(lfn, 'Target', surl)
return S_OK()
def getTargetSURL(self, lfn):
""" target SURL getter
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter(lfn, 'Target')
def getFailReason(self, lfn):
""" get fail reason for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter(lfn, 'Reason')
def getRetries(self, lfn):
""" get number of attepmts made to transfer file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter(lfn, 'Retries')
def getTransferTime(self, lfn):
""" get duration of transfer for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter(lfn, 'Duration')
def getFailed(self):
""" get list of wrongly transferred LFNs
:param self: self reference
"""
return S_OK([
lfn for lfn in self.fileDict
if self.fileDict[lfn].get('Status', '') in self.failedStates
])
def getDone(self):
""" get list of succesfully transferred LFNs
:param self: self reference
"""
return S_OK([
lfn for lfn in self.fileDict
if self.fileDict[lfn].get('Status', '') in self.successfulStates
])
def __setFileParameter(self, lfn, paramName, paramValue):
""" set :paramName: to :paramValue: for :lfn: file
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
:param mixed paramValue: a new parameter value
"""
self.setLFN(lfn)
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter(self, lfn, paramName):
""" get value of :paramName: for file :lfn:
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
"""
if lfn not in self.fileDict:
return S_ERROR("Supplied file not set")
if paramName not in self.fileDict[lfn]:
return S_ERROR("%s not set for file" % paramName)
return S_OK(self.fileDict[lfn][paramName])
####################################################################
#
# Methods for submission
#
def submit(self, monitor=False, printOutput=True):
""" submit FTS job
:param self: self reference
:param bool monitor: flag to monitor progress of FTS job
:param bool printOutput: flag to print output of execution to stdout
"""
res = self.__isSubmissionValid()
if not res['OK']:
return res
res = self.__createSURLPairFile()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = {'ftsGUID': self.ftsGUID, 'ftsServer': self.ftsServer}
print "Submitted %s @ %s" % (self.ftsGUID, self.ftsServer)
if monitor:
self.monitor(untilTerminal=True, printOutput=printOutput)
return S_OK(resDict)
def __isSubmissionValid(self):
""" check validity of job before submission
:param self: self reference
"""
if not self.fileDict:
return S_ERROR("No files set")
if not self.sourceValid:
return S_ERROR("SourceSE not valid")
if not self.targetValid:
return S_ERROR("TargetSE not valid")
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR("FTSServer not valid")
self.resolveSource()
self.resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR("No files to submit")
return S_OK()
def __getCatalogObject(self):
""" CatalogInterface instance facade
:param self: self reference
"""
try:
if not self.oCatalog:
self.oCatalog = CatalogInterface()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache(self, lfns=None, overwrite=False):
""" update replica cache for list of :lfns:
:param self: self reference
:param mixed lfns: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [
lfn for lfn in lfns
if (lfn not in self.catalogReplicas) or overwrite
]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogReplicas(toUpdate)
if not res['OK']:
return S_ERROR("Failed to update replica cache: %s" %
res['Message'])
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache(self, lfns=None, overwrite=False):
""" update metadata cache for list of LFNs
:param self: self reference
:param list lnfs: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [
lfn for lfn in lfns
if (lfn not in self.catalogMetadata) or overwrite
]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogFileMetadata(toUpdate)
if not res['OK']:
return S_ERROR("Failed to get source catalog metadata: %s" %
res['Message'])
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def resolveSource(self):
""" resolve source SE eligible for submission
:param self: self reference
"""
toResolve = [lfn for lfn in self.fileDict]
if not toResolve:
return S_OK()
res = self.__updateMetadataCache(toResolve)
if not res['OK']:
return res
res = self.__updateReplicaCache(toResolve)
if not res['OK']:
return res
for lfn in toResolve:
if self.fileDict[lfn].get("Status", "") == "Failed":
continue
replicas = self.catalogReplicas.get(lfn, {})
if self.sourceSE not in replicas:
gLogger.warn(
"resolveSource: skipping %s - not replicas at SourceSE %s"
% (lfn, self.sourceSE))
self.__setFileParameter(lfn, 'Reason',
"No replica at SourceSE")
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.oSourceSE.getPfnForProtocol(replicas[self.sourceSE],
'SRM2',
withPort=True)
if not res['OK']:
gLogger.warn("resolveSource: skipping %s - %s" %
(lfn, res["Message"]))
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.setSourceSURL(lfn, res['Value'])
if not res['OK']:
gLogger.warn("resolveSource: skipping %s - %s" %
(lfn, res["Message"]))
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
toResolve = {}
for lfn in self.fileDict:
if "Source" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Source']] = lfn
if not toResolve:
return S_ERROR("No eligible Source files")
res = self.oSourceSE.getFileMetadata(toResolve.keys())
if not res['OK']:
return S_ERROR("Failed to check source file metadata")
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
if re.search('File does not exist', error):
gLogger.warn(
"resolveSource: skipping %s - source file does not exists"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Source file does not exist")
self.__setFileParameter(lfn, 'Status', 'Failed')
else:
gLogger.warn(
"resolveSource: skipping %s - failed to get source metadata"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Failed to get Source metadata")
self.__setFileParameter(lfn, 'Status', 'Failed')
for pfn, metadata in res['Value']['Successful'].items():
lfn = toResolve[pfn]
if metadata['Unavailable']:
gLogger.warn(
"resolveSource: skipping %s - source file unavailable" %
lfn)
self.__setFileParameter(lfn, 'Reason',
"Source file Unavailable")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif metadata['Lost']:
gLogger.warn("resolveSource: skipping %s - source file lost" %
lfn)
self.__setFileParameter(lfn, 'Reason', "Source file Lost")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif not metadata['Cached']:
gLogger.warn(
"resolveSource: source file %s not cached, prestaging..." %
lfn)
stage = self.replicaManager.prestageStorageFile(
pfn, self.sourceSE, singleFile=True)
if not stage["OK"]:
gLogger.warn("resolveSource: skipping %s - %s" %
(lfn, stage["Message"]))
self.__setFileParameter(lfn, 'Reason', stage["Message"])
self.__setFileParameter(lfn, 'Status', 'Failed')
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
gLogger.warn(
"resolveSource: skipping %s - source file size mismatch" %
lfn)
self.__setFileParameter(lfn, 'Reason', "Source size mismatch")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \
not ( compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ) ):
gLogger.warn(
"resolveSource: skipping %s - source file checksum mismatch"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Source checksum mismatch")
self.__setFileParameter(lfn, 'Status', 'Failed')
return S_OK()
def resolveTarget(self):
""" find target SE eligible for submission
:param self: self reference
"""
toResolve = [lfn for lfn in self.fileDict]
if not toResolve:
return S_OK()
res = self.__updateReplicaCache(toResolve)
if not res['OK']:
return res
for lfn in toResolve:
if (self.fileDict[lfn].get('Status') == 'Failed'):
continue
res = self.oTargetSE.getPfnForLfn(lfn)
if not res['OK']:
gLogger.warn(
"resolveTarget: skipping %s - failed to create target pfn"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Failed to create Target")
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.oTargetSE.getPfnForProtocol(res['Value'],
'SRM2',
withPort=True)
if not res['OK']:
gLogger.warn("resolveTarget: skipping %s - %s" %
(lfn, res["Message"]))
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.setTargetSURL(lfn, res['Value'])
if not res['OK']:
gLogger.warn("resolveTarget: skipping %s - %s" %
(lfn, res["Message"]))
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
toResolve = {}
for lfn in self.fileDict:
if "Target" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Target']] = lfn
if not toResolve:
return S_ERROR("No eligible Target files")
res = self.oTargetSE.exists(toResolve.keys())
if not res['OK']:
return S_ERROR("Failed to check target existence")
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
toRemove = []
for pfn, exists in res['Value']['Successful'].items():
if exists:
lfn = toResolve[pfn]
res = self.getSourceSURL(lfn)
if not res['OK']:
gLogger.warn("resolveTarget: skipping %s - target exists" %
lfn)
self.__setFileParameter(lfn, 'Reason', "Target exists")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif res['Value'] == pfn:
gLogger.warn(
"resolveTarget: skipping %s - source and target pfns are teh same"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Source and Target the same")
self.__setFileParameter(lfn, 'Status', 'Failed')
else:
toRemove.append(pfn)
if toRemove:
self.oTargetSE.removeFile(toRemove)
return S_OK()
def __filesToSubmit(self):
"""
check if there is at least one file to submit
:return: S_OK if at least one file is present, S_ERROR otherwise
"""
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get('Status')
source = self.fileDict[lfn].get('Source')
target = self.fileDict[lfn].get('Target')
if (lfnStatus != 'Failed') and (lfnStatus !=
'Done') and source and target:
return S_OK()
return S_ERROR()
def __createSURLPairFile(self):
""" create LFNs file for glite-transfer-submit command
This file consists one line for each fiel to be transferred:
sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM]
:param self: self reference
"""
fd, fileName = tempfile.mkstemp()
surlFile = os.fdopen(fd, 'w')
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get('Status')
source = self.fileDict[lfn].get('Source')
target = self.fileDict[lfn].get('Target')
if (lfnStatus not in ('Failed', 'Done')) and source and target:
cksmStr = ""
## add chsmType:cksm only if cksmType is specified, else let FTS decide by itself
if self.__cksmTest and self.__cksmType:
if lfn in self.catalogMetadata and "Checksum" in self.catalogMetadata[
lfn]:
cksmStr = " %s:%s" % (self.__cksmType, self.
catalogMetadata[lfn]["Checksum"])
surlFile.write("%s %s%s\n" % (source, target, cksmStr))
surlFile.close()
self.surlFile = fileName
return S_OK()
def __submitFTSTransfer(self):
""" create and execute glite-transfer-submit CLI command
:param self: self reference
"""
comm = [
'glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile,
'-o'
]
if self.targetToken:
comm.append('-t')
comm.append(self.targetToken)
if self.sourceToken:
comm.append('-S')
comm.append(self.sourceToken)
if self.__cksmTest:
comm.append("--compare-checksums")
res = executeGridCommand('', comm)
os.remove(self.surlFile)
if not res['OK']:
return res
returnCode, output, errStr = res['Value']
if not returnCode == 0:
return S_ERROR(errStr)
guid = output.replace('\n', '')
if not checkGuid(guid):
return S_ERROR('Wrong GUID format returned')
self.ftsGUID = guid
#if self.priority != 3:
# comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)]
# executeGridCommand('',comm)
return res
def __resolveFTSServer(self):
"""
resolve FTS server to use, it should be the closest one from target SE
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR("Source SE not set")
if not self.targetSE:
return S_ERROR("Target SE not set")
res = getSitesForSE(self.sourceSE, 'LCG')
if not res['OK'] or not res['Value']:
return S_ERROR("Could not determine source site")
sourceSite = res['Value'][0]
res = getSitesForSE(self.targetSE, 'LCG')
if not res['OK'] or not res['Value']:
return S_ERROR("Could not determine target site")
targetSite = res['Value'][0]
if (sourceSite == 'LCG.CERN.ch') or (targetSite == 'LCG.CERN.ch'):
ep = 'LCG.CERN.ch'
else:
# Target site FTS server should be used
ep = targetSite
try:
configPath = '/Resources/FTSEndpoints/%s' % ep
endpointURL = gConfig.getValue(configPath)
if not endpointURL:
errStr = "FTSRequest.__resolveFTSServer: Failed to find FTS endpoint, check CS entry for '%s'." % ep
return S_ERROR(errStr)
self.ftsServer = endpointURL
return S_OK(endpointURL)
except Exception, x:
return S_ERROR(
'FTSRequest.__resolveFTSServer: Failed to obtain endpoint details from CS'
)
class FTSRequest:
def __init__( self ):
self.gridEnv = '/afs/cern.ch/project/gd/LCG-share/3.2.8-0/etc/profile.d/grid-env'
self.finalStates = ['Canceled', 'Failed', 'Hold', 'Finished', 'FinishedDirty']
self.failedStates = ['Canceled', 'Failed', 'Hold', 'FinishedDirty']
self.successfulStates = ['Finished', 'Done']
self.fileStates = ['Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed', 'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting']
self.newlyCompletedFiles = []
self.newlyFailedFiles = []
self.statusSummary = {}
self.requestStatus = 'Unknown'
self.fileDict = {}
self.catalogReplicas = {}
self.catalogMetadata = {}
self.oCatalog = None
self.submitTime = ''
self.ftsGUID = ''
self.ftsServer = ''
self.priority = 3
self.isTerminal = False
self.percentageComplete = 0.0
self.sourceSE = ''
self.sourceValid = False
self.sourceToken = ''
self.targetSE = ''
self.targetValid = False
self.targetToken = ''
self.dumpStr = ''
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE( self, se ):
if se == self.targetSE:
return S_ERROR( "SourceSE is TargetSE" )
self.sourceSE = se
self.oSourceSE = StorageElement( self.sourceSE )
return self.__checkSourceSE()
def getSourceSE( self ):
if not self.sourceSE:
return S_ERROR( "Source SE not defined" )
return S_OK( self.sourceSE )
def setSourceToken( self, token ):
self.sourceToken = token
return S_OK()
def getSourceToken( self ):
if not self.sourceToken:
return S_ERROR( "Source token not defined" )
return S_OK( self.sourceToken )
def __checkSourceSE( self ):
if not self.sourceSE:
return S_ERROR( "SourceSE not set" )
res = self.oSourceSE.isValid( 'Read' )
if not res['OK']:
return S_ERROR( "SourceSE not available for reading" )
res = self.__getSESpaceToken( self.oSourceSE )
if not res['OK']:
gLogger.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message'] )
return S_ERROR( "SourceSE does not support FTS transfers" )
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE( self, se ):
if se == self.sourceSE:
return S_ERROR( "TargetSE is SourceSE" )
self.targetSE = se
self.oTargetSE = StorageElement( self.targetSE )
return self.__checkTargetSE()
def getTargetSE( self ):
if not self.targetSE:
return S_ERROR( "Target SE not defined" )
return S_OK( self.targetSE )
def setTargetToken( self, token ):
self.targetToken = token
return S_OK()
def getTargetToken( self ):
if not self.targetToken:
return S_ERROR( "Target token not defined" )
return S_OK( self.targetToken )
def __checkTargetSE( self ):
if not self.targetSE:
return S_ERROR( "TargetSE not set" )
res = self.oTargetSE.isValid( 'Write' )
if not res['OK']:
return S_ERROR( "TargetSE not available for writing" )
res = self.__getSESpaceToken( self.oTargetSE )
if not res['OK']:
gLogger.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message'] )
return S_ERROR( "SourceSE does not support FTS transfers" )
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
def __getSESpaceToken( self, oSE ):
res = oSE.getStorageParameters( "SRM2" )
if not res['OK']:
return res
return S_OK( res['Value'].get( 'SpaceToken' ) )
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID( self, guid ):
if not checkGuid( guid ):
return S_ERROR( "Incorrect GUID format" )
self.ftsGUID = guid
return S_OK()
def getFTSGUID( self ):
if not self.ftsGUID:
return S_ERROR( "FTSGUID not set" )
return S_OK( self.ftsGUID )
def setFTSServer( self, server ):
self.ftsServer = server
return S_OK()
def getFTSServer( self ):
if not self.ftsServer:
return S_ERROR( "FTSServer not set" )
return S_OK( self.ftsServer )
def setPriority( self, priority ):
if not type( priority ) in [types.IntType, types.LongType]:
return S_ERROR( "Priority must be integer" )
if priority < 0:
priority = 0
elif priority > 5:
priority = 5
self.priority = priority
return S_OK( self.priority )
def getPriority( self ):
return S_OK( self.priority )
def getPercentageComplete( self ):
completedFiles = 0
totalFiles = 0
for state in ( self.statusSummary.keys() ):
if state in self.successfulStates:
completedFiles += self.statusSummary[state]
totalFiles += self.statusSummary[state]
self.percentageComplete = ( float( completedFiles ) * 100.0 ) / float( totalFiles )
return S_OK( self.percentageComplete )
def isRequestTerminal( self ):
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK( self.isTerminal )
def getStatus( self ):
return S_OK( self.requestStatus )
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN( self, lfn ):
if not self.fileDict.has_key( lfn ):
self.fileDict[lfn] = {}
return S_OK()
def setSourceSURL( self, lfn, surl ):
target = self.fileDict[lfn].get( 'Target' )
if target == surl:
return S_ERROR( "Source and target the same" )
self.__setFileParameter( lfn, 'Source', surl )
return S_OK()
def getSourceSURL( self, lfn ):
return self.__getFileParameter( lfn, 'Source' )
def setTargetSURL( self, lfn, surl ):
source = self.fileDict[lfn].get( 'Source' )
if source == surl:
return S_ERROR( "Source and target the same" )
self.__setFileParameter( lfn, 'Target', surl )
return S_OK()
def getTargetSURL( self, lfn ):
return self.__getFileParameter( lfn, 'Target' )
def getFailReason( self, lfn ):
return self.__getFileParameter( lfn, 'Reason' )
def getRetries( self, lfn ):
return self.__getFileParameter( lfn, 'Retries' )
def getTransferTime( self, lfn ):
return self.__getFileParameter( lfn, 'Duration' )
def getFailed( self ):
failed = []
for lfn in self.fileDict.keys():
status = self.fileDict[lfn].get( 'Status', '' )
if status in self.failedStates:
failed.append( lfn )
return S_OK( failed )
def getDone( self ):
done = []
for lfn in self.fileDict.keys():
status = self.fileDict[lfn].get( 'Status', '' )
if status in self.successfulStates:
done.append( lfn )
return S_OK( done )
def __setFileParameter( self, lfn, paramName, paramValue ):
self.setLFN( lfn )
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter( self, lfn, paramName ):
if not self.fileDict.has_key( lfn ):
return S_ERROR( "Supplied file not set" )
if not self.fileDict[lfn].has_key( paramName ):
return S_ERROR( "%s not set for file" % paramName )
return S_OK( self.fileDict[lfn][paramName] )
####################################################################
#
# Methods for submission
#
def submit( self, monitor = False, printOutput = True ):
res = self.__isSubmissionValid()
if not res['OK']:
return res
res = self.__createSURLPairFile()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = {'ftsGUID':self.ftsGUID, 'ftsServer':self.ftsServer}
print "Submitted %s @ %s" % ( self.ftsGUID, self.ftsServer )
if monitor:
self.monitor( untilTerminal = True, printOutput = printOutput )
return S_OK( resDict )
def __isSubmissionValid( self ):
if not self.fileDict:
return S_ERROR( "No files set" )
if not self.sourceValid:
return S_ERROR( "SourceSE not valid" )
if not self.targetValid:
return S_ERROR( "TargetSE not valid" )
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR( "FTSServer not valid" )
self.__resolveSource()
self.__resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR( "No files to submit" )
return S_OK()
def __getCatalogObject( self ):
try:
if not self.oCatalog:
self.oCatalog = CatalogInterface()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache( self, lfns = [], overwrite = False ):
if not lfns:
lfns = self.fileDict.keys()
toUpdate = []
for lfn in lfns:
if ( not lfn in self.catalogReplicas.keys() ) or overwrite:
toUpdate.append( lfn )
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogReplicas( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to update replica cache", res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache( self, lfns = [], overwrite = False ):
if not lfns:
lfns = self.fileDict.keys()
toUpdate = []
for lfn in lfns:
if ( not lfn in self.catalogMetadata.keys() ) or overwrite:
toUpdate.append( lfn )
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogFileMetadata( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to get source catalog metadata", res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def __resolveSource( self ):
toResolve = []
for lfn in self.fileDict.keys():
if ( not self.fileDict[lfn].has_key( 'Source' ) ) and ( self.fileDict[lfn].get( 'Status' ) != 'Failed' ):
toResolve.append( lfn )
if not toResolve:
return S_OK()
res = self.__updateMetadataCache( toResolve )
if not res['OK']:
return res
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
for lfn in toResolve:
if self.fileDict[lfn].get( 'Status' ) == 'Failed':
continue
replicas = self.catalogReplicas.get( lfn, {} )
if not replicas.has_key( self.sourceSE ):
self.__setFileParameter( lfn, 'Reason', "No replica at SourceSE" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.oSourceSE.getPfnForProtocol( replicas[self.sourceSE], 'SRM2', withPort = True )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setSourceSURL( lfn, res['Value'] )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict.keys():
if self.fileDict[lfn].has_key( 'Source' ):
toResolve[self.fileDict[lfn]['Source']] = lfn
if not toResolve:
return S_ERROR( "No eligible Source files" )
res = self.oSourceSE.getFileMetadata( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check source file metadata" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
if re.search( 'File does not exist', error ):
self.__setFileParameter( lfn, 'Reason', "Source file does not exist" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
self.__setFileParameter( lfn, 'Reason', "Failed to get Source metadata" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for pfn, metadata in res['Value']['Successful'].items():
lfn = toResolve[pfn]
if metadata['Unavailable']:
self.__setFileParameter( lfn, 'Reason', "Source file Unavailable" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif metadata['Lost']:
self.__setFileParameter( lfn, 'Reason', "Source file Lost" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif not metadata['Cached']:
self.__setFileParameter( lfn, 'Reason', "Source file not Cached" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
self.__setFileParameter( lfn, 'Reason', "Source size mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and not ( compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ) ):
self.__setFileParameter( lfn, 'Reason', "Source checksum mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
return S_OK()
def __resolveTarget( self ):
toResolve = []
for lfn in self.fileDict.keys():
if not self.fileDict[lfn].has_key( 'Target' ) and ( self.fileDict[lfn].get( 'Status' ) != 'Failed' ):
toResolve.append( lfn )
if not toResolve:
return S_OK()
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
atTarget = []
for lfn in sortList( toResolve ):
if self.fileDict[lfn].get( 'Status' ) == 'Failed':
continue
replicas = self.catalogReplicas.get( lfn, {} )
if replicas.has_key( self.targetSE ):
self.__setFileParameter( lfn, 'Reason', "File already at Target" )
self.__setFileParameter( lfn, 'Status', 'Done' )
atTarget.append( lfn )
for lfn in toResolve:
if ( self.fileDict[lfn].get( 'Status' ) == 'Failed' ) or ( lfn in atTarget ):
continue
res = self.oTargetSE.getPfnForLfn( lfn )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', "Failed to create Target" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.oTargetSE.getPfnForProtocol( res['Value'], 'SRM2', withPort = True )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setTargetSURL( lfn, res['Value'] )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict.keys():
if self.fileDict[lfn].has_key( 'Target' ):
toResolve[self.fileDict[lfn]['Target']] = lfn
if not toResolve:
return S_ERROR( "No eligible Target files" )
res = self.oTargetSE.exists( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check target existence" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toRemove = []
for pfn, exists in res['Value']['Successful'].items():
if exists:
lfn = toResolve[pfn]
res = self.getSourceSURL( lfn )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', "Target exists" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif res['Value'] == pfn:
self.__setFileParameter( lfn, 'Reason', "Source and Target the same" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRemove.append( pfn )
if toRemove:
self.oTargetSE.removeFile( toRemove )
return S_OK()
def __filesToSubmit( self ):
for lfn in self.fileDict.keys():
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if ( lfnStatus != 'Failed' ) and ( lfnStatus != 'Done' ) and source and target:
return S_OK()
return S_ERROR()
def __createSURLPairFile( self ):
fd, fileName = tempfile.mkstemp()
surlFile = os.fdopen( fd, 'w' )
for lfn in self.fileDict.keys():
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if ( lfnStatus != 'Failed' ) and ( lfnStatus != 'Done' ) and source and target:
surlString = '%s %s\n' % ( source, target )
surlFile.write( surlString )
surlFile.close()
self.surlFile = fileName
return S_OK()
def __submitFTSTransfer( self ):
comm = ['glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile, '-o']
if self.targetToken:
comm.append( '-t' )
comm.append( self.targetToken )
if self.sourceToken:
comm.append( '-S' )
comm.append( self.sourceToken )
res = executeGridCommand( '', comm, self.gridEnv )
os.remove( self.surlFile )
if not res['OK']:
return res
returnCode, output, errStr = res['Value']
if not returnCode == 0:
return S_ERROR( errStr )
guid = output.replace( '\n', '' )
if not checkGuid( guid ):
return S_ERROR( 'Wrong GUID format returned' )
self.ftsGUID = guid
#if self.priority != 3:
# comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)]
# executeGridCommand('',comm,self.gridEnv)
return res
def __resolveFTSServer( self ):
if not self.sourceSE:
return S_ERROR( "Source SE not set" )
if not self.targetSE:
return S_ERROR( "Target SE not set" )
res = getSitesForSE( self.sourceSE, 'LCG' )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine source site" )
sourceSite = res['Value'][0]
res = getSitesForSE( self.targetSE, 'LCG' )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine target site" )
targetSite = res['Value'][0]
if ( sourceSite == 'LCG.CERN.ch' ) or ( targetSite == 'LCG.CERN.ch' ):
ep = 'LCG.CERN.ch'
else:
# Target site FTS server should be used
ep = targetSite
try:
configPath = '/Resources/FTSEndpoints/%s' % ep
endpointURL = gConfig.getValue( configPath )
if not endpointURL:
errStr = "FTSRequest.__resolveFTSEndpoint: Failed to find FTS endpoint, check CS entry for '%s'." % ep
return S_ERROR( errStr )
self.ftsServer = endpointURL
return S_OK( endpointURL )
except Exception, x:
return S_ERROR( 'FTSRequest.__resolveFTSEndpoint: Failed to obtain endpoint details from CS' )
class FTSRequest:
def __init__(self):
self.gridEnv = '/afs/cern.ch/project/gd/LCG-share/3.2.8-0/etc/profile.d/grid-env'
self.finalStates = [
'Canceled', 'Failed', 'Hold', 'Finished', 'FinishedDirty'
]
self.failedStates = ['Canceled', 'Failed', 'Hold', 'FinishedDirty']
self.successfulStates = ['Finished', 'Done']
self.fileStates = [
'Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed',
'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting'
]
self.newlyCompletedFiles = []
self.newlyFailedFiles = []
self.statusSummary = {}
self.requestStatus = 'Unknown'
self.fileDict = {}
self.catalogReplicas = {}
self.catalogMetadata = {}
self.oCatalog = None
self.submitTime = ''
self.ftsGUID = ''
self.ftsServer = ''
self.priority = 3
self.isTerminal = False
self.percentageComplete = 0.0
self.sourceSE = ''
self.sourceValid = False
self.sourceToken = ''
self.targetSE = ''
self.targetValid = False
self.targetToken = ''
self.dumpStr = ''
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE(self, se):
if se == self.targetSE:
return S_ERROR("SourceSE is TargetSE")
self.sourceSE = se
self.oSourceSE = StorageElement(self.sourceSE)
return self.__checkSourceSE()
def getSourceSE(self):
if not self.sourceSE:
return S_ERROR("Source SE not defined")
return S_OK(self.sourceSE)
def setSourceToken(self, token):
self.sourceToken = token
return S_OK()
def getSourceToken(self):
if not self.sourceToken:
return S_ERROR("Source token not defined")
return S_OK(self.sourceToken)
def __checkSourceSE(self):
if not self.sourceSE:
return S_ERROR("SourceSE not set")
res = self.oSourceSE.isValid('Read')
if not res['OK']:
return S_ERROR("SourceSE not available for reading")
res = self.__getSESpaceToken(self.oSourceSE)
if not res['OK']:
gLogger.error(
"FTSRequest failed to get SRM Space Token for SourceSE",
res['Message'])
return S_ERROR("SourceSE does not support FTS transfers")
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE(self, se):
if se == self.sourceSE:
return S_ERROR("TargetSE is SourceSE")
self.targetSE = se
self.oTargetSE = StorageElement(self.targetSE)
return self.__checkTargetSE()
def getTargetSE(self):
if not self.targetSE:
return S_ERROR("Target SE not defined")
return S_OK(self.targetSE)
def setTargetToken(self, token):
self.targetToken = token
return S_OK()
def getTargetToken(self):
if not self.targetToken:
return S_ERROR("Target token not defined")
return S_OK(self.targetToken)
def __checkTargetSE(self):
if not self.targetSE:
return S_ERROR("TargetSE not set")
res = self.oTargetSE.isValid('Write')
if not res['OK']:
return S_ERROR("TargetSE not available for writing")
res = self.__getSESpaceToken(self.oTargetSE)
if not res['OK']:
gLogger.error(
"FTSRequest failed to get SRM Space Token for TargetSE",
res['Message'])
return S_ERROR("SourceSE does not support FTS transfers")
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
def __getSESpaceToken(self, oSE):
res = oSE.getStorageParameters("SRM2")
if not res['OK']:
return res
return S_OK(res['Value'].get('SpaceToken'))
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID(self, guid):
if not checkGuid(guid):
return S_ERROR("Incorrect GUID format")
self.ftsGUID = guid
return S_OK()
def getFTSGUID(self):
if not self.ftsGUID:
return S_ERROR("FTSGUID not set")
return S_OK(self.ftsGUID)
def setFTSServer(self, server):
self.ftsServer = server
return S_OK()
def getFTSServer(self):
if not self.ftsServer:
return S_ERROR("FTSServer not set")
return S_OK(self.ftsServer)
def setPriority(self, priority):
if not type(priority) in [types.IntType, types.LongType]:
return S_ERROR("Priority must be integer")
if priority < 0:
priority = 0
elif priority > 5:
priority = 5
self.priority = priority
return S_OK(self.priority)
def getPriority(self):
return S_OK(self.priority)
def getPercentageComplete(self):
completedFiles = 0
totalFiles = 0
for state in (self.statusSummary.keys()):
if state in self.successfulStates:
completedFiles += self.statusSummary[state]
totalFiles += self.statusSummary[state]
self.percentageComplete = (float(completedFiles) *
100.0) / float(totalFiles)
return S_OK(self.percentageComplete)
def isRequestTerminal(self):
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK(self.isTerminal)
def getStatus(self):
return S_OK(self.requestStatus)
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN(self, lfn):
if not self.fileDict.has_key(lfn):
self.fileDict[lfn] = {}
return S_OK()
def setSourceSURL(self, lfn, surl):
target = self.fileDict[lfn].get('Target')
if target == surl:
return S_ERROR("Source and target the same")
self.__setFileParameter(lfn, 'Source', surl)
return S_OK()
def getSourceSURL(self, lfn):
return self.__getFileParameter(lfn, 'Source')
def setTargetSURL(self, lfn, surl):
source = self.fileDict[lfn].get('Source')
if source == surl:
return S_ERROR("Source and target the same")
self.__setFileParameter(lfn, 'Target', surl)
return S_OK()
def getTargetSURL(self, lfn):
return self.__getFileParameter(lfn, 'Target')
def getFailReason(self, lfn):
return self.__getFileParameter(lfn, 'Reason')
def getRetries(self, lfn):
return self.__getFileParameter(lfn, 'Retries')
def getTransferTime(self, lfn):
return self.__getFileParameter(lfn, 'Duration')
def getFailed(self):
failed = []
for lfn in self.fileDict.keys():
status = self.fileDict[lfn].get('Status', '')
if status in self.failedStates:
failed.append(lfn)
return S_OK(failed)
def getDone(self):
done = []
for lfn in self.fileDict.keys():
status = self.fileDict[lfn].get('Status', '')
if status in self.successfulStates:
done.append(lfn)
return S_OK(done)
def __setFileParameter(self, lfn, paramName, paramValue):
self.setLFN(lfn)
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter(self, lfn, paramName):
if not self.fileDict.has_key(lfn):
return S_ERROR("Supplied file not set")
if not self.fileDict[lfn].has_key(paramName):
return S_ERROR("%s not set for file" % paramName)
return S_OK(self.fileDict[lfn][paramName])
####################################################################
#
# Methods for submission
#
def submit(self, monitor=False, printOutput=True):
res = self.__isSubmissionValid()
if not res['OK']:
return res
res = self.__createSURLPairFile()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = {'ftsGUID': self.ftsGUID, 'ftsServer': self.ftsServer}
print "Submitted %s @ %s" % (self.ftsGUID, self.ftsServer)
if monitor:
self.monitor(untilTerminal=True, printOutput=printOutput)
return S_OK(resDict)
def __isSubmissionValid(self):
if not self.fileDict:
return S_ERROR("No files set")
if not self.sourceValid:
return S_ERROR("SourceSE not valid")
if not self.targetValid:
return S_ERROR("TargetSE not valid")
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR("FTSServer not valid")
self.__resolveSource()
self.__resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR("No files to submit")
return S_OK()
def __getCatalogObject(self):
try:
if not self.oCatalog:
self.oCatalog = CatalogInterface()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache(self, lfns=[], overwrite=False):
if not lfns:
lfns = self.fileDict.keys()
toUpdate = []
for lfn in lfns:
if (not lfn in self.catalogReplicas.keys()) or overwrite:
toUpdate.append(lfn)
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogReplicas(toUpdate)
if not res['OK']:
return S_ERROR("Failed to update replica cache", res['Message'])
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache(self, lfns=[], overwrite=False):
if not lfns:
lfns = self.fileDict.keys()
toUpdate = []
for lfn in lfns:
if (not lfn in self.catalogMetadata.keys()) or overwrite:
toUpdate.append(lfn)
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogFileMetadata(toUpdate)
if not res['OK']:
return S_ERROR("Failed to get source catalog metadata",
res['Message'])
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def __resolveSource(self):
toResolve = []
for lfn in self.fileDict.keys():
if (not self.fileDict[lfn].has_key('Source')) and (
self.fileDict[lfn].get('Status') != 'Failed'):
toResolve.append(lfn)
if not toResolve:
return S_OK()
res = self.__updateMetadataCache(toResolve)
if not res['OK']:
return res
res = self.__updateReplicaCache(toResolve)
if not res['OK']:
return res
for lfn in toResolve:
if self.fileDict[lfn].get('Status') == 'Failed':
continue
replicas = self.catalogReplicas.get(lfn, {})
if not replicas.has_key(self.sourceSE):
self.__setFileParameter(lfn, 'Reason',
"No replica at SourceSE")
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.oSourceSE.getPfnForProtocol(replicas[self.sourceSE],
'SRM2',
withPort=True)
if not res['OK']:
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.setSourceSURL(lfn, res['Value'])
if not res['OK']:
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
toResolve = {}
for lfn in self.fileDict.keys():
if self.fileDict[lfn].has_key('Source'):
toResolve[self.fileDict[lfn]['Source']] = lfn
if not toResolve:
return S_ERROR("No eligible Source files")
res = self.oSourceSE.getFileMetadata(toResolve.keys())
if not res['OK']:
return S_ERROR("Failed to check source file metadata")
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
if re.search('File does not exist', error):
self.__setFileParameter(lfn, 'Reason',
"Source file does not exist")
self.__setFileParameter(lfn, 'Status', 'Failed')
else:
self.__setFileParameter(lfn, 'Reason',
"Failed to get Source metadata")
self.__setFileParameter(lfn, 'Status', 'Failed')
for pfn, metadata in res['Value']['Successful'].items():
lfn = toResolve[pfn]
if metadata['Unavailable']:
self.__setFileParameter(lfn, 'Reason',
"Source file Unavailable")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif metadata['Lost']:
self.__setFileParameter(lfn, 'Reason', "Source file Lost")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif not metadata['Cached']:
self.__setFileParameter(lfn, 'Reason',
"Source file not Cached")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
self.__setFileParameter(lfn, 'Reason', "Source size mismatch")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif self.catalogMetadata[lfn]['Checksum'] and metadata[
'Checksum'] and not (compareAdler(
metadata['Checksum'],
self.catalogMetadata[lfn]['Checksum'])):
self.__setFileParameter(lfn, 'Reason',
"Source checksum mismatch")
self.__setFileParameter(lfn, 'Status', 'Failed')
return S_OK()
def __resolveTarget(self):
toResolve = []
for lfn in self.fileDict.keys():
if not self.fileDict[lfn].has_key('Target') and (
self.fileDict[lfn].get('Status') != 'Failed'):
toResolve.append(lfn)
if not toResolve:
return S_OK()
res = self.__updateReplicaCache(toResolve)
if not res['OK']:
return res
atTarget = []
for lfn in sortList(toResolve):
if self.fileDict[lfn].get('Status') == 'Failed':
continue
replicas = self.catalogReplicas.get(lfn, {})
if replicas.has_key(self.targetSE):
self.__setFileParameter(lfn, 'Reason',
"File already at Target")
self.__setFileParameter(lfn, 'Status', 'Done')
atTarget.append(lfn)
for lfn in toResolve:
if (self.fileDict[lfn].get('Status') == 'Failed') or (lfn
in atTarget):
continue
res = self.oTargetSE.getPfnForLfn(lfn)
if not res['OK']:
self.__setFileParameter(lfn, 'Reason',
"Failed to create Target")
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.oTargetSE.getPfnForProtocol(res['Value'],
'SRM2',
withPort=True)
if not res['OK']:
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.setTargetSURL(lfn, res['Value'])
if not res['OK']:
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
toResolve = {}
for lfn in self.fileDict.keys():
if self.fileDict[lfn].has_key('Target'):
toResolve[self.fileDict[lfn]['Target']] = lfn
if not toResolve:
return S_ERROR("No eligible Target files")
res = self.oTargetSE.exists(toResolve.keys())
if not res['OK']:
return S_ERROR("Failed to check target existence")
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
toRemove = []
for pfn, exists in res['Value']['Successful'].items():
if exists:
lfn = toResolve[pfn]
res = self.getSourceSURL(lfn)
if not res['OK']:
self.__setFileParameter(lfn, 'Reason', "Target exists")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif res['Value'] == pfn:
self.__setFileParameter(lfn, 'Reason',
"Source and Target the same")
self.__setFileParameter(lfn, 'Status', 'Failed')
else:
toRemove.append(pfn)
if toRemove:
self.oTargetSE.removeFile(toRemove)
return S_OK()
def __filesToSubmit(self):
for lfn in self.fileDict.keys():
lfnStatus = self.fileDict[lfn].get('Status')
source = self.fileDict[lfn].get('Source')
target = self.fileDict[lfn].get('Target')
if (lfnStatus != 'Failed') and (lfnStatus !=
'Done') and source and target:
return S_OK()
return S_ERROR()
def __createSURLPairFile(self):
fd, fileName = tempfile.mkstemp()
surlFile = os.fdopen(fd, 'w')
for lfn in self.fileDict.keys():
lfnStatus = self.fileDict[lfn].get('Status')
source = self.fileDict[lfn].get('Source')
target = self.fileDict[lfn].get('Target')
if (lfnStatus != 'Failed') and (lfnStatus !=
'Done') and source and target:
surlString = '%s %s\n' % (source, target)
surlFile.write(surlString)
surlFile.close()
self.surlFile = fileName
return S_OK()
def __submitFTSTransfer(self):
comm = [
'glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile,
'-o'
]
if self.targetToken:
comm.append('-t')
comm.append(self.targetToken)
if self.sourceToken:
comm.append('-S')
comm.append(self.sourceToken)
res = executeGridCommand('', comm, self.gridEnv)
os.remove(self.surlFile)
if not res['OK']:
return res
returnCode, output, errStr = res['Value']
if not returnCode == 0:
return S_ERROR(errStr)
guid = output.replace('\n', '')
if not checkGuid(guid):
return S_ERROR('Wrong GUID format returned')
self.ftsGUID = guid
#if self.priority != 3:
# comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)]
# executeGridCommand('',comm,self.gridEnv)
return res
def __resolveFTSServer(self):
if not self.sourceSE:
return S_ERROR("Source SE not set")
if not self.targetSE:
return S_ERROR("Target SE not set")
res = getSitesForSE(self.sourceSE, 'LCG')
if not res['OK'] or not res['Value']:
return S_ERROR("Could not determine source site")
sourceSite = res['Value'][0]
res = getSitesForSE(self.targetSE, 'LCG')
if not res['OK'] or not res['Value']:
return S_ERROR("Could not determine target site")
targetSite = res['Value'][0]
if (sourceSite == 'LCG.CERN.ch') or (targetSite == 'LCG.CERN.ch'):
ep = 'LCG.CERN.ch'
else:
# Target site FTS server should be used
ep = targetSite
try:
configPath = '/Resources/FTSEndpoints/%s' % ep
endpointURL = gConfig.getValue(configPath)
if not endpointURL:
errStr = "FTSRequest.__resolveFTSEndpoint: Failed to find FTS endpoint, check CS entry for '%s'." % ep
return S_ERROR(errStr)
self.ftsServer = endpointURL
return S_OK(endpointURL)
except Exception, x:
return S_ERROR(
'FTSRequest.__resolveFTSEndpoint: Failed to obtain endpoint details from CS'
)