def __removeStorageDirectory( self, directory, storageElement ):
""" wipe out all contents from :directory: at :storageElement:
:param self: self reference
:param str directory: path
:param str storageElement: SE name
"""
self.log.info( 'Removing the contents of %s at %s' % ( directory, storageElement ) )
se = StorageElement( storageElement )
res = returnSingleResult( se.exists( directory ) )
if not res['OK']:
self.log.error( "Failed to obtain existance of directory", res['Message'] )
return res
exists = res['Value']
if not exists:
self.log.info( "The directory %s does not exist at %s " % ( directory, storageElement ) )
return S_OK()
res = returnSingleResult( se.removeDirectory( directory, recursive = True ) )
if not res['OK']:
self.log.error( "Failed to remove storage directory", res['Message'] )
return res
self.log.info( "Successfully removed %d files from %s at %s" % ( res['Value']['FilesRemoved'],
directory,
storageElement ) )
return S_OK()
def __getStoragePathExists(self, lfnPaths, storageElement):
gLogger.info('Determining the existance of %d files at %s' %
(len(lfnPaths), storageElement))
se = StorageElement(storageElement)
res = se.getPfnForLfn(lfnPaths)
if not res['OK']:
gLogger.error("Failed to get PFNs for LFNs", res['Message'])
return res
for lfnPath, error in res['Value']['Failed'].items():
gLogger.error('Failed to obtain PFN from LFN',
'%s %s' % (lfnPath, error))
if res['Value']['Failed']:
return S_ERROR('Failed to obtain PFNs from LFNs')
lfnPfns = res['Value']['Successful']
pfnLfns = {}
for lfn, pfn in lfnPfns.items():
pfnLfns[pfn] = lfn
res = se.exists(pfnLfns)
if not res['OK']:
gLogger.error("Failed to obtain existance of paths",
res['Message'])
return res
for lfnPath, error in res['Value']['Failed'].items():
gLogger.error('Failed to determine existance of path',
'%s %s' % (lfnPath, error))
if res['Value']['Failed']:
return S_ERROR('Failed to determine existance of paths')
pathExists = res['Value']['Successful']
resDict = {}
for pfn, exists in pathExists.items():
if exists:
resDict[pfnLfns[pfn]] = pfn
return S_OK(resDict)
def __getStoragePathExists( self, lfnPaths, storageElement ):
gLogger.info( 'Determining the existance of %d files at %s' % ( len( lfnPaths ), storageElement ) )
se = StorageElement( storageElement )
res = se.getPfnForLfn( lfnPaths )
if not res['OK']:
gLogger.error( "Failed to get PFNs for LFNs", res['Message'] )
return res
for lfnPath, error in res['Value']['Failed'].items():
gLogger.error( 'Failed to obtain PFN from LFN', '%s %s' % ( lfnPath, error ) )
if res['Value']['Failed']:
return S_ERROR( 'Failed to obtain PFNs from LFNs' )
lfnPfns = res['Value']['Successful']
pfnLfns = {}
for lfn, pfn in lfnPfns.items():
pfnLfns[pfn] = lfn
res = se.exists( pfnLfns )
if not res['OK']:
gLogger.error( "Failed to obtain existance of paths", res['Message'] )
return res
for lfnPath, error in res['Value']['Failed'].items():
gLogger.error( 'Failed to determine existance of path', '%s %s' % ( lfnPath, error ) )
if res['Value']['Failed']:
return S_ERROR( 'Failed to determine existance of paths' )
pathExists = res['Value']['Successful']
resDict = {}
for pfn, exists in pathExists.items():
if exists:
resDict[pfnLfns[pfn]] = pfn
return S_OK( resDict )
def __removeStorageDirectory(self, directory, storageElement):
""" wipe out all contents from :directory: at :storageElement:
:param self: self reference
:param str directory: path
:param str storageElement: SE name
"""
self.log.info('Removing the contents of %s at %s' %
(directory, storageElement))
se = StorageElement(storageElement)
res = returnSingleResult(se.exists(directory))
if not res['OK']:
self.log.error("Failed to obtain existance of directory",
res['Message'])
return res
exists = res['Value']
if not exists:
self.log.info("The directory %s does not exist at %s " %
(directory, storageElement))
return S_OK()
res = returnSingleResult(se.removeDirectory(directory, recursive=True))
if not res['OK']:
self.log.error("Failed to remove storage directory",
res['Message'])
return res
self.log.info(
"Successfully removed %d files from %s at %s" %
(res['Value']['FilesRemoved'], directory, storageElement))
return S_OK()
def __getStoragePathExists( self, lfnPaths, storageElement ):
gLogger.info( 'Determining the existance of %d files at %s' % ( len( lfnPaths ), storageElement ) )
se = StorageElement( storageElement )
res = se.exists( lfnPaths )
if not res['OK']:
gLogger.error( "Failed to obtain existance of paths", res['Message'] )
return res
for lfnPath, error in res['Value']['Failed'].items():
gLogger.error( 'Failed to determine existance of path', '%s %s' % ( lfnPath, error ) )
if res['Value']['Failed']:
return S_ERROR( 'Failed to determine existance of paths' )
pathExists = res['Value']['Successful']
resDict = {}
for lfn, exists in pathExists.items():
if exists:
resDict[lfn] = True
return S_OK( resDict )
def removeStorageDirectoryFromSE( directory, storageElement ):
"""
Delete directory on selected storage element
"""
se = StorageElement( storageElement, False )
res = returnSingleResult( se.exists( directory ) )
if not res['OK']:
return S_ERROR( "Failed to obtain existence of directory" + res['Message'] )
exists = res['Value']
if not exists:
return S_OK( "The directory %s does not exist at %s " % ( directory, storageElement ) )
res = returnSingleResult( se.removeDirectory( directory, recursive = True ) )
if not res['OK']:
return S_ERROR( "Failed to remove storage directory" + res['Message'] )
return S_OK()
def removeStorageDirectoryFromSE( directory, storageElement ):
"""
Delete directory on selected storage element
"""
se = StorageElement( storageElement, False )
res = returnSingleResult( se.exists( directory ) )
if not res['OK']:
return S_ERROR( "Failed to obtain existence of directory" + res['Message'] )
exists = res['Value']
if not exists:
return S_OK( "The directory %s does not exist at %s " % ( directory, storageElement ) )
res = returnSingleResult( se.removeDirectory( directory, recursive = True ) )
if not res['OK']:
return S_ERROR( "Failed to remove storage directory" + res['Message'] )
return S_OK()
def removeStorageDirectoryFromSE(directory, storageElement):
"""
Delete directory on selected storage element
"""
from DIRAC.Resources.Storage.StorageElement import StorageElement
se = StorageElement(storageElement, False)
res = returnSingleResult(se.exists(directory))
if not res["OK"]:
return S_ERROR("Failed to obtain existence of directory" + res["Message"])
exists = res["Value"]
if not exists:
return S_OK("The directory %s does not exist at %s " % (directory, storageElement))
res = returnSingleResult(se.removeDirectory(directory, recursive=True))
if not res["OK"]:
return S_ERROR("Failed to remove storage directory" + res["Message"])
return S_OK()
def __getStoragePathExists(self, lfnPaths, storageElement):
gLogger.info('Determining the existance of %d files at %s' %
(len(lfnPaths), storageElement))
se = StorageElement(storageElement)
res = se.exists(lfnPaths)
if not res['OK']:
gLogger.error("Failed to obtain existance of paths",
res['Message'])
return res
for lfnPath, error in res['Value']['Failed'].items():
gLogger.error('Failed to determine existance of path',
'%s %s' % (lfnPath, error))
if res['Value']['Failed']:
return S_ERROR('Failed to determine existance of paths')
pathExists = res['Value']['Successful']
resDict = {}
for lfn, exists in pathExists.items():
if exists:
resDict[lfn] = True
return S_OK(resDict)
def __removeStorageDirectory( self, directory, storageElement ):
""" wipe out all contents from :directory: at :storageElement:
:param self: self reference
:param str directory: path
:param str storageElement: SE name
"""
self.log.info( 'Removing the contents of %s at %s' % ( directory, storageElement ) )
se = StorageElement( storageElement )
res = se.getPfnForLfn( [directory] )
if not res['OK']:
self.log.error( "Failed to get PFN for directory", res['Message'] )
return res
if directory in res['Value']['Failed']:
self.log.verbose( 'Failed to obtain directory PFN from LFN', '%s %s' % ( directory, res['Value']['Failed'][directory] ) )
return S_ERROR( 'Failed to obtain directory PFN from LFNs' )
storageDirectory = res['Value']['Successful'][directory]
res = Utils.executeSingleFileOrDirWrapper( se.exists( storageDirectory ) )
if not res['OK']:
self.log.error( "Failed to obtain existance of directory", res['Message'] )
return res
exists = res['Value']
if not exists:
self.log.info( "The directory %s does not exist at %s " % ( directory, storageElement ) )
return S_OK()
res = Utils.executeSingleFileOrDirWrapper( se.removeDirectory( storageDirectory, recursive = True ) )
if not res['OK']:
self.log.error( "Failed to remove storage directory", res['Message'] )
return res
self.log.info( "Successfully removed %d files from %s at %s" % ( res['Value']['FilesRemoved'],
directory,
storageElement ) )
return S_OK()
class FTSRequest( object ):
"""
.. class:: FTSRequest
Helper class for FTS job submission and monitoring.
"""
# # default checksum type
__defaultCksmType = "ADLER32"
# # flag to disablr/enable checksum test, default: disabled
__cksmTest = False
def __init__( self ):
"""c'tor
:param self: self reference
"""
self.log = gLogger.getSubLogger( self.__class__.__name__, True )
# # final states tuple
self.finalStates = ( 'Canceled', 'Failed', 'Hold',
'Finished', 'FinishedDirty' )
# # failed states tuple
self.failedStates = ( 'Canceled', 'Failed',
'Hold', 'FinishedDirty' )
# # successful states tuple
self.successfulStates = ( 'Finished', 'Done' )
# # all file states tuple
self.fileStates = ( 'Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed',
'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting' )
self.statusSummary = {}
# # request status
self.requestStatus = 'Unknown'
# # dict for FTS job files
self.fileDict = {}
# # dict for replicas information
self.catalogReplicas = {}
# # dict for metadata information
self.catalogMetadata = {}
# # dict for files that failed to register
self.failedRegistrations = {}
# # placehoder for FileCatalog reference
self.oCatalog = None
# # submit timestamp
self.submitTime = ''
# # placeholder FTS job GUID
self.ftsGUID = ''
# # placeholder for FTS server URL
self.ftsServer = ''
# # flag marking FTS job completness
self.isTerminal = False
# # completness percentage
self.percentageComplete = 0.0
# # source SE name
self.sourceSE = ''
# # flag marking source SE validity
self.sourceValid = False
# # source space token
self.sourceToken = ''
# # target SE name
self.targetSE = ''
# # flag marking target SE validity
self.targetValid = False
# # target space token
self.targetToken = ''
# # placeholder for target StorageElement
self.oTargetSE = None
# # placeholder for source StorageElement
self.oSourceSE = None
# # checksum type, set it to default
self.__cksmType = self.__defaultCksmType
# # disable checksum test by default
self.__cksmTest = False
# # statuses that prevent submitting to FTS
self.noSubmitStatus = ( 'Failed', 'Done', 'Staging' )
# # were sources resolved?
self.sourceResolved = False
# # Number of file transfers actually submitted
self.submittedFiles = 0
self.transferTime = 0
self.submitCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/SubmitCommand', 'glite-transfer-submit' )
self.monitorCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/MonitorCommand', 'glite-transfer-status' )
self.ftsJob = None
self.ftsFiles = []
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE( self, se ):
""" set SE for source
:param self: self reference
:param str se: source SE name
"""
if se == self.targetSE:
return S_ERROR( "SourceSE is TargetSE" )
self.sourceSE = se
self.oSourceSE = StorageElement( self.sourceSE )
return self.__checkSourceSE()
def __checkSourceSE( self ):
""" check source SE availability
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR( "SourceSE not set" )
res = self.oSourceSE.isValid( 'Read' )
if not res['OK']:
return S_ERROR( "SourceSE not available for reading" )
res = self.__getSESpaceToken( self.oSourceSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message'] )
return S_ERROR( "SourceSE does not support FTS transfers" )
if self.__cksmTest:
res = self.oSourceSE.getChecksumType()
if not res["OK"]:
self.log.error( "Unable to get checksum type for SourceSE %s: %s" % ( self.sourceSE,
res["Message"] ) )
cksmType = res["Value"]
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at SourceSE %s, disabling checksum test" % ( cksmType,
self.sourceSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE( self, se ):
""" set target SE
:param self: self reference
:param str se: target SE name
"""
if se == self.sourceSE:
return S_ERROR( "TargetSE is SourceSE" )
self.targetSE = se
self.oTargetSE = StorageElement( self.targetSE )
return self.__checkTargetSE()
def setTargetToken( self, token ):
""" target space token setter
:param self: self reference
:param str token: target space token
"""
self.targetToken = token
return S_OK()
def __checkTargetSE( self ):
""" check target SE availability
:param self: self reference
"""
if not self.targetSE:
return S_ERROR( "TargetSE not set" )
res = self.oTargetSE.isValid( 'Write' )
if not res['OK']:
return S_ERROR( "TargetSE not available for writing" )
res = self.__getSESpaceToken( self.oTargetSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message'] )
return S_ERROR( "TargetSE does not support FTS transfers" )
# # check checksum types
if self.__cksmTest:
res = self.oTargetSE.getChecksumType()
if not res["OK"]:
self.log.error( "Unable to get checksum type for TargetSE %s: %s" % ( self.targetSE,
res["Message"] ) )
cksmType = res["Value"]
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at TargetSE %s, disabling checksum test" % ( cksmType,
self.targetSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
@staticmethod
def __getSESpaceToken( oSE ):
""" get space token from StorageElement instance
:param self: self reference
:param StorageElement oSE: StorageElement instance
"""
res = oSE.getStorageParameters( "SRM2" )
if not res['OK']:
return res
return S_OK( res['Value'].get( 'SpaceToken' ) )
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID( self, guid ):
""" FTS job GUID setter
:param self: self reference
:param str guid: string containg GUID
"""
if not checkGuid( guid ):
return S_ERROR( "Incorrect GUID format" )
self.ftsGUID = guid
return S_OK()
def setFTSServer( self, server ):
""" FTS server setter
:param self: self reference
:param str server: FTS server URL
"""
self.ftsServer = server
return S_OK()
def isRequestTerminal( self ):
""" check if FTS job has terminated
:param self: self reference
"""
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK( self.isTerminal )
def setCksmTest( self, cksmTest = False ):
""" set cksm test
:param self: self reference
:param bool cksmTest: flag to enable/disable checksum test
"""
self.__cksmTest = bool( cksmTest )
return S_OK( self.__cksmTest )
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN( self, lfn ):
""" add LFN :lfn: to :fileDict:
:param self: self reference
:param str lfn: LFN to add to
"""
self.fileDict.setdefault( lfn, {'Status':'Waiting'} )
return S_OK()
def setSourceSURL( self, lfn, surl ):
""" source SURL setter
:param self: self reference
:param str lfn: LFN
:param str surl: source SURL
"""
target = self.fileDict[lfn].get( 'Target' )
if target == surl:
return S_ERROR( "Source and target the same" )
return self.__setFileParameter( lfn, 'Source', surl )
def getSourceSURL( self, lfn ):
""" get source SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Source' )
def setTargetSURL( self, lfn, surl ):
""" set target SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
:param str surl: target SURL
"""
source = self.fileDict[lfn].get( 'Source' )
if source == surl:
return S_ERROR( "Source and target the same" )
return self.__setFileParameter( lfn, 'Target', surl )
def getFailReason( self, lfn ):
""" get fail reason for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Reason' )
def getRetries( self, lfn ):
""" get number of attepmts made to transfer file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Retries' )
def getTransferTime( self, lfn ):
""" get duration of transfer for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Duration' )
def getFailed( self ):
""" get list of wrongly transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.failedStates ] )
def getStaging( self ):
""" get files set for prestaging """
return S_OK( [lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) == 'Staging'] )
def getDone( self ):
""" get list of succesfully transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.successfulStates ] )
def __setFileParameter( self, lfn, paramName, paramValue ):
""" set :paramName: to :paramValue: for :lfn: file
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
:param mixed paramValue: a new parameter value
"""
self.setLFN( lfn )
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter( self, lfn, paramName ):
""" get value of :paramName: for file :lfn:
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
"""
if lfn not in self.fileDict:
return S_ERROR( "Supplied file not set" )
if paramName not in self.fileDict[lfn]:
return S_ERROR( "%s not set for file" % paramName )
return S_OK( self.fileDict[lfn][paramName] )
####################################################################
#
# Methods for submission
#
def submit( self, monitor = False, printOutput = True ):
""" submit FTS job
:param self: self reference
:param bool monitor: flag to monitor progress of FTS job
:param bool printOutput: flag to print output of execution to stdout
"""
res = self.__prepareForSubmission()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = { 'ftsGUID' : self.ftsGUID, 'ftsServer' : self.ftsServer, 'submittedFiles' : self.submittedFiles }
if monitor or printOutput:
gLogger.always( "Submitted %s@%s" % ( self.ftsGUID, self.ftsServer ) )
if monitor:
self.monitor( untilTerminal = True, printOutput = printOutput, full = False )
return S_OK( resDict )
def __prepareForSubmission( self ):
""" check validity of job before submission
:param self: self reference
"""
if not self.fileDict:
return S_ERROR( "No files set" )
if not self.sourceValid:
return S_ERROR( "SourceSE not valid" )
if not self.targetValid:
return S_ERROR( "TargetSE not valid" )
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR( "FTSServer not valid" )
self.resolveSource()
self.resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR( "No files to submit" )
return S_OK()
def __getCatalogObject( self ):
""" CatalogInterface instance facade
:param self: self reference
"""
try:
if not self.oCatalog:
self.oCatalog = FileCatalog()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache( self, lfns = None, overwrite = False ):
""" update replica cache for list of :lfns:
:param self: self reference
:param mixed lfns: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if ( lfn not in self.catalogReplicas ) or overwrite ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getReplicas( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to update replica cache: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache( self, lfns = None ):
""" update metadata cache for list of LFNs
:param self: self reference
:param list lnfs: list of LFNs
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if lfn not in self.catalogMetadata ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getFileMetadata( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to get source catalog metadata: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def resolveSource( self ):
""" resolve source SE eligible for submission
:param self: self reference
"""
# Avoid resolving sources twice
if self.sourceResolved:
return S_OK()
# Only resolve files that need a transfer
toResolve = [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( "Status", "" ) != "Failed" ]
if not toResolve:
return S_OK()
res = self.__updateMetadataCache( toResolve )
if not res['OK']:
return res
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
# Define the source URLs
for lfn in toResolve:
replicas = self.catalogReplicas.get( lfn, {} )
if self.sourceSE not in replicas:
gLogger.warn( "resolveSource: skipping %s - not replicas at SourceSE %s" % ( lfn, self.sourceSE ) )
self.__setFileParameter( lfn, 'Reason', "No replica at SourceSE" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
# Fix first the PFN
pfn = self.oSourceSE.getPfnForLfn( lfn ).get( 'Value', {} ).get( 'Successful', {} ).get( lfn, replicas[self.sourceSE] )
res = returnSingleResult( self.oSourceSE.getPfnForProtocol( pfn, protocol = 'SRM2', withPort = True ) )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setSourceSURL( lfn, res['Value'] )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict:
if "Source" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Source']] = lfn
if not toResolve:
return S_ERROR( "No eligible Source files" )
# Get metadata of the sources, to check for existance, availability and caching
res = self.oSourceSE.getFileMetadata( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check source file metadata" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
if re.search( 'File does not exist', error ):
gLogger.warn( "resolveSource: skipping %s - source file does not exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file does not exist" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
gLogger.warn( "resolveSource: skipping %s - failed to get source metadata" % lfn )
self.__setFileParameter( lfn, 'Reason', "Failed to get Source metadata" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toStage = []
nbStagedFiles = 0
for pfn, metadata in res['Value']['Successful'].items():
lfn = toResolve[pfn]
lfnStatus = self.fileDict.get( lfn, {} ).get( 'Status' )
if metadata['Unavailable']:
gLogger.warn( "resolveSource: skipping %s - source file unavailable" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Unavailable" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif metadata['Lost']:
gLogger.warn( "resolveSource: skipping %s - source file lost" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Lost" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif not metadata['Cached']:
if lfnStatus != 'Staging':
toStage.append( pfn )
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
gLogger.warn( "resolveSource: skipping %s - source file size mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source size mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \
not compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ):
gLogger.warn( "resolveSource: skipping %s - source file checksum mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source checksum mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif lfnStatus == 'Staging':
# file that was staging is now cached
self.__setFileParameter( lfn, 'Status', 'Waiting' )
nbStagedFiles += 1
# Some files were being staged
if nbStagedFiles:
self.log.info( 'resolveSource: %d files have been staged' % nbStagedFiles )
# Launching staging of files not in cache
if toStage:
gLogger.warn( "resolveSource: %s source files not cached, prestaging..." % len( toStage ) )
stage = self.oSourceSE.prestageFile( toStage )
if not stage["OK"]:
gLogger.error( "resolveSource: error is prestaging - %s" % stage["Message"] )
for pfn in toStage:
lfn = toResolve[pfn]
self.__setFileParameter( lfn, 'Reason', stage["Message"] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
for pfn in toStage:
lfn = toResolve[pfn]
if pfn in stage['Value']['Successful']:
self.__setFileParameter( lfn, 'Status', 'Staging' )
elif pfn in stage['Value']['Failed']:
self.__setFileParameter( lfn, 'Reason', stage['Value']['Failed'][pfn] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
self.sourceResolved = True
return S_OK()
def resolveTarget( self ):
""" find target SE eligible for submission
:param self: self reference
"""
toResolve = [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status' ) not in self.noSubmitStatus ]
if not toResolve:
return S_OK()
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
for lfn in toResolve:
res = self.oTargetSE.getPfnForLfn( lfn )
if not res['OK'] or lfn not in res['Value']['Successful']:
gLogger.warn( "resolveTarget: skipping %s - failed to create target pfn" % lfn )
self.__setFileParameter( lfn, 'Reason', "Failed to create Target" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
pfn = res['Value']['Successful'][lfn]
res = self.oTargetSE.getPfnForProtocol( pfn, protocol = 'SRM2', withPort = True )
if not res['OK'] or pfn not in res['Value']['Successful']:
reason = res.get( 'Message', res.get( 'Value', {} ).get( 'Failed', {} ).get( pfn ) )
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, reason ) )
self.__setFileParameter( lfn, 'Reason', reason )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
pfn = res['Value']['Successful'][pfn]
res = self.setTargetSURL( lfn, pfn )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict:
if "Target" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Target']] = lfn
if not toResolve:
return S_ERROR( "No eligible Target files" )
res = self.oTargetSE.exists( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check target existence" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toRemove = []
for pfn, exists in res['Value']['Successful'].items():
if exists:
lfn = toResolve[pfn]
res = self.getSourceSURL( lfn )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - target exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Target exists" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif res['Value'] == pfn:
gLogger.warn( "resolveTarget: skipping %s - source and target pfns are the same" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source and Target the same" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRemove.append( pfn )
if toRemove:
self.oTargetSE.removeFile( toRemove )
return S_OK()
def __filesToSubmit( self ):
"""
check if there is at least one file to submit
:return: S_OK if at least one file is present, S_ERROR otherwise
"""
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if lfnStatus not in self.noSubmitStatus and source and target:
return S_OK()
return S_ERROR()
def __createFTSFiles( self ):
""" create LFNs file for glite-transfer-submit command
This file consists one line for each fiel to be transferred:
sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM]
:param self: self reference
"""
self.__updateMetadataCache()
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
if lfnStatus not in self.noSubmitStatus:
cksmStr = ""
# # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself
if self.__cksmTest and self.__cksmType:
checkSum = self.catalogMetadata.get( lfn, {} ).get( 'Checksum' )
if checkSum:
cksmStr = " %s:%s" % ( self.__cksmType, intAdlerToHex( hexAdlerToInt( checkSum ) ) )
ftsFile = FTSFile()
ftsFile.LFN = lfn
ftsFile.SourceSURL = self.fileDict[lfn].get( 'Source' )
ftsFile.TargetSURL = self.fileDict[lfn].get( 'Target' )
ftsFile.SourceSE = self.sourceSE
ftsFile.TargetSE = self.targetSE
ftsFile.Status = self.fileDict[lfn].get( 'Status' )
ftsFile.Checksum = cksmStr
ftsFile.Size = self.catalogMetadata.get( lfn, {} ).get( 'Size' )
self.ftsFiles.append( ftsFile )
self.submittedFiles += 1
return S_OK()
def __createFTSJob( self, guid = None ):
self.__createFTSFiles()
ftsJob = FTSJob()
ftsJob.RequestID = 0
ftsJob.OperationID = 0
ftsJob.SourceSE = self.sourceSE
ftsJob.TargetSE = self.targetSE
ftsJob.SourceToken = self.sourceToken
ftsJob.TargetToken = self.targetToken
ftsJob.FTSServer = self.ftsServer
if guid:
ftsJob.FTSGUID = guid
for ftsFile in self.ftsFiles:
ftsFile.Attempt += 1
ftsFile.Error = ""
ftsJob.addFile( ftsFile )
self.ftsJob = ftsJob
def __submitFTSTransfer( self ):
""" create and execute glite-transfer-submit CLI command
:param self: self reference
"""
log = gLogger.getSubLogger( 'Submit' )
self.__createFTSJob()
submit = self.ftsJob.submitFTS2( command = self.submitCommand )
if not submit["OK"]:
log.error( "unable to submit FTSJob: %s" % submit["Message"] )
return submit
log.info( "FTSJob '%s'@'%s' has been submitted" % ( self.ftsJob.FTSGUID, self.ftsJob.FTSServer ) )
# # update statuses for job files
for ftsFile in self.ftsJob:
ftsFile.FTSGUID = self.ftsJob.FTSGUID
ftsFile.Status = "Submitted"
ftsFile.Attempt += 1
log.info( "FTSJob '%s'@'%s' has been submitted" % ( self.ftsJob.FTSGUID, self.ftsJob.FTSServer ) )
self.ftsGUID = self.ftsJob.FTSGUID
return S_OK()
def __resolveFTSServer( self ):
"""
resolve FTS server to use, it should be the closest one from target SE
:param self: self reference
"""
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTSServersForSites
if not self.targetSE:
return S_ERROR( "Target SE not set" )
res = getSitesForSE( self.targetSE )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine target site" )
targetSites = res['Value']
targetSite = ''
for targetSite in targetSites:
targetFTS = getFTSServersForSites( [targetSite] )
if targetFTS['OK']:
ftsTarget = targetFTS['Value'][targetSite]
if ftsTarget:
self.ftsServer = ftsTarget
return S_OK( self.ftsServer )
else:
return targetFTS
return S_ERROR( 'No FTS server found for %s' % targetSite )
####################################################################
#
# Methods for monitoring
#
def summary( self, untilTerminal = False, printOutput = False ):
""" summary of FTS job
:param self: self reference
:param bool untilTerminal: flag to monitor FTS job to its final state
:param bool printOutput: flag to print out monitoring information to the stdout
"""
res = self.__isSummaryValid()
if not res['OK']:
return res
while not self.isTerminal:
res = self.__parseOutput( full = True )
if not res['OK']:
return res
if untilTerminal:
self.__print()
self.isRequestTerminal()
if res['Value'] or ( not untilTerminal ):
break
time.sleep( 1 )
if untilTerminal:
print ""
if printOutput and ( not untilTerminal ):
return self.dumpSummary( printOutput = printOutput )
return S_OK()
def monitor( self, untilTerminal = False, printOutput = False, full = True ):
""" monitor FTS job
:param self: self reference
:param bool untilTerminal: flag to monitor FTS job to its final state
:param bool printOutput: flag to print out monitoring information to the stdout
"""
if not self.ftsJob:
self.resolveSource()
self.__createFTSJob( self.ftsGUID )
res = self.__isSummaryValid()
if not res['OK']:
return res
if untilTerminal:
res = self.summary( untilTerminal = untilTerminal, printOutput = printOutput )
if not res['OK']:
return res
res = self.__parseOutput( full = full )
if not res['OK']:
return res
if untilTerminal:
self.finalize()
if printOutput:
self.dump()
return res
def dumpSummary( self, printOutput = False ):
""" get FTS job summary as str
:param self: self reference
:param bool printOutput: print summary to stdout
"""
outStr = ''
for status in sorted( self.statusSummary ):
if self.statusSummary[status]:
outStr = '%s\t%-10s : %-10s\n' % ( outStr, status, str( self.statusSummary[status] ) )
outStr = outStr.rstrip( '\n' )
if printOutput:
print outStr
return S_OK( outStr )
def __print( self ):
""" print progress bar of FTS job completeness to stdout
:param self: self reference
"""
width = 100
bits = int( ( width * self.percentageComplete ) / 100 )
outStr = "|%s>%s| %.1f%s %s %s" % ( "="*bits, " "*( width - bits ),
self.percentageComplete, "%",
self.requestStatus, " "*10 )
sys.stdout.write( "%s\r" % ( outStr ) )
sys.stdout.flush()
def dump( self ):
""" print FTS job parameters and files to stdout
:param self: self reference
"""
print "%-10s : %-10s" % ( "Status", self.requestStatus )
print "%-10s : %-10s" % ( "Source", self.sourceSE )
print "%-10s : %-10s" % ( "Target", self.targetSE )
print "%-10s : %-128s" % ( "Server", self.ftsServer )
print "%-10s : %-128s" % ( "GUID", self.ftsGUID )
for lfn in sorted( self.fileDict ):
print "\n %-15s : %-128s" % ( 'LFN', lfn )
for key in ['Source', 'Target', 'Status', 'Reason', 'Duration']:
print " %-15s : %-128s" % ( key, str( self.fileDict[lfn].get( key ) ) )
return S_OK()
def __isSummaryValid( self ):
""" check validity of FTS job summary report
:param self: self reference
"""
if not self.ftsServer:
return S_ERROR( "FTSServer not set" )
if not self.ftsGUID:
return S_ERROR( "FTSGUID not set" )
return S_OK()
def __parseOutput( self, full = False ):
""" execute glite-transfer-status command and parse its output
:param self: self reference
:param bool full: glite-transfer-status verbosity level, when set, collect information of files as well
"""
monitor = self.ftsJob.monitorFTS2( command = self.monitorCommand, full = full )
if not monitor['OK']:
return monitor
self.percentageComplete = self.ftsJob.Completeness
self.requestStatus = self.ftsJob.Status
self.submitTime = self.ftsJob.SubmitTime
statusSummary = monitor['Value']
if statusSummary:
for state in statusSummary:
self.statusSummary[state] = statusSummary[state]
self.transferTime = 0
for ftsFile in self.ftsJob:
lfn = ftsFile.LFN
self.__setFileParameter( lfn, 'Status', ftsFile.Status )
self.__setFileParameter( lfn, 'Reason', ftsFile.Error )
self.__setFileParameter( lfn, 'Duration', ftsFile._duration )
targetURL = self.__getFileParameter( lfn, 'Target' )
if not targetURL['OK']:
self.__setFileParameter( lfn, 'Target', ftsFile.TargetSURL )
self.transferTime += int( ftsFile._duration )
return S_OK()
####################################################################
#
# Methods for finalization
#
def finalize( self ):
""" finalize FTS job
:param self: self reference
"""
self.__updateMetadataCache()
transEndTime = dateTime()
regStartTime = time.time()
res = self.getTransferStatistics()
transDict = res['Value']
res = self.__registerSuccessful( transDict['transLFNs'] )
regSuc, regTotal = res['Value']
regTime = time.time() - regStartTime
if self.sourceSE and self.targetSE:
self.__sendAccounting( regSuc, regTotal, regTime, transEndTime, transDict )
return S_OK()
def getTransferStatistics( self ):
""" collect information of Transfers that can be used by Accounting
:param self: self reference
"""
transDict = { 'transTotal': len( self.fileDict ),
'transLFNs': [],
'transOK': 0,
'transSize': 0 }
for lfn in self.fileDict:
if self.fileDict[lfn].get( 'Status' ) in self.successfulStates:
if self.fileDict[lfn].get( 'Duration', 0 ):
transDict['transLFNs'].append( lfn )
transDict['transOK'] += 1
if lfn in self.catalogMetadata:
transDict['transSize'] += self.catalogMetadata[lfn].get( 'Size', 0 )
return S_OK( transDict )
def getFailedRegistrations( self ):
""" get failed registrations dict
:param self: self reference
"""
return S_OK( self.failedRegistrations )
def __registerSuccessful( self, transLFNs ):
""" register successfully transferred files to the catalogs,
fill failedRegistrations dict for files that failed to register
:param self: self reference
:param list transLFNs: LFNs in FTS job
"""
self.failedRegistrations = {}
toRegister = {}
for lfn in transLFNs:
res = returnSingleResult( self.oTargetSE.getPfnForProtocol( self.fileDict[lfn].get( 'Target' ), protocol = 'SRM2', withPort = False ) )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRegister[lfn] = { 'PFN' : res['Value'], 'SE' : self.targetSE }
if not toRegister:
return S_OK( ( 0, 0 ) )
res = self.__getCatalogObject()
if not res['OK']:
for lfn in toRegister:
self.failedRegistrations = toRegister
self.log.error( 'Failed to get Catalog Object', res['Message'] )
return S_OK( ( 0, len( toRegister ) ) )
res = self.oCatalog.addReplica( toRegister )
if not res['OK']:
self.failedRegistrations = toRegister
self.log.error( 'Failed to get Catalog Object', res['Message'] )
return S_OK( ( 0, len( toRegister ) ) )
for lfn, error in res['Value']['Failed'].items():
self.failedRegistrations[lfn] = toRegister[lfn]
self.log.error( 'Registration of Replica failed', '%s : %s' % ( lfn, str( error ) ) )
return S_OK( ( len( res['Value']['Successful'] ), len( toRegister ) ) )
def __sendAccounting( self, regSuc, regTotal, regTime, transEndTime, transDict ):
""" send accounting record
:param self: self reference
:param regSuc: number of files successfully registered
:param regTotal: number of files attepted to register
:param regTime: time stamp at the end of registration
:param transEndTime: time stamp at the end of FTS job
:param dict transDict: dict holding couters for files being transerred, their sizes and successfull transfers
"""
oAccounting = DataOperation()
oAccounting.setEndTime( transEndTime )
oAccounting.setStartTime( self.submitTime )
accountingDict = {}
accountingDict['OperationType'] = 'replicateAndRegister'
result = getProxyInfo()
if not result['OK']:
userName = '******'
else:
userName = result['Value'].get( 'username', 'unknown' )
accountingDict['User'] = userName
accountingDict['Protocol'] = 'FTS' if 'fts3' not in self.ftsServer else 'FTS3'
accountingDict['RegistrationTime'] = regTime
accountingDict['RegistrationOK'] = regSuc
accountingDict['RegistrationTotal'] = regTotal
accountingDict['TransferOK'] = transDict['transOK']
accountingDict['TransferTotal'] = transDict['transTotal']
accountingDict['TransferSize'] = transDict['transSize']
accountingDict['FinalStatus'] = self.requestStatus
accountingDict['Source'] = self.sourceSE
accountingDict['Destination'] = self.targetSE
accountingDict['TransferTime'] = self.transferTime
oAccounting.setValuesFromDict( accountingDict )
self.log.verbose( "Attempting to commit accounting message..." )
oAccounting.commit()
self.log.verbose( "...committed." )
return S_OK()
def __getStorageDirectoryContents(self, lfnDir, storageElement):
""" Obtians the contents of the supplied directory on the storage
"""
gLogger.info('Obtaining the contents for %s directories at %s' %
(len(lfnDir), storageElement))
se = StorageElement(storageElement)
res = se.exists(lfnDir)
if not res['OK']:
gLogger.error("Failed to obtain existance of directories",
res['Message'])
return res
for directory, error in res['Value']['Failed'].items():
gLogger.error('Failed to determine existance of directory',
'%s %s' % (directory, error))
if res['Value']['Failed']:
return S_ERROR('Failed to determine existance of directory')
directoryExists = res['Value']['Successful']
activeDirs = []
for directory in sorted(directoryExists):
exists = directoryExists[directory]
if exists:
activeDirs.append(directory)
allFiles = {}
while len(activeDirs) > 0:
currentDir = activeDirs[0]
res = se.listDirectory(currentDir)
activeDirs.remove(currentDir)
if not res['OK']:
gLogger.error('Failed to get directory contents',
res['Message'])
return res
elif currentDir in res['Value']['Failed']:
gLogger.error(
'Failed to get directory contents',
'%s %s' % (currentDir, res['Value']['Failed'][currentDir]))
return S_ERROR(res['Value']['Failed'][currentDir])
else:
dirContents = res['Value']['Successful'][currentDir]
activeDirs.extend(
se.getLFNFromURL(dirContents['SubDirs']).get(
'Value', {}).get('Successful', []))
fileURLMetadata = dirContents['Files']
fileMetadata = {}
res = se.getLFNFromURL(fileURLMetadata)
if not res['OK']:
gLogger.error('Failed to get directory content LFNs',
res['Message'])
return res
for url, error in res['Value']['Failed'].items():
gLogger.error("Failed to get LFN for URL",
"%s %s" % (url, error))
if res['Value']['Failed']:
return S_ERROR("Failed to get LFNs for PFNs")
urlLfns = res['Value']['Successful']
for urlLfn, lfn in urlLfns.items():
fileMetadata[lfn] = fileURLMetadata[urlLfn]
allFiles.update(fileMetadata)
zeroSizeFiles = []
for lfn in sorted(allFiles):
if os.path.basename(lfn) == 'dirac_directory':
allFiles.pop(lfn)
else:
metadata = allFiles[lfn]
if metadata['Size'] == 0:
zeroSizeFiles.append(
(lfn, 'deprecatedUrl', storageElement, 'PFNZeroSize'))
if zeroSizeFiles:
self.__reportProblematicReplicas(zeroSizeFiles, storageElement,
'PFNZeroSize')
gLogger.info('Obtained at total of %s files for directories at %s' %
(len(allFiles), storageElement))
return S_OK(allFiles)
def getStorageDirectoryContents(self, lfnDir, storageElement):
""" This takes the supplied lfn directories and recursively obtains the files in the supplied storage element
"""
gLogger.info('Obtaining the contents for %s directories at %s' %
(len(lfnDir), storageElement))
se = StorageElement(storageElement)
res = se.exists(lfnDir)
if not res['OK']:
gLogger.error(
"Failed to obtain existance of directories", res['Message'])
return res
for directory, error in res['Value']['Failed'].iteritems():
gLogger.error('Failed to determine existance of directory',
'%s %s' % (directory, error))
if res['Value']['Failed']:
return S_ERROR(errno.ENOENT, 'Failed to determine existance of directory')
directoryExists = res['Value']['Successful']
activeDirs = []
for directory in sorted(directoryExists):
exists = directoryExists[directory]
if exists:
activeDirs.append(directory)
allFiles = {}
while len(activeDirs) > 0:
currentDir = activeDirs[0]
res = se.listDirectory(currentDir)
activeDirs.remove(currentDir)
if not res['OK']:
gLogger.error('Failed to get directory contents', res['Message'])
return res
elif currentDir in res['Value']['Failed']:
gLogger.error('Failed to get directory contents', '%s %s' %
(currentDir, res['Value']['Failed'][currentDir]))
return S_ERROR(errno.ENOENT, res['Value']['Failed'][currentDir])
else:
dirContents = res['Value']['Successful'][currentDir]
activeDirs.extend(se.getLFNFromURL(dirContents['SubDirs']).get(
'Value', {}).get('Successful', []))
fileURLMetadata = dirContents['Files']
fileMetadata = {}
res = se.getLFNFromURL(fileURLMetadata)
if not res['OK']:
gLogger.error('Failed to get directory content LFNs', res['Message'])
return res
for url, error in res['Value']['Failed'].iteritems():
gLogger.error("Failed to get LFN for URL", "%s %s" % (url, error))
if res['Value']['Failed']:
return S_ERROR(errno.ENOENT, "Failed to get LFNs for PFNs")
urlLfns = res['Value']['Successful']
for urlLfn, lfn in urlLfns.iteritems():
fileMetadata[lfn] = fileURLMetadata[urlLfn]
allFiles.update(fileMetadata)
zeroSizeFiles = []
for lfn in sorted(allFiles):
if os.path.basename(lfn) == 'dirac_directory':
allFiles.pop(lfn)
else:
metadata = allFiles[lfn]
if not metadata['Size']:
zeroSizeFiles.append(
(lfn, 'deprecatedUrl', storageElement, 'PFNZeroSize'))
if zeroSizeFiles:
self.dic.reportProblematicReplicas(
zeroSizeFiles, storageElement, 'PFNZeroSize')
gLogger.info('Obtained at total of %s files for directories at %s' %
(len(allFiles), storageElement))
return S_OK(allFiles)
class FTSRequest( object ):
"""
.. class:: FTSRequest
Helper class for FTS job submission and monitoring.
"""
# # default checksum type
__defaultCksmType = "ADLER32"
# # flag to disablr/enable checksum test, default: disabled
__cksmTest = False
def __init__( self ):
"""c'tor
:param self: self reference
"""
self.log = gLogger.getSubLogger( self.__class__.__name__, True )
# # final states tuple
self.finalStates = ( 'Canceled', 'Failed', 'Hold',
'Finished', 'FinishedDirty' )
# # failed states tuple
self.failedStates = ( 'Canceled', 'Failed',
'Hold', 'FinishedDirty' )
# # successful states tuple
self.successfulStates = ( 'Finished', 'Done' )
# # all file states tuple
self.fileStates = ( 'Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed',
'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting' )
self.newlyCompletedFiles = []
self.newlyFailedFiles = []
self.statusSummary = {}
# # request status
self.requestStatus = 'Unknown'
# # dict for FTS job files
self.fileDict = {}
# # dict for replicas information
self.catalogReplicas = {}
# # dict for metadata information
self.catalogMetadata = {}
# # dict for files that failed to register
self.failedRegistrations = {}
# # placehoder for CatalogInterface reference
self.oCatalog = None
# # submit timestamp
self.submitTime = ''
# # placeholder FTS job GUID
self.ftsGUID = ''
# # placeholder for FTS server URL
self.ftsServer = ''
# # not used
self.priority = 3
# # flag marking FTS job completness
self.isTerminal = False
# # completness percentage
self.percentageComplete = 0.0
# # source SE name
self.sourceSE = ''
# # flag marking source SE validity
self.sourceValid = False
# # source space token
self.sourceToken = ''
# # target SE name
self.targetSE = ''
# # flag marking target SE validity
self.targetValid = False
# # target space token
self.targetToken = ''
# # whatever
self.dumpStr = ''
# # placeholder for surl file
self.surlFile = None
# # placeholder for target StorageElement
self.oTargetSE = None
# # placeholder for source StorageElement
self.oSourceSE = None
# # checksum type, set it to default
self.__cksmType = self.__defaultCksmType
# # disable checksum test by default
self.__cksmTest = False
# # replica manager handler
self.replicaManager = ReplicaManager()
# # statuses that prevent submitting to FTS
self.noSubmitStatus = ( 'Failed', 'Done', 'Staging' )
# # were sources resolved?
self.sourceResolved = False
# # Number of file transfers actually submitted
self.submittedFiles = 0
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE( self, se ):
""" set SE for source
:param self: self reference
:param str se: source SE name
"""
if se == self.targetSE:
return S_ERROR( "SourceSE is TargetSE" )
self.sourceSE = se
self.oSourceSE = StorageElement( self.sourceSE )
return self.__checkSourceSE()
def getSourceSE( self ):
""" source SE getter
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR( "Source SE not defined" )
return S_OK( self.sourceSE )
def setSourceToken( self, token ):
""" set source space token
:param self: self reference
:param str token: source space token
"""
self.sourceToken = token
return S_OK()
def getSourceToken( self ):
""" source space token getter
:param self: self reference
"""
if not self.sourceToken:
return S_ERROR( "Source token not defined" )
return S_OK( self.sourceToken )
def __checkSourceSE( self ):
""" check source SE availability
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR( "SourceSE not set" )
res = self.oSourceSE.isValid( 'Read' )
if not res['OK']:
return S_ERROR( "SourceSE not available for reading" )
res = self.__getSESpaceToken( self.oSourceSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message'] )
return S_ERROR( "SourceSE does not support FTS transfers" )
if self.__cksmTest:
res = self.oSourceSE.getChecksumType()
if not res["OK"]:
self.log.error( "Unable to get checksum type for SourceSE %s: %s" % ( self.sourceSE,
res["Message"] ) )
cksmType = res["Value"]
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at SourceSE %s, disabling checksum test" % ( cksmType,
self.sourceSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE( self, se ):
""" set target SE
:param self: self reference
:param str se: target SE name
"""
if se == self.sourceSE:
return S_ERROR( "TargetSE is SourceSE" )
self.targetSE = se
self.oTargetSE = StorageElement( self.targetSE )
return self.__checkTargetSE()
def getTargetSE( self ):
""" target SE getter
:param self: self reference
"""
if not self.targetSE:
return S_ERROR( "Target SE not defined" )
return S_OK( self.targetSE )
def setTargetToken( self, token ):
""" target space token setter
:param self: self reference
:param str token: target space token
"""
self.targetToken = token
return S_OK()
def getTargetToken( self ):
""" target space token getter
:param self: self reference
"""
if not self.targetToken:
return S_ERROR( "Target token not defined" )
return S_OK( self.targetToken )
def __checkTargetSE( self ):
""" check target SE availability
:param self: self reference
"""
if not self.targetSE:
return S_ERROR( "TargetSE not set" )
res = self.oTargetSE.isValid( 'Write' )
if not res['OK']:
return S_ERROR( "TargetSE not available for writing" )
res = self.__getSESpaceToken( self.oTargetSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message'] )
return S_ERROR( "TargetSE does not support FTS transfers" )
# # check checksum types
if self.__cksmTest:
res = self.oTargetSE.getChecksumType()
if not res["OK"]:
self.log.error( "Unable to get checksum type for TargetSE %s: %s" % ( self.targetSE,
res["Message"] ) )
cksmType = res["Value"]
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at TargetSE %s, disabling checksum test" % ( cksmType,
self.targetSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
@staticmethod
def __getSESpaceToken( oSE ):
""" get space token from StorageElement instance
:param self: self reference
:param StorageElement oSE: StorageElement instance
"""
res = oSE.getStorageParameters( "SRM2" )
if not res['OK']:
return res
return S_OK( res['Value'].get( 'SpaceToken' ) )
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID( self, guid ):
""" FTS job GUID setter
:param self: self reference
:param str guid: string containg GUID
"""
if not checkGuid( guid ):
return S_ERROR( "Incorrect GUID format" )
self.ftsGUID = guid
return S_OK()
def getFTSGUID( self ):
""" FTS job GUID getter
:param self: self refenece
"""
if not self.ftsGUID:
return S_ERROR( "FTSGUID not set" )
return S_OK( self.ftsGUID )
def setFTSServer( self, server ):
""" FTS server setter
:param self: self reference
:param str server: FTS server URL
"""
self.ftsServer = server
return S_OK()
def getFTSServer( self ):
""" FTS server getter
:param self: self reference
"""
if not self.ftsServer:
return S_ERROR( "FTSServer not set" )
return S_OK( self.ftsServer )
def setPriority( self, priority ):
""" set priority for FTS job
:param self: self reference
:param int priority: a new priority
"""
if not type( priority ) in ( IntType, LongType ):
return S_ERROR( "Priority must be integer" )
if priority < 0:
priority = 0
elif priority > 5:
priority = 5
self.priority = priority
return S_OK( self.priority )
def getPriority( self ):
""" FTS job priority getter
:param self: self reference
"""
return S_OK( self.priority )
def getPercentageComplete( self ):
""" get completness percentage
:param self: self reference
"""
completedFiles = 0
totalFiles = 0
for state in self.statusSummary:
if state in self.successfulStates:
completedFiles += self.statusSummary[state]
totalFiles += self.statusSummary[state]
self.percentageComplete = ( float( completedFiles ) * 100.0 ) / float( totalFiles )
return S_OK( self.percentageComplete )
def isRequestTerminal( self ):
""" check if FTS job has terminated
:param self: self reference
"""
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK( self.isTerminal )
def getStatus( self ):
""" get FTS job status
:param self: self reference
"""
return S_OK( self.requestStatus )
def setCksmType( self, cksm = None ):
""" set checksum type to use
:param self: self reference
:param mixed cksm: checksum type, should be one of 'Adler32', 'md5', 'sha1', None
"""
if str( cksm ).upper() not in ( "ADLER32", "MD5", "SHA1", "NONE" ):
return S_ERROR( "Not supported checksum type: %s" % str( cksm ) )
if not cksm:
self.__cksmType = None
return S_OK( False )
self.__cksmType = str( cksm ).upper()
return S_OK( True )
def getCksmType( self ):
""" get checksum type
:param self: self reference
"""
return S_OK( self.__cksmType )
def setCksmTest( self, cksmTest = False ):
""" set cksm test
:param self: self reference
:param bool cksmTest: flag to enable/disable checksum test
"""
self.__cksmTest = bool( cksmTest )
return S_OK( self.__cksmTest )
def getCksmTest( self ):
""" get cksm test flag
:param self: self reference
"""
return S_OK( self.__cksmTest )
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN( self, lfn ):
""" add LFN :lfn: to :fileDict:
:param self: self reference
:param str lfn: LFN to add to
"""
self.fileDict.setdefault( lfn, {'Status':'Waiting'} )
return S_OK()
def setStatus( self, lfn, status ):
""" set status of a file """
return( self.__setFileParameter( lfn, 'Status', status ) )
def setSourceSURL( self, lfn, surl ):
""" source SURL setter
:param self: self reference
:param str lfn: LFN
:param str surl: source SURL
"""
target = self.fileDict[lfn].get( 'Target' )
if target == surl:
return S_ERROR( "Source and target the same" )
return( self.__setFileParameter( lfn, 'Source', surl ) )
def getSourceSURL( self, lfn ):
""" get source SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Source' )
def setTargetSURL( self, lfn, surl ):
""" set target SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
:param str surl: target SURL
"""
source = self.fileDict[lfn].get( 'Source' )
if source == surl:
return S_ERROR( "Source and target the same" )
return( self.__setFileParameter( lfn, 'Target', surl ) )
def getTargetSURL( self, lfn ):
""" target SURL getter
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Target' )
def getFailReason( self, lfn ):
""" get fail reason for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Reason' )
def getRetries( self, lfn ):
""" get number of attepmts made to transfer file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Retries' )
def getTransferTime( self, lfn ):
""" get duration of transfer for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Duration' )
def getFailed( self ):
""" get list of wrongly transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.failedStates ] )
def getStaging( self ):
""" get files set for prestaging """
return S_OK( [lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) == 'Staging'] )
def getDone( self ):
""" get list of succesfully transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.successfulStates ] )
def __setFileParameter( self, lfn, paramName, paramValue ):
""" set :paramName: to :paramValue: for :lfn: file
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
:param mixed paramValue: a new parameter value
"""
self.setLFN( lfn )
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter( self, lfn, paramName ):
""" get value of :paramName: for file :lfn:
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
"""
if lfn not in self.fileDict:
return S_ERROR( "Supplied file not set" )
if paramName not in self.fileDict[lfn]:
return S_ERROR( "%s not set for file" % paramName )
return S_OK( self.fileDict[lfn][paramName] )
####################################################################
#
# Methods for submission
#
def submit( self, monitor = False, printOutput = True ):
""" submit FTS job
:param self: self reference
:param bool monitor: flag to monitor progress of FTS job
:param bool printOutput: flag to print output of execution to stdout
"""
res = self.__isSubmissionValid()
if not res['OK']:
return res
res = self.__createSURLPairFile()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = { 'ftsGUID' : self.ftsGUID, 'ftsServer' : self.ftsServer, 'submittedFiles' : self.submittedFiles }
# print "Submitted %s @ %s" % ( self.ftsGUID, self.ftsServer )
if monitor:
self.monitor( untilTerminal = True, printOutput = printOutput )
return S_OK( resDict )
def __isSubmissionValid( self ):
""" check validity of job before submission
:param self: self reference
"""
if not self.fileDict:
return S_ERROR( "No files set" )
if not self.sourceValid:
return S_ERROR( "SourceSE not valid" )
if not self.targetValid:
return S_ERROR( "TargetSE not valid" )
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR( "FTSServer not valid" )
self.resolveSource()
self.resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR( "No files to submit" )
return S_OK()
def __getCatalogObject( self ):
""" CatalogInterface instance facade
:param self: self reference
"""
try:
if not self.oCatalog:
self.oCatalog = CatalogInterface()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache( self, lfns = None, overwrite = False ):
""" update replica cache for list of :lfns:
:param self: self reference
:param mixed lfns: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if ( lfn not in self.catalogReplicas ) or overwrite ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogReplicas( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to update replica cache: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache( self, lfns = None, overwrite = False ):
""" update metadata cache for list of LFNs
:param self: self reference
:param list lnfs: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if ( lfn not in self.catalogMetadata ) or overwrite ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogFileMetadata( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to get source catalog metadata: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def resolveSource( self ):
""" resolve source SE eligible for submission
:param self: self reference
"""
# Avoid resolving sources twice
if self.sourceResolved:
return S_OK()
# Only resolve files that need a transfer
toResolve = [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( "Status", "" ) != "Failed" ]
if not toResolve:
return S_OK()
res = self.__updateMetadataCache( toResolve )
if not res['OK']:
return res
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
# Define the source URLs
for lfn in toResolve:
replicas = self.catalogReplicas.get( lfn, {} )
if self.sourceSE not in replicas:
gLogger.warn( "resolveSource: skipping %s - not replicas at SourceSE %s" % ( lfn, self.sourceSE ) )
self.__setFileParameter( lfn, 'Reason', "No replica at SourceSE" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.oSourceSE.getPfnForProtocol( replicas[self.sourceSE], 'SRM2', withPort = True )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setSourceSURL( lfn, res['Value'] )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict:
if "Source" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Source']] = lfn
if not toResolve:
return S_ERROR( "No eligible Source files" )
# Get metadata of the sources, to check for existance, availability and caching
res = self.oSourceSE.getFileMetadata( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check source file metadata" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
if re.search( 'File does not exist', error ):
gLogger.warn( "resolveSource: skipping %s - source file does not exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file does not exist" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
gLogger.warn( "resolveSource: skipping %s - failed to get source metadata" % lfn )
self.__setFileParameter( lfn, 'Reason', "Failed to get Source metadata" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toStage = []
nbStagedFiles = 0
for pfn, metadata in res['Value']['Successful'].items():
lfn = toResolve[pfn]
lfnStatus = self.fileDict.get( lfn, {} ).get( 'Status' )
if metadata['Unavailable']:
gLogger.warn( "resolveSource: skipping %s - source file unavailable" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Unavailable" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif metadata['Lost']:
gLogger.warn( "resolveSource: skipping %s - source file lost" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Lost" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif not metadata['Cached']:
if lfnStatus != 'Staging':
toStage.append( pfn )
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
gLogger.warn( "resolveSource: skipping %s - source file size mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source size mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \
not ( compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ) ):
gLogger.warn( "resolveSource: skipping %s - source file checksum mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source checksum mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif lfnStatus == 'Staging':
# file that was staging is now cached
self.__setFileParameter( lfn, 'Status', 'Waiting' )
nbStagedFiles += 1
# Some files were being staged
if nbStagedFiles:
self.log.info( 'resolveSource: %d files have been staged' % nbStagedFiles )
# Launching staging of files not in cache
if toStage:
gLogger.warn( "resolveSource: %s source files not cached, prestaging..." % len( toStage ) )
stage = self.replicaManager.prestageStorageFile( toStage, self.sourceSE )
if not stage["OK"]:
gLogger.error( "resolveSource: error is prestaging - %s" % stage["Message"] )
for pfn in toStage:
lfn = toResolve[pfn]
self.__setFileParameter( lfn, 'Reason', stage["Message"] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
for pfn in toStage:
lfn = toResolve[pfn]
if pfn in stage['Value']['Successful']:
self.__setFileParameter( lfn, 'Status', 'Staging' )
elif pfn in stage['Value']['Failed']:
self.__setFileParameter( lfn, 'Reason', stage['Value']['Failed'][pfn] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
self.sourceResolved = True
return S_OK()
def resolveTarget( self ):
""" find target SE eligible for submission
:param self: self reference
"""
toResolve = [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status' ) not in self.noSubmitStatus ]
if not toResolve:
return S_OK()
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
for lfn in toResolve:
res = self.oTargetSE.getPfnForLfn( lfn )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - failed to create target pfn" % lfn )
self.__setFileParameter( lfn, 'Reason', "Failed to create Target" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.oTargetSE.getPfnForProtocol( res['Value'], 'SRM2', withPort = True )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setTargetSURL( lfn, res['Value'] )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict:
if "Target" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Target']] = lfn
if not toResolve:
return S_ERROR( "No eligible Target files" )
res = self.oTargetSE.exists( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check target existence" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toRemove = []
for pfn, exists in res['Value']['Successful'].items():
if exists:
lfn = toResolve[pfn]
res = self.getSourceSURL( lfn )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - target exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Target exists" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif res['Value'] == pfn:
gLogger.warn( "resolveTarget: skipping %s - source and target pfns are the same" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source and Target the same" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRemove.append( pfn )
if toRemove:
self.oTargetSE.removeFile( toRemove )
return S_OK()
def __filesToSubmit( self ):
"""
check if there is at least one file to submit
:return: S_OK if at least one file is present, S_ERROR otherwise
"""
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if lfnStatus not in self.noSubmitStatus and source and target:
return S_OK()
return S_ERROR()
def __createSURLPairFile( self ):
""" create LFNs file for glite-transfer-submit command
This file consists one line for each fiel to be transferred:
sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM]
:param self: self reference
"""
fd, fileName = tempfile.mkstemp()
surlFile = os.fdopen( fd, 'w' )
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if lfnStatus not in self.noSubmitStatus and source and target:
cksmStr = ""
# # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself
if self.__cksmTest and self.__cksmType:
checkSum = self.catalogMetadata.get( lfn, {} ).get( 'Checksum' )
if checkSum:
cksmStr = " %s:%s" % ( self.__cksmType, intAdlerToHex( hexAdlerToInt( checkSum ) ) )
surlFile.write( "%s %s%s\n" % ( source, target, cksmStr ) )
self.submittedFiles += 1
surlFile.close()
self.surlFile = fileName
return S_OK()
def __submitFTSTransfer( self ):
""" create and execute glite-transfer-submit CLI command
:param self: self reference
"""
comm = [ 'glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile, '-o' ]
if self.targetToken:
comm += [ '-t', self.targetToken ]
if self.sourceToken:
comm += [ '-S', self.sourceToken ]
if self.__cksmTest:
comm.append( "--compare-checksums" )
gLogger.verbose( 'Executing %s' % ' '.join( comm ) )
res = executeGridCommand( '', comm )
os.remove( self.surlFile )
if not res['OK']:
return res
returnCode, output, errStr = res['Value']
if not returnCode == 0:
return S_ERROR( errStr )
guid = output.replace( '\n', '' )
if not checkGuid( guid ):
return S_ERROR( 'Wrong GUID format returned' )
self.ftsGUID = guid
# if self.priority != 3:
# comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)]
# executeGridCommand('',comm)
return res
def __getFTSServer( self, site ):
try:
configPath = '/Resources/FTSEndpoints/%s' % site
endpointURL = gConfig.getValue( configPath )
if not endpointURL:
errStr = "FTSRequest.__getFTSServer: Failed to find FTS endpoint, check CS entry for '%s'." % site
return S_ERROR( errStr )
return S_OK( endpointURL )
except Exception, x:
return S_ERROR( 'FTSRequest.__getFTSServer: Failed to obtain endpoint details from CS' )
class FTSRequest:
def __init__( self ):
self.gridEnv = '/afs/cern.ch/project/gd/LCG-share/3.2.8-0/etc/profile.d/grid-env'
self.finalStates = ['Canceled', 'Failed', 'Hold', 'Finished', 'FinishedDirty']
self.failedStates = ['Canceled', 'Failed', 'Hold', 'FinishedDirty']
self.successfulStates = ['Finished', 'Done']
self.fileStates = ['Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed', 'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting']
self.newlyCompletedFiles = []
self.newlyFailedFiles = []
self.statusSummary = {}
self.requestStatus = 'Unknown'
self.fileDict = {}
self.catalogReplicas = {}
self.catalogMetadata = {}
self.oCatalog = None
self.submitTime = ''
self.ftsGUID = ''
self.ftsServer = ''
self.priority = 3
self.isTerminal = False
self.percentageComplete = 0.0
self.sourceSE = ''
self.sourceValid = False
self.sourceToken = ''
self.targetSE = ''
self.targetValid = False
self.targetToken = ''
self.dumpStr = ''
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE( self, se ):
if se == self.targetSE:
return S_ERROR( "SourceSE is TargetSE" )
self.sourceSE = se
self.oSourceSE = StorageElement( self.sourceSE )
return self.__checkSourceSE()
def getSourceSE( self ):
if not self.sourceSE:
return S_ERROR( "Source SE not defined" )
return S_OK( self.sourceSE )
def setSourceToken( self, token ):
self.sourceToken = token
return S_OK()
def getSourceToken( self ):
if not self.sourceToken:
return S_ERROR( "Source token not defined" )
return S_OK( self.sourceToken )
def __checkSourceSE( self ):
if not self.sourceSE:
return S_ERROR( "SourceSE not set" )
res = self.oSourceSE.isValid( 'Read' )
if not res['OK']:
return S_ERROR( "SourceSE not available for reading" )
res = self.__getSESpaceToken( self.oSourceSE )
if not res['OK']:
gLogger.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message'] )
return S_ERROR( "SourceSE does not support FTS transfers" )
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE( self, se ):
if se == self.sourceSE:
return S_ERROR( "TargetSE is SourceSE" )
self.targetSE = se
self.oTargetSE = StorageElement( self.targetSE )
return self.__checkTargetSE()
def getTargetSE( self ):
if not self.targetSE:
return S_ERROR( "Target SE not defined" )
return S_OK( self.targetSE )
def setTargetToken( self, token ):
self.targetToken = token
return S_OK()
def getTargetToken( self ):
if not self.targetToken:
return S_ERROR( "Target token not defined" )
return S_OK( self.targetToken )
def __checkTargetSE( self ):
if not self.targetSE:
return S_ERROR( "TargetSE not set" )
res = self.oTargetSE.isValid( 'Write' )
if not res['OK']:
return S_ERROR( "TargetSE not available for writing" )
res = self.__getSESpaceToken( self.oTargetSE )
if not res['OK']:
gLogger.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message'] )
return S_ERROR( "SourceSE does not support FTS transfers" )
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
def __getSESpaceToken( self, oSE ):
res = oSE.getStorageParameters( "SRM2" )
if not res['OK']:
return res
return S_OK( res['Value'].get( 'SpaceToken' ) )
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID( self, guid ):
if not checkGuid( guid ):
return S_ERROR( "Incorrect GUID format" )
self.ftsGUID = guid
return S_OK()
def getFTSGUID( self ):
if not self.ftsGUID:
return S_ERROR( "FTSGUID not set" )
return S_OK( self.ftsGUID )
def setFTSServer( self, server ):
self.ftsServer = server
return S_OK()
def getFTSServer( self ):
if not self.ftsServer:
return S_ERROR( "FTSServer not set" )
return S_OK( self.ftsServer )
def setPriority( self, priority ):
if not type( priority ) in [types.IntType, types.LongType]:
return S_ERROR( "Priority must be integer" )
if priority < 0:
priority = 0
elif priority > 5:
priority = 5
self.priority = priority
return S_OK( self.priority )
def getPriority( self ):
return S_OK( self.priority )
def getPercentageComplete( self ):
completedFiles = 0
totalFiles = 0
for state in ( self.statusSummary.keys() ):
if state in self.successfulStates:
completedFiles += self.statusSummary[state]
totalFiles += self.statusSummary[state]
self.percentageComplete = ( float( completedFiles ) * 100.0 ) / float( totalFiles )
return S_OK( self.percentageComplete )
def isRequestTerminal( self ):
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK( self.isTerminal )
def getStatus( self ):
return S_OK( self.requestStatus )
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN( self, lfn ):
if not self.fileDict.has_key( lfn ):
self.fileDict[lfn] = {}
return S_OK()
def setSourceSURL( self, lfn, surl ):
target = self.fileDict[lfn].get( 'Target' )
if target == surl:
return S_ERROR( "Source and target the same" )
self.__setFileParameter( lfn, 'Source', surl )
return S_OK()
def getSourceSURL( self, lfn ):
return self.__getFileParameter( lfn, 'Source' )
def setTargetSURL( self, lfn, surl ):
source = self.fileDict[lfn].get( 'Source' )
if source == surl:
return S_ERROR( "Source and target the same" )
self.__setFileParameter( lfn, 'Target', surl )
return S_OK()
def getTargetSURL( self, lfn ):
return self.__getFileParameter( lfn, 'Target' )
def getFailReason( self, lfn ):
return self.__getFileParameter( lfn, 'Reason' )
def getRetries( self, lfn ):
return self.__getFileParameter( lfn, 'Retries' )
def getTransferTime( self, lfn ):
return self.__getFileParameter( lfn, 'Duration' )
def getFailed( self ):
failed = []
for lfn in self.fileDict.keys():
status = self.fileDict[lfn].get( 'Status', '' )
if status in self.failedStates:
failed.append( lfn )
return S_OK( failed )
def getDone( self ):
done = []
for lfn in self.fileDict.keys():
status = self.fileDict[lfn].get( 'Status', '' )
if status in self.successfulStates:
done.append( lfn )
return S_OK( done )
def __setFileParameter( self, lfn, paramName, paramValue ):
self.setLFN( lfn )
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter( self, lfn, paramName ):
if not self.fileDict.has_key( lfn ):
return S_ERROR( "Supplied file not set" )
if not self.fileDict[lfn].has_key( paramName ):
return S_ERROR( "%s not set for file" % paramName )
return S_OK( self.fileDict[lfn][paramName] )
####################################################################
#
# Methods for submission
#
def submit( self, monitor = False, printOutput = True ):
res = self.__isSubmissionValid()
if not res['OK']:
return res
res = self.__createSURLPairFile()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = {'ftsGUID':self.ftsGUID, 'ftsServer':self.ftsServer}
print "Submitted %s @ %s" % ( self.ftsGUID, self.ftsServer )
if monitor:
self.monitor( untilTerminal = True, printOutput = printOutput )
return S_OK( resDict )
def __isSubmissionValid( self ):
if not self.fileDict:
return S_ERROR( "No files set" )
if not self.sourceValid:
return S_ERROR( "SourceSE not valid" )
if not self.targetValid:
return S_ERROR( "TargetSE not valid" )
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR( "FTSServer not valid" )
self.__resolveSource()
self.__resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR( "No files to submit" )
return S_OK()
def __getCatalogObject( self ):
try:
if not self.oCatalog:
self.oCatalog = CatalogInterface()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache( self, lfns = [], overwrite = False ):
if not lfns:
lfns = self.fileDict.keys()
toUpdate = []
for lfn in lfns:
if ( not lfn in self.catalogReplicas.keys() ) or overwrite:
toUpdate.append( lfn )
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogReplicas( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to update replica cache", res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache( self, lfns = [], overwrite = False ):
if not lfns:
lfns = self.fileDict.keys()
toUpdate = []
for lfn in lfns:
if ( not lfn in self.catalogMetadata.keys() ) or overwrite:
toUpdate.append( lfn )
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogFileMetadata( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to get source catalog metadata", res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def __resolveSource( self ):
toResolve = []
for lfn in self.fileDict.keys():
if ( not self.fileDict[lfn].has_key( 'Source' ) ) and ( self.fileDict[lfn].get( 'Status' ) != 'Failed' ):
toResolve.append( lfn )
if not toResolve:
return S_OK()
res = self.__updateMetadataCache( toResolve )
if not res['OK']:
return res
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
for lfn in toResolve:
if self.fileDict[lfn].get( 'Status' ) == 'Failed':
continue
replicas = self.catalogReplicas.get( lfn, {} )
if not replicas.has_key( self.sourceSE ):
self.__setFileParameter( lfn, 'Reason', "No replica at SourceSE" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.oSourceSE.getPfnForProtocol( replicas[self.sourceSE], 'SRM2', withPort = True )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setSourceSURL( lfn, res['Value'] )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict.keys():
if self.fileDict[lfn].has_key( 'Source' ):
toResolve[self.fileDict[lfn]['Source']] = lfn
if not toResolve:
return S_ERROR( "No eligible Source files" )
res = self.oSourceSE.getFileMetadata( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check source file metadata" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
if re.search( 'File does not exist', error ):
self.__setFileParameter( lfn, 'Reason', "Source file does not exist" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
self.__setFileParameter( lfn, 'Reason', "Failed to get Source metadata" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for pfn, metadata in res['Value']['Successful'].items():
lfn = toResolve[pfn]
if metadata['Unavailable']:
self.__setFileParameter( lfn, 'Reason', "Source file Unavailable" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif metadata['Lost']:
self.__setFileParameter( lfn, 'Reason', "Source file Lost" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif not metadata['Cached']:
self.__setFileParameter( lfn, 'Reason', "Source file not Cached" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
self.__setFileParameter( lfn, 'Reason', "Source size mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and not ( compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ) ):
self.__setFileParameter( lfn, 'Reason', "Source checksum mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
return S_OK()
def __resolveTarget( self ):
toResolve = []
for lfn in self.fileDict.keys():
if not self.fileDict[lfn].has_key( 'Target' ) and ( self.fileDict[lfn].get( 'Status' ) != 'Failed' ):
toResolve.append( lfn )
if not toResolve:
return S_OK()
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
atTarget = []
for lfn in sortList( toResolve ):
if self.fileDict[lfn].get( 'Status' ) == 'Failed':
continue
replicas = self.catalogReplicas.get( lfn, {} )
if replicas.has_key( self.targetSE ):
self.__setFileParameter( lfn, 'Reason', "File already at Target" )
self.__setFileParameter( lfn, 'Status', 'Done' )
atTarget.append( lfn )
for lfn in toResolve:
if ( self.fileDict[lfn].get( 'Status' ) == 'Failed' ) or ( lfn in atTarget ):
continue
res = self.oTargetSE.getPfnForLfn( lfn )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', "Failed to create Target" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.oTargetSE.getPfnForProtocol( res['Value'], 'SRM2', withPort = True )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setTargetSURL( lfn, res['Value'] )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict.keys():
if self.fileDict[lfn].has_key( 'Target' ):
toResolve[self.fileDict[lfn]['Target']] = lfn
if not toResolve:
return S_ERROR( "No eligible Target files" )
res = self.oTargetSE.exists( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check target existence" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toRemove = []
for pfn, exists in res['Value']['Successful'].items():
if exists:
lfn = toResolve[pfn]
res = self.getSourceSURL( lfn )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', "Target exists" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif res['Value'] == pfn:
self.__setFileParameter( lfn, 'Reason', "Source and Target the same" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRemove.append( pfn )
if toRemove:
self.oTargetSE.removeFile( toRemove )
return S_OK()
def __filesToSubmit( self ):
for lfn in self.fileDict.keys():
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if ( lfnStatus != 'Failed' ) and ( lfnStatus != 'Done' ) and source and target:
return S_OK()
return S_ERROR()
def __createSURLPairFile( self ):
fd, fileName = tempfile.mkstemp()
surlFile = os.fdopen( fd, 'w' )
for lfn in self.fileDict.keys():
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if ( lfnStatus != 'Failed' ) and ( lfnStatus != 'Done' ) and source and target:
surlString = '%s %s\n' % ( source, target )
surlFile.write( surlString )
surlFile.close()
self.surlFile = fileName
return S_OK()
def __submitFTSTransfer( self ):
comm = ['glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile, '-o']
if self.targetToken:
comm.append( '-t' )
comm.append( self.targetToken )
if self.sourceToken:
comm.append( '-S' )
comm.append( self.sourceToken )
res = executeGridCommand( '', comm, self.gridEnv )
os.remove( self.surlFile )
if not res['OK']:
return res
returnCode, output, errStr = res['Value']
if not returnCode == 0:
return S_ERROR( errStr )
guid = output.replace( '\n', '' )
if not checkGuid( guid ):
return S_ERROR( 'Wrong GUID format returned' )
self.ftsGUID = guid
#if self.priority != 3:
# comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)]
# executeGridCommand('',comm,self.gridEnv)
return res
def __resolveFTSServer( self ):
if not self.sourceSE:
return S_ERROR( "Source SE not set" )
if not self.targetSE:
return S_ERROR( "Target SE not set" )
res = getSitesForSE( self.sourceSE, 'LCG' )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine source site" )
sourceSite = res['Value'][0]
res = getSitesForSE( self.targetSE, 'LCG' )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine target site" )
targetSite = res['Value'][0]
if ( sourceSite == 'LCG.CERN.ch' ) or ( targetSite == 'LCG.CERN.ch' ):
ep = 'LCG.CERN.ch'
else:
# Target site FTS server should be used
ep = targetSite
try:
configPath = '/Resources/FTSEndpoints/%s' % ep
endpointURL = gConfig.getValue( configPath )
if not endpointURL:
errStr = "FTSRequest.__resolveFTSEndpoint: Failed to find FTS endpoint, check CS entry for '%s'." % ep
return S_ERROR( errStr )
self.ftsServer = endpointURL
return S_OK( endpointURL )
except Exception, x:
return S_ERROR( 'FTSRequest.__resolveFTSEndpoint: Failed to obtain endpoint details from CS' )
def __getStorageDirectoryContents(self, lfnDir, storageElement):
""" Obtians the contents of the supplied directory on the storage
"""
gLogger.info('Obtaining the contents for %s directories at %s' %
(len(lfnDir), storageElement))
se = StorageElement(storageElement)
res = se.getPfnForLfn(lfnDir)
if not res['OK']:
gLogger.error("Failed to get PFNs for directories", res['Message'])
return res
for directory, error in res['Value']['Failed'].items():
gLogger.error('Failed to obtain directory PFN from LFNs',
'%s %s' % (directory, error))
if res['Value']['Failed']:
return S_ERROR('Failed to obtain directory PFN from LFNs')
storageDirectories = res['Value']['Successful'].values()
res = se.exists(storageDirectories)
if not res['OK']:
gLogger.error("Failed to obtain existance of directories",
res['Message'])
return res
for directory, error in res['Value']['Failed'].items():
gLogger.error('Failed to determine existance of directory',
'%s %s' % (directory, error))
if res['Value']['Failed']:
return S_ERROR('Failed to determine existance of directory')
directoryExists = res['Value']['Successful']
activeDirs = []
for directory in sortList(directoryExists.keys()):
exists = directoryExists[directory]
if exists:
activeDirs.append(directory)
allFiles = {}
while len(activeDirs) > 0:
currentDir = activeDirs[0]
res = se.listDirectory(currentDir)
activeDirs.remove(currentDir)
if not res['OK']:
gLogger.error('Failed to get directory contents',
res['Message'])
return res
elif res['Value']['Failed'].has_key(currentDir):
gLogger.error(
'Failed to get directory contents',
'%s %s' % (currentDir, res['Value']['Failed'][currentDir]))
return S_ERROR(res['Value']['Failed'][currentDir])
else:
dirContents = res['Value']['Successful'][currentDir]
activeDirs.extend(dirContents['SubDirs'])
fileMetadata = dirContents['Files']
# RF_NOTE This ugly trick is needed because se.getPfnPath does not follow the Successful/Failed convention
# res = { "Successful" : {}, "Failed" : {} }
# for pfn in fileMetadata:
# inRes = se.getPfnPath( pfn )
# if inRes["OK"]:
# res["Successful"][pfn] = inRes["Value"]
# else:
# res["Failed"][pfn] = inRes["Message"]
res = se.getLfnForPfn(fileMetadata.keys())
if not res['OK']:
gLogger.error('Failed to get directory content LFNs',
res['Message'])
return res
for pfn, error in res['Value']['Failed'].items():
gLogger.error("Failed to get LFN for PFN",
"%s %s" % (pfn, error))
if res['Value']['Failed']:
return S_ERROR("Failed to get LFNs for PFNs")
pfnLfns = res['Value']['Successful']
for pfn, lfn in pfnLfns.items():
fileMetadata[pfn]['LFN'] = lfn
allFiles.update(fileMetadata)
zeroSizeFiles = []
lostFiles = []
unavailableFiles = []
for pfn in sortList(allFiles.keys()):
if os.path.basename(pfn) == 'dirac_directory':
allFiles.pop(pfn)
else:
metadata = allFiles[pfn]
if metadata['Size'] == 0:
zeroSizeFiles.append(
(metadata['LFN'], pfn, storageElement, 'PFNZeroSize'))
# if metadata['Lost']:
# lostFiles.append((metadata['LFN'],pfn,storageElement,'PFNLost'))
# if metadata['Unavailable']:
# unavailableFiles.append((metadata['LFN'],pfn,storageElement,'PFNUnavailable'))
if zeroSizeFiles:
self.__reportProblematicReplicas(zeroSizeFiles, storageElement,
'PFNZeroSize')
if lostFiles:
self.__reportProblematicReplicas(lostFiles, storageElement,
'PFNLost')
if unavailableFiles:
self.__reportProblematicReplicas(unavailableFiles, storageElement,
'PFNUnavailable')
gLogger.info('Obtained at total of %s files for directories at %s' %
(len(allFiles), storageElement))
return S_OK(allFiles)
class FTSRequest( object ):
"""
.. class:: FTSRequest
Helper class for FTS job submission and monitoring.
"""
# # default checksum type
__defaultCksmType = "ADLER32"
# # flag to disablr/enable checksum test, default: disabled
__cksmTest = False
def __init__( self ):
"""c'tor
:param self: self reference
"""
self.log = gLogger.getSubLogger( self.__class__.__name__, True )
# # final states tuple
self.finalStates = ( 'Canceled', 'Failed', 'Hold',
'Finished', 'FinishedDirty' )
# # failed states tuple
self.failedStates = ( 'Canceled', 'Failed',
'Hold', 'FinishedDirty' )
# # successful states tuple
self.successfulStates = ( 'Finished', 'Done' )
# # all file states tuple
self.fileStates = ( 'Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed',
'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting' )
self.statusSummary = {}
# # request status
self.requestStatus = 'Unknown'
# # dict for FTS job files
self.fileDict = {}
# # dict for replicas information
self.catalogReplicas = {}
# # dict for metadata information
self.catalogMetadata = {}
# # dict for files that failed to register
self.failedRegistrations = {}
# # placehoder for FileCatalog reference
self.oCatalog = None
# # submit timestamp
self.submitTime = ''
# # placeholder FTS job GUID
self.ftsGUID = ''
# # placeholder for FTS server URL
self.ftsServer = ''
# # flag marking FTS job completness
self.isTerminal = False
# # completness percentage
self.percentageComplete = 0.0
# # source SE name
self.sourceSE = ''
# # flag marking source SE validity
self.sourceValid = False
# # source space token
self.sourceToken = ''
# # target SE name
self.targetSE = ''
# # flag marking target SE validity
self.targetValid = False
# # target space token
self.targetToken = ''
# # placeholder for target StorageElement
self.oTargetSE = None
# # placeholder for source StorageElement
self.oSourceSE = None
# # checksum type, set it to default
self.__cksmType = self.__defaultCksmType
# # disable checksum test by default
self.__cksmTest = False
# # statuses that prevent submitting to FTS
self.noSubmitStatus = ( 'Failed', 'Done', 'Staging' )
# # were sources resolved?
self.sourceResolved = False
# # Number of file transfers actually submitted
self.submittedFiles = 0
self.transferTime = 0
self.submitCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/SubmitCommand', 'glite-transfer-submit' )
self.monitorCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/MonitorCommand', 'glite-transfer-status' )
self.ftsVersion = Operations().getValue( 'DataManagement/FTSVersion', 'FTS2' )
self.ftsJob = None
self.ftsFiles = []
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE( self, se ):
""" set SE for source
:param self: self reference
:param str se: source SE name
"""
if se == self.targetSE:
return S_ERROR( "SourceSE is TargetSE" )
self.sourceSE = se
self.oSourceSE = StorageElement( self.sourceSE )
return self.__checkSourceSE()
def __checkSourceSE( self ):
""" check source SE availability
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR( "SourceSE not set" )
res = self.oSourceSE.isValid( 'Read' )
if not res['OK']:
return S_ERROR( "SourceSE not available for reading" )
res = self.__getSESpaceToken( self.oSourceSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message'] )
return S_ERROR( "SourceSE does not support FTS transfers" )
if self.__cksmTest:
cksmType = self.oSourceSE.checksumType()
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at SourceSE %s, disabling checksum test" % ( cksmType,
self.sourceSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE( self, se ):
""" set target SE
:param self: self reference
:param str se: target SE name
"""
if se == self.sourceSE:
return S_ERROR( "TargetSE is SourceSE" )
self.targetSE = se
self.oTargetSE = StorageElement( self.targetSE )
return self.__checkTargetSE()
def setTargetToken( self, token ):
""" target space token setter
:param self: self reference
:param str token: target space token
"""
self.targetToken = token
return S_OK()
def __checkTargetSE( self ):
""" check target SE availability
:param self: self reference
"""
if not self.targetSE:
return S_ERROR( "TargetSE not set" )
res = self.oTargetSE.isValid( 'Write' )
if not res['OK']:
return S_ERROR( "TargetSE not available for writing" )
res = self.__getSESpaceToken( self.oTargetSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message'] )
return S_ERROR( "TargetSE does not support FTS transfers" )
# # check checksum types
if self.__cksmTest:
cksmType = self.oTargetSE.checksumType()
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at TargetSE %s, disabling checksum test" % ( cksmType,
self.targetSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
@staticmethod
def __getSESpaceToken( oSE ):
""" get space token from StorageElement instance
:param self: self reference
:param StorageElement oSE: StorageElement instance
"""
res = oSE.getStorageParameters( protocol = 'srm' )
if not res['OK']:
return res
return S_OK( res['Value'].get( 'SpaceToken' ) )
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID( self, guid ):
""" FTS job GUID setter
:param self: self reference
:param str guid: string containg GUID
"""
if not checkGuid( guid ):
return S_ERROR( "Incorrect GUID format" )
self.ftsGUID = guid
return S_OK()
def setFTSServer( self, server ):
""" FTS server setter
:param self: self reference
:param str server: FTS server URL
"""
self.ftsServer = server
return S_OK()
def isRequestTerminal( self ):
""" check if FTS job has terminated
:param self: self reference
"""
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK( self.isTerminal )
def setCksmTest( self, cksmTest = False ):
""" set cksm test
:param self: self reference
:param bool cksmTest: flag to enable/disable checksum test
"""
self.__cksmTest = bool( cksmTest )
return S_OK( self.__cksmTest )
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN( self, lfn ):
""" add LFN :lfn: to :fileDict:
:param self: self reference
:param str lfn: LFN to add to
"""
self.fileDict.setdefault( lfn, {'Status':'Waiting'} )
return S_OK()
def setSourceSURL( self, lfn, surl ):
""" source SURL setter
:param self: self reference
:param str lfn: LFN
:param str surl: source SURL
"""
target = self.fileDict[lfn].get( 'Target' )
if target == surl:
return S_ERROR( "Source and target the same" )
return self.__setFileParameter( lfn, 'Source', surl )
def getSourceSURL( self, lfn ):
""" get source SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Source' )
def setTargetSURL( self, lfn, surl ):
""" set target SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
:param str surl: target SURL
"""
source = self.fileDict[lfn].get( 'Source' )
if source == surl:
return S_ERROR( "Source and target the same" )
return self.__setFileParameter( lfn, 'Target', surl )
def getFailReason( self, lfn ):
""" get fail reason for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Reason' )
def getRetries( self, lfn ):
""" get number of attepmts made to transfer file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Retries' )
def getTransferTime( self, lfn ):
""" get duration of transfer for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Duration' )
def getFailed( self ):
""" get list of wrongly transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.failedStates ] )
def getStaging( self ):
""" get files set for prestaging """
return S_OK( [lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) == 'Staging'] )
def getDone( self ):
""" get list of succesfully transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.successfulStates ] )
def __setFileParameter( self, lfn, paramName, paramValue ):
""" set :paramName: to :paramValue: for :lfn: file
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
:param mixed paramValue: a new parameter value
"""
self.setLFN( lfn )
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter( self, lfn, paramName ):
""" get value of :paramName: for file :lfn:
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
"""
if lfn not in self.fileDict:
return S_ERROR( "Supplied file not set" )
if paramName not in self.fileDict[lfn]:
return S_ERROR( "%s not set for file" % paramName )
return S_OK( self.fileDict[lfn][paramName] )
####################################################################
#
# Methods for submission
#
def submit( self, monitor = False, printOutput = True ):
""" submit FTS job
:param self: self reference
:param bool monitor: flag to monitor progress of FTS job
:param bool printOutput: flag to print output of execution to stdout
"""
res = self.__prepareForSubmission()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = { 'ftsGUID' : self.ftsGUID, 'ftsServer' : self.ftsServer, 'submittedFiles' : self.submittedFiles }
if monitor or printOutput:
gLogger.always( "Submitted %s@%s" % ( self.ftsGUID, self.ftsServer ) )
if monitor:
self.monitor( untilTerminal = True, printOutput = printOutput, full = False )
return S_OK( resDict )
def __prepareForSubmission( self ):
""" check validity of job before submission
:param self: self reference
"""
if not self.fileDict:
return S_ERROR( "No files set" )
if not self.sourceValid:
return S_ERROR( "SourceSE not valid" )
if not self.targetValid:
return S_ERROR( "TargetSE not valid" )
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR( "FTSServer not valid" )
self.resolveSource()
self.resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR( "No files to submit" )
return S_OK()
def __getCatalogObject( self ):
""" CatalogInterface instance facade
:param self: self reference
"""
try:
if not self.oCatalog:
self.oCatalog = FileCatalog()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache( self, lfns = None, overwrite = False ):
""" update replica cache for list of :lfns:
:param self: self reference
:param mixed lfns: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if ( lfn not in self.catalogReplicas ) or overwrite ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getReplicas( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to update replica cache: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache( self, lfns = None ):
""" update metadata cache for list of LFNs
:param self: self reference
:param list lnfs: list of LFNs
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if lfn not in self.catalogMetadata ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getFileMetadata( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to get source catalog metadata: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def resolveSource( self ):
""" resolve source SE eligible for submission
:param self: self reference
"""
# Avoid resolving sources twice
if self.sourceResolved:
return S_OK()
# Only resolve files that need a transfer
toResolve = [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( "Status", "" ) != "Failed" ]
if not toResolve:
return S_OK()
res = self.__updateMetadataCache( toResolve )
if not res['OK']:
return res
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
# Define the source URLs
for lfn in toResolve:
replicas = self.catalogReplicas.get( lfn, {} )
if self.sourceSE not in replicas:
gLogger.warn( "resolveSource: skipping %s - not replicas at SourceSE %s" % ( lfn, self.sourceSE ) )
self.__setFileParameter( lfn, 'Reason', "No replica at SourceSE" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = returnSingleResult( self.oSourceSE.getURL( lfn, protocol = 'srm' ) )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setSourceSURL( lfn, res['Value'] )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = []
for lfn in self.fileDict:
if "Source" in self.fileDict[lfn]:
toResolve.append( lfn )
if not toResolve:
return S_ERROR( "No eligible Source files" )
# Get metadata of the sources, to check for existance, availability and caching
res = self.oSourceSE.getFileMetadata( toResolve )
if not res['OK']:
return S_ERROR( "Failed to check source file metadata" )
for lfn, error in res['Value']['Failed'].items():
if re.search( 'File does not exist', error ):
gLogger.warn( "resolveSource: skipping %s - source file does not exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file does not exist" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
gLogger.warn( "resolveSource: skipping %s - failed to get source metadata" % lfn )
self.__setFileParameter( lfn, 'Reason', "Failed to get Source metadata" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toStage = []
nbStagedFiles = 0
for lfn, metadata in res['Value']['Successful'].items():
lfnStatus = self.fileDict.get( lfn, {} ).get( 'Status' )
if metadata.get( 'Unavailable', False ):
gLogger.warn( "resolveSource: skipping %s - source file unavailable" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Unavailable" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif metadata.get( 'Lost', False ):
gLogger.warn( "resolveSource: skipping %s - source file lost" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Lost" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif not metadata.get( 'Cached', metadata['Accessible'] ):
if lfnStatus != 'Staging':
toStage.append( lfn )
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
gLogger.warn( "resolveSource: skipping %s - source file size mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source size mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \
not compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ):
gLogger.warn( "resolveSource: skipping %s - source file checksum mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source checksum mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif lfnStatus == 'Staging':
# file that was staging is now cached
self.__setFileParameter( lfn, 'Status', 'Waiting' )
nbStagedFiles += 1
# Some files were being staged
if nbStagedFiles:
self.log.info( 'resolveSource: %d files have been staged' % nbStagedFiles )
# Launching staging of files not in cache
if toStage:
gLogger.warn( "resolveSource: %s source files not cached, prestaging..." % len( toStage ) )
stage = self.oSourceSE.prestageFile( toStage )
if not stage["OK"]:
gLogger.error( "resolveSource: error is prestaging", stage["Message"] )
for lfn in toStage:
self.__setFileParameter( lfn, 'Reason', stage["Message"] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
for lfn in toStage:
if lfn in stage['Value']['Successful']:
self.__setFileParameter( lfn, 'Status', 'Staging' )
elif lfn in stage['Value']['Failed']:
self.__setFileParameter( lfn, 'Reason', stage['Value']['Failed'][lfn] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
self.sourceResolved = True
return S_OK()
def resolveTarget( self ):
""" find target SE eligible for submission
:param self: self reference
"""
toResolve = [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status' ) not in self.noSubmitStatus ]
if not toResolve:
return S_OK()
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
for lfn in toResolve:
res = returnSingleResult( self.oTargetSE.getURL( lfn, protocol = 'srm' ) )
if not res['OK']:
reason = res.get( 'Message', res['Message'] )
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, reason ) )
self.__setFileParameter( lfn, 'Reason', reason )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setTargetSURL( lfn, res['Value'] )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = []
for lfn in self.fileDict:
if "Target" in self.fileDict[lfn]:
toResolve.append( lfn )
if not toResolve:
return S_ERROR( "No eligible Target files" )
res = self.oTargetSE.exists( toResolve )
if not res['OK']:
return S_ERROR( "Failed to check target existence" )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toRemove = []
for lfn, exists in res['Value']['Successful'].items():
if exists:
res = self.getSourceSURL( lfn )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - target exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Target exists" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif res['Value'] == self.fileDict[lfn]['Target']:
gLogger.warn( "resolveTarget: skipping %s - source and target pfns are the same" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source and Target the same" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRemove.append( lfn )
if toRemove:
self.oTargetSE.removeFile( toRemove )
return S_OK()
def __filesToSubmit( self ):
"""
check if there is at least one file to submit
:return: S_OK if at least one file is present, S_ERROR otherwise
"""
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if lfnStatus not in self.noSubmitStatus and source and target:
return S_OK()
return S_ERROR()
def __createFTSFiles( self ):
""" create LFNs file for glite-transfer-submit command
This file consists one line for each fiel to be transferred:
sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM]
:param self: self reference
"""
self.__updateMetadataCache()
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
if lfnStatus not in self.noSubmitStatus:
cksmStr = ""
# # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself
if self.__cksmTest and self.__cksmType:
checkSum = self.catalogMetadata.get( lfn, {} ).get( 'Checksum' )
if checkSum:
cksmStr = " %s:%s" % ( self.__cksmType, intAdlerToHex( hexAdlerToInt( checkSum ) ) )
ftsFile = FTSFile()
ftsFile.LFN = lfn
ftsFile.SourceSURL = self.fileDict[lfn].get( 'Source' )
ftsFile.TargetSURL = self.fileDict[lfn].get( 'Target' )
ftsFile.SourceSE = self.sourceSE
ftsFile.TargetSE = self.targetSE
ftsFile.Status = self.fileDict[lfn].get( 'Status' )
ftsFile.Checksum = cksmStr
ftsFile.Size = self.catalogMetadata.get( lfn, {} ).get( 'Size' )
self.ftsFiles.append( ftsFile )
self.submittedFiles += 1
return S_OK()
def __createFTSJob( self, guid = None ):
self.__createFTSFiles()
ftsJob = FTSJob()
ftsJob.RequestID = 0
ftsJob.OperationID = 0
ftsJob.SourceSE = self.sourceSE
ftsJob.TargetSE = self.targetSE
ftsJob.SourceToken = self.sourceToken
ftsJob.TargetToken = self.targetToken
ftsJob.FTSServer = self.ftsServer
if guid:
ftsJob.FTSGUID = guid
for ftsFile in self.ftsFiles:
ftsFile.Attempt += 1
ftsFile.Error = ""
ftsJob.addFile( ftsFile )
self.ftsJob = ftsJob
def __submitFTSTransfer( self ):
""" create and execute glite-transfer-submit CLI command
:param self: self reference
"""
log = gLogger.getSubLogger( 'Submit' )
self.__createFTSJob()
submit = self.ftsJob.submitFTS( self.ftsVersion, command = self.submitCommand )
if not submit["OK"]:
log.error( "unable to submit FTSJob: %s" % submit["Message"] )
return submit
log.info( "FTSJob '%s'@'%s' has been submitted" % ( self.ftsJob.FTSGUID, self.ftsJob.FTSServer ) )
# # update statuses for job files
for ftsFile in self.ftsJob:
ftsFile.FTSGUID = self.ftsJob.FTSGUID
ftsFile.Status = "Submitted"
ftsFile.Attempt += 1
log.info( "FTSJob '%s'@'%s' has been submitted" % ( self.ftsJob.FTSGUID, self.ftsJob.FTSServer ) )
self.ftsGUID = self.ftsJob.FTSGUID
return S_OK()
def __resolveFTSServer( self ):
"""
resolve FTS server to use, it should be the closest one from target SE
:param self: self reference
"""
if self.ftsVersion.upper() == 'FTS2':
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTS2ServersForSites
if not self.targetSE:
return S_ERROR( "Target SE not set" )
res = getSitesForSE( self.targetSE )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine target site" )
targetSites = res['Value']
targetSite = ''
for targetSite in targetSites:
targetFTS = getFTS2ServersForSites( [targetSite] )
if targetFTS['OK']:
ftsTarget = targetFTS['Value'][targetSite]
if ftsTarget:
self.ftsServer = ftsTarget
return S_OK( self.ftsServer )
else:
return targetFTS
elif self.ftsVersion.upper() == 'FTS3':
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTS3Servers
res = getFTS3Servers()
if not res['OK']:
return res
ftsServerList = res['Value']
if ftsServerList:
# Here we take the first one, regardless of the policy...
# Unclean but all this will disapear after refactoring the fts code
self.ftsServer = ftsServerList[0]
return S_OK( self.ftsServer )
else:
return S_ERROR( 'Unknown FTS version %s' % self.ftsVersion )
return S_ERROR( 'No FTS server found for %s' % targetSite )
####################################################################
#
# Methods for monitoring
#
def summary( self, untilTerminal = False, printOutput = False ):
""" summary of FTS job
:param self: self reference
:param bool untilTerminal: flag to monitor FTS job to its final state
:param bool printOutput: flag to print out monitoring information to the stdout
"""
res = self.__isSummaryValid()
if not res['OK']:
return res
while not self.isTerminal:
res = self.__parseOutput( full = True )
if not res['OK']:
return res
if untilTerminal:
self.__print()
self.isRequestTerminal()
if res['Value'] or ( not untilTerminal ):
break
time.sleep( 1 )
if untilTerminal:
print ""
if printOutput and ( not untilTerminal ):
return self.dumpSummary( printOutput = printOutput )
return S_OK()
def monitor( self, untilTerminal = False, printOutput = False, full = True ):
""" monitor FTS job
:param self: self reference
:param bool untilTerminal: flag to monitor FTS job to its final state
:param bool printOutput: flag to print out monitoring information to the stdout
"""
if not self.ftsJob:
self.resolveSource()
self.__createFTSJob( self.ftsGUID )
res = self.__isSummaryValid()
if not res['OK']:
return res
if untilTerminal:
res = self.summary( untilTerminal = untilTerminal, printOutput = printOutput )
if not res['OK']:
return res
res = self.__parseOutput( full = full )
if not res['OK']:
return res
if untilTerminal:
self.finalize()
if printOutput:
self.dump()
return res
def dumpSummary( self, printOutput = False ):
""" get FTS job summary as str
:param self: self reference
:param bool printOutput: print summary to stdout
"""
outStr = ''
for status in sorted( self.statusSummary ):
if self.statusSummary[status]:
outStr = '%s\t%-10s : %-10s\n' % ( outStr, status, str( self.statusSummary[status] ) )
outStr = outStr.rstrip( '\n' )
if printOutput:
print outStr
return S_OK( outStr )
def __print( self ):
""" print progress bar of FTS job completeness to stdout
:param self: self reference
"""
width = 100
bits = int( ( width * self.percentageComplete ) / 100 )
outStr = "|%s>%s| %.1f%s %s %s" % ( "="*bits, " "*( width - bits ),
self.percentageComplete, "%",
self.requestStatus, " "*10 )
sys.stdout.write( "%s\r" % ( outStr ) )
sys.stdout.flush()
def dump( self ):
""" print FTS job parameters and files to stdout
:param self: self reference
"""
print "%-10s : %-10s" % ( "Status", self.requestStatus )
print "%-10s : %-10s" % ( "Source", self.sourceSE )
print "%-10s : %-10s" % ( "Target", self.targetSE )
print "%-10s : %-128s" % ( "Server", self.ftsServer )
print "%-10s : %-128s" % ( "GUID", self.ftsGUID )
for lfn in sorted( self.fileDict ):
print "\n %-15s : %-128s" % ( 'LFN', lfn )
for key in ['Source', 'Target', 'Status', 'Reason', 'Duration']:
print " %-15s : %-128s" % ( key, str( self.fileDict[lfn].get( key ) ) )
return S_OK()
def __isSummaryValid( self ):
""" check validity of FTS job summary report
:param self: self reference
"""
if not self.ftsServer:
return S_ERROR( "FTSServer not set" )
if not self.ftsGUID:
return S_ERROR( "FTSGUID not set" )
return S_OK()
def __parseOutput( self, full = False ):
""" execute glite-transfer-status command and parse its output
:param self: self reference
:param bool full: glite-transfer-status verbosity level, when set, collect information of files as well
"""
monitor = self.ftsJob.monitorFTS( self.ftsVersion, command = self.monitorCommand, full = full )
if not monitor['OK']:
return monitor
self.percentageComplete = self.ftsJob.Completeness
self.requestStatus = self.ftsJob.Status
self.submitTime = self.ftsJob.SubmitTime
statusSummary = monitor['Value']
if statusSummary:
for state in statusSummary:
self.statusSummary[state] = statusSummary[state]
self.transferTime = 0
for ftsFile in self.ftsJob:
lfn = ftsFile.LFN
self.__setFileParameter( lfn, 'Status', ftsFile.Status )
self.__setFileParameter( lfn, 'Reason', ftsFile.Error )
self.__setFileParameter( lfn, 'Duration', ftsFile._duration )
targetURL = self.__getFileParameter( lfn, 'Target' )
if not targetURL['OK']:
self.__setFileParameter( lfn, 'Target', ftsFile.TargetSURL )
sourceURL = self.__getFileParameter( lfn, 'Source' )
if not sourceURL['OK']:
self.__setFileParameter( lfn, 'Source', ftsFile.SourceSURL )
self.transferTime += int( ftsFile._duration )
return S_OK()
####################################################################
#
# Methods for finalization
#
def finalize( self ):
""" finalize FTS job
:param self: self reference
"""
self.__updateMetadataCache()
transEndTime = dateTime()
regStartTime = time.time()
res = self.getTransferStatistics()
transDict = res['Value']
res = self.__registerSuccessful( transDict['transLFNs'] )
regSuc, regTotal = res['Value']
regTime = time.time() - regStartTime
if self.sourceSE and self.targetSE:
self.__sendAccounting( regSuc, regTotal, regTime, transEndTime, transDict )
return S_OK()
def getTransferStatistics( self ):
""" collect information of Transfers that can be used by Accounting
:param self: self reference
"""
transDict = { 'transTotal': len( self.fileDict ),
'transLFNs': [],
'transOK': 0,
'transSize': 0 }
for lfn in self.fileDict:
if self.fileDict[lfn].get( 'Status' ) in self.successfulStates:
if self.fileDict[lfn].get( 'Duration', 0 ):
transDict['transLFNs'].append( lfn )
transDict['transOK'] += 1
if lfn in self.catalogMetadata:
transDict['transSize'] += self.catalogMetadata[lfn].get( 'Size', 0 )
return S_OK( transDict )
def getFailedRegistrations( self ):
""" get failed registrations dict
:param self: self reference
"""
return S_OK( self.failedRegistrations )
def __registerSuccessful( self, transLFNs ):
""" register successfully transferred files to the catalogs,
fill failedRegistrations dict for files that failed to register
:param self: self reference
:param list transLFNs: LFNs in FTS job
"""
self.failedRegistrations = {}
toRegister = {}
for lfn in transLFNs:
res = returnSingleResult( self.oTargetSE.getURL( self.fileDict[lfn].get( 'Target' ), protocol = 'srm' ) )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRegister[lfn] = { 'PFN' : res['Value'], 'SE' : self.targetSE }
if not toRegister:
return S_OK( ( 0, 0 ) )
res = self.__getCatalogObject()
if not res['OK']:
for lfn in toRegister:
self.failedRegistrations = toRegister
self.log.error( 'Failed to get Catalog Object', res['Message'] )
return S_OK( ( 0, len( toRegister ) ) )
res = self.oCatalog.addReplica( toRegister )
if not res['OK']:
self.failedRegistrations = toRegister
self.log.error( 'Failed to get Catalog Object', res['Message'] )
return S_OK( ( 0, len( toRegister ) ) )
for lfn, error in res['Value']['Failed'].items():
self.failedRegistrations[lfn] = toRegister[lfn]
self.log.error( 'Registration of Replica failed', '%s : %s' % ( lfn, str( error ) ) )
return S_OK( ( len( res['Value']['Successful'] ), len( toRegister ) ) )
def __sendAccounting( self, regSuc, regTotal, regTime, transEndTime, transDict ):
""" send accounting record
:param self: self reference
:param regSuc: number of files successfully registered
:param regTotal: number of files attepted to register
:param regTime: time stamp at the end of registration
:param transEndTime: time stamp at the end of FTS job
:param dict transDict: dict holding couters for files being transerred, their sizes and successfull transfers
"""
oAccounting = DataOperation()
oAccounting.setEndTime( transEndTime )
oAccounting.setStartTime( self.submitTime )
accountingDict = {}
accountingDict['OperationType'] = 'replicateAndRegister'
result = getProxyInfo()
if not result['OK']:
userName = '******'
else:
userName = result['Value'].get( 'username', 'unknown' )
accountingDict['User'] = userName
accountingDict['Protocol'] = 'FTS' if 'fts3' not in self.ftsServer else 'FTS3'
accountingDict['RegistrationTime'] = regTime
accountingDict['RegistrationOK'] = regSuc
accountingDict['RegistrationTotal'] = regTotal
accountingDict['TransferOK'] = transDict['transOK']
accountingDict['TransferTotal'] = transDict['transTotal']
accountingDict['TransferSize'] = transDict['transSize']
accountingDict['FinalStatus'] = self.requestStatus
accountingDict['Source'] = self.sourceSE
accountingDict['Destination'] = self.targetSE
accountingDict['TransferTime'] = self.transferTime
oAccounting.setValuesFromDict( accountingDict )
self.log.verbose( "Attempting to commit accounting message..." )
oAccounting.commit()
self.log.verbose( "...committed." )
return S_OK()
class FTSRequest:
def __init__(self):
self.gridEnv = '/afs/cern.ch/project/gd/LCG-share/3.2.8-0/etc/profile.d/grid-env'
self.finalStates = [
'Canceled', 'Failed', 'Hold', 'Finished', 'FinishedDirty'
]
self.failedStates = ['Canceled', 'Failed', 'Hold', 'FinishedDirty']
self.successfulStates = ['Finished', 'Done']
self.fileStates = [
'Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed',
'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting'
]
self.newlyCompletedFiles = []
self.newlyFailedFiles = []
self.statusSummary = {}
self.requestStatus = 'Unknown'
self.fileDict = {}
self.catalogReplicas = {}
self.catalogMetadata = {}
self.oCatalog = None
self.submitTime = ''
self.ftsGUID = ''
self.ftsServer = ''
self.priority = 3
self.isTerminal = False
self.percentageComplete = 0.0
self.sourceSE = ''
self.sourceValid = False
self.sourceToken = ''
self.targetSE = ''
self.targetValid = False
self.targetToken = ''
self.dumpStr = ''
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE(self, se):
if se == self.targetSE:
return S_ERROR("SourceSE is TargetSE")
self.sourceSE = se
self.oSourceSE = StorageElement(self.sourceSE)
return self.__checkSourceSE()
def getSourceSE(self):
if not self.sourceSE:
return S_ERROR("Source SE not defined")
return S_OK(self.sourceSE)
def setSourceToken(self, token):
self.sourceToken = token
return S_OK()
def getSourceToken(self):
if not self.sourceToken:
return S_ERROR("Source token not defined")
return S_OK(self.sourceToken)
def __checkSourceSE(self):
if not self.sourceSE:
return S_ERROR("SourceSE not set")
res = self.oSourceSE.isValid('Read')
if not res['OK']:
return S_ERROR("SourceSE not available for reading")
res = self.__getSESpaceToken(self.oSourceSE)
if not res['OK']:
gLogger.error(
"FTSRequest failed to get SRM Space Token for SourceSE",
res['Message'])
return S_ERROR("SourceSE does not support FTS transfers")
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE(self, se):
if se == self.sourceSE:
return S_ERROR("TargetSE is SourceSE")
self.targetSE = se
self.oTargetSE = StorageElement(self.targetSE)
return self.__checkTargetSE()
def getTargetSE(self):
if not self.targetSE:
return S_ERROR("Target SE not defined")
return S_OK(self.targetSE)
def setTargetToken(self, token):
self.targetToken = token
return S_OK()
def getTargetToken(self):
if not self.targetToken:
return S_ERROR("Target token not defined")
return S_OK(self.targetToken)
def __checkTargetSE(self):
if not self.targetSE:
return S_ERROR("TargetSE not set")
res = self.oTargetSE.isValid('Write')
if not res['OK']:
return S_ERROR("TargetSE not available for writing")
res = self.__getSESpaceToken(self.oTargetSE)
if not res['OK']:
gLogger.error(
"FTSRequest failed to get SRM Space Token for TargetSE",
res['Message'])
return S_ERROR("SourceSE does not support FTS transfers")
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
def __getSESpaceToken(self, oSE):
res = oSE.getStorageParameters("SRM2")
if not res['OK']:
return res
return S_OK(res['Value'].get('SpaceToken'))
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID(self, guid):
if not checkGuid(guid):
return S_ERROR("Incorrect GUID format")
self.ftsGUID = guid
return S_OK()
def getFTSGUID(self):
if not self.ftsGUID:
return S_ERROR("FTSGUID not set")
return S_OK(self.ftsGUID)
def setFTSServer(self, server):
self.ftsServer = server
return S_OK()
def getFTSServer(self):
if not self.ftsServer:
return S_ERROR("FTSServer not set")
return S_OK(self.ftsServer)
def setPriority(self, priority):
if not type(priority) in [types.IntType, types.LongType]:
return S_ERROR("Priority must be integer")
if priority < 0:
priority = 0
elif priority > 5:
priority = 5
self.priority = priority
return S_OK(self.priority)
def getPriority(self):
return S_OK(self.priority)
def getPercentageComplete(self):
completedFiles = 0
totalFiles = 0
for state in (self.statusSummary.keys()):
if state in self.successfulStates:
completedFiles += self.statusSummary[state]
totalFiles += self.statusSummary[state]
self.percentageComplete = (float(completedFiles) *
100.0) / float(totalFiles)
return S_OK(self.percentageComplete)
def isRequestTerminal(self):
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK(self.isTerminal)
def getStatus(self):
return S_OK(self.requestStatus)
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN(self, lfn):
if not self.fileDict.has_key(lfn):
self.fileDict[lfn] = {}
return S_OK()
def setSourceSURL(self, lfn, surl):
target = self.fileDict[lfn].get('Target')
if target == surl:
return S_ERROR("Source and target the same")
self.__setFileParameter(lfn, 'Source', surl)
return S_OK()
def getSourceSURL(self, lfn):
return self.__getFileParameter(lfn, 'Source')
def setTargetSURL(self, lfn, surl):
source = self.fileDict[lfn].get('Source')
if source == surl:
return S_ERROR("Source and target the same")
self.__setFileParameter(lfn, 'Target', surl)
return S_OK()
def getTargetSURL(self, lfn):
return self.__getFileParameter(lfn, 'Target')
def getFailReason(self, lfn):
return self.__getFileParameter(lfn, 'Reason')
def getRetries(self, lfn):
return self.__getFileParameter(lfn, 'Retries')
def getTransferTime(self, lfn):
return self.__getFileParameter(lfn, 'Duration')
def getFailed(self):
failed = []
for lfn in self.fileDict.keys():
status = self.fileDict[lfn].get('Status', '')
if status in self.failedStates:
failed.append(lfn)
return S_OK(failed)
def getDone(self):
done = []
for lfn in self.fileDict.keys():
status = self.fileDict[lfn].get('Status', '')
if status in self.successfulStates:
done.append(lfn)
return S_OK(done)
def __setFileParameter(self, lfn, paramName, paramValue):
self.setLFN(lfn)
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter(self, lfn, paramName):
if not self.fileDict.has_key(lfn):
return S_ERROR("Supplied file not set")
if not self.fileDict[lfn].has_key(paramName):
return S_ERROR("%s not set for file" % paramName)
return S_OK(self.fileDict[lfn][paramName])
####################################################################
#
# Methods for submission
#
def submit(self, monitor=False, printOutput=True):
res = self.__isSubmissionValid()
if not res['OK']:
return res
res = self.__createSURLPairFile()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = {'ftsGUID': self.ftsGUID, 'ftsServer': self.ftsServer}
print "Submitted %s @ %s" % (self.ftsGUID, self.ftsServer)
if monitor:
self.monitor(untilTerminal=True, printOutput=printOutput)
return S_OK(resDict)
def __isSubmissionValid(self):
if not self.fileDict:
return S_ERROR("No files set")
if not self.sourceValid:
return S_ERROR("SourceSE not valid")
if not self.targetValid:
return S_ERROR("TargetSE not valid")
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR("FTSServer not valid")
self.__resolveSource()
self.__resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR("No files to submit")
return S_OK()
def __getCatalogObject(self):
try:
if not self.oCatalog:
self.oCatalog = CatalogInterface()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache(self, lfns=[], overwrite=False):
if not lfns:
lfns = self.fileDict.keys()
toUpdate = []
for lfn in lfns:
if (not lfn in self.catalogReplicas.keys()) or overwrite:
toUpdate.append(lfn)
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogReplicas(toUpdate)
if not res['OK']:
return S_ERROR("Failed to update replica cache", res['Message'])
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache(self, lfns=[], overwrite=False):
if not lfns:
lfns = self.fileDict.keys()
toUpdate = []
for lfn in lfns:
if (not lfn in self.catalogMetadata.keys()) or overwrite:
toUpdate.append(lfn)
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogFileMetadata(toUpdate)
if not res['OK']:
return S_ERROR("Failed to get source catalog metadata",
res['Message'])
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def __resolveSource(self):
toResolve = []
for lfn in self.fileDict.keys():
if (not self.fileDict[lfn].has_key('Source')) and (
self.fileDict[lfn].get('Status') != 'Failed'):
toResolve.append(lfn)
if not toResolve:
return S_OK()
res = self.__updateMetadataCache(toResolve)
if not res['OK']:
return res
res = self.__updateReplicaCache(toResolve)
if not res['OK']:
return res
for lfn in toResolve:
if self.fileDict[lfn].get('Status') == 'Failed':
continue
replicas = self.catalogReplicas.get(lfn, {})
if not replicas.has_key(self.sourceSE):
self.__setFileParameter(lfn, 'Reason',
"No replica at SourceSE")
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.oSourceSE.getPfnForProtocol(replicas[self.sourceSE],
'SRM2',
withPort=True)
if not res['OK']:
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.setSourceSURL(lfn, res['Value'])
if not res['OK']:
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
toResolve = {}
for lfn in self.fileDict.keys():
if self.fileDict[lfn].has_key('Source'):
toResolve[self.fileDict[lfn]['Source']] = lfn
if not toResolve:
return S_ERROR("No eligible Source files")
res = self.oSourceSE.getFileMetadata(toResolve.keys())
if not res['OK']:
return S_ERROR("Failed to check source file metadata")
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
if re.search('File does not exist', error):
self.__setFileParameter(lfn, 'Reason',
"Source file does not exist")
self.__setFileParameter(lfn, 'Status', 'Failed')
else:
self.__setFileParameter(lfn, 'Reason',
"Failed to get Source metadata")
self.__setFileParameter(lfn, 'Status', 'Failed')
for pfn, metadata in res['Value']['Successful'].items():
lfn = toResolve[pfn]
if metadata['Unavailable']:
self.__setFileParameter(lfn, 'Reason',
"Source file Unavailable")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif metadata['Lost']:
self.__setFileParameter(lfn, 'Reason', "Source file Lost")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif not metadata['Cached']:
self.__setFileParameter(lfn, 'Reason',
"Source file not Cached")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
self.__setFileParameter(lfn, 'Reason', "Source size mismatch")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif self.catalogMetadata[lfn]['Checksum'] and metadata[
'Checksum'] and not (compareAdler(
metadata['Checksum'],
self.catalogMetadata[lfn]['Checksum'])):
self.__setFileParameter(lfn, 'Reason',
"Source checksum mismatch")
self.__setFileParameter(lfn, 'Status', 'Failed')
return S_OK()
def __resolveTarget(self):
toResolve = []
for lfn in self.fileDict.keys():
if not self.fileDict[lfn].has_key('Target') and (
self.fileDict[lfn].get('Status') != 'Failed'):
toResolve.append(lfn)
if not toResolve:
return S_OK()
res = self.__updateReplicaCache(toResolve)
if not res['OK']:
return res
atTarget = []
for lfn in sortList(toResolve):
if self.fileDict[lfn].get('Status') == 'Failed':
continue
replicas = self.catalogReplicas.get(lfn, {})
if replicas.has_key(self.targetSE):
self.__setFileParameter(lfn, 'Reason',
"File already at Target")
self.__setFileParameter(lfn, 'Status', 'Done')
atTarget.append(lfn)
for lfn in toResolve:
if (self.fileDict[lfn].get('Status') == 'Failed') or (lfn
in atTarget):
continue
res = self.oTargetSE.getPfnForLfn(lfn)
if not res['OK']:
self.__setFileParameter(lfn, 'Reason',
"Failed to create Target")
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.oTargetSE.getPfnForProtocol(res['Value'],
'SRM2',
withPort=True)
if not res['OK']:
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.setTargetSURL(lfn, res['Value'])
if not res['OK']:
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
toResolve = {}
for lfn in self.fileDict.keys():
if self.fileDict[lfn].has_key('Target'):
toResolve[self.fileDict[lfn]['Target']] = lfn
if not toResolve:
return S_ERROR("No eligible Target files")
res = self.oTargetSE.exists(toResolve.keys())
if not res['OK']:
return S_ERROR("Failed to check target existence")
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
toRemove = []
for pfn, exists in res['Value']['Successful'].items():
if exists:
lfn = toResolve[pfn]
res = self.getSourceSURL(lfn)
if not res['OK']:
self.__setFileParameter(lfn, 'Reason', "Target exists")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif res['Value'] == pfn:
self.__setFileParameter(lfn, 'Reason',
"Source and Target the same")
self.__setFileParameter(lfn, 'Status', 'Failed')
else:
toRemove.append(pfn)
if toRemove:
self.oTargetSE.removeFile(toRemove)
return S_OK()
def __filesToSubmit(self):
for lfn in self.fileDict.keys():
lfnStatus = self.fileDict[lfn].get('Status')
source = self.fileDict[lfn].get('Source')
target = self.fileDict[lfn].get('Target')
if (lfnStatus != 'Failed') and (lfnStatus !=
'Done') and source and target:
return S_OK()
return S_ERROR()
def __createSURLPairFile(self):
fd, fileName = tempfile.mkstemp()
surlFile = os.fdopen(fd, 'w')
for lfn in self.fileDict.keys():
lfnStatus = self.fileDict[lfn].get('Status')
source = self.fileDict[lfn].get('Source')
target = self.fileDict[lfn].get('Target')
if (lfnStatus != 'Failed') and (lfnStatus !=
'Done') and source and target:
surlString = '%s %s\n' % (source, target)
surlFile.write(surlString)
surlFile.close()
self.surlFile = fileName
return S_OK()
def __submitFTSTransfer(self):
comm = [
'glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile,
'-o'
]
if self.targetToken:
comm.append('-t')
comm.append(self.targetToken)
if self.sourceToken:
comm.append('-S')
comm.append(self.sourceToken)
res = executeGridCommand('', comm, self.gridEnv)
os.remove(self.surlFile)
if not res['OK']:
return res
returnCode, output, errStr = res['Value']
if not returnCode == 0:
return S_ERROR(errStr)
guid = output.replace('\n', '')
if not checkGuid(guid):
return S_ERROR('Wrong GUID format returned')
self.ftsGUID = guid
#if self.priority != 3:
# comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)]
# executeGridCommand('',comm,self.gridEnv)
return res
def __resolveFTSServer(self):
if not self.sourceSE:
return S_ERROR("Source SE not set")
if not self.targetSE:
return S_ERROR("Target SE not set")
res = getSitesForSE(self.sourceSE, 'LCG')
if not res['OK'] or not res['Value']:
return S_ERROR("Could not determine source site")
sourceSite = res['Value'][0]
res = getSitesForSE(self.targetSE, 'LCG')
if not res['OK'] or not res['Value']:
return S_ERROR("Could not determine target site")
targetSite = res['Value'][0]
if (sourceSite == 'LCG.CERN.ch') or (targetSite == 'LCG.CERN.ch'):
ep = 'LCG.CERN.ch'
else:
# Target site FTS server should be used
ep = targetSite
try:
configPath = '/Resources/FTSEndpoints/%s' % ep
endpointURL = gConfig.getValue(configPath)
if not endpointURL:
errStr = "FTSRequest.__resolveFTSEndpoint: Failed to find FTS endpoint, check CS entry for '%s'." % ep
return S_ERROR(errStr)
self.ftsServer = endpointURL
return S_OK(endpointURL)
except Exception, x:
return S_ERROR(
'FTSRequest.__resolveFTSEndpoint: Failed to obtain endpoint details from CS'
)
class FTSRequest(object):
"""
.. class:: FTSRequest
Helper class for FTS job submission and monitoring.
"""
## default checksum type
__defaultCksmType = "ADLER32"
## flag to disablr/enable checksum test, default: disabled
__cksmTest = False
def __init__(self):
"""c'tor
:param self: self reference
"""
self.log = gLogger.getSubLogger(self.__class__.__name__, True)
## final states tuple
self.finalStates = ('Canceled', 'Failed', 'Hold', 'Finished',
'FinishedDirty')
## failed states tuple
self.failedStates = ('Canceled', 'Failed', 'Hold', 'FinishedDirty')
## successful states tuple
self.successfulStates = ('Finished', 'Done')
## all file states tuple
self.fileStates = ('Done', 'Active', 'Pending', 'Ready', 'Canceled',
'Failed', 'Finishing', 'Finished', 'Submitted',
'Hold', 'Waiting')
self.newlyCompletedFiles = []
self.newlyFailedFiles = []
self.statusSummary = {}
## request status
self.requestStatus = 'Unknown'
## dict for FTS job files
self.fileDict = {}
## dict for replicas information
self.catalogReplicas = {}
## dict for metadata information
self.catalogMetadata = {}
## dict for files that failed to register
self.failedRegistrations = {}
## placehoder for CatalogInterface reference
self.oCatalog = None
## submit timestamp
self.submitTime = ''
## placeholder FTS job GUID
self.ftsGUID = ''
## placeholder for FTS server URL
self.ftsServer = ''
## not used
self.priority = 3
## flag marking FTS job completness
self.isTerminal = False
## completness percentage
self.percentageComplete = 0.0
## source SE name
self.sourceSE = ''
## flag marking source SE validity
self.sourceValid = False
## source space token
self.sourceToken = ''
## target SE name
self.targetSE = ''
## flag marking target SE validity
self.targetValid = False
## target space token
self.targetToken = ''
## whatever
self.dumpStr = ''
## placeholder for surl file
self.surlFile = None
## placeholder for target StorageElement
self.oTargetSE = None
## placeholder for source StorageElement
self.oSourceSE = None
## checksum type, set it to default
self.__cksmType = self.__defaultCksmType
## disable checksum test by default
self.__cksmTest = False
## replica manager handler
self.replicaManager = ReplicaManager()
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE(self, se):
""" set SE for source
:param self: self reference
:param str se: source SE name
"""
if se == self.targetSE:
return S_ERROR("SourceSE is TargetSE")
self.sourceSE = se
self.oSourceSE = StorageElement(self.sourceSE)
return self.__checkSourceSE()
def getSourceSE(self):
""" source SE getter
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR("Source SE not defined")
return S_OK(self.sourceSE)
def setSourceToken(self, token):
""" set source space token
:param self: self reference
:param str token: source space token
"""
self.sourceToken = token
return S_OK()
def getSourceToken(self):
""" source space token getter
:param self: self reference
"""
if not self.sourceToken:
return S_ERROR("Source token not defined")
return S_OK(self.sourceToken)
def __checkSourceSE(self):
""" check source SE availability
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR("SourceSE not set")
res = self.oSourceSE.isValid('Read')
if not res['OK']:
return S_ERROR("SourceSE not available for reading")
res = self.__getSESpaceToken(self.oSourceSE)
if not res['OK']:
self.log.error(
"FTSRequest failed to get SRM Space Token for SourceSE",
res['Message'])
return S_ERROR("SourceSE does not support FTS transfers")
if self.__cksmTest:
res = self.oSourceSE.getChecksumType()
if not res["OK"]:
self.log.error(
"Unable to get checksum type for SourceSE %s: %s" %
(self.sourceSE, res["Message"]))
cksmType = res["Value"]
if cksmType in ("NONE", "NULL"):
self.log.warn(
"Checksum type set to %s at SourceSE %s, disabling checksum test"
% (cksmType, self.sourceSE))
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn(
"Checksum type mismatch, disabling checksum test")
self.__cksmTest = False
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE(self, se):
""" set target SE
:param self: self reference
:param str se: target SE name
"""
if se == self.sourceSE:
return S_ERROR("TargetSE is SourceSE")
self.targetSE = se
self.oTargetSE = StorageElement(self.targetSE)
return self.__checkTargetSE()
def getTargetSE(self):
""" target SE getter
:param self: self reference
"""
if not self.targetSE:
return S_ERROR("Target SE not defined")
return S_OK(self.targetSE)
def setTargetToken(self, token):
""" target space token setter
:param self: self reference
:param str token: target space token
"""
self.targetToken = token
return S_OK()
def getTargetToken(self):
""" target space token getter
:param self: self reference
"""
if not self.targetToken:
return S_ERROR("Target token not defined")
return S_OK(self.targetToken)
def __checkTargetSE(self):
""" check target SE availability
:param self: self reference
"""
if not self.targetSE:
return S_ERROR("TargetSE not set")
res = self.oTargetSE.isValid('Write')
if not res['OK']:
return S_ERROR("TargetSE not available for writing")
res = self.__getSESpaceToken(self.oTargetSE)
if not res['OK']:
self.log.error(
"FTSRequest failed to get SRM Space Token for TargetSE",
res['Message'])
return S_ERROR("TargetSE does not support FTS transfers")
## check checksum types
if self.__cksmTest:
res = self.oTargetSE.getChecksumType()
if not res["OK"]:
self.log.error(
"Unable to get checksum type for TargetSE %s: %s" %
(self.targetSE, res["Message"]))
cksmType = res["Value"]
if cksmType in ("NONE", "NULL"):
self.log.warn(
"Checksum type set to %s at TargetSE %s, disabling checksum test"
% (cksmType, self.targetSE))
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn(
"Checksum type mismatch, disabling checksum test")
self.__cksmTest = False
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
@staticmethod
def __getSESpaceToken(oSE):
""" get space token from StorageElement instance
:param self: self reference
:param StorageElement oSE: StorageElement instance
"""
res = oSE.getStorageParameters("SRM2")
if not res['OK']:
return res
return S_OK(res['Value'].get('SpaceToken'))
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID(self, guid):
""" FTS job GUID setter
:param self: self reference
:param str guid: string containg GUID
"""
if not checkGuid(guid):
return S_ERROR("Incorrect GUID format")
self.ftsGUID = guid
return S_OK()
def getFTSGUID(self):
""" FTS job GUID getter
:param self: self refenece
"""
if not self.ftsGUID:
return S_ERROR("FTSGUID not set")
return S_OK(self.ftsGUID)
def setFTSServer(self, server):
""" FTS server setter
:param self: self reference
:param str server: FTS server URL
"""
self.ftsServer = server
return S_OK()
def getFTSServer(self):
""" FTS server getter
:param self: self reference
"""
if not self.ftsServer:
return S_ERROR("FTSServer not set")
return S_OK(self.ftsServer)
def setPriority(self, priority):
""" set priority for FTS job
:param self: self reference
:param int priority: a new priority
"""
if not type(priority) in (IntType, LongType):
return S_ERROR("Priority must be integer")
if priority < 0:
priority = 0
elif priority > 5:
priority = 5
self.priority = priority
return S_OK(self.priority)
def getPriority(self):
""" FTS job priority getter
:param self: self reference
"""
return S_OK(self.priority)
def getPercentageComplete(self):
""" get completness percentage
:param self: self reference
"""
completedFiles = 0
totalFiles = 0
for state in (self.statusSummary.keys()):
if state in self.successfulStates:
completedFiles += self.statusSummary[state]
totalFiles += self.statusSummary[state]
self.percentageComplete = (float(completedFiles) *
100.0) / float(totalFiles)
return S_OK(self.percentageComplete)
def isRequestTerminal(self):
""" check if FTS job has terminated
:param self: self reference
"""
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK(self.isTerminal)
def getStatus(self):
""" get FTS job status
:param self: self reference
"""
return S_OK(self.requestStatus)
def setCksmType(self, cksm=None):
""" set checksum type to use
:param self: self reference
:param mixed cksm: checksum type, should be one of 'Adler32', 'md5', 'sha1', None
"""
if str(cksm).upper() not in ("ADLER32", "MD5", "SHA1", "NONE"):
return S_ERROR("Not supported checksum type: %s" % str(cksm))
if not cksm:
self.__cksmType = None
return S_OK(False)
self.__cksmType = str(cksm).upper()
return S_OK(True)
def getCksmType(self):
""" get checksum type
:param self: self reference
"""
return S_OK(self.__cksmType)
def setCksmTest(self, cksmTest=False):
""" set cksm test
:param self: self reference
:param bool cksmTest: flag to enable/disable checksum test
"""
self.__cksmTest = bool(cksmTest)
return S_OK(self.__cksmTest)
def getCksmTest(self):
""" get cksm test flag
:param self: self reference
"""
return S_OK(self.__cksmTest)
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN(self, lfn):
""" add LFN :lfn: to :fileDict:
:param self: self reference
:param str lfn: LFN to add to
"""
if lfn not in self.fileDict:
self.fileDict[lfn] = {}
return S_OK()
def setSourceSURL(self, lfn, surl):
""" source SURL setter
:param self: self reference
:param str lfn: LFN
:param str surl: source SURL
"""
target = self.fileDict[lfn].get('Target')
if target == surl:
return S_ERROR("Source and target the same")
self.__setFileParameter(lfn, 'Source', surl)
return S_OK()
def getSourceSURL(self, lfn):
""" get source SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter(lfn, 'Source')
def setTargetSURL(self, lfn, surl):
""" set target SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
:param str surl: target SURL
"""
source = self.fileDict[lfn].get('Source')
if source == surl:
return S_ERROR("Source and target the same")
self.__setFileParameter(lfn, 'Target', surl)
return S_OK()
def getTargetSURL(self, lfn):
""" target SURL getter
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter(lfn, 'Target')
def getFailReason(self, lfn):
""" get fail reason for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter(lfn, 'Reason')
def getRetries(self, lfn):
""" get number of attepmts made to transfer file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter(lfn, 'Retries')
def getTransferTime(self, lfn):
""" get duration of transfer for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter(lfn, 'Duration')
def getFailed(self):
""" get list of wrongly transferred LFNs
:param self: self reference
"""
return S_OK([
lfn for lfn in self.fileDict
if self.fileDict[lfn].get('Status', '') in self.failedStates
])
def getDone(self):
""" get list of succesfully transferred LFNs
:param self: self reference
"""
return S_OK([
lfn for lfn in self.fileDict
if self.fileDict[lfn].get('Status', '') in self.successfulStates
])
def __setFileParameter(self, lfn, paramName, paramValue):
""" set :paramName: to :paramValue: for :lfn: file
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
:param mixed paramValue: a new parameter value
"""
self.setLFN(lfn)
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter(self, lfn, paramName):
""" get value of :paramName: for file :lfn:
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
"""
if lfn not in self.fileDict:
return S_ERROR("Supplied file not set")
if paramName not in self.fileDict[lfn]:
return S_ERROR("%s not set for file" % paramName)
return S_OK(self.fileDict[lfn][paramName])
####################################################################
#
# Methods for submission
#
def submit(self, monitor=False, printOutput=True):
""" submit FTS job
:param self: self reference
:param bool monitor: flag to monitor progress of FTS job
:param bool printOutput: flag to print output of execution to stdout
"""
res = self.__isSubmissionValid()
if not res['OK']:
return res
res = self.__createSURLPairFile()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = {'ftsGUID': self.ftsGUID, 'ftsServer': self.ftsServer}
print "Submitted %s @ %s" % (self.ftsGUID, self.ftsServer)
if monitor:
self.monitor(untilTerminal=True, printOutput=printOutput)
return S_OK(resDict)
def __isSubmissionValid(self):
""" check validity of job before submission
:param self: self reference
"""
if not self.fileDict:
return S_ERROR("No files set")
if not self.sourceValid:
return S_ERROR("SourceSE not valid")
if not self.targetValid:
return S_ERROR("TargetSE not valid")
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR("FTSServer not valid")
self.resolveSource()
self.resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR("No files to submit")
return S_OK()
def __getCatalogObject(self):
""" CatalogInterface instance facade
:param self: self reference
"""
try:
if not self.oCatalog:
self.oCatalog = CatalogInterface()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache(self, lfns=None, overwrite=False):
""" update replica cache for list of :lfns:
:param self: self reference
:param mixed lfns: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [
lfn for lfn in lfns
if (lfn not in self.catalogReplicas) or overwrite
]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogReplicas(toUpdate)
if not res['OK']:
return S_ERROR("Failed to update replica cache: %s" %
res['Message'])
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache(self, lfns=None, overwrite=False):
""" update metadata cache for list of LFNs
:param self: self reference
:param list lnfs: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [
lfn for lfn in lfns
if (lfn not in self.catalogMetadata) or overwrite
]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogFileMetadata(toUpdate)
if not res['OK']:
return S_ERROR("Failed to get source catalog metadata: %s" %
res['Message'])
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def resolveSource(self):
""" resolve source SE eligible for submission
:param self: self reference
"""
toResolve = [lfn for lfn in self.fileDict]
if not toResolve:
return S_OK()
res = self.__updateMetadataCache(toResolve)
if not res['OK']:
return res
res = self.__updateReplicaCache(toResolve)
if not res['OK']:
return res
for lfn in toResolve:
if self.fileDict[lfn].get("Status", "") == "Failed":
continue
replicas = self.catalogReplicas.get(lfn, {})
if self.sourceSE not in replicas:
gLogger.warn(
"resolveSource: skipping %s - not replicas at SourceSE %s"
% (lfn, self.sourceSE))
self.__setFileParameter(lfn, 'Reason',
"No replica at SourceSE")
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.oSourceSE.getPfnForProtocol(replicas[self.sourceSE],
'SRM2',
withPort=True)
if not res['OK']:
gLogger.warn("resolveSource: skipping %s - %s" %
(lfn, res["Message"]))
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.setSourceSURL(lfn, res['Value'])
if not res['OK']:
gLogger.warn("resolveSource: skipping %s - %s" %
(lfn, res["Message"]))
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
toResolve = {}
for lfn in self.fileDict:
if "Source" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Source']] = lfn
if not toResolve:
return S_ERROR("No eligible Source files")
res = self.oSourceSE.getFileMetadata(toResolve.keys())
if not res['OK']:
return S_ERROR("Failed to check source file metadata")
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
if re.search('File does not exist', error):
gLogger.warn(
"resolveSource: skipping %s - source file does not exists"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Source file does not exist")
self.__setFileParameter(lfn, 'Status', 'Failed')
else:
gLogger.warn(
"resolveSource: skipping %s - failed to get source metadata"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Failed to get Source metadata")
self.__setFileParameter(lfn, 'Status', 'Failed')
for pfn, metadata in res['Value']['Successful'].items():
lfn = toResolve[pfn]
if metadata['Unavailable']:
gLogger.warn(
"resolveSource: skipping %s - source file unavailable" %
lfn)
self.__setFileParameter(lfn, 'Reason',
"Source file Unavailable")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif metadata['Lost']:
gLogger.warn("resolveSource: skipping %s - source file lost" %
lfn)
self.__setFileParameter(lfn, 'Reason', "Source file Lost")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif not metadata['Cached']:
gLogger.warn(
"resolveSource: source file %s not cached, prestaging..." %
lfn)
stage = self.replicaManager.prestageStorageFile(
pfn, self.sourceSE, singleFile=True)
if not stage["OK"]:
gLogger.warn("resolveSource: skipping %s - %s" %
(lfn, stage["Message"]))
self.__setFileParameter(lfn, 'Reason', stage["Message"])
self.__setFileParameter(lfn, 'Status', 'Failed')
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
gLogger.warn(
"resolveSource: skipping %s - source file size mismatch" %
lfn)
self.__setFileParameter(lfn, 'Reason', "Source size mismatch")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \
not ( compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ) ):
gLogger.warn(
"resolveSource: skipping %s - source file checksum mismatch"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Source checksum mismatch")
self.__setFileParameter(lfn, 'Status', 'Failed')
return S_OK()
def resolveTarget(self):
""" find target SE eligible for submission
:param self: self reference
"""
toResolve = [lfn for lfn in self.fileDict]
if not toResolve:
return S_OK()
res = self.__updateReplicaCache(toResolve)
if not res['OK']:
return res
for lfn in toResolve:
if (self.fileDict[lfn].get('Status') == 'Failed'):
continue
res = self.oTargetSE.getPfnForLfn(lfn)
if not res['OK']:
gLogger.warn(
"resolveTarget: skipping %s - failed to create target pfn"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Failed to create Target")
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.oTargetSE.getPfnForProtocol(res['Value'],
'SRM2',
withPort=True)
if not res['OK']:
gLogger.warn("resolveTarget: skipping %s - %s" %
(lfn, res["Message"]))
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.setTargetSURL(lfn, res['Value'])
if not res['OK']:
gLogger.warn("resolveTarget: skipping %s - %s" %
(lfn, res["Message"]))
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
toResolve = {}
for lfn in self.fileDict:
if "Target" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Target']] = lfn
if not toResolve:
return S_ERROR("No eligible Target files")
res = self.oTargetSE.exists(toResolve.keys())
if not res['OK']:
return S_ERROR("Failed to check target existence")
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
toRemove = []
for pfn, exists in res['Value']['Successful'].items():
if exists:
lfn = toResolve[pfn]
res = self.getSourceSURL(lfn)
if not res['OK']:
gLogger.warn("resolveTarget: skipping %s - target exists" %
lfn)
self.__setFileParameter(lfn, 'Reason', "Target exists")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif res['Value'] == pfn:
gLogger.warn(
"resolveTarget: skipping %s - source and target pfns are teh same"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Source and Target the same")
self.__setFileParameter(lfn, 'Status', 'Failed')
else:
toRemove.append(pfn)
if toRemove:
self.oTargetSE.removeFile(toRemove)
return S_OK()
def __filesToSubmit(self):
"""
check if there is at least one file to submit
:return: S_OK if at least one file is present, S_ERROR otherwise
"""
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get('Status')
source = self.fileDict[lfn].get('Source')
target = self.fileDict[lfn].get('Target')
if (lfnStatus != 'Failed') and (lfnStatus !=
'Done') and source and target:
return S_OK()
return S_ERROR()
def __createSURLPairFile(self):
""" create LFNs file for glite-transfer-submit command
This file consists one line for each fiel to be transferred:
sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM]
:param self: self reference
"""
fd, fileName = tempfile.mkstemp()
surlFile = os.fdopen(fd, 'w')
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get('Status')
source = self.fileDict[lfn].get('Source')
target = self.fileDict[lfn].get('Target')
if (lfnStatus not in ('Failed', 'Done')) and source and target:
cksmStr = ""
## add chsmType:cksm only if cksmType is specified, else let FTS decide by itself
if self.__cksmTest and self.__cksmType:
if lfn in self.catalogMetadata and "Checksum" in self.catalogMetadata[
lfn]:
cksmStr = " %s:%s" % (self.__cksmType, self.
catalogMetadata[lfn]["Checksum"])
surlFile.write("%s %s%s\n" % (source, target, cksmStr))
surlFile.close()
self.surlFile = fileName
return S_OK()
def __submitFTSTransfer(self):
""" create and execute glite-transfer-submit CLI command
:param self: self reference
"""
comm = [
'glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile,
'-o'
]
if self.targetToken:
comm.append('-t')
comm.append(self.targetToken)
if self.sourceToken:
comm.append('-S')
comm.append(self.sourceToken)
if self.__cksmTest:
comm.append("--compare-checksums")
res = executeGridCommand('', comm)
os.remove(self.surlFile)
if not res['OK']:
return res
returnCode, output, errStr = res['Value']
if not returnCode == 0:
return S_ERROR(errStr)
guid = output.replace('\n', '')
if not checkGuid(guid):
return S_ERROR('Wrong GUID format returned')
self.ftsGUID = guid
#if self.priority != 3:
# comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)]
# executeGridCommand('',comm)
return res
def __resolveFTSServer(self):
"""
resolve FTS server to use, it should be the closest one from target SE
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR("Source SE not set")
if not self.targetSE:
return S_ERROR("Target SE not set")
res = getSitesForSE(self.sourceSE, 'LCG')
if not res['OK'] or not res['Value']:
return S_ERROR("Could not determine source site")
sourceSite = res['Value'][0]
res = getSitesForSE(self.targetSE, 'LCG')
if not res['OK'] or not res['Value']:
return S_ERROR("Could not determine target site")
targetSite = res['Value'][0]
if (sourceSite == 'LCG.CERN.ch') or (targetSite == 'LCG.CERN.ch'):
ep = 'LCG.CERN.ch'
else:
# Target site FTS server should be used
ep = targetSite
try:
configPath = '/Resources/FTSEndpoints/%s' % ep
endpointURL = gConfig.getValue(configPath)
if not endpointURL:
errStr = "FTSRequest.__resolveFTSServer: Failed to find FTS endpoint, check CS entry for '%s'." % ep
return S_ERROR(errStr)
self.ftsServer = endpointURL
return S_OK(endpointURL)
except Exception, x:
return S_ERROR(
'FTSRequest.__resolveFTSServer: Failed to obtain endpoint details from CS'
)
def __getStorageDirectoryContents( self, lfnDir, storageElement ):
""" Obtians the contents of the supplied directory on the storage
"""
gLogger.info( 'Obtaining the contents for %s directories at %s' % ( len( lfnDir ), storageElement ) )
se = StorageElement( storageElement )
res = se.getPfnForLfn( lfnDir )
if not res['OK']:
gLogger.error( "Failed to get PFNs for directories", res['Message'] )
return res
for directory, error in res['Value']['Failed'].items():
gLogger.error( 'Failed to obtain directory PFN from LFNs', '%s %s' % ( directory, error ) )
if res['Value']['Failed']:
return S_ERROR( 'Failed to obtain directory PFN from LFNs' )
storageDirectories = res['Value']['Successful'].values()
res = se.exists( storageDirectories )
if not res['OK']:
gLogger.error( "Failed to obtain existance of directories", res['Message'] )
return res
for directory, error in res['Value']['Failed'].items():
gLogger.error( 'Failed to determine existance of directory', '%s %s' % ( directory, error ) )
if res['Value']['Failed']:
return S_ERROR( 'Failed to determine existance of directory' )
directoryExists = res['Value']['Successful']
activeDirs = []
for directory in sortList( directoryExists.keys() ):
exists = directoryExists[directory]
if exists:
activeDirs.append( directory )
allFiles = {}
while len( activeDirs ) > 0:
currentDir = activeDirs[0]
res = se.listDirectory( currentDir )
activeDirs.remove( currentDir )
if not res['OK']:
gLogger.error( 'Failed to get directory contents', res['Message'] )
return res
elif res['Value']['Failed'].has_key( currentDir ):
gLogger.error( 'Failed to get directory contents', '%s %s' % ( currentDir, res['Value']['Failed'][currentDir] ) )
return S_ERROR( res['Value']['Failed'][currentDir] )
else:
dirContents = res['Value']['Successful'][currentDir]
activeDirs.extend( dirContents['SubDirs'] )
fileMetadata = dirContents['Files']
# RF_NOTE This ugly trick is needed because se.getPfnPath does not follow the Successful/Failed convention
# res = { "Successful" : {}, "Failed" : {} }
# for pfn in fileMetadata:
# inRes = se.getPfnPath( pfn )
# if inRes["OK"]:
# res["Successful"][pfn] = inRes["Value"]
# else:
# res["Failed"][pfn] = inRes["Message"]
res = se.getLfnForPfn( fileMetadata.keys() )
if not res['OK']:
gLogger.error( 'Failed to get directory content LFNs', res['Message'] )
return res
for pfn, error in res['Value']['Failed'].items():
gLogger.error( "Failed to get LFN for PFN", "%s %s" % ( pfn, error ) )
if res['Value']['Failed']:
return S_ERROR( "Failed to get LFNs for PFNs" )
pfnLfns = res['Value']['Successful']
for pfn, lfn in pfnLfns.items():
fileMetadata[pfn]['LFN'] = lfn
allFiles.update( fileMetadata )
zeroSizeFiles = []
lostFiles = []
unavailableFiles = []
for pfn in sortList( allFiles.keys() ):
if os.path.basename( pfn ) == 'dirac_directory':
allFiles.pop( pfn )
else:
metadata = allFiles[pfn]
if metadata['Size'] == 0:
zeroSizeFiles.append( ( metadata['LFN'], pfn, storageElement, 'PFNZeroSize' ) )
# if metadata['Lost']:
# lostFiles.append((metadata['LFN'],pfn,storageElement,'PFNLost'))
# if metadata['Unavailable']:
# unavailableFiles.append((metadata['LFN'],pfn,storageElement,'PFNUnavailable'))
if zeroSizeFiles:
self.__reportProblematicReplicas( zeroSizeFiles, storageElement, 'PFNZeroSize' )
if lostFiles:
self.__reportProblematicReplicas( lostFiles, storageElement, 'PFNLost' )
if unavailableFiles:
self.__reportProblematicReplicas( unavailableFiles, storageElement, 'PFNUnavailable' )
gLogger.info( 'Obtained at total of %s files for directories at %s' % ( len( allFiles ), storageElement ) )
return S_OK( allFiles )
