def __insertRegisterOperation( self, request, operation, toRegister ):
""" add RegisterReplica operation
:param Request request: request instance
:param Operation transferOp: 'ReplicateAndRegister' operation for this FTSJob
:param list toRegister: [ FTSDB.FTSFile, ... ] - files that failed to register
"""
log = self.log.getSubLogger( "req_%s/%s/registerFiles" % ( request.RequestID, request.RequestName ) )
byTarget = {}
for ftsFile in toRegister:
if ftsFile.TargetSE not in byTarget:
byTarget.setdefault( ftsFile.TargetSE, [] )
byTarget[ftsFile.TargetSE].append( ftsFile )
log.info( "will create %s 'RegisterReplica' operations" % len( byTarget ) )
for target, ftsFileList in byTarget.iteritems():
log.info( "creating 'RegisterReplica' operation for targetSE %s with %s files..." % ( target,
len( ftsFileList ) ) )
registerOperation = Operation()
registerOperation.Type = "RegisterReplica"
registerOperation.Status = "Waiting"
registerOperation.TargetSE = target
targetSE = StorageElement( target )
for ftsFile in ftsFileList:
opFile = File()
opFile.LFN = ftsFile.LFN
pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = self.registrationProtocols ) )
if not pfn["OK"]:
continue
opFile.PFN = pfn["Value"]
registerOperation.addFile( opFile )
request.insertBefore( registerOperation, operation )
return S_OK()
def __insertRegisterOperation( self, request, operation, toRegister ):
""" add RegisterReplica operation
:param Request request: request instance
:param Operation transferOp: 'ReplicateAndRegister' operation for this FTSJob
:param list toRegister: [ FTSDB.FTSFile, ... ] - files that failed to register
"""
log = self.log.getSubLogger( "req_%s/%s/registerFiles" % ( request.RequestID, request.RequestName ) )
byTarget = {}
for ftsFile in toRegister:
if ftsFile.TargetSE not in byTarget:
byTarget.setdefault( ftsFile.TargetSE, [] )
byTarget[ftsFile.TargetSE].append( ftsFile )
log.info( "will create %s 'RegisterReplica' operations" % len( byTarget ) )
for target, ftsFileList in byTarget.iteritems():
log.info( "creating 'RegisterReplica' operation for targetSE %s with %s files..." % ( target,
len( ftsFileList ) ) )
registerOperation = Operation()
registerOperation.Type = "RegisterReplica"
registerOperation.Status = "Waiting"
registerOperation.TargetSE = target
targetSE = StorageElement( target )
for ftsFile in ftsFileList:
opFile = File()
opFile.LFN = ftsFile.LFN
pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = self.registrationProtocols ) )
if not pfn["OK"]:
continue
opFile.PFN = pfn["Value"]
registerOperation.addFile( opFile )
request.insertBefore( registerOperation, operation )
return S_OK()
def finalize(self):
""" register successfully transferred files """
if self.Status not in FTSJob.FINALSTATES:
return S_OK()
if not len(self):
return S_ERROR("Empty job in finalize")
startTime = time.time()
targetSE = StorageElement(self.TargetSE)
toRegister = [
ftsFile for ftsFile in self if ftsFile.Status == "Finished"
]
toRegisterDict = {}
for ftsFile in toRegister:
pfn = returnSingleResult(
targetSE.getURL(ftsFile.LFN, protocol='srm'))
if pfn["OK"]:
pfn = pfn["Value"]
toRegisterDict[ftsFile.LFN] = {"PFN": pfn, "SE": self.TargetSE}
else:
self._log.error("Error getting SRM URL", pfn['Message'])
if toRegisterDict:
self._regTotal += len(toRegisterDict)
register = self._fc.addReplica(toRegisterDict)
self._regTime += time.time() - startTime
if not register["OK"]:
self._log.error('Error registering replica',
register['Message'])
for ftsFile in toRegister:
ftsFile.Error = "AddCatalogReplicaFailed"
return register
register = register["Value"]
self._regSuccess += len(register.get('Successful', {}))
if self._regSuccess:
self._log.info('Successfully registered %d replicas' %
self._regSuccess)
failedFiles = register.get("Failed", {})
errorReason = {}
for lfn, reason in failedFiles.items():
errorReason.setdefault(str(reason), []).append(lfn)
for reason in errorReason:
self._log.error(
'Error registering %d replicas' % len(errorReason[reason]),
reason)
for ftsFile in toRegister:
if ftsFile.LFN in failedFiles:
ftsFile.Error = "AddCatalogReplicaFailed"
else:
statuses = set([ftsFile.Status for ftsFile in self])
self._log.warn( "No replicas to register for FTSJob (%s) - Files status: '%s'" % \
( self.Status, ','.join( sorted( statuses ) ) ) )
return S_OK()
def finalize( self ):
""" register successfully transferred files """
if self.Status not in FTSJob.FINALSTATES:
return S_OK()
if not len( self ):
return S_ERROR( "Empty job in finalize" )
startTime = time.time()
targetSE = StorageElement( self.TargetSE )
toRegister = [ ftsFile for ftsFile in self if ftsFile.Status == "Finished" ]
toRegisterDict = {}
for ftsFile in toRegister:
pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = 'srm' ) )
if pfn["OK"]:
pfn = pfn["Value"]
toRegisterDict[ ftsFile.LFN ] = { "PFN": pfn, "SE": self.TargetSE }
else:
self._log.error( "Error getting SRM URL", pfn['Message'] )
if toRegisterDict:
self._regTotal += len( toRegisterDict )
register = self._fc.addReplica( toRegisterDict )
self._regTime += time.time() - startTime
if not register["OK"]:
self._log.error( 'Error registering replica', register['Message'] )
for ftsFile in toRegister:
ftsFile.Error = "AddCatalogReplicaFailed"
return register
register = register["Value"]
self._regSuccess += len( register.get( 'Successful', {} ) )
if self._regSuccess:
self._log.info( 'Successfully registered %d replicas' % self._regSuccess )
failedFiles = register.get( "Failed", {} )
errorReason = {}
for lfn, reason in failedFiles.items():
errorReason.setdefault( str( reason ), [] ).append( lfn )
for reason in errorReason:
self._log.error( 'Error registering %d replicas' % len( errorReason[reason] ), reason )
for ftsFile in toRegister:
if ftsFile.LFN in failedFiles:
ftsFile.Error = "AddCatalogReplicaFailed"
else:
statuses = set( [ftsFile.Status for ftsFile in self] )
self._log.warn( "No replicas to register for FTSJob (%s) - Files status: '%s'" % \
( self.Status, ','.join( sorted( statuses ) ) ) )
return S_OK()
def _setRegistrationRequest(self, lfn, targetSE, fileDict, catalog):
""" Sets a registration request
:param str lfn: LFN
:param list se: list of SE (or just string)
:param list catalog: list (or string) of catalogs to use
:param dict fileDict: file metadata
"""
self.log.info('Setting registration request for %s at %s.' %
(lfn, targetSE))
if not isinstance(catalog, list):
catalog = [catalog]
for cat in catalog:
register = Operation()
register.Type = "RegisterFile"
register.Catalog = cat
register.TargetSE = targetSE
regFile = File()
regFile.LFN = lfn
regFile.Checksum = fileDict.get("Checksum", "")
regFile.ChecksumType = fileDict.get("ChecksumType",
self.defaultChecksumType)
regFile.Size = fileDict.get("Size", 0)
regFile.GUID = fileDict.get("GUID", "")
se = StorageElement(targetSE)
pfn = se.getURL(lfn, self.registrationProtocols)
if not pfn["OK"] or lfn not in pfn["Value"]['Successful']:
self.log.error(
"Unable to get PFN for LFN", "%s" %
pfn.get('Message',
pfn.get('Value', {}).get('Failed', {}).get(lfn)))
return pfn
regFile.PFN = pfn["Value"]['Successful'][lfn]
register.addFile(regFile)
self.request.addOperation(register)
return S_OK()
def __generateLocation( self, sbPath ):
"""
Generate the location string
"""
if self.__useLocalStorage:
return S_OK( ( self.__localSEName, sbPath ) )
# It's external storage
storageElement = StorageElement( self.__externalSEName )
res = storageElement.isValid()
if not res['OK']:
errStr = "Failed to instantiate destination StorageElement"
gLogger.error( errStr, self.__externalSEName )
return S_ERROR( errStr )
result = storageElement.getURL( sbPath )
if not result['OK'] or sbPath not in result['Value']['Successful']:
errStr = "Failed to generate PFN"
gLogger.error( errStr, self.__externalSEName )
return S_ERROR( errStr )
destPfn = result['Value']['Successful'][sbPath]
return S_OK( ( self.__externalSEName, destPfn ) )
def __generateLocation(self, sbPath):
"""
Generate the location string
"""
if self.__useLocalStorage:
return S_OK((self.__localSEName, sbPath))
# It's external storage
storageElement = StorageElement(self.__externalSEName)
res = storageElement.isValid()
if not res['OK']:
errStr = "Failed to instantiate destination StorageElement"
gLogger.error(errStr, self.__externalSEName)
return S_ERROR(errStr)
result = storageElement.getURL(sbPath)
if not result['OK'] or sbPath not in result['Value']['Successful']:
errStr = "Failed to generate PFN"
gLogger.error(errStr, self.__externalSEName)
return S_ERROR(errStr)
destPfn = result['Value']['Successful'][sbPath]
return S_OK((self.__externalSEName, destPfn))
def _setRegistrationRequest(self, lfn, targetSE, fileDict, catalog):
"""Sets a registration request
:param str lfn: LFN
:param list se: list of SE (or just string)
:param list catalog: list (or string) of catalogs to use
:param dict fileDict: file metadata
"""
self.log.info("Setting registration request",
"for %s at %s." % (lfn, targetSE))
if not isinstance(catalog, list):
catalog = [catalog]
for cat in catalog:
register = Operation()
register.Type = "RegisterFile"
register.Catalog = cat
register.TargetSE = targetSE
regFile = File()
regFile.LFN = lfn
regFile.Checksum = fileDict.get("Checksum", "")
regFile.ChecksumType = fileDict.get("ChecksumType",
self.defaultChecksumType)
regFile.Size = fileDict.get("Size", 0)
regFile.GUID = fileDict.get("GUID", "")
se = StorageElement(targetSE)
res = returnSingleResult(se.getURL(lfn,
self.registrationProtocols))
if not res["OK"]:
self.log.error("Unable to get PFN for LFN", res["Message"])
return res
regFile.PFN = res["Value"]
register.addFile(regFile)
self.request.addOperation(register)
return S_OK()
def _setRegistrationRequest( self, lfn, targetSE, fileDict, catalog ):
""" Sets a registration request
:param str lfn: LFN
:param list se: list of SE (or just string)
:param list catalog: list (or string) of catalogs to use
:param dict fileDict: file metadata
"""
self.log.info( 'Setting registration request for %s at %s.' % ( lfn, targetSE ) )
if not isinstance( catalog, list ):
catalog = [catalog]
for cat in catalog:
register = Operation()
register.Type = "RegisterFile"
register.Catalog = cat
register.TargetSE = targetSE
regFile = File()
regFile.LFN = lfn
regFile.Checksum = fileDict.get( "Checksum", "" )
regFile.ChecksumType = fileDict.get( "ChecksumType", self.defaultChecksumType )
regFile.Size = fileDict.get( "Size", 0 )
regFile.GUID = fileDict.get( "GUID", "" )
se = StorageElement( targetSE )
pfn = se.getURL( lfn, self.registrationProtocols )
if not pfn["OK"] or lfn not in pfn["Value"]['Successful']:
self.log.error( "Unable to get PFN for LFN", "%s" % pfn.get( 'Message', pfn.get( 'Value', {} ).get( 'Failed', {} ).get( lfn ) ) )
return pfn
regFile.PFN = pfn["Value"]['Successful'][lfn]
register.addFile( regFile )
self.request.addOperation( register )
return S_OK()
class FTSRequest( object ):
"""
.. class:: FTSRequest
Helper class for FTS job submission and monitoring.
"""
# # default checksum type
__defaultCksmType = "ADLER32"
# # flag to disablr/enable checksum test, default: disabled
__cksmTest = False
def __init__( self ):
"""c'tor
:param self: self reference
"""
self.log = gLogger.getSubLogger( self.__class__.__name__, True )
# # final states tuple
self.finalStates = ( 'Canceled', 'Failed', 'Hold',
'Finished', 'FinishedDirty' )
# # failed states tuple
self.failedStates = ( 'Canceled', 'Failed',
'Hold', 'FinishedDirty' )
# # successful states tuple
self.successfulStates = ( 'Finished', 'Done' )
# # all file states tuple
self.fileStates = ( 'Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed',
'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting' )
self.statusSummary = {}
# # request status
self.requestStatus = 'Unknown'
# # dict for FTS job files
self.fileDict = {}
# # dict for replicas information
self.catalogReplicas = {}
# # dict for metadata information
self.catalogMetadata = {}
# # dict for files that failed to register
self.failedRegistrations = {}
# # placehoder for FileCatalog reference
self.oCatalog = None
# # submit timestamp
self.submitTime = ''
# # placeholder FTS job GUID
self.ftsGUID = ''
# # placeholder for FTS server URL
self.ftsServer = ''
# # flag marking FTS job completness
self.isTerminal = False
# # completness percentage
self.percentageComplete = 0.0
# # source SE name
self.sourceSE = ''
# # flag marking source SE validity
self.sourceValid = False
# # source space token
self.sourceToken = ''
# # target SE name
self.targetSE = ''
# # flag marking target SE validity
self.targetValid = False
# # target space token
self.targetToken = ''
# # placeholder for target StorageElement
self.oTargetSE = None
# # placeholder for source StorageElement
self.oSourceSE = None
# # checksum type, set it to default
self.__cksmType = self.__defaultCksmType
# # disable checksum test by default
self.__cksmTest = False
# # statuses that prevent submitting to FTS
self.noSubmitStatus = ( 'Failed', 'Done', 'Staging' )
# # were sources resolved?
self.sourceResolved = False
# # Number of file transfers actually submitted
self.submittedFiles = 0
self.transferTime = 0
self.submitCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/SubmitCommand', 'glite-transfer-submit' )
self.monitorCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/MonitorCommand', 'glite-transfer-status' )
self.ftsVersion = Operations().getValue( 'DataManagement/FTSVersion', 'FTS2' )
self.ftsJob = None
self.ftsFiles = []
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE( self, se ):
""" set SE for source
:param self: self reference
:param str se: source SE name
"""
if se == self.targetSE:
return S_ERROR( "SourceSE is TargetSE" )
self.sourceSE = se
self.oSourceSE = StorageElement( self.sourceSE )
return self.__checkSourceSE()
def __checkSourceSE( self ):
""" check source SE availability
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR( "SourceSE not set" )
res = self.oSourceSE.isValid( 'Read' )
if not res['OK']:
return S_ERROR( "SourceSE not available for reading" )
res = self.__getSESpaceToken( self.oSourceSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message'] )
return S_ERROR( "SourceSE does not support FTS transfers" )
if self.__cksmTest:
cksmType = self.oSourceSE.checksumType()
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at SourceSE %s, disabling checksum test" % ( cksmType,
self.sourceSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE( self, se ):
""" set target SE
:param self: self reference
:param str se: target SE name
"""
if se == self.sourceSE:
return S_ERROR( "TargetSE is SourceSE" )
self.targetSE = se
self.oTargetSE = StorageElement( self.targetSE )
return self.__checkTargetSE()
def setTargetToken( self, token ):
""" target space token setter
:param self: self reference
:param str token: target space token
"""
self.targetToken = token
return S_OK()
def __checkTargetSE( self ):
""" check target SE availability
:param self: self reference
"""
if not self.targetSE:
return S_ERROR( "TargetSE not set" )
res = self.oTargetSE.isValid( 'Write' )
if not res['OK']:
return S_ERROR( "TargetSE not available for writing" )
res = self.__getSESpaceToken( self.oTargetSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message'] )
return S_ERROR( "TargetSE does not support FTS transfers" )
# # check checksum types
if self.__cksmTest:
cksmType = self.oTargetSE.checksumType()
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at TargetSE %s, disabling checksum test" % ( cksmType,
self.targetSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
@staticmethod
def __getSESpaceToken( oSE ):
""" get space token from StorageElement instance
:param self: self reference
:param StorageElement oSE: StorageElement instance
"""
res = oSE.getStorageParameters( protocol = 'srm' )
if not res['OK']:
return res
return S_OK( res['Value'].get( 'SpaceToken' ) )
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID( self, guid ):
""" FTS job GUID setter
:param self: self reference
:param str guid: string containg GUID
"""
if not checkGuid( guid ):
return S_ERROR( "Incorrect GUID format" )
self.ftsGUID = guid
return S_OK()
def setFTSServer( self, server ):
""" FTS server setter
:param self: self reference
:param str server: FTS server URL
"""
self.ftsServer = server
return S_OK()
def isRequestTerminal( self ):
""" check if FTS job has terminated
:param self: self reference
"""
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK( self.isTerminal )
def setCksmTest( self, cksmTest = False ):
""" set cksm test
:param self: self reference
:param bool cksmTest: flag to enable/disable checksum test
"""
self.__cksmTest = bool( cksmTest )
return S_OK( self.__cksmTest )
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN( self, lfn ):
""" add LFN :lfn: to :fileDict:
:param self: self reference
:param str lfn: LFN to add to
"""
self.fileDict.setdefault( lfn, {'Status':'Waiting'} )
return S_OK()
def setSourceSURL( self, lfn, surl ):
""" source SURL setter
:param self: self reference
:param str lfn: LFN
:param str surl: source SURL
"""
target = self.fileDict[lfn].get( 'Target' )
if target == surl:
return S_ERROR( "Source and target the same" )
return self.__setFileParameter( lfn, 'Source', surl )
def getSourceSURL( self, lfn ):
""" get source SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Source' )
def setTargetSURL( self, lfn, surl ):
""" set target SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
:param str surl: target SURL
"""
source = self.fileDict[lfn].get( 'Source' )
if source == surl:
return S_ERROR( "Source and target the same" )
return self.__setFileParameter( lfn, 'Target', surl )
def getFailReason( self, lfn ):
""" get fail reason for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Reason' )
def getRetries( self, lfn ):
""" get number of attepmts made to transfer file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Retries' )
def getTransferTime( self, lfn ):
""" get duration of transfer for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Duration' )
def getFailed( self ):
""" get list of wrongly transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.failedStates ] )
def getStaging( self ):
""" get files set for prestaging """
return S_OK( [lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) == 'Staging'] )
def getDone( self ):
""" get list of succesfully transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.successfulStates ] )
def __setFileParameter( self, lfn, paramName, paramValue ):
""" set :paramName: to :paramValue: for :lfn: file
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
:param mixed paramValue: a new parameter value
"""
self.setLFN( lfn )
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter( self, lfn, paramName ):
""" get value of :paramName: for file :lfn:
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
"""
if lfn not in self.fileDict:
return S_ERROR( "Supplied file not set" )
if paramName not in self.fileDict[lfn]:
return S_ERROR( "%s not set for file" % paramName )
return S_OK( self.fileDict[lfn][paramName] )
####################################################################
#
# Methods for submission
#
def submit( self, monitor = False, printOutput = True ):
""" submit FTS job
:param self: self reference
:param bool monitor: flag to monitor progress of FTS job
:param bool printOutput: flag to print output of execution to stdout
"""
res = self.__prepareForSubmission()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = { 'ftsGUID' : self.ftsGUID, 'ftsServer' : self.ftsServer, 'submittedFiles' : self.submittedFiles }
if monitor or printOutput:
gLogger.always( "Submitted %s@%s" % ( self.ftsGUID, self.ftsServer ) )
if monitor:
self.monitor( untilTerminal = True, printOutput = printOutput, full = False )
return S_OK( resDict )
def __prepareForSubmission( self ):
""" check validity of job before submission
:param self: self reference
"""
if not self.fileDict:
return S_ERROR( "No files set" )
if not self.sourceValid:
return S_ERROR( "SourceSE not valid" )
if not self.targetValid:
return S_ERROR( "TargetSE not valid" )
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR( "FTSServer not valid" )
self.resolveSource()
self.resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR( "No files to submit" )
return S_OK()
def __getCatalogObject( self ):
""" CatalogInterface instance facade
:param self: self reference
"""
try:
if not self.oCatalog:
self.oCatalog = FileCatalog()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache( self, lfns = None, overwrite = False ):
""" update replica cache for list of :lfns:
:param self: self reference
:param mixed lfns: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if ( lfn not in self.catalogReplicas ) or overwrite ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getReplicas( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to update replica cache: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache( self, lfns = None ):
""" update metadata cache for list of LFNs
:param self: self reference
:param list lnfs: list of LFNs
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if lfn not in self.catalogMetadata ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getFileMetadata( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to get source catalog metadata: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def resolveSource( self ):
""" resolve source SE eligible for submission
:param self: self reference
"""
# Avoid resolving sources twice
if self.sourceResolved:
return S_OK()
# Only resolve files that need a transfer
toResolve = [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( "Status", "" ) != "Failed" ]
if not toResolve:
return S_OK()
res = self.__updateMetadataCache( toResolve )
if not res['OK']:
return res
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
# Define the source URLs
for lfn in toResolve:
replicas = self.catalogReplicas.get( lfn, {} )
if self.sourceSE not in replicas:
gLogger.warn( "resolveSource: skipping %s - not replicas at SourceSE %s" % ( lfn, self.sourceSE ) )
self.__setFileParameter( lfn, 'Reason', "No replica at SourceSE" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = returnSingleResult( self.oSourceSE.getURL( lfn, protocol = 'srm' ) )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setSourceSURL( lfn, res['Value'] )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = []
for lfn in self.fileDict:
if "Source" in self.fileDict[lfn]:
toResolve.append( lfn )
if not toResolve:
return S_ERROR( "No eligible Source files" )
# Get metadata of the sources, to check for existance, availability and caching
res = self.oSourceSE.getFileMetadata( toResolve )
if not res['OK']:
return S_ERROR( "Failed to check source file metadata" )
for lfn, error in res['Value']['Failed'].items():
if re.search( 'File does not exist', error ):
gLogger.warn( "resolveSource: skipping %s - source file does not exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file does not exist" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
gLogger.warn( "resolveSource: skipping %s - failed to get source metadata" % lfn )
self.__setFileParameter( lfn, 'Reason', "Failed to get Source metadata" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toStage = []
nbStagedFiles = 0
for lfn, metadata in res['Value']['Successful'].items():
lfnStatus = self.fileDict.get( lfn, {} ).get( 'Status' )
if metadata.get( 'Unavailable', False ):
gLogger.warn( "resolveSource: skipping %s - source file unavailable" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Unavailable" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif metadata.get( 'Lost', False ):
gLogger.warn( "resolveSource: skipping %s - source file lost" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Lost" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif not metadata.get( 'Cached', metadata['Accessible'] ):
if lfnStatus != 'Staging':
toStage.append( lfn )
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
gLogger.warn( "resolveSource: skipping %s - source file size mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source size mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \
not compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ):
gLogger.warn( "resolveSource: skipping %s - source file checksum mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source checksum mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif lfnStatus == 'Staging':
# file that was staging is now cached
self.__setFileParameter( lfn, 'Status', 'Waiting' )
nbStagedFiles += 1
# Some files were being staged
if nbStagedFiles:
self.log.info( 'resolveSource: %d files have been staged' % nbStagedFiles )
# Launching staging of files not in cache
if toStage:
gLogger.warn( "resolveSource: %s source files not cached, prestaging..." % len( toStage ) )
stage = self.oSourceSE.prestageFile( toStage )
if not stage["OK"]:
gLogger.error( "resolveSource: error is prestaging", stage["Message"] )
for lfn in toStage:
self.__setFileParameter( lfn, 'Reason', stage["Message"] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
for lfn in toStage:
if lfn in stage['Value']['Successful']:
self.__setFileParameter( lfn, 'Status', 'Staging' )
elif lfn in stage['Value']['Failed']:
self.__setFileParameter( lfn, 'Reason', stage['Value']['Failed'][lfn] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
self.sourceResolved = True
return S_OK()
def resolveTarget( self ):
""" find target SE eligible for submission
:param self: self reference
"""
toResolve = [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status' ) not in self.noSubmitStatus ]
if not toResolve:
return S_OK()
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
for lfn in toResolve:
res = returnSingleResult( self.oTargetSE.getURL( lfn, protocol = 'srm' ) )
if not res['OK']:
reason = res.get( 'Message', res['Message'] )
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, reason ) )
self.__setFileParameter( lfn, 'Reason', reason )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setTargetSURL( lfn, res['Value'] )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = []
for lfn in self.fileDict:
if "Target" in self.fileDict[lfn]:
toResolve.append( lfn )
if not toResolve:
return S_ERROR( "No eligible Target files" )
res = self.oTargetSE.exists( toResolve )
if not res['OK']:
return S_ERROR( "Failed to check target existence" )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toRemove = []
for lfn, exists in res['Value']['Successful'].items():
if exists:
res = self.getSourceSURL( lfn )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - target exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Target exists" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif res['Value'] == self.fileDict[lfn]['Target']:
gLogger.warn( "resolveTarget: skipping %s - source and target pfns are the same" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source and Target the same" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRemove.append( lfn )
if toRemove:
self.oTargetSE.removeFile( toRemove )
return S_OK()
def __filesToSubmit( self ):
"""
check if there is at least one file to submit
:return: S_OK if at least one file is present, S_ERROR otherwise
"""
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if lfnStatus not in self.noSubmitStatus and source and target:
return S_OK()
return S_ERROR()
def __createFTSFiles( self ):
""" create LFNs file for glite-transfer-submit command
This file consists one line for each fiel to be transferred:
sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM]
:param self: self reference
"""
self.__updateMetadataCache()
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
if lfnStatus not in self.noSubmitStatus:
cksmStr = ""
# # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself
if self.__cksmTest and self.__cksmType:
checkSum = self.catalogMetadata.get( lfn, {} ).get( 'Checksum' )
if checkSum:
cksmStr = " %s:%s" % ( self.__cksmType, intAdlerToHex( hexAdlerToInt( checkSum ) ) )
ftsFile = FTSFile()
ftsFile.LFN = lfn
ftsFile.SourceSURL = self.fileDict[lfn].get( 'Source' )
ftsFile.TargetSURL = self.fileDict[lfn].get( 'Target' )
ftsFile.SourceSE = self.sourceSE
ftsFile.TargetSE = self.targetSE
ftsFile.Status = self.fileDict[lfn].get( 'Status' )
ftsFile.Checksum = cksmStr
ftsFile.Size = self.catalogMetadata.get( lfn, {} ).get( 'Size' )
self.ftsFiles.append( ftsFile )
self.submittedFiles += 1
return S_OK()
def __createFTSJob( self, guid = None ):
self.__createFTSFiles()
ftsJob = FTSJob()
ftsJob.RequestID = 0
ftsJob.OperationID = 0
ftsJob.SourceSE = self.sourceSE
ftsJob.TargetSE = self.targetSE
ftsJob.SourceToken = self.sourceToken
ftsJob.TargetToken = self.targetToken
ftsJob.FTSServer = self.ftsServer
if guid:
ftsJob.FTSGUID = guid
for ftsFile in self.ftsFiles:
ftsFile.Attempt += 1
ftsFile.Error = ""
ftsJob.addFile( ftsFile )
self.ftsJob = ftsJob
def __submitFTSTransfer( self ):
""" create and execute glite-transfer-submit CLI command
:param self: self reference
"""
log = gLogger.getSubLogger( 'Submit' )
self.__createFTSJob()
submit = self.ftsJob.submitFTS( self.ftsVersion, command = self.submitCommand )
if not submit["OK"]:
log.error( "unable to submit FTSJob: %s" % submit["Message"] )
return submit
log.info( "FTSJob '%s'@'%s' has been submitted" % ( self.ftsJob.FTSGUID, self.ftsJob.FTSServer ) )
# # update statuses for job files
for ftsFile in self.ftsJob:
ftsFile.FTSGUID = self.ftsJob.FTSGUID
ftsFile.Status = "Submitted"
ftsFile.Attempt += 1
log.info( "FTSJob '%s'@'%s' has been submitted" % ( self.ftsJob.FTSGUID, self.ftsJob.FTSServer ) )
self.ftsGUID = self.ftsJob.FTSGUID
return S_OK()
def __resolveFTSServer( self ):
"""
resolve FTS server to use, it should be the closest one from target SE
:param self: self reference
"""
if self.ftsVersion.upper() == 'FTS2':
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTS2ServersForSites
if not self.targetSE:
return S_ERROR( "Target SE not set" )
res = getSitesForSE( self.targetSE )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine target site" )
targetSites = res['Value']
targetSite = ''
for targetSite in targetSites:
targetFTS = getFTS2ServersForSites( [targetSite] )
if targetFTS['OK']:
ftsTarget = targetFTS['Value'][targetSite]
if ftsTarget:
self.ftsServer = ftsTarget
return S_OK( self.ftsServer )
else:
return targetFTS
elif self.ftsVersion.upper() == 'FTS3':
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTS3Servers
res = getFTS3Servers()
if not res['OK']:
return res
ftsServerList = res['Value']
if ftsServerList:
# Here we take the first one, regardless of the policy...
# Unclean but all this will disapear after refactoring the fts code
self.ftsServer = ftsServerList[0]
return S_OK( self.ftsServer )
else:
return S_ERROR( 'Unknown FTS version %s' % self.ftsVersion )
return S_ERROR( 'No FTS server found for %s' % targetSite )
####################################################################
#
# Methods for monitoring
#
def summary( self, untilTerminal = False, printOutput = False ):
""" summary of FTS job
:param self: self reference
:param bool untilTerminal: flag to monitor FTS job to its final state
:param bool printOutput: flag to print out monitoring information to the stdout
"""
res = self.__isSummaryValid()
if not res['OK']:
return res
while not self.isTerminal:
res = self.__parseOutput( full = True )
if not res['OK']:
return res
if untilTerminal:
self.__print()
self.isRequestTerminal()
if res['Value'] or ( not untilTerminal ):
break
time.sleep( 1 )
if untilTerminal:
print ""
if printOutput and ( not untilTerminal ):
return self.dumpSummary( printOutput = printOutput )
return S_OK()
def monitor( self, untilTerminal = False, printOutput = False, full = True ):
""" monitor FTS job
:param self: self reference
:param bool untilTerminal: flag to monitor FTS job to its final state
:param bool printOutput: flag to print out monitoring information to the stdout
"""
if not self.ftsJob:
self.resolveSource()
self.__createFTSJob( self.ftsGUID )
res = self.__isSummaryValid()
if not res['OK']:
return res
if untilTerminal:
res = self.summary( untilTerminal = untilTerminal, printOutput = printOutput )
if not res['OK']:
return res
res = self.__parseOutput( full = full )
if not res['OK']:
return res
if untilTerminal:
self.finalize()
if printOutput:
self.dump()
return res
def dumpSummary( self, printOutput = False ):
""" get FTS job summary as str
:param self: self reference
:param bool printOutput: print summary to stdout
"""
outStr = ''
for status in sorted( self.statusSummary ):
if self.statusSummary[status]:
outStr = '%s\t%-10s : %-10s\n' % ( outStr, status, str( self.statusSummary[status] ) )
outStr = outStr.rstrip( '\n' )
if printOutput:
print outStr
return S_OK( outStr )
def __print( self ):
""" print progress bar of FTS job completeness to stdout
:param self: self reference
"""
width = 100
bits = int( ( width * self.percentageComplete ) / 100 )
outStr = "|%s>%s| %.1f%s %s %s" % ( "="*bits, " "*( width - bits ),
self.percentageComplete, "%",
self.requestStatus, " "*10 )
sys.stdout.write( "%s\r" % ( outStr ) )
sys.stdout.flush()
def dump( self ):
""" print FTS job parameters and files to stdout
:param self: self reference
"""
print "%-10s : %-10s" % ( "Status", self.requestStatus )
print "%-10s : %-10s" % ( "Source", self.sourceSE )
print "%-10s : %-10s" % ( "Target", self.targetSE )
print "%-10s : %-128s" % ( "Server", self.ftsServer )
print "%-10s : %-128s" % ( "GUID", self.ftsGUID )
for lfn in sorted( self.fileDict ):
print "\n %-15s : %-128s" % ( 'LFN', lfn )
for key in ['Source', 'Target', 'Status', 'Reason', 'Duration']:
print " %-15s : %-128s" % ( key, str( self.fileDict[lfn].get( key ) ) )
return S_OK()
def __isSummaryValid( self ):
""" check validity of FTS job summary report
:param self: self reference
"""
if not self.ftsServer:
return S_ERROR( "FTSServer not set" )
if not self.ftsGUID:
return S_ERROR( "FTSGUID not set" )
return S_OK()
def __parseOutput( self, full = False ):
""" execute glite-transfer-status command and parse its output
:param self: self reference
:param bool full: glite-transfer-status verbosity level, when set, collect information of files as well
"""
monitor = self.ftsJob.monitorFTS( self.ftsVersion, command = self.monitorCommand, full = full )
if not monitor['OK']:
return monitor
self.percentageComplete = self.ftsJob.Completeness
self.requestStatus = self.ftsJob.Status
self.submitTime = self.ftsJob.SubmitTime
statusSummary = monitor['Value']
if statusSummary:
for state in statusSummary:
self.statusSummary[state] = statusSummary[state]
self.transferTime = 0
for ftsFile in self.ftsJob:
lfn = ftsFile.LFN
self.__setFileParameter( lfn, 'Status', ftsFile.Status )
self.__setFileParameter( lfn, 'Reason', ftsFile.Error )
self.__setFileParameter( lfn, 'Duration', ftsFile._duration )
targetURL = self.__getFileParameter( lfn, 'Target' )
if not targetURL['OK']:
self.__setFileParameter( lfn, 'Target', ftsFile.TargetSURL )
sourceURL = self.__getFileParameter( lfn, 'Source' )
if not sourceURL['OK']:
self.__setFileParameter( lfn, 'Source', ftsFile.SourceSURL )
if ftsFile._duration:
self.transferTime += int( ftsFile._duration )
return S_OK()
####################################################################
#
# Methods for finalization
#
def finalize( self ):
""" finalize FTS job
:param self: self reference
"""
self.__updateMetadataCache()
transEndTime = dateTime()
regStartTime = time.time()
res = self.getTransferStatistics()
transDict = res['Value']
res = self.__registerSuccessful( transDict['transLFNs'] )
regSuc, regTotal = res['Value']
regTime = time.time() - regStartTime
if self.sourceSE and self.targetSE:
self.__sendAccounting( regSuc, regTotal, regTime, transEndTime, transDict )
return S_OK()
def getTransferStatistics( self ):
""" collect information of Transfers that can be used by Accounting
:param self: self reference
"""
transDict = { 'transTotal': len( self.fileDict ),
'transLFNs': [],
'transOK': 0,
'transSize': 0 }
for lfn in self.fileDict:
if self.fileDict[lfn].get( 'Status' ) in self.successfulStates:
if self.fileDict[lfn].get( 'Duration', 0 ):
transDict['transLFNs'].append( lfn )
transDict['transOK'] += 1
if lfn in self.catalogMetadata:
transDict['transSize'] += self.catalogMetadata[lfn].get( 'Size', 0 )
return S_OK( transDict )
def getFailedRegistrations( self ):
""" get failed registrations dict
:param self: self reference
"""
return S_OK( self.failedRegistrations )
def __registerSuccessful( self, transLFNs ):
""" register successfully transferred files to the catalogs,
fill failedRegistrations dict for files that failed to register
:param self: self reference
:param list transLFNs: LFNs in FTS job
"""
self.failedRegistrations = {}
toRegister = {}
for lfn in transLFNs:
res = returnSingleResult( self.oTargetSE.getURL( self.fileDict[lfn].get( 'Target' ), protocol = 'srm' ) )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRegister[lfn] = { 'PFN' : res['Value'], 'SE' : self.targetSE }
if not toRegister:
return S_OK( ( 0, 0 ) )
res = self.__getCatalogObject()
if not res['OK']:
for lfn in toRegister:
self.failedRegistrations = toRegister
self.log.error( 'Failed to get Catalog Object', res['Message'] )
return S_OK( ( 0, len( toRegister ) ) )
res = self.oCatalog.addReplica( toRegister )
if not res['OK']:
self.failedRegistrations = toRegister
self.log.error( 'Failed to get Catalog Object', res['Message'] )
return S_OK( ( 0, len( toRegister ) ) )
for lfn, error in res['Value']['Failed'].items():
self.failedRegistrations[lfn] = toRegister[lfn]
self.log.error( 'Registration of Replica failed', '%s : %s' % ( lfn, str( error ) ) )
return S_OK( ( len( res['Value']['Successful'] ), len( toRegister ) ) )
def __sendAccounting( self, regSuc, regTotal, regTime, transEndTime, transDict ):
""" send accounting record
:param self: self reference
:param regSuc: number of files successfully registered
:param regTotal: number of files attepted to register
:param regTime: time stamp at the end of registration
:param transEndTime: time stamp at the end of FTS job
:param dict transDict: dict holding couters for files being transerred, their sizes and successfull transfers
"""
oAccounting = DataOperation()
oAccounting.setEndTime( transEndTime )
oAccounting.setStartTime( self.submitTime )
accountingDict = {}
accountingDict['OperationType'] = 'replicateAndRegister'
result = getProxyInfo()
if not result['OK']:
userName = '******'
else:
userName = result['Value'].get( 'username', 'unknown' )
accountingDict['User'] = userName
accountingDict['Protocol'] = 'FTS' if 'fts3' not in self.ftsServer else 'FTS3'
accountingDict['RegistrationTime'] = regTime
accountingDict['RegistrationOK'] = regSuc
accountingDict['RegistrationTotal'] = regTotal
accountingDict['TransferOK'] = transDict['transOK']
accountingDict['TransferTotal'] = transDict['transTotal']
accountingDict['TransferSize'] = transDict['transSize']
accountingDict['FinalStatus'] = self.requestStatus
accountingDict['Source'] = self.sourceSE
accountingDict['Destination'] = self.targetSE
accountingDict['TransferTime'] = self.transferTime
oAccounting.setValuesFromDict( accountingDict )
self.log.verbose( "Attempting to commit accounting message..." )
oAccounting.commit()
self.log.verbose( "...committed." )
return S_OK()
def _callback(self):
"""" After a Transfer operation, we have to update the matching Request in the
RMS, and add the registration operation just before the ReplicateAndRegister one
NOTE: we don't use ReqProxy when putting the request back to avoid operational hell
"""
log = self._log.getSubLogger("callback", child=True)
# In case there is no Request associated to the Transfer
# we do not do the callback. Not really advised, but there is a feature
# request to use the FTS3 system without RMS
if self.rmsReqID == -1:
return S_OK()
# Now we check the status of the Request.
# in principle, it should be scheduled
res = self.reqClient.getRequestStatus(self.rmsReqID)
if not res['OK']:
log.error("Could not get request status", res)
return res
status = res['Value']
# If it is not scheduled, something went wrong
# and we will not modify it
if status != 'Scheduled':
# If the Request is in a final state, just leave it,
# and we consider our job done.
# (typically happens when the callback had already been done but not persisted to the FTS3DB)
if status in rmsRequest.FINAL_STATES:
log.warn(
"Request with id %s is not Scheduled (%s), but okay it is in a Final State" %
(self.rmsReqID, status))
return S_OK()
# If the Request is not in a final state, then something really wrong is going on,
# and we do not do anything, keep ourselves pending
else:
return S_ERROR("Request with id %s is not Scheduled:%s" % (self.rmsReqID, status))
res = self._updateRmsOperationStatus()
if not res['OK']:
return res
ftsFilesByTarget = res['Value']['ftsFilesByTarget']
request = res['Value']['request']
operation = res['Value']['operation']
log.info("will create %s 'RegisterReplica' operations" % len(ftsFilesByTarget))
for target, ftsFileList in ftsFilesByTarget.iteritems():
log.info(
"creating 'RegisterReplica' operation for targetSE %s with %s files..." %
(target, len(ftsFileList)))
registerOperation = rmsOperation()
registerOperation.Type = "RegisterReplica"
registerOperation.Status = "Waiting"
registerOperation.TargetSE = target
if operation.Catalog:
registerOperation.Catalog = operation.Catalog
targetSE = StorageElement(target, vo=self.vo)
for ftsFile in ftsFileList:
opFile = rmsFile()
opFile.LFN = ftsFile.lfn
opFile.Checksum = ftsFile.checksum
# TODO: are we really ever going to change type... ?
opFile.ChecksumType = 'ADLER32'
opFile.Size = ftsFile.size
res = returnSingleResult(targetSE.getURL(ftsFile.lfn, protocol='srm'))
# This should never happen !
if not res["OK"]:
log.error("Could not get url", res['Message'])
continue
opFile.PFN = res["Value"]
registerOperation.addFile(opFile)
request.insertBefore(registerOperation, operation)
return self.reqClient.putRequest(request, useFailoverProxy=False, retryMainService=3)
class FTSRequest( object ):
"""
.. class:: FTSRequest
Helper class for FTS job submission and monitoring.
"""
# # default checksum type
__defaultCksmType = "ADLER32"
# # flag to disablr/enable checksum test, default: disabled
__cksmTest = False
def __init__( self ):
"""c'tor
:param self: self reference
"""
self.log = gLogger.getSubLogger( self.__class__.__name__, True )
# # final states tuple
self.finalStates = ( 'Canceled', 'Failed', 'Hold',
'Finished', 'FinishedDirty' )
# # failed states tuple
self.failedStates = ( 'Canceled', 'Failed',
'Hold', 'FinishedDirty' )
# # successful states tuple
self.successfulStates = ( 'Finished', 'Done' )
# # all file states tuple
self.fileStates = ( 'Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed',
'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting' )
self.statusSummary = {}
# # request status
self.requestStatus = 'Unknown'
# # dict for FTS job files
self.fileDict = {}
# # dict for replicas information
self.catalogReplicas = {}
# # dict for metadata information
self.catalogMetadata = {}
# # dict for files that failed to register
self.failedRegistrations = {}
# # placehoder for FileCatalog reference
self.oCatalog = None
# # submit timestamp
self.submitTime = ''
# # placeholder FTS job GUID
self.ftsGUID = ''
# # placeholder for FTS server URL
self.ftsServer = ''
# # flag marking FTS job completness
self.isTerminal = False
# # completness percentage
self.percentageComplete = 0.0
# # source SE name
self.sourceSE = ''
# # flag marking source SE validity
self.sourceValid = False
# # source space token
self.sourceToken = ''
# # target SE name
self.targetSE = ''
# # flag marking target SE validity
self.targetValid = False
# # target space token
self.targetToken = ''
# # placeholder for target StorageElement
self.oTargetSE = None
# # placeholder for source StorageElement
self.oSourceSE = None
# # checksum type, set it to default
self.__cksmType = self.__defaultCksmType
# # disable checksum test by default
self.__cksmTest = False
# # statuses that prevent submitting to FTS
self.noSubmitStatus = ( 'Failed', 'Done', 'Staging' )
# # were sources resolved?
self.sourceResolved = False
# # Number of file transfers actually submitted
self.submittedFiles = 0
self.transferTime = 0
self.submitCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/SubmitCommand', 'glite-transfer-submit' )
self.monitorCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/MonitorCommand', 'glite-transfer-status' )
self.ftsVersion = Operations().getValue( 'DataManagement/FTSVersion', 'FTS2' )
self.ftsJob = None
self.ftsFiles = []
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE( self, se ):
""" set SE for source
:param self: self reference
:param str se: source SE name
"""
if se == self.targetSE:
return S_ERROR( "SourceSE is TargetSE" )
self.sourceSE = se
self.oSourceSE = StorageElement( self.sourceSE )
return self.__checkSourceSE()
def __checkSourceSE( self ):
""" check source SE availability
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR( "SourceSE not set" )
res = self.oSourceSE.isValid( 'Read' )
if not res['OK']:
return S_ERROR( "SourceSE not available for reading" )
res = self.__getSESpaceToken( self.oSourceSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message'] )
return S_ERROR( "SourceSE does not support FTS transfers" )
if self.__cksmTest:
cksmType = self.oSourceSE.checksumType()
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at SourceSE %s, disabling checksum test" % ( cksmType,
self.sourceSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE( self, se ):
""" set target SE
:param self: self reference
:param str se: target SE name
"""
if se == self.sourceSE:
return S_ERROR( "TargetSE is SourceSE" )
self.targetSE = se
self.oTargetSE = StorageElement( self.targetSE )
return self.__checkTargetSE()
def setTargetToken( self, token ):
""" target space token setter
:param self: self reference
:param str token: target space token
"""
self.targetToken = token
return S_OK()
def __checkTargetSE( self ):
""" check target SE availability
:param self: self reference
"""
if not self.targetSE:
return S_ERROR( "TargetSE not set" )
res = self.oTargetSE.isValid( 'Write' )
if not res['OK']:
return S_ERROR( "TargetSE not available for writing" )
res = self.__getSESpaceToken( self.oTargetSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message'] )
return S_ERROR( "TargetSE does not support FTS transfers" )
# # check checksum types
if self.__cksmTest:
cksmType = self.oTargetSE.checksumType()
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at TargetSE %s, disabling checksum test" % ( cksmType,
self.targetSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
@staticmethod
def __getSESpaceToken( oSE ):
""" get space token from StorageElement instance
:param self: self reference
:param StorageElement oSE: StorageElement instance
"""
res = oSE.getStorageParameters( protocol = 'srm' )
if not res['OK']:
return res
return S_OK( res['Value'].get( 'SpaceToken' ) )
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID( self, guid ):
""" FTS job GUID setter
:param self: self reference
:param str guid: string containg GUID
"""
if not checkGuid( guid ):
return S_ERROR( "Incorrect GUID format" )
self.ftsGUID = guid
return S_OK()
def setFTSServer( self, server ):
""" FTS server setter
:param self: self reference
:param str server: FTS server URL
"""
self.ftsServer = server
return S_OK()
def isRequestTerminal( self ):
""" check if FTS job has terminated
:param self: self reference
"""
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK( self.isTerminal )
def setCksmTest( self, cksmTest = False ):
""" set cksm test
:param self: self reference
:param bool cksmTest: flag to enable/disable checksum test
"""
self.__cksmTest = bool( cksmTest )
return S_OK( self.__cksmTest )
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN( self, lfn ):
""" add LFN :lfn: to :fileDict:
:param self: self reference
:param str lfn: LFN to add to
"""
self.fileDict.setdefault( lfn, {'Status':'Waiting'} )
return S_OK()
def setSourceSURL( self, lfn, surl ):
""" source SURL setter
:param self: self reference
:param str lfn: LFN
:param str surl: source SURL
"""
target = self.fileDict[lfn].get( 'Target' )
if target == surl:
return S_ERROR( "Source and target the same" )
return self.__setFileParameter( lfn, 'Source', surl )
def getSourceSURL( self, lfn ):
""" get source SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Source' )
def setTargetSURL( self, lfn, surl ):
""" set target SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
:param str surl: target SURL
"""
source = self.fileDict[lfn].get( 'Source' )
if source == surl:
return S_ERROR( "Source and target the same" )
return self.__setFileParameter( lfn, 'Target', surl )
def getFailReason( self, lfn ):
""" get fail reason for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Reason' )
def getRetries( self, lfn ):
""" get number of attepmts made to transfer file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Retries' )
def getTransferTime( self, lfn ):
""" get duration of transfer for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Duration' )
def getFailed( self ):
""" get list of wrongly transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.failedStates ] )
def getStaging( self ):
""" get files set for prestaging """
return S_OK( [lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) == 'Staging'] )
def getDone( self ):
""" get list of succesfully transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.successfulStates ] )
def __setFileParameter( self, lfn, paramName, paramValue ):
""" set :paramName: to :paramValue: for :lfn: file
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
:param mixed paramValue: a new parameter value
"""
self.setLFN( lfn )
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter( self, lfn, paramName ):
""" get value of :paramName: for file :lfn:
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
"""
if lfn not in self.fileDict:
return S_ERROR( "Supplied file not set" )
if paramName not in self.fileDict[lfn]:
return S_ERROR( "%s not set for file" % paramName )
return S_OK( self.fileDict[lfn][paramName] )
####################################################################
#
# Methods for submission
#
def submit( self, monitor = False, printOutput = True ):
""" submit FTS job
:param self: self reference
:param bool monitor: flag to monitor progress of FTS job
:param bool printOutput: flag to print output of execution to stdout
"""
res = self.__prepareForSubmission()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = { 'ftsGUID' : self.ftsGUID, 'ftsServer' : self.ftsServer, 'submittedFiles' : self.submittedFiles }
if monitor or printOutput:
gLogger.always( "Submitted %s@%s" % ( self.ftsGUID, self.ftsServer ) )
if monitor:
self.monitor( untilTerminal = True, printOutput = printOutput, full = False )
return S_OK( resDict )
def __prepareForSubmission( self ):
""" check validity of job before submission
:param self: self reference
"""
if not self.fileDict:
return S_ERROR( "No files set" )
if not self.sourceValid:
return S_ERROR( "SourceSE not valid" )
if not self.targetValid:
return S_ERROR( "TargetSE not valid" )
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR( "FTSServer not valid" )
self.resolveSource()
self.resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR( "No files to submit" )
return S_OK()
def __getCatalogObject( self ):
""" CatalogInterface instance facade
:param self: self reference
"""
try:
if not self.oCatalog:
self.oCatalog = FileCatalog()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache( self, lfns = None, overwrite = False ):
""" update replica cache for list of :lfns:
:param self: self reference
:param mixed lfns: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if ( lfn not in self.catalogReplicas ) or overwrite ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getReplicas( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to update replica cache: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache( self, lfns = None ):
""" update metadata cache for list of LFNs
:param self: self reference
:param list lnfs: list of LFNs
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if lfn not in self.catalogMetadata ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getFileMetadata( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to get source catalog metadata: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def resolveSource( self ):
""" resolve source SE eligible for submission
:param self: self reference
"""
# Avoid resolving sources twice
if self.sourceResolved:
return S_OK()
# Only resolve files that need a transfer
toResolve = [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( "Status", "" ) != "Failed" ]
if not toResolve:
return S_OK()
res = self.__updateMetadataCache( toResolve )
if not res['OK']:
return res
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
# Define the source URLs
for lfn in toResolve:
replicas = self.catalogReplicas.get( lfn, {} )
if self.sourceSE not in replicas:
gLogger.warn( "resolveSource: skipping %s - not replicas at SourceSE %s" % ( lfn, self.sourceSE ) )
self.__setFileParameter( lfn, 'Reason', "No replica at SourceSE" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = returnSingleResult( self.oSourceSE.getURL( lfn, protocol = 'srm' ) )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setSourceSURL( lfn, res['Value'] )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = []
for lfn in self.fileDict:
if "Source" in self.fileDict[lfn]:
toResolve.append( lfn )
if not toResolve:
return S_ERROR( "No eligible Source files" )
# Get metadata of the sources, to check for existance, availability and caching
res = self.oSourceSE.getFileMetadata( toResolve )
if not res['OK']:
return S_ERROR( "Failed to check source file metadata" )
for lfn, error in res['Value']['Failed'].items():
if re.search( 'File does not exist', error ):
gLogger.warn( "resolveSource: skipping %s - source file does not exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file does not exist" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
gLogger.warn( "resolveSource: skipping %s - failed to get source metadata" % lfn )
self.__setFileParameter( lfn, 'Reason', "Failed to get Source metadata" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toStage = []
nbStagedFiles = 0
for lfn, metadata in res['Value']['Successful'].items():
lfnStatus = self.fileDict.get( lfn, {} ).get( 'Status' )
if metadata.get( 'Unavailable', False ):
gLogger.warn( "resolveSource: skipping %s - source file unavailable" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Unavailable" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif metadata.get( 'Lost', False ):
gLogger.warn( "resolveSource: skipping %s - source file lost" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Lost" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif not metadata.get( 'Cached', metadata['Accessible'] ):
if lfnStatus != 'Staging':
toStage.append( lfn )
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
gLogger.warn( "resolveSource: skipping %s - source file size mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source size mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \
not compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ):
gLogger.warn( "resolveSource: skipping %s - source file checksum mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source checksum mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif lfnStatus == 'Staging':
# file that was staging is now cached
self.__setFileParameter( lfn, 'Status', 'Waiting' )
nbStagedFiles += 1
# Some files were being staged
if nbStagedFiles:
self.log.info( 'resolveSource: %d files have been staged' % nbStagedFiles )
# Launching staging of files not in cache
if toStage:
gLogger.warn( "resolveSource: %s source files not cached, prestaging..." % len( toStage ) )
stage = self.oSourceSE.prestageFile( toStage )
if not stage["OK"]:
gLogger.error( "resolveSource: error is prestaging", stage["Message"] )
for lfn in toStage:
self.__setFileParameter( lfn, 'Reason', stage["Message"] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
for lfn in toStage:
if lfn in stage['Value']['Successful']:
self.__setFileParameter( lfn, 'Status', 'Staging' )
elif lfn in stage['Value']['Failed']:
self.__setFileParameter( lfn, 'Reason', stage['Value']['Failed'][lfn] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
self.sourceResolved = True
return S_OK()
def resolveTarget( self ):
""" find target SE eligible for submission
:param self: self reference
"""
toResolve = [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status' ) not in self.noSubmitStatus ]
if not toResolve:
return S_OK()
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
for lfn in toResolve:
res = returnSingleResult( self.oTargetSE.getURL( lfn, protocol = 'srm' ) )
if not res['OK']:
reason = res.get( 'Message', res['Message'] )
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, reason ) )
self.__setFileParameter( lfn, 'Reason', reason )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setTargetSURL( lfn, res['Value'] )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = []
for lfn in self.fileDict:
if "Target" in self.fileDict[lfn]:
toResolve.append( lfn )
if not toResolve:
return S_ERROR( "No eligible Target files" )
res = self.oTargetSE.exists( toResolve )
if not res['OK']:
return S_ERROR( "Failed to check target existence" )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toRemove = []
for lfn, exists in res['Value']['Successful'].items():
if exists:
res = self.getSourceSURL( lfn )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - target exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Target exists" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif res['Value'] == self.fileDict[lfn]['Target']:
gLogger.warn( "resolveTarget: skipping %s - source and target pfns are the same" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source and Target the same" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRemove.append( lfn )
if toRemove:
self.oTargetSE.removeFile( toRemove )
return S_OK()
def __filesToSubmit( self ):
"""
check if there is at least one file to submit
:return: S_OK if at least one file is present, S_ERROR otherwise
"""
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if lfnStatus not in self.noSubmitStatus and source and target:
return S_OK()
return S_ERROR()
def __createFTSFiles( self ):
""" create LFNs file for glite-transfer-submit command
This file consists one line for each fiel to be transferred:
sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM]
:param self: self reference
"""
self.__updateMetadataCache()
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
if lfnStatus not in self.noSubmitStatus:
cksmStr = ""
# # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself
if self.__cksmTest and self.__cksmType:
checkSum = self.catalogMetadata.get( lfn, {} ).get( 'Checksum' )
if checkSum:
cksmStr = " %s:%s" % ( self.__cksmType, intAdlerToHex( hexAdlerToInt( checkSum ) ) )
ftsFile = FTSFile()
ftsFile.LFN = lfn
ftsFile.SourceSURL = self.fileDict[lfn].get( 'Source' )
ftsFile.TargetSURL = self.fileDict[lfn].get( 'Target' )
ftsFile.SourceSE = self.sourceSE
ftsFile.TargetSE = self.targetSE
ftsFile.Status = self.fileDict[lfn].get( 'Status' )
ftsFile.Checksum = cksmStr
ftsFile.Size = self.catalogMetadata.get( lfn, {} ).get( 'Size' )
self.ftsFiles.append( ftsFile )
self.submittedFiles += 1
return S_OK()
def __createFTSJob( self, guid = None ):
self.__createFTSFiles()
ftsJob = FTSJob()
ftsJob.RequestID = 0
ftsJob.OperationID = 0
ftsJob.SourceSE = self.sourceSE
ftsJob.TargetSE = self.targetSE
ftsJob.SourceToken = self.sourceToken
ftsJob.TargetToken = self.targetToken
ftsJob.FTSServer = self.ftsServer
if guid:
ftsJob.FTSGUID = guid
for ftsFile in self.ftsFiles:
ftsFile.Attempt += 1
ftsFile.Error = ""
ftsJob.addFile( ftsFile )
self.ftsJob = ftsJob
def __submitFTSTransfer( self ):
""" create and execute glite-transfer-submit CLI command
:param self: self reference
"""
log = gLogger.getSubLogger( 'Submit' )
self.__createFTSJob()
submit = self.ftsJob.submitFTS( self.ftsVersion, command = self.submitCommand )
if not submit["OK"]:
log.error( "unable to submit FTSJob: %s" % submit["Message"] )
return submit
log.info( "FTSJob '%s'@'%s' has been submitted" % ( self.ftsJob.FTSGUID, self.ftsJob.FTSServer ) )
# # update statuses for job files
for ftsFile in self.ftsJob:
ftsFile.FTSGUID = self.ftsJob.FTSGUID
ftsFile.Status = "Submitted"
ftsFile.Attempt += 1
log.info( "FTSJob '%s'@'%s' has been submitted" % ( self.ftsJob.FTSGUID, self.ftsJob.FTSServer ) )
self.ftsGUID = self.ftsJob.FTSGUID
return S_OK()
def __resolveFTSServer( self ):
"""
resolve FTS server to use, it should be the closest one from target SE
:param self: self reference
"""
if self.ftsVersion.upper() == 'FTS2':
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTS2ServersForSites
if not self.targetSE:
return S_ERROR( "Target SE not set" )
res = getSitesForSE( self.targetSE )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine target site" )
targetSites = res['Value']
targetSite = ''
for targetSite in targetSites:
targetFTS = getFTS2ServersForSites( [targetSite] )
if targetFTS['OK']:
ftsTarget = targetFTS['Value'][targetSite]
if ftsTarget:
self.ftsServer = ftsTarget
return S_OK( self.ftsServer )
else:
return targetFTS
elif self.ftsVersion.upper() == 'FTS3':
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTS3Servers
res = getFTS3Servers()
if not res['OK']:
return res
ftsServerList = res['Value']
if ftsServerList:
# Here we take the first one, regardless of the policy...
# Unclean but all this will disapear after refactoring the fts code
self.ftsServer = ftsServerList[0]
return S_OK( self.ftsServer )
else:
return S_ERROR( 'Unknown FTS version %s' % self.ftsVersion )
return S_ERROR( 'No FTS server found for %s' % targetSite )
####################################################################
#
# Methods for monitoring
#
def summary( self, untilTerminal = False, printOutput = False ):
""" summary of FTS job
:param self: self reference
:param bool untilTerminal: flag to monitor FTS job to its final state
:param bool printOutput: flag to print out monitoring information to the stdout
"""
res = self.__isSummaryValid()
if not res['OK']:
return res
while not self.isTerminal:
res = self.__parseOutput( full = True )
if not res['OK']:
return res
if untilTerminal:
self.__print()
self.isRequestTerminal()
if res['Value'] or ( not untilTerminal ):
break
time.sleep( 1 )
if untilTerminal:
print ""
if printOutput and ( not untilTerminal ):
return self.dumpSummary( printOutput = printOutput )
return S_OK()
def monitor( self, untilTerminal = False, printOutput = False, full = True ):
""" monitor FTS job
:param self: self reference
:param bool untilTerminal: flag to monitor FTS job to its final state
:param bool printOutput: flag to print out monitoring information to the stdout
"""
if not self.ftsJob:
self.resolveSource()
self.__createFTSJob( self.ftsGUID )
res = self.__isSummaryValid()
if not res['OK']:
return res
if untilTerminal:
res = self.summary( untilTerminal = untilTerminal, printOutput = printOutput )
if not res['OK']:
return res
res = self.__parseOutput( full = full )
if not res['OK']:
return res
if untilTerminal:
self.finalize()
if printOutput:
self.dump()
return res
def dumpSummary( self, printOutput = False ):
""" get FTS job summary as str
:param self: self reference
:param bool printOutput: print summary to stdout
"""
outStr = ''
for status in sorted( self.statusSummary ):
if self.statusSummary[status]:
outStr = '%s\t%-10s : %-10s\n' % ( outStr, status, str( self.statusSummary[status] ) )
outStr = outStr.rstrip( '\n' )
if printOutput:
print outStr
return S_OK( outStr )
def __print( self ):
""" print progress bar of FTS job completeness to stdout
:param self: self reference
"""
width = 100
bits = int( ( width * self.percentageComplete ) / 100 )
outStr = "|%s>%s| %.1f%s %s %s" % ( "="*bits, " "*( width - bits ),
self.percentageComplete, "%",
self.requestStatus, " "*10 )
sys.stdout.write( "%s\r" % ( outStr ) )
sys.stdout.flush()
def dump( self ):
""" print FTS job parameters and files to stdout
:param self: self reference
"""
print "%-10s : %-10s" % ( "Status", self.requestStatus )
print "%-10s : %-10s" % ( "Source", self.sourceSE )
print "%-10s : %-10s" % ( "Target", self.targetSE )
print "%-10s : %-128s" % ( "Server", self.ftsServer )
print "%-10s : %-128s" % ( "GUID", self.ftsGUID )
for lfn in sorted( self.fileDict ):
print "\n %-15s : %-128s" % ( 'LFN', lfn )
for key in ['Source', 'Target', 'Status', 'Reason', 'Duration']:
print " %-15s : %-128s" % ( key, str( self.fileDict[lfn].get( key ) ) )
return S_OK()
def __isSummaryValid( self ):
""" check validity of FTS job summary report
:param self: self reference
"""
if not self.ftsServer:
return S_ERROR( "FTSServer not set" )
if not self.ftsGUID:
return S_ERROR( "FTSGUID not set" )
return S_OK()
def __parseOutput( self, full = False ):
""" execute glite-transfer-status command and parse its output
:param self: self reference
:param bool full: glite-transfer-status verbosity level, when set, collect information of files as well
"""
monitor = self.ftsJob.monitorFTS( self.ftsVersion, command = self.monitorCommand, full = full )
if not monitor['OK']:
return monitor
self.percentageComplete = self.ftsJob.Completeness
self.requestStatus = self.ftsJob.Status
self.submitTime = self.ftsJob.SubmitTime
statusSummary = monitor['Value']
if statusSummary:
for state in statusSummary:
self.statusSummary[state] = statusSummary[state]
self.transferTime = 0
for ftsFile in self.ftsJob:
lfn = ftsFile.LFN
self.__setFileParameter( lfn, 'Status', ftsFile.Status )
self.__setFileParameter( lfn, 'Reason', ftsFile.Error )
self.__setFileParameter( lfn, 'Duration', ftsFile._duration )
targetURL = self.__getFileParameter( lfn, 'Target' )
if not targetURL['OK']:
self.__setFileParameter( lfn, 'Target', ftsFile.TargetSURL )
sourceURL = self.__getFileParameter( lfn, 'Source' )
if not sourceURL['OK']:
self.__setFileParameter( lfn, 'Source', ftsFile.SourceSURL )
self.transferTime += int( ftsFile._duration )
return S_OK()
####################################################################
#
# Methods for finalization
#
def finalize( self ):
""" finalize FTS job
:param self: self reference
"""
self.__updateMetadataCache()
transEndTime = dateTime()
regStartTime = time.time()
res = self.getTransferStatistics()
transDict = res['Value']
res = self.__registerSuccessful( transDict['transLFNs'] )
regSuc, regTotal = res['Value']
regTime = time.time() - regStartTime
if self.sourceSE and self.targetSE:
self.__sendAccounting( regSuc, regTotal, regTime, transEndTime, transDict )
return S_OK()
def getTransferStatistics( self ):
""" collect information of Transfers that can be used by Accounting
:param self: self reference
"""
transDict = { 'transTotal': len( self.fileDict ),
'transLFNs': [],
'transOK': 0,
'transSize': 0 }
for lfn in self.fileDict:
if self.fileDict[lfn].get( 'Status' ) in self.successfulStates:
if self.fileDict[lfn].get( 'Duration', 0 ):
transDict['transLFNs'].append( lfn )
transDict['transOK'] += 1
if lfn in self.catalogMetadata:
transDict['transSize'] += self.catalogMetadata[lfn].get( 'Size', 0 )
return S_OK( transDict )
def getFailedRegistrations( self ):
""" get failed registrations dict
:param self: self reference
"""
return S_OK( self.failedRegistrations )
def __registerSuccessful( self, transLFNs ):
""" register successfully transferred files to the catalogs,
fill failedRegistrations dict for files that failed to register
:param self: self reference
:param list transLFNs: LFNs in FTS job
"""
self.failedRegistrations = {}
toRegister = {}
for lfn in transLFNs:
res = returnSingleResult( self.oTargetSE.getURL( self.fileDict[lfn].get( 'Target' ), protocol = 'srm' ) )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRegister[lfn] = { 'PFN' : res['Value'], 'SE' : self.targetSE }
if not toRegister:
return S_OK( ( 0, 0 ) )
res = self.__getCatalogObject()
if not res['OK']:
for lfn in toRegister:
self.failedRegistrations = toRegister
self.log.error( 'Failed to get Catalog Object', res['Message'] )
return S_OK( ( 0, len( toRegister ) ) )
res = self.oCatalog.addReplica( toRegister )
if not res['OK']:
self.failedRegistrations = toRegister
self.log.error( 'Failed to get Catalog Object', res['Message'] )
return S_OK( ( 0, len( toRegister ) ) )
for lfn, error in res['Value']['Failed'].items():
self.failedRegistrations[lfn] = toRegister[lfn]
self.log.error( 'Registration of Replica failed', '%s : %s' % ( lfn, str( error ) ) )
return S_OK( ( len( res['Value']['Successful'] ), len( toRegister ) ) )
def __sendAccounting( self, regSuc, regTotal, regTime, transEndTime, transDict ):
""" send accounting record
:param self: self reference
:param regSuc: number of files successfully registered
:param regTotal: number of files attepted to register
:param regTime: time stamp at the end of registration
:param transEndTime: time stamp at the end of FTS job
:param dict transDict: dict holding couters for files being transerred, their sizes and successfull transfers
"""
oAccounting = DataOperation()
oAccounting.setEndTime( transEndTime )
oAccounting.setStartTime( self.submitTime )
accountingDict = {}
accountingDict['OperationType'] = 'replicateAndRegister'
result = getProxyInfo()
if not result['OK']:
userName = '******'
else:
userName = result['Value'].get( 'username', 'unknown' )
accountingDict['User'] = userName
accountingDict['Protocol'] = 'FTS' if 'fts3' not in self.ftsServer else 'FTS3'
accountingDict['RegistrationTime'] = regTime
accountingDict['RegistrationOK'] = regSuc
accountingDict['RegistrationTotal'] = regTotal
accountingDict['TransferOK'] = transDict['transOK']
accountingDict['TransferTotal'] = transDict['transTotal']
accountingDict['TransferSize'] = transDict['transSize']
accountingDict['FinalStatus'] = self.requestStatus
accountingDict['Source'] = self.sourceSE
accountingDict['Destination'] = self.targetSE
accountingDict['TransferTime'] = self.transferTime
oAccounting.setValuesFromDict( accountingDict )
self.log.verbose( "Attempting to commit accounting message..." )
oAccounting.commit()
self.log.verbose( "...committed." )
return S_OK()
# initial list
protocols = ['root', 'http', 'gsiftp', 'srm']
fileList = []
fileDFCList = []
fileDarkList = []
fileUnlinkedList = []
inputFileList = []
if inputSwitch:
inputFile = open(INPUT, 'r')
tmpList = inputFile.readlines()
for i in tmpList:
inputFileList.append(i.rstrip('\n').rstrip('/'))
else:
inputFileList.append(INPUT.rstrip('/'))
HEAD0 = se.getURL(inputFileList[0], protocols)['Value']['Successful'][
inputFileList[0]][0:-len(inputFileList[0])]
HEAD = re.sub('\?', '\\?', HEAD0)
# print HEAD0
# special for SRM
if HEAD[0:6] == ("srm://"):
gLogger.notice(
"Notice: This is a SRM SE. Cannot scan more than 2000 files! Please use srm mode if you need to scan and register more than 2000 files in a directory.\n"
)
def resolveUrl(dfcUrl):
pfnUrl = se.getURL(dfcUrl, protocols)['Value']['Successful'][dfcUrl]
return pfnUrl
def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=None):
"""Build a job for transfer
Some attributes of the job are expected to be set
* sourceSE
* targetSE
* multiHopSE (optional)
* activity (optional)
* priority (optional)
* filesToSubmit
* operationID (optional, used as metadata for the job)
Note that, because of FTS limitations (and also because it anyway would be "not very smart"),
multiHop can only use non-SRM disk storage as hops.
:param pinTime: pining time in case staging is needed
:param allLFNs: list of LFNs to transfer
:param failedLFNs: set of LFNs in filesToSubmit for which there was a problem
:param target_spacetoken: the space token of the target
:param protocols: list of protocols to restrict the protocol choice for the transfer
:return: S_OK( (job object, list of ftsFileIDs in the job))
"""
log = gLogger.getSubLogger(f"constructTransferJob/{self.operationID}/{self.sourceSE}_{self.targetSE}")
isMultiHop = False
# Check if it is a multiHop transfer
if self.multiHopSE:
if len(allLFNs) != 1:
log.debug("Multihop job has %s files while only 1 allowed" % len(allLFNs))
return S_ERROR(errno.E2BIG, "Trying multihop job with more than one file !")
allHops = [(self.sourceSE, self.multiHopSE), (self.multiHopSE, self.targetSE)]
isMultiHop = True
else:
allHops = [(self.sourceSE, self.targetSE)]
nbOfHops = len(allHops)
res = self.__fetchSpaceToken(self.sourceSE, self.vo)
if not res["OK"]:
return res
source_spacetoken = res["Value"]
failedLFNs = set()
copy_pin_lifetime = None
bring_online = None
archive_timeout = None
transfers = []
fileIDsInTheJob = set()
for hopId, (hopSrcSEName, hopDstSEName) in enumerate(allHops, start=1):
# Again, this is relevant only for the very initial source
# but code factorization is more important
hopSrcIsTape = self.__isTapeSE(hopSrcSEName, self.vo)
dstSE = StorageElement(hopDstSEName, vo=self.vo)
srcSE = StorageElement(hopSrcSEName, vo=self.vo)
# getting all the (source, dest) surls
res = dstSE.generateTransferURLsBetweenSEs(allLFNs, srcSE, protocols=protocols)
if not res["OK"]:
return res
for lfn, reason in res["Value"]["Failed"].items():
failedLFNs.add(lfn)
log.error("Could not get source SURL", "%s %s" % (lfn, reason))
allSrcDstSURLs = res["Value"]["Successful"]
srcProto, destProto = res["Value"]["Protocols"]
# If the source is a tape SE, we should set the
# copy_pin_lifetime and bring_online params
# In case of multihop, this is relevant only for the
# original source, but again, code factorization is more important
if hopSrcIsTape:
copy_pin_lifetime = pinTime
bring_online = srcSE.options.get("BringOnlineTimeout", BRING_ONLINE_TIMEOUT)
# If the destination is a tape, and the protocol supports it,
# check if we want to have an archive timeout
# In case of multihop, this is relevant only for the
# final target, but again, code factorization is more important
dstIsTape = self.__isTapeSE(hopDstSEName, self.vo)
if dstIsTape and destProto in dstSE.localStageProtocolList:
archive_timeout = dstSE.options.get("ArchiveTimeout")
# This contains the staging URLs if they are different from the transfer URLs
# (CTA...)
allStageURLs = dict()
# In case we are transfering from a tape system, and the stage protocol
# is not the same as the transfer protocol, we generate the staging URLs
# to do a multihop transfer. See below.
if hopSrcIsTape and srcProto not in srcSE.localStageProtocolList:
isMultiHop = True
# As of version 3.10, FTS can only handle one file per multi hop
# job. If we are here, that means that we need one, so make sure that
# we only have a single file to transfer (this should have been checked
# at the job construction step in FTS3Operation).
# This test is important, because multiple files would result in the source
# being deleted !
if len(allLFNs) != 1:
log.debug("Multihop job has %s files while only 1 allowed" % len(allLFNs))
return S_ERROR(errno.E2BIG, "Trying multihop job with more than one file !")
res = srcSE.getURL(allSrcDstSURLs, protocol=srcSE.localStageProtocolList)
if not res["OK"]:
return res
for lfn, reason in res["Value"]["Failed"].items():
failedLFNs.add(lfn)
log.error("Could not get stage SURL", "%s %s" % (lfn, reason))
allSrcDstSURLs.pop(lfn)
allStageURLs = res["Value"]["Successful"]
for ftsFile in self.filesToSubmit:
if ftsFile.lfn in failedLFNs:
log.debug("Not preparing transfer for file %s" % ftsFile.lfn)
continue
sourceSURL, targetSURL = allSrcDstSURLs[ftsFile.lfn]
stageURL = allStageURLs.get(ftsFile.lfn)
if sourceSURL == targetSURL:
log.error("sourceSURL equals to targetSURL", "%s" % ftsFile.lfn)
ftsFile.error = "sourceSURL equals to targetSURL"
ftsFile.status = "Defunct"
continue
ftsFileID = getattr(ftsFile, "fileID")
# Under normal circumstances, we simply submit an fts transfer as such:
# * srcProto://myFile -> destProto://myFile
#
# Even in case of the source storage being a tape system, it works fine.
# However, if the staging and transfer protocols are different (which might be the case for CTA),
# we use the multihop machinery to submit two sequential fts transfers:
# one to stage, one to transfer.
# It looks like such
# * stageProto://myFile -> stageProto://myFile
# * srcProto://myFile -> destProto://myFile
if stageURL:
# We do not set a fileID in the metadata
# such that we do not update the DB when monitoring
stageTrans_metadata = {"desc": "PreStage %s" % ftsFileID}
# If we use an activity, also set it as file metadata
# for WLCG monitoring purposes
# https://its.cern.ch/jira/projects/DOMATPC/issues/DOMATPC-14?
if self.activity:
stageTrans_metadata["activity"] = self.activity
stageTrans = fts3.new_transfer(
stageURL,
stageURL,
checksum="ADLER32:%s" % ftsFile.checksum,
filesize=ftsFile.size,
metadata=stageTrans_metadata,
activity=self.activity,
)
transfers.append(stageTrans)
# If it is the last hop only, we set the fileID metadata
# for monitoring
if hopId == nbOfHops:
trans_metadata = {"desc": "Transfer %s" % ftsFileID, "fileID": ftsFileID}
else:
trans_metadata = {"desc": "MultiHop %s" % ftsFileID}
# If we use an activity, also set it as file metadata
# for WLCG monitoring purposes
# https://its.cern.ch/jira/projects/DOMATPC/issues/DOMATPC-14?
if self.activity:
trans_metadata["activity"] = self.activity
# because of an xroot bug (https://github.com/xrootd/xrootd/issues/1433)
# the checksum needs to be lowercase. It does not impact the other
# protocol, so it's fine to put it here.
# I only add it in this transfer and not the "staging" one above because it
# impacts only root -> root transfers
trans = fts3.new_transfer(
sourceSURL,
targetSURL,
checksum="ADLER32:%s" % ftsFile.checksum.lower(),
filesize=ftsFile.size,
metadata=trans_metadata,
activity=self.activity,
)
transfers.append(trans)
fileIDsInTheJob.add(ftsFileID)
if not transfers:
log.error("No transfer possible!")
return S_ERROR(errno.ENODATA, "No transfer possible")
# We add a few metadata to the fts job so that we can reuse them later on without
# querying our DB.
# source and target SE are just used for accounting purpose
job_metadata = {
"operationID": self.operationID,
"rmsReqID": self.rmsReqID,
"sourceSE": self.sourceSE,
"targetSE": self.targetSE,
}
if self.activity:
job_metadata["activity"] = self.activity
job = fts3.new_job(
transfers=transfers,
overwrite=True,
source_spacetoken=source_spacetoken,
spacetoken=target_spacetoken,
bring_online=bring_online,
copy_pin_lifetime=copy_pin_lifetime,
retry=3,
verify_checksum="target", # Only check target vs specified, since we verify the source earlier
multihop=isMultiHop,
metadata=job_metadata,
priority=self.priority,
archive_timeout=archive_timeout,
)
return S_OK((job, fileIDsInTheJob))
def _constructTransferJob(self,
pinTime,
allLFNs,
target_spacetoken,
protocols=None):
""" Build a job for transfer
Some attributes of the job are expected to be set
* sourceSE
* targetSE
* activity (optional)
* priority (optional)
* filesToSubmit
* operationID (optional, used as metadata for the job)
:param pinTime: pining time in case staging is needed
:param allLFNs: list of LFNs to transfer
:param failedLFNs: set of LFNs in filesToSubmit for which there was a problem
:param target_spacetoken: the space token of the target
:param protocols: list of protocols to restrict the protocol choice for the transfer
:return: S_OK( (job object, list of ftsFileIDs in the job))
"""
log = gLogger.getSubLogger(
"constructTransferJob/%s/%s_%s" %
(self.operationID, self.sourceSE, self.targetSE), True)
res = self.__fetchSpaceToken(self.sourceSE, self.vo)
if not res['OK']:
return res
source_spacetoken = res['Value']
failedLFNs = set()
dstSE = StorageElement(self.targetSE, vo=self.vo)
srcSE = StorageElement(self.sourceSE, vo=self.vo)
# If the source is not a tape SE, we should set the
# copy_pin_lifetime and bring_online params to None,
# otherwise they will do an extra useless queue in FTS
sourceIsTape = self.__isTapeSE(self.sourceSE, self.vo)
copy_pin_lifetime = pinTime if sourceIsTape else None
bring_online = BRING_ONLINE_TIMEOUT if sourceIsTape else None
# getting all the (source, dest) surls
res = dstSE.generateTransferURLsBetweenSEs(allLFNs,
srcSE,
protocols=protocols)
if not res['OK']:
return res
for lfn, reason in res['Value']['Failed'].items():
failedLFNs.add(lfn)
log.error("Could not get source SURL", "%s %s" % (lfn, reason))
allSrcDstSURLs = res['Value']['Successful']
# This contains the staging URLs if they are different from the transfer URLs
# (CTA...)
allStageURLs = dict()
# In case we are transfering from a tape system, and the stage protocol
# is not the same as the transfer protocol, we generate the staging URLs
# to do a multihop transfer. See below.
if sourceIsTape:
srcProto, _destProto = res['Value']['Protocols']
if srcProto not in srcSE.localStageProtocolList:
# As of version 3.10, FTS can only handle one file per multi hop
# job. If we are here, that means that we need one, so make sure that
# we only have a single file to transfer (this should have been checked
# at the job construction step in FTS3Operation).
# This test is important, because multiple files would result in the source
# being deleted !
if len(allLFNs) != 1:
log.debug(
"Multihop job has %s files while only 1 allowed" %
len(allLFNs))
return S_ERROR(
errno.E2BIG,
"Trying multihop job with more than one file !")
res = srcSE.getURL(allSrcDstSURLs,
protocol=srcSE.localStageProtocolList)
if not res['OK']:
return res
for lfn, reason in res['Value']['Failed'].items():
failedLFNs.add(lfn)
log.error("Could not get stage SURL",
"%s %s" % (lfn, reason))
allSrcDstSURLs.pop(lfn)
allStageURLs = res['Value']['Successful']
transfers = []
fileIDsInTheJob = []
for ftsFile in self.filesToSubmit:
if ftsFile.lfn in failedLFNs:
log.debug("Not preparing transfer for file %s" % ftsFile.lfn)
continue
sourceSURL, targetSURL = allSrcDstSURLs[ftsFile.lfn]
stageURL = allStageURLs.get(ftsFile.lfn)
if sourceSURL == targetSURL:
log.error("sourceSURL equals to targetSURL",
"%s" % ftsFile.lfn)
ftsFile.error = "sourceSURL equals to targetSURL"
ftsFile.status = 'Defunct'
continue
ftsFileID = getattr(ftsFile, 'fileID')
# Under normal circumstances, we simply submit an fts transfer as such:
# * srcProto://myFile -> destProto://myFile
#
# Even in case of the source storage being a tape system, it works fine.
# However, if the staging and transfer protocols are different (which might be the case for CTA),
# we use the multihop machinery to submit two sequential fts transfers:
# one to stage, one to transfer.
# It looks like such
# * stageProto://myFile -> stageProto://myFile
# * srcProto://myFile -> destProto://myFile
if stageURL:
# We do not set a fileID in the metadata
# such that we do not update the DB when monitoring
stageTrans_metadata = {'desc': 'PreStage %s' % ftsFileID}
stageTrans = fts3.new_transfer(stageURL,
stageURL,
checksum='ADLER32:%s' %
ftsFile.checksum,
filesize=ftsFile.size,
metadata=stageTrans_metadata,
activity=self.activity)
transfers.append(stageTrans)
trans_metadata = {
'desc': 'Transfer %s' % ftsFileID,
'fileID': ftsFileID
}
trans = fts3.new_transfer(sourceSURL,
targetSURL,
checksum='ADLER32:%s' % ftsFile.checksum,
filesize=ftsFile.size,
metadata=trans_metadata,
activity=self.activity)
transfers.append(trans)
fileIDsInTheJob.append(ftsFileID)
if not transfers:
log.error("No transfer possible!")
return S_ERROR("No transfer possible")
# We add a few metadata to the fts job so that we can reuse them later on without
# querying our DB.
# source and target SE are just used for accounting purpose
job_metadata = {
'operationID': self.operationID,
'rmsReqID': self.rmsReqID,
'sourceSE': self.sourceSE,
'targetSE': self.targetSE
}
job = fts3.new_job(
transfers=transfers,
overwrite=True,
source_spacetoken=source_spacetoken,
spacetoken=target_spacetoken,
bring_online=bring_online,
copy_pin_lifetime=copy_pin_lifetime,
retry=3,
verify_checksum=
'target', # Only check target vs specified, since we verify the source earlier
multihop=bool(
allStageURLs), # if we have stage urls, then we need multihop
metadata=job_metadata,
priority=self.priority)
return S_OK((job, fileIDsInTheJob))
def resolvePFNNotRegistered(self, problematicDict):
""" This takes the problematic dictionary returned by the integrity DB and resolved the PFNNotRegistered prognosis
"""
lfn = problematicDict['LFN']
seName = problematicDict['SE']
fileID = problematicDict['FileID']
se = StorageElement(seName)
res = returnSingleResult(self.fc.exists(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if not res['Value']:
# The file does not exist in the catalog
res = returnSingleResult(se.removeFile(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
return self.__updateCompletedFiles('PFNNotRegistered', fileID)
res = returnSingleResult(se.getFileMetadata(lfn))
if (not res['OK']) and (re.search('File does not exist',
res['Message'])):
gLogger.info("PFNNotRegistered replica (%d) found to be missing." %
fileID)
return self.__updateCompletedFiles('PFNNotRegistered', fileID)
elif not res['OK']:
return self.__returnProblematicError(fileID, res)
storageMetadata = res['Value']
if storageMetadata['Lost']:
gLogger.info(
"PFNNotRegistered replica (%d) found to be Lost. Updating prognosis"
% fileID)
return self.changeProblematicPrognosis(fileID, 'PFNLost')
if storageMetadata['Unavailable']:
gLogger.info(
"PFNNotRegistered replica (%d) found to be Unavailable. Updating retry count"
% fileID)
return self.incrementProblematicRetry(fileID)
# HACK until we can obtain the space token descriptions through GFAL
site = seName.split('_')[0].split('-')[0]
if not storageMetadata['Cached']:
if lfn.endswith('.raw'):
seName = '%s-RAW' % site
else:
seName = '%s-RDST' % site
elif storageMetadata['Migrated']:
if lfn.startswith('/lhcb/data'):
seName = '%s_M-DST' % site
else:
seName = '%s_MC_M-DST' % site
else:
if lfn.startswith('/lhcb/data'):
seName = '%s-DST' % site
else:
seName = '%s_MC-DST' % site
problematicDict['SE'] = seName
res = returnSingleResult(se.getURL(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
problematicDict['PFN'] = res['Value']
res = returnSingleResult(self.fc.addReplica({lfn: problematicDict}))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
res = returnSingleResult(self.fc.getFileMetadata(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if res['Value']['Size'] != storageMetadata['Size']:
gLogger.info(
"PFNNotRegistered replica (%d) found with catalog size mismatch. Updating prognosis"
% fileID)
return self.changeProblematicPrognosis(fileID,
'CatalogPFNSizeMismatch')
return self.__updateCompletedFiles('PFNNotRegistered', fileID)
def _callback(self):
"""" After a Transfer operation, we have to update the matching Request in the
RMS, and add the registration operation just before the ReplicateAndRegister one
NOTE: we don't use ReqProxy when putting the request back to avoid operational hell
"""
log = self._log.getSubLogger("callback", child=True)
# In case there is no Request associated to the Transfer
# we do not do the callback. Not really advised, but there is a feature
# request to use the FTS3 system without RMS
if self.rmsReqID == -1:
return S_OK()
# Now we check the status of the Request.
# If it is not scheduled, we do not perform the callback
res = self.reqClient.getRequestStatus(self.rmsReqID)
if not res['OK']:
log.error("Could not get request status", res)
return res
status = res['Value']
if status != 'Scheduled':
return S_ERROR("Request with id %s is not Scheduled:%s" %
(self.rmsReqID, status))
res = self._updateRmsOperationStatus()
if not res['OK']:
return res
ftsFilesByTarget = res['Value']['ftsFilesByTarget']
request = res['Value']['request']
operation = res['Value']['operation']
log.info("will create %s 'RegisterReplica' operations" %
len(ftsFilesByTarget))
for target, ftsFileList in ftsFilesByTarget.iteritems():
log.info(
"creating 'RegisterReplica' operation for targetSE %s with %s files..."
% (target, len(ftsFileList)))
registerOperation = rmsOperation()
registerOperation.Type = "RegisterReplica"
registerOperation.Status = "Waiting"
registerOperation.TargetSE = target
if operation.Catalog:
registerOperation.Catalog = operation.Catalog
targetSE = StorageElement(target, vo=self.vo)
for ftsFile in ftsFileList:
opFile = rmsFile()
opFile.LFN = ftsFile.lfn
opFile.Checksum = ftsFile.checksum
# TODO: are we really ever going to change type... ?
opFile.ChecksumType = 'ADLER32'
opFile.Size = ftsFile.size
res = returnSingleResult(
targetSE.getURL(ftsFile.lfn, protocol='srm'))
# This should never happen !
if not res["OK"]:
log.error("Could not get url", res['Message'])
continue
opFile.PFN = res["Value"]
registerOperation.addFile(opFile)
request.insertBefore(registerOperation, operation)
return self.reqClient.putRequest(request,
useFailoverProxy=False,
retryMainService=3)
def _callback(self):
"""" After a Transfer operation, we have to update the matching Request in the
RMS, and add the registration operation just before the ReplicateAndRegister one
NOTE: we don't use ReqProxy when putting the request back to avoid operational hell
"""
log = self._log.getSubLogger("callback", child=True)
# In case there is no Request associated to the Transfer
# we do not do the callback. Not really advised, but there is a feature
# request to use the FTS3 system without RMS
if self.rmsReqID == -1:
return S_OK()
# Now we check the status of the Request.
# in principle, it should be scheduled
res = self.reqClient.getRequestStatus(self.rmsReqID)
if not res['OK']:
log.error("Could not get request status", res)
return res
status = res['Value']
# If it is not scheduled, something went wrong
# and we will not modify it
if status != 'Scheduled':
# If the Request is in a final state, just leave it,
# and we consider our job done.
# (typically happens when the callback had already been done but not persisted to the FTS3DB)
if status in rmsRequest.FINAL_STATES:
log.warn(
"Request with id %s is not Scheduled (%s), but okay it is in a Final State"
% (self.rmsReqID, status))
return S_OK()
# If the Request is not in a final state, then something really wrong is going on,
# and we do not do anything, keep ourselves pending
else:
return S_ERROR("Request with id %s is not Scheduled:%s" %
(self.rmsReqID, status))
res = self._updateRmsOperationStatus()
if not res['OK']:
return res
ftsFilesByTarget = res['Value']['ftsFilesByTarget']
request = res['Value']['request']
operation = res['Value']['operation']
registrationProtocols = DMSHelpers(
vo=self._vo).getRegistrationProtocols()
log.info("will create %s 'RegisterReplica' operations" %
len(ftsFilesByTarget))
for target, ftsFileList in ftsFilesByTarget.iteritems():
log.info(
"creating 'RegisterReplica' operation for targetSE %s with %s files..."
% (target, len(ftsFileList)))
registerOperation = rmsOperation()
registerOperation.Type = "RegisterReplica"
registerOperation.Status = "Waiting"
registerOperation.TargetSE = target
if operation.Catalog:
registerOperation.Catalog = operation.Catalog
targetSE = StorageElement(target, vo=self.vo)
for ftsFile in ftsFileList:
opFile = rmsFile()
opFile.LFN = ftsFile.lfn
opFile.Checksum = ftsFile.checksum
# TODO: are we really ever going to change type... ?
opFile.ChecksumType = 'ADLER32'
opFile.Size = ftsFile.size
res = returnSingleResult(
targetSE.getURL(ftsFile.lfn,
protocol=registrationProtocols))
# This should never happen !
if not res["OK"]:
log.error("Could not get url", res['Message'])
continue
opFile.PFN = res["Value"]
registerOperation.addFile(opFile)
request.insertBefore(registerOperation, operation)
return self.reqClient.putRequest(request,
useFailoverProxy=False,
retryMainService=3)
def resolvePFNNotRegistered( self, problematicDict ):
""" This takes the problematic dictionary returned by the integrity DB and resolved the PFNNotRegistered prognosis
"""
lfn = problematicDict['LFN']
seName = problematicDict['SE']
fileID = problematicDict['FileID']
se = StorageElement( seName )
res = returnSingleResult( self.fc.exists( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if not res['Value']:
# The file does not exist in the catalog
res = returnSingleResult( se.removeFile( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
return self.__updateCompletedFiles( 'PFNNotRegistered', fileID )
res = returnSingleResult( se.getFileMetadata( lfn ) )
if ( not res['OK'] ) and ( re.search( 'File does not exist', res['Message'] ) ):
gLogger.info( "PFNNotRegistered replica (%d) found to be missing." % fileID )
return self.__updateCompletedFiles( 'PFNNotRegistered', fileID )
elif not res['OK']:
return self.__returnProblematicError( fileID, res )
storageMetadata = res['Value']
if storageMetadata['Lost']:
gLogger.info( "PFNNotRegistered replica (%d) found to be Lost. Updating prognosis" % fileID )
return self.changeProblematicPrognosis( fileID, 'PFNLost' )
if storageMetadata['Unavailable']:
gLogger.info( "PFNNotRegistered replica (%d) found to be Unavailable. Updating retry count" % fileID )
return self.incrementProblematicRetry( fileID )
# HACK until we can obtain the space token descriptions through GFAL
site = seName.split( '_' )[0].split( '-' )[0]
if not storageMetadata['Cached']:
if lfn.endswith( '.raw' ):
seName = '%s-RAW' % site
else:
seName = '%s-RDST' % site
elif storageMetadata['Migrated']:
if lfn.startswith( '/lhcb/data' ):
seName = '%s_M-DST' % site
else:
seName = '%s_MC_M-DST' % site
else:
if lfn.startswith( '/lhcb/data' ):
seName = '%s-DST' % site
else:
seName = '%s_MC-DST' % site
problematicDict['SE'] = seName
res = returnSingleResult( se.getURL( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
problematicDict['PFN'] = res['Value']
res = returnSingleResult( self.fc.addReplica( {lfn:problematicDict} ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
res = returnSingleResult( self.fc.getFileMetadata( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if res['Value']['Size'] != storageMetadata['Size']:
gLogger.info( "PFNNotRegistered replica (%d) found with catalog size mismatch. Updating prognosis" % fileID )
return self.changeProblematicPrognosis( fileID, 'CatalogPFNSizeMismatch' )
return self.__updateCompletedFiles( 'PFNNotRegistered', fileID )