def _run(self):
"""Check for migration bit, set file done when migrated."""
self.waitingFiles = self.getWaitingFilesList()
self.log.notice("Waiting files:", len(self.waitingFiles))
targetSESet = set(self.operation.targetSEList)
self.log.notice("Target SEs:", ",".join(targetSESet))
migrated = True
for opFile in self.waitingFiles:
self.log.notice("Checking:", opFile.LFN)
for targetSE in targetSESet:
se = StorageElement(targetSE)
if not se.status()["TapeSE"]:
migrated = True and migrated
continue
metaData = returnSingleResult(se.getFileMetadata(opFile.LFN))
self.log.debug("MetaData: %s" % pformat(metaData))
if not metaData["OK"]:
self.log.error("Failed to get metadata:", "%s: %s" % (opFile.LFN, metaData["Message"]))
migrated = False
continue
migrated = metaData["Value"].get("Migrated", 0) == 1 and migrated
if migrated:
self.log.notice("File has been migrated:", opFile.LFN)
opFile.Status = "Done"
else:
self.log.notice("File has NOT been migrated:", opFile.LFN)
now = datetime.datetime.utcnow().replace(microsecond=0)
extraDelay = datetime.timedelta(minutes=20)
self.request.NotBefore = now + extraDelay
def checkSourceSE( self, sourceSE, lfn, catalogMetadata ):
""" filter out SourceSE where PFN is not existing
:param self: self reference
:param str lfn: logical file name
:param dict catalogMetadata: catalog metadata
"""
se = self.seCache.get( sourceSE, None )
if not se:
se = StorageElement( sourceSE, "SRM2" )
self.seCache[sourceSE] = se
pfn = se.getPfnForLfn( lfn )
if not pfn["OK"]:
self.log.warn("checkSourceSE: unable to create pfn for %s lfn: %s" % ( lfn, pfn["Message"] ) )
return pfn
pfn = pfn["Value"]
seMetadata = se.getFileMetadata( pfn, singleFile=True )
if not seMetadata["OK"]:
self.log.warn("checkSourceSE: %s" % seMetadata["Message"] )
return S_ERROR("checkSourceSE: failed to get metadata")
seMetadata = seMetadata["Value"]
catalogChecksum = catalogMetadata["Checksum"].replace("x", "0" ).zfill(8) if "Checksum" in catalogMetadata else None
storageChecksum = seMetadata["Checksum"].replace("x", "0").zfill(8) if "Checksum" in seMetadata else None
if catalogChecksum != storageChecksum:
self.log.warn( "checkSourceSE: %s checksum mismatch catalogue:%s %s:%s" % ( lfn,
catalogChecksum,
sourceSE,
storageChecksum ) )
return S_ERROR("checkSourceSE: checksum mismatch")
## if we're here everything is OK
return S_OK()
def _run(self):
"""Check for migration bit, set file done when migrated."""
self.waitingFiles = self.getWaitingFilesList()
self.log.notice('Waiting files:', len(self.waitingFiles))
targetSESet = set(self.operation.targetSEList)
self.log.notice('Target SEs:', ','.join(targetSESet))
migrated = True
for opFile in self.waitingFiles:
self.log.notice('Checking:', opFile.LFN)
for targetSE in targetSESet:
se = StorageElement(targetSE)
if not se.status()['TapeSE']:
migrated = True and migrated
continue
metaData = returnSingleResult(se.getFileMetadata(opFile.LFN))
self.log.debug('MetaData: %s' % pformat(metaData))
if not metaData['OK']:
self.log.error(
'Failed to get metadata:',
'%s: %s' % (opFile.LFN, metaData['Message']))
migrated = False
continue
migrated = metaData['Value'].get('Migrated',
0) == 1 and migrated
if migrated:
self.log.notice('File has been migrated:', opFile.LFN)
opFile.Status = 'Done'
else:
self.log.notice('File has NOT been migrated:', opFile.LFN)
now = datetime.datetime.utcnow().replace(microsecond=0)
extraDelay = datetime.timedelta(minutes=20)
self.request.NotBefore = now + extraDelay
def _filterReplicas( self, opFile ):
""" filter out banned/invalid source SEs """
ret = { "Valid" : [], "Banned" : [], "Bad" : [] }
replicas = self.replicaManager().getActiveReplicas( opFile.LFN )
if not replicas["OK"]:
self.log.error( replicas["Message"] )
reNotExists = re.compile( "not such file or directory" )
replicas = replicas["Value"]
failed = replicas["Failed"].get( opFile.LFN , "" )
if reNotExists.match( failed.lower() ):
opFile.Status = "Failed"
opFile.Error = failed
return S_ERROR( failed )
replicas = replicas["Successful"][opFile.LFN] if opFile.LFN in replicas["Successful"] else {}
for repSEName in replicas:
seRead = self.rssSEStatus( repSEName, "ReadAccess" )
if not seRead["OK"]:
self.log.error( seRead["Message"] )
ret["Banned"].append( repSEName )
continue
if not seRead["Value"]:
self.log.error( "StorageElement '%s' is banned for reading" % ( repSEName ) )
repSE = self.seCache.get( repSEName, None )
if not repSE:
repSE = StorageElement( repSEName, "SRM2" )
self.seCache[repSE] = repSE
pfn = repSE.getPfnForLfn( opFile.LFN )
if not pfn["OK"]:
self.log.warn( "unable to create pfn for %s lfn: %s" % ( opFile.LFN, pfn["Message"] ) )
ret["Banned"].append( repSEName )
continue
pfn = pfn["Value"]
repSEMetadata = repSE.getFileMetadata( pfn, singleFile = True )
if not repSEMetadata["OK"]:
self.log.warn( repSEMetadata["Message"] )
ret["Banned"].append( repSEName )
continue
repSEMetadata = repSEMetadata["Value"]
seChecksum = repSEMetadata["Checksum"].replace( "x", "0" ).zfill( 8 ) if "Checksum" in repSEMetadata else None
if opFile.Checksum and opFile.Checksum != seChecksum:
self.log.warn( " %s checksum mismatch: %s %s:%s" % ( opFile.LFN,
opFile.Checksum,
repSE,
seChecksum ) )
ret["Bad"].append( repSEName )
continue
# # if we're here repSE is OK
ret["Valid"].append( repSEName )
return S_OK( ret )
def resolvePFNNotRegistered(self, problematicDict):
""" This takes the problematic dictionary returned by the integrity DB and resolved the PFNNotRegistered prognosis
"""
lfn = problematicDict['LFN']
seName = problematicDict['SE']
fileID = problematicDict['FileID']
se = StorageElement(seName)
res = returnSingleResult(self.fc.exists(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if not res['Value']:
# The file does not exist in the catalog
res = returnSingleResult(se.removeFile(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
return self.__updateCompletedFiles('PFNNotRegistered', fileID)
res = returnSingleResult(se.getFileMetadata(lfn))
if (not res['OK']) and (re.search('File does not exist',
res['Message'])):
gLogger.info("PFNNotRegistered replica (%d) found to be missing." %
fileID)
return self.__updateCompletedFiles('PFNNotRegistered', fileID)
elif not res['OK']:
return self.__returnProblematicError(fileID, res)
storageMetadata = res['Value']
if storageMetadata['Lost']:
gLogger.info(
"PFNNotRegistered replica (%d) found to be Lost. Updating prognosis"
% fileID)
return self.changeProblematicPrognosis(fileID, 'PFNLost')
if storageMetadata['Unavailable']:
gLogger.info(
"PFNNotRegistered replica (%d) found to be Unavailable. Updating retry count"
% fileID)
return self.incrementProblematicRetry(fileID)
# HACK until we can obtain the space token descriptions through GFAL
site = seName.split('_')[0].split('-')[0]
if not storageMetadata['Cached']:
if lfn.endswith('.raw'):
seName = '%s-RAW' % site
else:
seName = '%s-RDST' % site
elif storageMetadata['Migrated']:
if lfn.startswith('/lhcb/data'):
seName = '%s_M-DST' % site
else:
seName = '%s_MC_M-DST' % site
else:
if lfn.startswith('/lhcb/data'):
seName = '%s-DST' % site
else:
seName = '%s_MC-DST' % site
problematicDict['SE'] = seName
res = returnSingleResult(se.getURL(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
problematicDict['PFN'] = res['Value']
res = returnSingleResult(self.fc.addReplica({lfn: problematicDict}))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
res = returnSingleResult(self.fc.getFileMetadata(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if res['Value']['Size'] != storageMetadata['Size']:
gLogger.info(
"PFNNotRegistered replica (%d) found with catalog size mismatch. Updating prognosis"
% fileID)
return self.changeProblematicPrognosis(fileID,
'CatalogPFNSizeMismatch')
return self.__updateCompletedFiles('PFNNotRegistered', fileID)
def compareChecksum(self, lfns):
"""compare the checksum of the file in the FC and the checksum of the physical replicas.
Returns a dictionary containing 3 sub-dictionaries: one with files with missing PFN, one with
files with all replicas corrupted, and one with files with some replicas corrupted and at least
one good replica
"""
retDict = {'AllReplicasCorrupted': {},
'SomeReplicasCorrupted': {},
'MissingReplica': {},
'MissingAllReplicas': {},
'NoReplicas': {}}
chunkSize = 100
replicas = {}
setLfns = set(lfns)
cachedLfns = setLfns & set(self.cachedReplicas)
for lfn in cachedLfns:
replicas[lfn] = self.cachedReplicas[lfn]
lfnsLeft = list(setLfns - cachedLfns)
if lfnsLeft:
self.__write("Get replicas for %d files (chunks of %d): " %
(len(lfnsLeft), chunkSize))
for lfnChunk in breakListIntoChunks(lfnsLeft, chunkSize):
self.__write('.')
replicasRes = self.fileCatalog.getReplicas(lfnChunk)
if not replicasRes['OK']:
gLogger.error("error: %s" % replicasRes['Message'])
return S_ERROR(errno.ENOENT, "error: %s" % replicasRes['Message'])
replicasRes = replicasRes['Value']
if replicasRes['Failed']:
retDict['NoReplicas'].update(replicasRes['Failed'])
replicas.update(replicasRes['Successful'])
self.__write("Get FC metadata for %d files to be checked: " % len(lfns))
metadata = {}
for lfnChunk in breakListIntoChunks(replicas, chunkSize):
self.__write('.')
res = self.fileCatalog.getFileMetadata(lfnChunk)
if not res['OK']:
return S_ERROR(errno.ENOENT, "error %s" % res['Message'])
metadata.update(res['Value']['Successful'])
gLogger.notice("Check existence and compare checksum file by file...")
csDict = {}
seFiles = {}
# Reverse the LFN->SE dictionary
nReps = 0
for lfn in replicas:
csDict.setdefault(lfn, {})['LFCChecksum'] = metadata.get(
lfn, {}).get('Checksum')
for se in replicas[lfn]:
seFiles.setdefault(se, []).append(lfn)
nReps += 1
gLogger.notice('Getting checksum of %d replicas in %d SEs' %
(nReps, len(seFiles)))
checkSum = {}
lfnNotExisting = {}
lfnNoInfo = {}
logLevel = gLogger.getLevel()
gLogger.setLevel('FATAL')
for num, se in enumerate(sorted(seFiles)):
self.__write('\n%d. At %s (%d files): ' % (num, se, len(seFiles[se])))
oSe = StorageElement(se)
notFound = 0
for surlChunk in breakListIntoChunks(seFiles[se], chunkSize):
self.__write('.')
metadata = oSe.getFileMetadata(surlChunk)
if not metadata['OK']:
gLogger.error("Error: getFileMetadata returns %s. Ignore those replicas" % (
metadata['Message']))
# Remove from list of replicas as we don't know whether it is OK or
# not
for lfn in seFiles[se]:
lfnNoInfo.setdefault(lfn, []).append(se)
else:
metadata = metadata['Value']
notFound += len(metadata['Failed'])
for lfn in metadata['Failed']:
lfnNotExisting.setdefault(lfn, []).append(se)
for lfn in metadata['Successful']:
checkSum.setdefault(
lfn, {})[se] = metadata['Successful'][lfn]['Checksum']
if notFound:
gLogger.error('%d files not found' % notFound)
gLogger.setLevel(logLevel)
gLogger.notice('Verifying checksum of %d files' % len(replicas))
for lfn in replicas:
# get the lfn checksum from the FC
replicaDict = replicas[lfn]
oneGoodReplica = False
allGoodReplicas = True
lfcChecksum = csDict[lfn].pop('LFCChecksum')
for se in replicaDict:
# If replica doesn't exist skip check
if se in lfnNotExisting.get(lfn, []):
allGoodReplicas = False
continue
if se in lfnNoInfo.get(lfn, []):
# If there is no info, a priori it could be good
oneGoodReplica = True
continue
# get the surls metadata and compare the checksum
surlChecksum = checkSum.get(lfn, {}).get(se, '')
if not surlChecksum or not compareAdler(lfcChecksum, surlChecksum):
# if lfcChecksum does not match surlChecksum
csDict[lfn][se] = {'PFNChecksum': surlChecksum}
gLogger.info("ERROR!! checksum mismatch at %s for LFN %s: LFC checksum: %s , PFN checksum : %s "
% (se, lfn, lfcChecksum, surlChecksum))
allGoodReplicas = False
else:
oneGoodReplica = True
if not oneGoodReplica:
if lfn in lfnNotExisting:
gLogger.info("=> All replicas are missing", lfn)
retDict['MissingAllReplicas'][lfn] = 'All'
else:
gLogger.info("=> All replicas have bad checksum", lfn)
retDict['AllReplicasCorrupted'][lfn] = csDict[lfn]
elif not allGoodReplicas:
if lfn in lfnNotExisting:
gLogger.info("=> At least one replica missing", lfn)
retDict['MissingReplica'][lfn] = lfnNotExisting[lfn]
else:
gLogger.info("=> At least one replica with good Checksum", lfn)
retDict['SomeReplicasCorrupted'][lfn] = csDict[lfn]
return S_OK(retDict)
class FTSRequest( object ):
"""
.. class:: FTSRequest
Helper class for FTS job submission and monitoring.
"""
# # default checksum type
__defaultCksmType = "ADLER32"
# # flag to disablr/enable checksum test, default: disabled
__cksmTest = False
def __init__( self ):
"""c'tor
:param self: self reference
"""
self.log = gLogger.getSubLogger( self.__class__.__name__, True )
# # final states tuple
self.finalStates = ( 'Canceled', 'Failed', 'Hold',
'Finished', 'FinishedDirty' )
# # failed states tuple
self.failedStates = ( 'Canceled', 'Failed',
'Hold', 'FinishedDirty' )
# # successful states tuple
self.successfulStates = ( 'Finished', 'Done' )
# # all file states tuple
self.fileStates = ( 'Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed',
'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting' )
self.newlyCompletedFiles = []
self.newlyFailedFiles = []
self.statusSummary = {}
# # request status
self.requestStatus = 'Unknown'
# # dict for FTS job files
self.fileDict = {}
# # dict for replicas information
self.catalogReplicas = {}
# # dict for metadata information
self.catalogMetadata = {}
# # dict for files that failed to register
self.failedRegistrations = {}
# # placehoder for CatalogInterface reference
self.oCatalog = None
# # submit timestamp
self.submitTime = ''
# # placeholder FTS job GUID
self.ftsGUID = ''
# # placeholder for FTS server URL
self.ftsServer = ''
# # not used
self.priority = 3
# # flag marking FTS job completness
self.isTerminal = False
# # completness percentage
self.percentageComplete = 0.0
# # source SE name
self.sourceSE = ''
# # flag marking source SE validity
self.sourceValid = False
# # source space token
self.sourceToken = ''
# # target SE name
self.targetSE = ''
# # flag marking target SE validity
self.targetValid = False
# # target space token
self.targetToken = ''
# # whatever
self.dumpStr = ''
# # placeholder for surl file
self.surlFile = None
# # placeholder for target StorageElement
self.oTargetSE = None
# # placeholder for source StorageElement
self.oSourceSE = None
# # checksum type, set it to default
self.__cksmType = self.__defaultCksmType
# # disable checksum test by default
self.__cksmTest = False
# # replica manager handler
self.replicaManager = ReplicaManager()
# # statuses that prevent submitting to FTS
self.noSubmitStatus = ( 'Failed', 'Done', 'Staging' )
# # were sources resolved?
self.sourceResolved = False
# # Number of file transfers actually submitted
self.submittedFiles = 0
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE( self, se ):
""" set SE for source
:param self: self reference
:param str se: source SE name
"""
if se == self.targetSE:
return S_ERROR( "SourceSE is TargetSE" )
self.sourceSE = se
self.oSourceSE = StorageElement( self.sourceSE )
return self.__checkSourceSE()
def getSourceSE( self ):
""" source SE getter
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR( "Source SE not defined" )
return S_OK( self.sourceSE )
def setSourceToken( self, token ):
""" set source space token
:param self: self reference
:param str token: source space token
"""
self.sourceToken = token
return S_OK()
def getSourceToken( self ):
""" source space token getter
:param self: self reference
"""
if not self.sourceToken:
return S_ERROR( "Source token not defined" )
return S_OK( self.sourceToken )
def __checkSourceSE( self ):
""" check source SE availability
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR( "SourceSE not set" )
res = self.oSourceSE.isValid( 'Read' )
if not res['OK']:
return S_ERROR( "SourceSE not available for reading" )
res = self.__getSESpaceToken( self.oSourceSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message'] )
return S_ERROR( "SourceSE does not support FTS transfers" )
if self.__cksmTest:
res = self.oSourceSE.getChecksumType()
if not res["OK"]:
self.log.error( "Unable to get checksum type for SourceSE %s: %s" % ( self.sourceSE,
res["Message"] ) )
cksmType = res["Value"]
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at SourceSE %s, disabling checksum test" % ( cksmType,
self.sourceSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE( self, se ):
""" set target SE
:param self: self reference
:param str se: target SE name
"""
if se == self.sourceSE:
return S_ERROR( "TargetSE is SourceSE" )
self.targetSE = se
self.oTargetSE = StorageElement( self.targetSE )
return self.__checkTargetSE()
def getTargetSE( self ):
""" target SE getter
:param self: self reference
"""
if not self.targetSE:
return S_ERROR( "Target SE not defined" )
return S_OK( self.targetSE )
def setTargetToken( self, token ):
""" target space token setter
:param self: self reference
:param str token: target space token
"""
self.targetToken = token
return S_OK()
def getTargetToken( self ):
""" target space token getter
:param self: self reference
"""
if not self.targetToken:
return S_ERROR( "Target token not defined" )
return S_OK( self.targetToken )
def __checkTargetSE( self ):
""" check target SE availability
:param self: self reference
"""
if not self.targetSE:
return S_ERROR( "TargetSE not set" )
res = self.oTargetSE.isValid( 'Write' )
if not res['OK']:
return S_ERROR( "TargetSE not available for writing" )
res = self.__getSESpaceToken( self.oTargetSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message'] )
return S_ERROR( "TargetSE does not support FTS transfers" )
# # check checksum types
if self.__cksmTest:
res = self.oTargetSE.getChecksumType()
if not res["OK"]:
self.log.error( "Unable to get checksum type for TargetSE %s: %s" % ( self.targetSE,
res["Message"] ) )
cksmType = res["Value"]
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at TargetSE %s, disabling checksum test" % ( cksmType,
self.targetSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
@staticmethod
def __getSESpaceToken( oSE ):
""" get space token from StorageElement instance
:param self: self reference
:param StorageElement oSE: StorageElement instance
"""
res = oSE.getStorageParameters( "SRM2" )
if not res['OK']:
return res
return S_OK( res['Value'].get( 'SpaceToken' ) )
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID( self, guid ):
""" FTS job GUID setter
:param self: self reference
:param str guid: string containg GUID
"""
if not checkGuid( guid ):
return S_ERROR( "Incorrect GUID format" )
self.ftsGUID = guid
return S_OK()
def getFTSGUID( self ):
""" FTS job GUID getter
:param self: self refenece
"""
if not self.ftsGUID:
return S_ERROR( "FTSGUID not set" )
return S_OK( self.ftsGUID )
def setFTSServer( self, server ):
""" FTS server setter
:param self: self reference
:param str server: FTS server URL
"""
self.ftsServer = server
return S_OK()
def getFTSServer( self ):
""" FTS server getter
:param self: self reference
"""
if not self.ftsServer:
return S_ERROR( "FTSServer not set" )
return S_OK( self.ftsServer )
def setPriority( self, priority ):
""" set priority for FTS job
:param self: self reference
:param int priority: a new priority
"""
if not type( priority ) in ( IntType, LongType ):
return S_ERROR( "Priority must be integer" )
if priority < 0:
priority = 0
elif priority > 5:
priority = 5
self.priority = priority
return S_OK( self.priority )
def getPriority( self ):
""" FTS job priority getter
:param self: self reference
"""
return S_OK( self.priority )
def getPercentageComplete( self ):
""" get completness percentage
:param self: self reference
"""
completedFiles = 0
totalFiles = 0
for state in self.statusSummary:
if state in self.successfulStates:
completedFiles += self.statusSummary[state]
totalFiles += self.statusSummary[state]
self.percentageComplete = ( float( completedFiles ) * 100.0 ) / float( totalFiles )
return S_OK( self.percentageComplete )
def isRequestTerminal( self ):
""" check if FTS job has terminated
:param self: self reference
"""
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK( self.isTerminal )
def getStatus( self ):
""" get FTS job status
:param self: self reference
"""
return S_OK( self.requestStatus )
def setCksmType( self, cksm = None ):
""" set checksum type to use
:param self: self reference
:param mixed cksm: checksum type, should be one of 'Adler32', 'md5', 'sha1', None
"""
if str( cksm ).upper() not in ( "ADLER32", "MD5", "SHA1", "NONE" ):
return S_ERROR( "Not supported checksum type: %s" % str( cksm ) )
if not cksm:
self.__cksmType = None
return S_OK( False )
self.__cksmType = str( cksm ).upper()
return S_OK( True )
def getCksmType( self ):
""" get checksum type
:param self: self reference
"""
return S_OK( self.__cksmType )
def setCksmTest( self, cksmTest = False ):
""" set cksm test
:param self: self reference
:param bool cksmTest: flag to enable/disable checksum test
"""
self.__cksmTest = bool( cksmTest )
return S_OK( self.__cksmTest )
def getCksmTest( self ):
""" get cksm test flag
:param self: self reference
"""
return S_OK( self.__cksmTest )
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN( self, lfn ):
""" add LFN :lfn: to :fileDict:
:param self: self reference
:param str lfn: LFN to add to
"""
self.fileDict.setdefault( lfn, {'Status':'Waiting'} )
return S_OK()
def setStatus( self, lfn, status ):
""" set status of a file """
return( self.__setFileParameter( lfn, 'Status', status ) )
def setSourceSURL( self, lfn, surl ):
""" source SURL setter
:param self: self reference
:param str lfn: LFN
:param str surl: source SURL
"""
target = self.fileDict[lfn].get( 'Target' )
if target == surl:
return S_ERROR( "Source and target the same" )
return( self.__setFileParameter( lfn, 'Source', surl ) )
def getSourceSURL( self, lfn ):
""" get source SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Source' )
def setTargetSURL( self, lfn, surl ):
""" set target SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
:param str surl: target SURL
"""
source = self.fileDict[lfn].get( 'Source' )
if source == surl:
return S_ERROR( "Source and target the same" )
return( self.__setFileParameter( lfn, 'Target', surl ) )
def getTargetSURL( self, lfn ):
""" target SURL getter
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Target' )
def getFailReason( self, lfn ):
""" get fail reason for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Reason' )
def getRetries( self, lfn ):
""" get number of attepmts made to transfer file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Retries' )
def getTransferTime( self, lfn ):
""" get duration of transfer for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Duration' )
def getFailed( self ):
""" get list of wrongly transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.failedStates ] )
def getStaging( self ):
""" get files set for prestaging """
return S_OK( [lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) == 'Staging'] )
def getDone( self ):
""" get list of succesfully transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.successfulStates ] )
def __setFileParameter( self, lfn, paramName, paramValue ):
""" set :paramName: to :paramValue: for :lfn: file
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
:param mixed paramValue: a new parameter value
"""
self.setLFN( lfn )
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter( self, lfn, paramName ):
""" get value of :paramName: for file :lfn:
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
"""
if lfn not in self.fileDict:
return S_ERROR( "Supplied file not set" )
if paramName not in self.fileDict[lfn]:
return S_ERROR( "%s not set for file" % paramName )
return S_OK( self.fileDict[lfn][paramName] )
####################################################################
#
# Methods for submission
#
def submit( self, monitor = False, printOutput = True ):
""" submit FTS job
:param self: self reference
:param bool monitor: flag to monitor progress of FTS job
:param bool printOutput: flag to print output of execution to stdout
"""
res = self.__isSubmissionValid()
if not res['OK']:
return res
res = self.__createSURLPairFile()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = { 'ftsGUID' : self.ftsGUID, 'ftsServer' : self.ftsServer, 'submittedFiles' : self.submittedFiles }
# print "Submitted %s @ %s" % ( self.ftsGUID, self.ftsServer )
if monitor:
self.monitor( untilTerminal = True, printOutput = printOutput )
return S_OK( resDict )
def __isSubmissionValid( self ):
""" check validity of job before submission
:param self: self reference
"""
if not self.fileDict:
return S_ERROR( "No files set" )
if not self.sourceValid:
return S_ERROR( "SourceSE not valid" )
if not self.targetValid:
return S_ERROR( "TargetSE not valid" )
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR( "FTSServer not valid" )
self.resolveSource()
self.resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR( "No files to submit" )
return S_OK()
def __getCatalogObject( self ):
""" CatalogInterface instance facade
:param self: self reference
"""
try:
if not self.oCatalog:
self.oCatalog = CatalogInterface()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache( self, lfns = None, overwrite = False ):
""" update replica cache for list of :lfns:
:param self: self reference
:param mixed lfns: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if ( lfn not in self.catalogReplicas ) or overwrite ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogReplicas( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to update replica cache: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache( self, lfns = None, overwrite = False ):
""" update metadata cache for list of LFNs
:param self: self reference
:param list lnfs: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if ( lfn not in self.catalogMetadata ) or overwrite ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogFileMetadata( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to get source catalog metadata: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def resolveSource( self ):
""" resolve source SE eligible for submission
:param self: self reference
"""
# Avoid resolving sources twice
if self.sourceResolved:
return S_OK()
# Only resolve files that need a transfer
toResolve = [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( "Status", "" ) != "Failed" ]
if not toResolve:
return S_OK()
res = self.__updateMetadataCache( toResolve )
if not res['OK']:
return res
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
# Define the source URLs
for lfn in toResolve:
replicas = self.catalogReplicas.get( lfn, {} )
if self.sourceSE not in replicas:
gLogger.warn( "resolveSource: skipping %s - not replicas at SourceSE %s" % ( lfn, self.sourceSE ) )
self.__setFileParameter( lfn, 'Reason', "No replica at SourceSE" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.oSourceSE.getPfnForProtocol( replicas[self.sourceSE], 'SRM2', withPort = True )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setSourceSURL( lfn, res['Value'] )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict:
if "Source" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Source']] = lfn
if not toResolve:
return S_ERROR( "No eligible Source files" )
# Get metadata of the sources, to check for existance, availability and caching
res = self.oSourceSE.getFileMetadata( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check source file metadata" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
if re.search( 'File does not exist', error ):
gLogger.warn( "resolveSource: skipping %s - source file does not exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file does not exist" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
gLogger.warn( "resolveSource: skipping %s - failed to get source metadata" % lfn )
self.__setFileParameter( lfn, 'Reason', "Failed to get Source metadata" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toStage = []
nbStagedFiles = 0
for pfn, metadata in res['Value']['Successful'].items():
lfn = toResolve[pfn]
lfnStatus = self.fileDict.get( lfn, {} ).get( 'Status' )
if metadata['Unavailable']:
gLogger.warn( "resolveSource: skipping %s - source file unavailable" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Unavailable" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif metadata['Lost']:
gLogger.warn( "resolveSource: skipping %s - source file lost" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Lost" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif not metadata['Cached']:
if lfnStatus != 'Staging':
toStage.append( pfn )
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
gLogger.warn( "resolveSource: skipping %s - source file size mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source size mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \
not ( compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ) ):
gLogger.warn( "resolveSource: skipping %s - source file checksum mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source checksum mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif lfnStatus == 'Staging':
# file that was staging is now cached
self.__setFileParameter( lfn, 'Status', 'Waiting' )
nbStagedFiles += 1
# Some files were being staged
if nbStagedFiles:
self.log.info( 'resolveSource: %d files have been staged' % nbStagedFiles )
# Launching staging of files not in cache
if toStage:
gLogger.warn( "resolveSource: %s source files not cached, prestaging..." % len( toStage ) )
stage = self.replicaManager.prestageStorageFile( toStage, self.sourceSE )
if not stage["OK"]:
gLogger.error( "resolveSource: error is prestaging - %s" % stage["Message"] )
for pfn in toStage:
lfn = toResolve[pfn]
self.__setFileParameter( lfn, 'Reason', stage["Message"] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
for pfn in toStage:
lfn = toResolve[pfn]
if pfn in stage['Value']['Successful']:
self.__setFileParameter( lfn, 'Status', 'Staging' )
elif pfn in stage['Value']['Failed']:
self.__setFileParameter( lfn, 'Reason', stage['Value']['Failed'][pfn] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
self.sourceResolved = True
return S_OK()
def resolveTarget( self ):
""" find target SE eligible for submission
:param self: self reference
"""
toResolve = [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status' ) not in self.noSubmitStatus ]
if not toResolve:
return S_OK()
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
for lfn in toResolve:
res = self.oTargetSE.getPfnForLfn( lfn )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - failed to create target pfn" % lfn )
self.__setFileParameter( lfn, 'Reason', "Failed to create Target" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.oTargetSE.getPfnForProtocol( res['Value'], 'SRM2', withPort = True )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setTargetSURL( lfn, res['Value'] )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict:
if "Target" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Target']] = lfn
if not toResolve:
return S_ERROR( "No eligible Target files" )
res = self.oTargetSE.exists( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check target existence" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toRemove = []
for pfn, exists in res['Value']['Successful'].items():
if exists:
lfn = toResolve[pfn]
res = self.getSourceSURL( lfn )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - target exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Target exists" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif res['Value'] == pfn:
gLogger.warn( "resolveTarget: skipping %s - source and target pfns are the same" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source and Target the same" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRemove.append( pfn )
if toRemove:
self.oTargetSE.removeFile( toRemove )
return S_OK()
def __filesToSubmit( self ):
"""
check if there is at least one file to submit
:return: S_OK if at least one file is present, S_ERROR otherwise
"""
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if lfnStatus not in self.noSubmitStatus and source and target:
return S_OK()
return S_ERROR()
def __createSURLPairFile( self ):
""" create LFNs file for glite-transfer-submit command
This file consists one line for each fiel to be transferred:
sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM]
:param self: self reference
"""
fd, fileName = tempfile.mkstemp()
surlFile = os.fdopen( fd, 'w' )
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if lfnStatus not in self.noSubmitStatus and source and target:
cksmStr = ""
# # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself
if self.__cksmTest and self.__cksmType:
checkSum = self.catalogMetadata.get( lfn, {} ).get( 'Checksum' )
if checkSum:
cksmStr = " %s:%s" % ( self.__cksmType, intAdlerToHex( hexAdlerToInt( checkSum ) ) )
surlFile.write( "%s %s%s\n" % ( source, target, cksmStr ) )
self.submittedFiles += 1
surlFile.close()
self.surlFile = fileName
return S_OK()
def __submitFTSTransfer( self ):
""" create and execute glite-transfer-submit CLI command
:param self: self reference
"""
comm = [ 'glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile, '-o' ]
if self.targetToken:
comm += [ '-t', self.targetToken ]
if self.sourceToken:
comm += [ '-S', self.sourceToken ]
if self.__cksmTest:
comm.append( "--compare-checksums" )
gLogger.verbose( 'Executing %s' % ' '.join( comm ) )
res = executeGridCommand( '', comm )
os.remove( self.surlFile )
if not res['OK']:
return res
returnCode, output, errStr = res['Value']
if not returnCode == 0:
return S_ERROR( errStr )
guid = output.replace( '\n', '' )
if not checkGuid( guid ):
return S_ERROR( 'Wrong GUID format returned' )
self.ftsGUID = guid
# if self.priority != 3:
# comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)]
# executeGridCommand('',comm)
return res
def __getFTSServer( self, site ):
try:
configPath = '/Resources/FTSEndpoints/%s' % site
endpointURL = gConfig.getValue( configPath )
if not endpointURL:
errStr = "FTSRequest.__getFTSServer: Failed to find FTS endpoint, check CS entry for '%s'." % site
return S_ERROR( errStr )
return S_OK( endpointURL )
except Exception, x:
return S_ERROR( 'FTSRequest.__getFTSServer: Failed to obtain endpoint details from CS' )
class FTSRequest:
def __init__( self ):
self.gridEnv = '/afs/cern.ch/project/gd/LCG-share/3.2.8-0/etc/profile.d/grid-env'
self.finalStates = ['Canceled', 'Failed', 'Hold', 'Finished', 'FinishedDirty']
self.failedStates = ['Canceled', 'Failed', 'Hold', 'FinishedDirty']
self.successfulStates = ['Finished', 'Done']
self.fileStates = ['Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed', 'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting']
self.newlyCompletedFiles = []
self.newlyFailedFiles = []
self.statusSummary = {}
self.requestStatus = 'Unknown'
self.fileDict = {}
self.catalogReplicas = {}
self.catalogMetadata = {}
self.oCatalog = None
self.submitTime = ''
self.ftsGUID = ''
self.ftsServer = ''
self.priority = 3
self.isTerminal = False
self.percentageComplete = 0.0
self.sourceSE = ''
self.sourceValid = False
self.sourceToken = ''
self.targetSE = ''
self.targetValid = False
self.targetToken = ''
self.dumpStr = ''
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE( self, se ):
if se == self.targetSE:
return S_ERROR( "SourceSE is TargetSE" )
self.sourceSE = se
self.oSourceSE = StorageElement( self.sourceSE )
return self.__checkSourceSE()
def getSourceSE( self ):
if not self.sourceSE:
return S_ERROR( "Source SE not defined" )
return S_OK( self.sourceSE )
def setSourceToken( self, token ):
self.sourceToken = token
return S_OK()
def getSourceToken( self ):
if not self.sourceToken:
return S_ERROR( "Source token not defined" )
return S_OK( self.sourceToken )
def __checkSourceSE( self ):
if not self.sourceSE:
return S_ERROR( "SourceSE not set" )
res = self.oSourceSE.isValid( 'Read' )
if not res['OK']:
return S_ERROR( "SourceSE not available for reading" )
res = self.__getSESpaceToken( self.oSourceSE )
if not res['OK']:
gLogger.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message'] )
return S_ERROR( "SourceSE does not support FTS transfers" )
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE( self, se ):
if se == self.sourceSE:
return S_ERROR( "TargetSE is SourceSE" )
self.targetSE = se
self.oTargetSE = StorageElement( self.targetSE )
return self.__checkTargetSE()
def getTargetSE( self ):
if not self.targetSE:
return S_ERROR( "Target SE not defined" )
return S_OK( self.targetSE )
def setTargetToken( self, token ):
self.targetToken = token
return S_OK()
def getTargetToken( self ):
if not self.targetToken:
return S_ERROR( "Target token not defined" )
return S_OK( self.targetToken )
def __checkTargetSE( self ):
if not self.targetSE:
return S_ERROR( "TargetSE not set" )
res = self.oTargetSE.isValid( 'Write' )
if not res['OK']:
return S_ERROR( "TargetSE not available for writing" )
res = self.__getSESpaceToken( self.oTargetSE )
if not res['OK']:
gLogger.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message'] )
return S_ERROR( "SourceSE does not support FTS transfers" )
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
def __getSESpaceToken( self, oSE ):
res = oSE.getStorageParameters( "SRM2" )
if not res['OK']:
return res
return S_OK( res['Value'].get( 'SpaceToken' ) )
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID( self, guid ):
if not checkGuid( guid ):
return S_ERROR( "Incorrect GUID format" )
self.ftsGUID = guid
return S_OK()
def getFTSGUID( self ):
if not self.ftsGUID:
return S_ERROR( "FTSGUID not set" )
return S_OK( self.ftsGUID )
def setFTSServer( self, server ):
self.ftsServer = server
return S_OK()
def getFTSServer( self ):
if not self.ftsServer:
return S_ERROR( "FTSServer not set" )
return S_OK( self.ftsServer )
def setPriority( self, priority ):
if not type( priority ) in [types.IntType, types.LongType]:
return S_ERROR( "Priority must be integer" )
if priority < 0:
priority = 0
elif priority > 5:
priority = 5
self.priority = priority
return S_OK( self.priority )
def getPriority( self ):
return S_OK( self.priority )
def getPercentageComplete( self ):
completedFiles = 0
totalFiles = 0
for state in ( self.statusSummary.keys() ):
if state in self.successfulStates:
completedFiles += self.statusSummary[state]
totalFiles += self.statusSummary[state]
self.percentageComplete = ( float( completedFiles ) * 100.0 ) / float( totalFiles )
return S_OK( self.percentageComplete )
def isRequestTerminal( self ):
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK( self.isTerminal )
def getStatus( self ):
return S_OK( self.requestStatus )
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN( self, lfn ):
if not self.fileDict.has_key( lfn ):
self.fileDict[lfn] = {}
return S_OK()
def setSourceSURL( self, lfn, surl ):
target = self.fileDict[lfn].get( 'Target' )
if target == surl:
return S_ERROR( "Source and target the same" )
self.__setFileParameter( lfn, 'Source', surl )
return S_OK()
def getSourceSURL( self, lfn ):
return self.__getFileParameter( lfn, 'Source' )
def setTargetSURL( self, lfn, surl ):
source = self.fileDict[lfn].get( 'Source' )
if source == surl:
return S_ERROR( "Source and target the same" )
self.__setFileParameter( lfn, 'Target', surl )
return S_OK()
def getTargetSURL( self, lfn ):
return self.__getFileParameter( lfn, 'Target' )
def getFailReason( self, lfn ):
return self.__getFileParameter( lfn, 'Reason' )
def getRetries( self, lfn ):
return self.__getFileParameter( lfn, 'Retries' )
def getTransferTime( self, lfn ):
return self.__getFileParameter( lfn, 'Duration' )
def getFailed( self ):
failed = []
for lfn in self.fileDict.keys():
status = self.fileDict[lfn].get( 'Status', '' )
if status in self.failedStates:
failed.append( lfn )
return S_OK( failed )
def getDone( self ):
done = []
for lfn in self.fileDict.keys():
status = self.fileDict[lfn].get( 'Status', '' )
if status in self.successfulStates:
done.append( lfn )
return S_OK( done )
def __setFileParameter( self, lfn, paramName, paramValue ):
self.setLFN( lfn )
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter( self, lfn, paramName ):
if not self.fileDict.has_key( lfn ):
return S_ERROR( "Supplied file not set" )
if not self.fileDict[lfn].has_key( paramName ):
return S_ERROR( "%s not set for file" % paramName )
return S_OK( self.fileDict[lfn][paramName] )
####################################################################
#
# Methods for submission
#
def submit( self, monitor = False, printOutput = True ):
res = self.__isSubmissionValid()
if not res['OK']:
return res
res = self.__createSURLPairFile()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = {'ftsGUID':self.ftsGUID, 'ftsServer':self.ftsServer}
print "Submitted %s @ %s" % ( self.ftsGUID, self.ftsServer )
if monitor:
self.monitor( untilTerminal = True, printOutput = printOutput )
return S_OK( resDict )
def __isSubmissionValid( self ):
if not self.fileDict:
return S_ERROR( "No files set" )
if not self.sourceValid:
return S_ERROR( "SourceSE not valid" )
if not self.targetValid:
return S_ERROR( "TargetSE not valid" )
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR( "FTSServer not valid" )
self.__resolveSource()
self.__resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR( "No files to submit" )
return S_OK()
def __getCatalogObject( self ):
try:
if not self.oCatalog:
self.oCatalog = CatalogInterface()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache( self, lfns = [], overwrite = False ):
if not lfns:
lfns = self.fileDict.keys()
toUpdate = []
for lfn in lfns:
if ( not lfn in self.catalogReplicas.keys() ) or overwrite:
toUpdate.append( lfn )
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogReplicas( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to update replica cache", res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache( self, lfns = [], overwrite = False ):
if not lfns:
lfns = self.fileDict.keys()
toUpdate = []
for lfn in lfns:
if ( not lfn in self.catalogMetadata.keys() ) or overwrite:
toUpdate.append( lfn )
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogFileMetadata( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to get source catalog metadata", res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def __resolveSource( self ):
toResolve = []
for lfn in self.fileDict.keys():
if ( not self.fileDict[lfn].has_key( 'Source' ) ) and ( self.fileDict[lfn].get( 'Status' ) != 'Failed' ):
toResolve.append( lfn )
if not toResolve:
return S_OK()
res = self.__updateMetadataCache( toResolve )
if not res['OK']:
return res
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
for lfn in toResolve:
if self.fileDict[lfn].get( 'Status' ) == 'Failed':
continue
replicas = self.catalogReplicas.get( lfn, {} )
if not replicas.has_key( self.sourceSE ):
self.__setFileParameter( lfn, 'Reason', "No replica at SourceSE" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.oSourceSE.getPfnForProtocol( replicas[self.sourceSE], 'SRM2', withPort = True )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setSourceSURL( lfn, res['Value'] )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict.keys():
if self.fileDict[lfn].has_key( 'Source' ):
toResolve[self.fileDict[lfn]['Source']] = lfn
if not toResolve:
return S_ERROR( "No eligible Source files" )
res = self.oSourceSE.getFileMetadata( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check source file metadata" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
if re.search( 'File does not exist', error ):
self.__setFileParameter( lfn, 'Reason', "Source file does not exist" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
self.__setFileParameter( lfn, 'Reason', "Failed to get Source metadata" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for pfn, metadata in res['Value']['Successful'].items():
lfn = toResolve[pfn]
if metadata['Unavailable']:
self.__setFileParameter( lfn, 'Reason', "Source file Unavailable" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif metadata['Lost']:
self.__setFileParameter( lfn, 'Reason', "Source file Lost" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif not metadata['Cached']:
self.__setFileParameter( lfn, 'Reason', "Source file not Cached" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
self.__setFileParameter( lfn, 'Reason', "Source size mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and not ( compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ) ):
self.__setFileParameter( lfn, 'Reason', "Source checksum mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
return S_OK()
def __resolveTarget( self ):
toResolve = []
for lfn in self.fileDict.keys():
if not self.fileDict[lfn].has_key( 'Target' ) and ( self.fileDict[lfn].get( 'Status' ) != 'Failed' ):
toResolve.append( lfn )
if not toResolve:
return S_OK()
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
atTarget = []
for lfn in sortList( toResolve ):
if self.fileDict[lfn].get( 'Status' ) == 'Failed':
continue
replicas = self.catalogReplicas.get( lfn, {} )
if replicas.has_key( self.targetSE ):
self.__setFileParameter( lfn, 'Reason', "File already at Target" )
self.__setFileParameter( lfn, 'Status', 'Done' )
atTarget.append( lfn )
for lfn in toResolve:
if ( self.fileDict[lfn].get( 'Status' ) == 'Failed' ) or ( lfn in atTarget ):
continue
res = self.oTargetSE.getPfnForLfn( lfn )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', "Failed to create Target" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.oTargetSE.getPfnForProtocol( res['Value'], 'SRM2', withPort = True )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setTargetSURL( lfn, res['Value'] )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict.keys():
if self.fileDict[lfn].has_key( 'Target' ):
toResolve[self.fileDict[lfn]['Target']] = lfn
if not toResolve:
return S_ERROR( "No eligible Target files" )
res = self.oTargetSE.exists( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check target existence" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toRemove = []
for pfn, exists in res['Value']['Successful'].items():
if exists:
lfn = toResolve[pfn]
res = self.getSourceSURL( lfn )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', "Target exists" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif res['Value'] == pfn:
self.__setFileParameter( lfn, 'Reason', "Source and Target the same" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRemove.append( pfn )
if toRemove:
self.oTargetSE.removeFile( toRemove )
return S_OK()
def __filesToSubmit( self ):
for lfn in self.fileDict.keys():
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if ( lfnStatus != 'Failed' ) and ( lfnStatus != 'Done' ) and source and target:
return S_OK()
return S_ERROR()
def __createSURLPairFile( self ):
fd, fileName = tempfile.mkstemp()
surlFile = os.fdopen( fd, 'w' )
for lfn in self.fileDict.keys():
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if ( lfnStatus != 'Failed' ) and ( lfnStatus != 'Done' ) and source and target:
surlString = '%s %s\n' % ( source, target )
surlFile.write( surlString )
surlFile.close()
self.surlFile = fileName
return S_OK()
def __submitFTSTransfer( self ):
comm = ['glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile, '-o']
if self.targetToken:
comm.append( '-t' )
comm.append( self.targetToken )
if self.sourceToken:
comm.append( '-S' )
comm.append( self.sourceToken )
res = executeGridCommand( '', comm, self.gridEnv )
os.remove( self.surlFile )
if not res['OK']:
return res
returnCode, output, errStr = res['Value']
if not returnCode == 0:
return S_ERROR( errStr )
guid = output.replace( '\n', '' )
if not checkGuid( guid ):
return S_ERROR( 'Wrong GUID format returned' )
self.ftsGUID = guid
#if self.priority != 3:
# comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)]
# executeGridCommand('',comm,self.gridEnv)
return res
def __resolveFTSServer( self ):
if not self.sourceSE:
return S_ERROR( "Source SE not set" )
if not self.targetSE:
return S_ERROR( "Target SE not set" )
res = getSitesForSE( self.sourceSE, 'LCG' )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine source site" )
sourceSite = res['Value'][0]
res = getSitesForSE( self.targetSE, 'LCG' )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine target site" )
targetSite = res['Value'][0]
if ( sourceSite == 'LCG.CERN.ch' ) or ( targetSite == 'LCG.CERN.ch' ):
ep = 'LCG.CERN.ch'
else:
# Target site FTS server should be used
ep = targetSite
try:
configPath = '/Resources/FTSEndpoints/%s' % ep
endpointURL = gConfig.getValue( configPath )
if not endpointURL:
errStr = "FTSRequest.__resolveFTSEndpoint: Failed to find FTS endpoint, check CS entry for '%s'." % ep
return S_ERROR( errStr )
self.ftsServer = endpointURL
return S_OK( endpointURL )
except Exception, x:
return S_ERROR( 'FTSRequest.__resolveFTSEndpoint: Failed to obtain endpoint details from CS' )
class FTSRequest( object ):
"""
.. class:: FTSRequest
Helper class for FTS job submission and monitoring.
"""
# # default checksum type
__defaultCksmType = "ADLER32"
# # flag to disablr/enable checksum test, default: disabled
__cksmTest = False
def __init__( self ):
"""c'tor
:param self: self reference
"""
self.log = gLogger.getSubLogger( self.__class__.__name__, True )
# # final states tuple
self.finalStates = ( 'Canceled', 'Failed', 'Hold',
'Finished', 'FinishedDirty' )
# # failed states tuple
self.failedStates = ( 'Canceled', 'Failed',
'Hold', 'FinishedDirty' )
# # successful states tuple
self.successfulStates = ( 'Finished', 'Done' )
# # all file states tuple
self.fileStates = ( 'Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed',
'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting' )
self.statusSummary = {}
# # request status
self.requestStatus = 'Unknown'
# # dict for FTS job files
self.fileDict = {}
# # dict for replicas information
self.catalogReplicas = {}
# # dict for metadata information
self.catalogMetadata = {}
# # dict for files that failed to register
self.failedRegistrations = {}
# # placehoder for FileCatalog reference
self.oCatalog = None
# # submit timestamp
self.submitTime = ''
# # placeholder FTS job GUID
self.ftsGUID = ''
# # placeholder for FTS server URL
self.ftsServer = ''
# # flag marking FTS job completness
self.isTerminal = False
# # completness percentage
self.percentageComplete = 0.0
# # source SE name
self.sourceSE = ''
# # flag marking source SE validity
self.sourceValid = False
# # source space token
self.sourceToken = ''
# # target SE name
self.targetSE = ''
# # flag marking target SE validity
self.targetValid = False
# # target space token
self.targetToken = ''
# # placeholder for target StorageElement
self.oTargetSE = None
# # placeholder for source StorageElement
self.oSourceSE = None
# # checksum type, set it to default
self.__cksmType = self.__defaultCksmType
# # disable checksum test by default
self.__cksmTest = False
# # statuses that prevent submitting to FTS
self.noSubmitStatus = ( 'Failed', 'Done', 'Staging' )
# # were sources resolved?
self.sourceResolved = False
# # Number of file transfers actually submitted
self.submittedFiles = 0
self.transferTime = 0
self.submitCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/SubmitCommand', 'glite-transfer-submit' )
self.monitorCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/MonitorCommand', 'glite-transfer-status' )
self.ftsVersion = Operations().getValue( 'DataManagement/FTSVersion', 'FTS2' )
self.ftsJob = None
self.ftsFiles = []
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE( self, se ):
""" set SE for source
:param self: self reference
:param str se: source SE name
"""
if se == self.targetSE:
return S_ERROR( "SourceSE is TargetSE" )
self.sourceSE = se
self.oSourceSE = StorageElement( self.sourceSE )
return self.__checkSourceSE()
def __checkSourceSE( self ):
""" check source SE availability
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR( "SourceSE not set" )
res = self.oSourceSE.isValid( 'Read' )
if not res['OK']:
return S_ERROR( "SourceSE not available for reading" )
res = self.__getSESpaceToken( self.oSourceSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message'] )
return S_ERROR( "SourceSE does not support FTS transfers" )
if self.__cksmTest:
cksmType = self.oSourceSE.checksumType()
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at SourceSE %s, disabling checksum test" % ( cksmType,
self.sourceSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE( self, se ):
""" set target SE
:param self: self reference
:param str se: target SE name
"""
if se == self.sourceSE:
return S_ERROR( "TargetSE is SourceSE" )
self.targetSE = se
self.oTargetSE = StorageElement( self.targetSE )
return self.__checkTargetSE()
def setTargetToken( self, token ):
""" target space token setter
:param self: self reference
:param str token: target space token
"""
self.targetToken = token
return S_OK()
def __checkTargetSE( self ):
""" check target SE availability
:param self: self reference
"""
if not self.targetSE:
return S_ERROR( "TargetSE not set" )
res = self.oTargetSE.isValid( 'Write' )
if not res['OK']:
return S_ERROR( "TargetSE not available for writing" )
res = self.__getSESpaceToken( self.oTargetSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message'] )
return S_ERROR( "TargetSE does not support FTS transfers" )
# # check checksum types
if self.__cksmTest:
cksmType = self.oTargetSE.checksumType()
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at TargetSE %s, disabling checksum test" % ( cksmType,
self.targetSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
@staticmethod
def __getSESpaceToken( oSE ):
""" get space token from StorageElement instance
:param self: self reference
:param StorageElement oSE: StorageElement instance
"""
res = oSE.getStorageParameters( protocol = 'srm' )
if not res['OK']:
return res
return S_OK( res['Value'].get( 'SpaceToken' ) )
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID( self, guid ):
""" FTS job GUID setter
:param self: self reference
:param str guid: string containg GUID
"""
if not checkGuid( guid ):
return S_ERROR( "Incorrect GUID format" )
self.ftsGUID = guid
return S_OK()
def setFTSServer( self, server ):
""" FTS server setter
:param self: self reference
:param str server: FTS server URL
"""
self.ftsServer = server
return S_OK()
def isRequestTerminal( self ):
""" check if FTS job has terminated
:param self: self reference
"""
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK( self.isTerminal )
def setCksmTest( self, cksmTest = False ):
""" set cksm test
:param self: self reference
:param bool cksmTest: flag to enable/disable checksum test
"""
self.__cksmTest = bool( cksmTest )
return S_OK( self.__cksmTest )
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN( self, lfn ):
""" add LFN :lfn: to :fileDict:
:param self: self reference
:param str lfn: LFN to add to
"""
self.fileDict.setdefault( lfn, {'Status':'Waiting'} )
return S_OK()
def setSourceSURL( self, lfn, surl ):
""" source SURL setter
:param self: self reference
:param str lfn: LFN
:param str surl: source SURL
"""
target = self.fileDict[lfn].get( 'Target' )
if target == surl:
return S_ERROR( "Source and target the same" )
return self.__setFileParameter( lfn, 'Source', surl )
def getSourceSURL( self, lfn ):
""" get source SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Source' )
def setTargetSURL( self, lfn, surl ):
""" set target SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
:param str surl: target SURL
"""
source = self.fileDict[lfn].get( 'Source' )
if source == surl:
return S_ERROR( "Source and target the same" )
return self.__setFileParameter( lfn, 'Target', surl )
def getFailReason( self, lfn ):
""" get fail reason for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Reason' )
def getRetries( self, lfn ):
""" get number of attepmts made to transfer file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Retries' )
def getTransferTime( self, lfn ):
""" get duration of transfer for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Duration' )
def getFailed( self ):
""" get list of wrongly transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.failedStates ] )
def getStaging( self ):
""" get files set for prestaging """
return S_OK( [lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) == 'Staging'] )
def getDone( self ):
""" get list of succesfully transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.successfulStates ] )
def __setFileParameter( self, lfn, paramName, paramValue ):
""" set :paramName: to :paramValue: for :lfn: file
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
:param mixed paramValue: a new parameter value
"""
self.setLFN( lfn )
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter( self, lfn, paramName ):
""" get value of :paramName: for file :lfn:
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
"""
if lfn not in self.fileDict:
return S_ERROR( "Supplied file not set" )
if paramName not in self.fileDict[lfn]:
return S_ERROR( "%s not set for file" % paramName )
return S_OK( self.fileDict[lfn][paramName] )
####################################################################
#
# Methods for submission
#
def submit( self, monitor = False, printOutput = True ):
""" submit FTS job
:param self: self reference
:param bool monitor: flag to monitor progress of FTS job
:param bool printOutput: flag to print output of execution to stdout
"""
res = self.__prepareForSubmission()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = { 'ftsGUID' : self.ftsGUID, 'ftsServer' : self.ftsServer, 'submittedFiles' : self.submittedFiles }
if monitor or printOutput:
gLogger.always( "Submitted %s@%s" % ( self.ftsGUID, self.ftsServer ) )
if monitor:
self.monitor( untilTerminal = True, printOutput = printOutput, full = False )
return S_OK( resDict )
def __prepareForSubmission( self ):
""" check validity of job before submission
:param self: self reference
"""
if not self.fileDict:
return S_ERROR( "No files set" )
if not self.sourceValid:
return S_ERROR( "SourceSE not valid" )
if not self.targetValid:
return S_ERROR( "TargetSE not valid" )
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR( "FTSServer not valid" )
self.resolveSource()
self.resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR( "No files to submit" )
return S_OK()
def __getCatalogObject( self ):
""" CatalogInterface instance facade
:param self: self reference
"""
try:
if not self.oCatalog:
self.oCatalog = FileCatalog()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache( self, lfns = None, overwrite = False ):
""" update replica cache for list of :lfns:
:param self: self reference
:param mixed lfns: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if ( lfn not in self.catalogReplicas ) or overwrite ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getReplicas( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to update replica cache: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache( self, lfns = None ):
""" update metadata cache for list of LFNs
:param self: self reference
:param list lnfs: list of LFNs
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if lfn not in self.catalogMetadata ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getFileMetadata( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to get source catalog metadata: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def resolveSource( self ):
""" resolve source SE eligible for submission
:param self: self reference
"""
# Avoid resolving sources twice
if self.sourceResolved:
return S_OK()
# Only resolve files that need a transfer
toResolve = [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( "Status", "" ) != "Failed" ]
if not toResolve:
return S_OK()
res = self.__updateMetadataCache( toResolve )
if not res['OK']:
return res
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
# Define the source URLs
for lfn in toResolve:
replicas = self.catalogReplicas.get( lfn, {} )
if self.sourceSE not in replicas:
gLogger.warn( "resolveSource: skipping %s - not replicas at SourceSE %s" % ( lfn, self.sourceSE ) )
self.__setFileParameter( lfn, 'Reason', "No replica at SourceSE" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = returnSingleResult( self.oSourceSE.getURL( lfn, protocol = 'srm' ) )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setSourceSURL( lfn, res['Value'] )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = []
for lfn in self.fileDict:
if "Source" in self.fileDict[lfn]:
toResolve.append( lfn )
if not toResolve:
return S_ERROR( "No eligible Source files" )
# Get metadata of the sources, to check for existance, availability and caching
res = self.oSourceSE.getFileMetadata( toResolve )
if not res['OK']:
return S_ERROR( "Failed to check source file metadata" )
for lfn, error in res['Value']['Failed'].items():
if re.search( 'File does not exist', error ):
gLogger.warn( "resolveSource: skipping %s - source file does not exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file does not exist" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
gLogger.warn( "resolveSource: skipping %s - failed to get source metadata" % lfn )
self.__setFileParameter( lfn, 'Reason', "Failed to get Source metadata" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toStage = []
nbStagedFiles = 0
for lfn, metadata in res['Value']['Successful'].items():
lfnStatus = self.fileDict.get( lfn, {} ).get( 'Status' )
if metadata.get( 'Unavailable', False ):
gLogger.warn( "resolveSource: skipping %s - source file unavailable" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Unavailable" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif metadata.get( 'Lost', False ):
gLogger.warn( "resolveSource: skipping %s - source file lost" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Lost" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif not metadata.get( 'Cached', metadata['Accessible'] ):
if lfnStatus != 'Staging':
toStage.append( lfn )
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
gLogger.warn( "resolveSource: skipping %s - source file size mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source size mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \
not compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ):
gLogger.warn( "resolveSource: skipping %s - source file checksum mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source checksum mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif lfnStatus == 'Staging':
# file that was staging is now cached
self.__setFileParameter( lfn, 'Status', 'Waiting' )
nbStagedFiles += 1
# Some files were being staged
if nbStagedFiles:
self.log.info( 'resolveSource: %d files have been staged' % nbStagedFiles )
# Launching staging of files not in cache
if toStage:
gLogger.warn( "resolveSource: %s source files not cached, prestaging..." % len( toStage ) )
stage = self.oSourceSE.prestageFile( toStage )
if not stage["OK"]:
gLogger.error( "resolveSource: error is prestaging", stage["Message"] )
for lfn in toStage:
self.__setFileParameter( lfn, 'Reason', stage["Message"] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
for lfn in toStage:
if lfn in stage['Value']['Successful']:
self.__setFileParameter( lfn, 'Status', 'Staging' )
elif lfn in stage['Value']['Failed']:
self.__setFileParameter( lfn, 'Reason', stage['Value']['Failed'][lfn] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
self.sourceResolved = True
return S_OK()
def resolveTarget( self ):
""" find target SE eligible for submission
:param self: self reference
"""
toResolve = [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status' ) not in self.noSubmitStatus ]
if not toResolve:
return S_OK()
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
for lfn in toResolve:
res = returnSingleResult( self.oTargetSE.getURL( lfn, protocol = 'srm' ) )
if not res['OK']:
reason = res.get( 'Message', res['Message'] )
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, reason ) )
self.__setFileParameter( lfn, 'Reason', reason )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setTargetSURL( lfn, res['Value'] )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = []
for lfn in self.fileDict:
if "Target" in self.fileDict[lfn]:
toResolve.append( lfn )
if not toResolve:
return S_ERROR( "No eligible Target files" )
res = self.oTargetSE.exists( toResolve )
if not res['OK']:
return S_ERROR( "Failed to check target existence" )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toRemove = []
for lfn, exists in res['Value']['Successful'].items():
if exists:
res = self.getSourceSURL( lfn )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - target exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Target exists" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif res['Value'] == self.fileDict[lfn]['Target']:
gLogger.warn( "resolveTarget: skipping %s - source and target pfns are the same" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source and Target the same" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRemove.append( lfn )
if toRemove:
self.oTargetSE.removeFile( toRemove )
return S_OK()
def __filesToSubmit( self ):
"""
check if there is at least one file to submit
:return: S_OK if at least one file is present, S_ERROR otherwise
"""
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if lfnStatus not in self.noSubmitStatus and source and target:
return S_OK()
return S_ERROR()
def __createFTSFiles( self ):
""" create LFNs file for glite-transfer-submit command
This file consists one line for each fiel to be transferred:
sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM]
:param self: self reference
"""
self.__updateMetadataCache()
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
if lfnStatus not in self.noSubmitStatus:
cksmStr = ""
# # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself
if self.__cksmTest and self.__cksmType:
checkSum = self.catalogMetadata.get( lfn, {} ).get( 'Checksum' )
if checkSum:
cksmStr = " %s:%s" % ( self.__cksmType, intAdlerToHex( hexAdlerToInt( checkSum ) ) )
ftsFile = FTSFile()
ftsFile.LFN = lfn
ftsFile.SourceSURL = self.fileDict[lfn].get( 'Source' )
ftsFile.TargetSURL = self.fileDict[lfn].get( 'Target' )
ftsFile.SourceSE = self.sourceSE
ftsFile.TargetSE = self.targetSE
ftsFile.Status = self.fileDict[lfn].get( 'Status' )
ftsFile.Checksum = cksmStr
ftsFile.Size = self.catalogMetadata.get( lfn, {} ).get( 'Size' )
self.ftsFiles.append( ftsFile )
self.submittedFiles += 1
return S_OK()
def __createFTSJob( self, guid = None ):
self.__createFTSFiles()
ftsJob = FTSJob()
ftsJob.RequestID = 0
ftsJob.OperationID = 0
ftsJob.SourceSE = self.sourceSE
ftsJob.TargetSE = self.targetSE
ftsJob.SourceToken = self.sourceToken
ftsJob.TargetToken = self.targetToken
ftsJob.FTSServer = self.ftsServer
if guid:
ftsJob.FTSGUID = guid
for ftsFile in self.ftsFiles:
ftsFile.Attempt += 1
ftsFile.Error = ""
ftsJob.addFile( ftsFile )
self.ftsJob = ftsJob
def __submitFTSTransfer( self ):
""" create and execute glite-transfer-submit CLI command
:param self: self reference
"""
log = gLogger.getSubLogger( 'Submit' )
self.__createFTSJob()
submit = self.ftsJob.submitFTS( self.ftsVersion, command = self.submitCommand )
if not submit["OK"]:
log.error( "unable to submit FTSJob: %s" % submit["Message"] )
return submit
log.info( "FTSJob '%s'@'%s' has been submitted" % ( self.ftsJob.FTSGUID, self.ftsJob.FTSServer ) )
# # update statuses for job files
for ftsFile in self.ftsJob:
ftsFile.FTSGUID = self.ftsJob.FTSGUID
ftsFile.Status = "Submitted"
ftsFile.Attempt += 1
log.info( "FTSJob '%s'@'%s' has been submitted" % ( self.ftsJob.FTSGUID, self.ftsJob.FTSServer ) )
self.ftsGUID = self.ftsJob.FTSGUID
return S_OK()
def __resolveFTSServer( self ):
"""
resolve FTS server to use, it should be the closest one from target SE
:param self: self reference
"""
if self.ftsVersion.upper() == 'FTS2':
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTS2ServersForSites
if not self.targetSE:
return S_ERROR( "Target SE not set" )
res = getSitesForSE( self.targetSE )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine target site" )
targetSites = res['Value']
targetSite = ''
for targetSite in targetSites:
targetFTS = getFTS2ServersForSites( [targetSite] )
if targetFTS['OK']:
ftsTarget = targetFTS['Value'][targetSite]
if ftsTarget:
self.ftsServer = ftsTarget
return S_OK( self.ftsServer )
else:
return targetFTS
elif self.ftsVersion.upper() == 'FTS3':
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTS3Servers
res = getFTS3Servers()
if not res['OK']:
return res
ftsServerList = res['Value']
if ftsServerList:
# Here we take the first one, regardless of the policy...
# Unclean but all this will disapear after refactoring the fts code
self.ftsServer = ftsServerList[0]
return S_OK( self.ftsServer )
else:
return S_ERROR( 'Unknown FTS version %s' % self.ftsVersion )
return S_ERROR( 'No FTS server found for %s' % targetSite )
####################################################################
#
# Methods for monitoring
#
def summary( self, untilTerminal = False, printOutput = False ):
""" summary of FTS job
:param self: self reference
:param bool untilTerminal: flag to monitor FTS job to its final state
:param bool printOutput: flag to print out monitoring information to the stdout
"""
res = self.__isSummaryValid()
if not res['OK']:
return res
while not self.isTerminal:
res = self.__parseOutput( full = True )
if not res['OK']:
return res
if untilTerminal:
self.__print()
self.isRequestTerminal()
if res['Value'] or ( not untilTerminal ):
break
time.sleep( 1 )
if untilTerminal:
print ""
if printOutput and ( not untilTerminal ):
return self.dumpSummary( printOutput = printOutput )
return S_OK()
def monitor( self, untilTerminal = False, printOutput = False, full = True ):
""" monitor FTS job
:param self: self reference
:param bool untilTerminal: flag to monitor FTS job to its final state
:param bool printOutput: flag to print out monitoring information to the stdout
"""
if not self.ftsJob:
self.resolveSource()
self.__createFTSJob( self.ftsGUID )
res = self.__isSummaryValid()
if not res['OK']:
return res
if untilTerminal:
res = self.summary( untilTerminal = untilTerminal, printOutput = printOutput )
if not res['OK']:
return res
res = self.__parseOutput( full = full )
if not res['OK']:
return res
if untilTerminal:
self.finalize()
if printOutput:
self.dump()
return res
def dumpSummary( self, printOutput = False ):
""" get FTS job summary as str
:param self: self reference
:param bool printOutput: print summary to stdout
"""
outStr = ''
for status in sorted( self.statusSummary ):
if self.statusSummary[status]:
outStr = '%s\t%-10s : %-10s\n' % ( outStr, status, str( self.statusSummary[status] ) )
outStr = outStr.rstrip( '\n' )
if printOutput:
print outStr
return S_OK( outStr )
def __print( self ):
""" print progress bar of FTS job completeness to stdout
:param self: self reference
"""
width = 100
bits = int( ( width * self.percentageComplete ) / 100 )
outStr = "|%s>%s| %.1f%s %s %s" % ( "="*bits, " "*( width - bits ),
self.percentageComplete, "%",
self.requestStatus, " "*10 )
sys.stdout.write( "%s\r" % ( outStr ) )
sys.stdout.flush()
def dump( self ):
""" print FTS job parameters and files to stdout
:param self: self reference
"""
print "%-10s : %-10s" % ( "Status", self.requestStatus )
print "%-10s : %-10s" % ( "Source", self.sourceSE )
print "%-10s : %-10s" % ( "Target", self.targetSE )
print "%-10s : %-128s" % ( "Server", self.ftsServer )
print "%-10s : %-128s" % ( "GUID", self.ftsGUID )
for lfn in sorted( self.fileDict ):
print "\n %-15s : %-128s" % ( 'LFN', lfn )
for key in ['Source', 'Target', 'Status', 'Reason', 'Duration']:
print " %-15s : %-128s" % ( key, str( self.fileDict[lfn].get( key ) ) )
return S_OK()
def __isSummaryValid( self ):
""" check validity of FTS job summary report
:param self: self reference
"""
if not self.ftsServer:
return S_ERROR( "FTSServer not set" )
if not self.ftsGUID:
return S_ERROR( "FTSGUID not set" )
return S_OK()
def __parseOutput( self, full = False ):
""" execute glite-transfer-status command and parse its output
:param self: self reference
:param bool full: glite-transfer-status verbosity level, when set, collect information of files as well
"""
monitor = self.ftsJob.monitorFTS( self.ftsVersion, command = self.monitorCommand, full = full )
if not monitor['OK']:
return monitor
self.percentageComplete = self.ftsJob.Completeness
self.requestStatus = self.ftsJob.Status
self.submitTime = self.ftsJob.SubmitTime
statusSummary = monitor['Value']
if statusSummary:
for state in statusSummary:
self.statusSummary[state] = statusSummary[state]
self.transferTime = 0
for ftsFile in self.ftsJob:
lfn = ftsFile.LFN
self.__setFileParameter( lfn, 'Status', ftsFile.Status )
self.__setFileParameter( lfn, 'Reason', ftsFile.Error )
self.__setFileParameter( lfn, 'Duration', ftsFile._duration )
targetURL = self.__getFileParameter( lfn, 'Target' )
if not targetURL['OK']:
self.__setFileParameter( lfn, 'Target', ftsFile.TargetSURL )
sourceURL = self.__getFileParameter( lfn, 'Source' )
if not sourceURL['OK']:
self.__setFileParameter( lfn, 'Source', ftsFile.SourceSURL )
self.transferTime += int( ftsFile._duration )
return S_OK()
####################################################################
#
# Methods for finalization
#
def finalize( self ):
""" finalize FTS job
:param self: self reference
"""
self.__updateMetadataCache()
transEndTime = dateTime()
regStartTime = time.time()
res = self.getTransferStatistics()
transDict = res['Value']
res = self.__registerSuccessful( transDict['transLFNs'] )
regSuc, regTotal = res['Value']
regTime = time.time() - regStartTime
if self.sourceSE and self.targetSE:
self.__sendAccounting( regSuc, regTotal, regTime, transEndTime, transDict )
return S_OK()
def getTransferStatistics( self ):
""" collect information of Transfers that can be used by Accounting
:param self: self reference
"""
transDict = { 'transTotal': len( self.fileDict ),
'transLFNs': [],
'transOK': 0,
'transSize': 0 }
for lfn in self.fileDict:
if self.fileDict[lfn].get( 'Status' ) in self.successfulStates:
if self.fileDict[lfn].get( 'Duration', 0 ):
transDict['transLFNs'].append( lfn )
transDict['transOK'] += 1
if lfn in self.catalogMetadata:
transDict['transSize'] += self.catalogMetadata[lfn].get( 'Size', 0 )
return S_OK( transDict )
def getFailedRegistrations( self ):
""" get failed registrations dict
:param self: self reference
"""
return S_OK( self.failedRegistrations )
def __registerSuccessful( self, transLFNs ):
""" register successfully transferred files to the catalogs,
fill failedRegistrations dict for files that failed to register
:param self: self reference
:param list transLFNs: LFNs in FTS job
"""
self.failedRegistrations = {}
toRegister = {}
for lfn in transLFNs:
res = returnSingleResult( self.oTargetSE.getURL( self.fileDict[lfn].get( 'Target' ), protocol = 'srm' ) )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRegister[lfn] = { 'PFN' : res['Value'], 'SE' : self.targetSE }
if not toRegister:
return S_OK( ( 0, 0 ) )
res = self.__getCatalogObject()
if not res['OK']:
for lfn in toRegister:
self.failedRegistrations = toRegister
self.log.error( 'Failed to get Catalog Object', res['Message'] )
return S_OK( ( 0, len( toRegister ) ) )
res = self.oCatalog.addReplica( toRegister )
if not res['OK']:
self.failedRegistrations = toRegister
self.log.error( 'Failed to get Catalog Object', res['Message'] )
return S_OK( ( 0, len( toRegister ) ) )
for lfn, error in res['Value']['Failed'].items():
self.failedRegistrations[lfn] = toRegister[lfn]
self.log.error( 'Registration of Replica failed', '%s : %s' % ( lfn, str( error ) ) )
return S_OK( ( len( res['Value']['Successful'] ), len( toRegister ) ) )
def __sendAccounting( self, regSuc, regTotal, regTime, transEndTime, transDict ):
""" send accounting record
:param self: self reference
:param regSuc: number of files successfully registered
:param regTotal: number of files attepted to register
:param regTime: time stamp at the end of registration
:param transEndTime: time stamp at the end of FTS job
:param dict transDict: dict holding couters for files being transerred, their sizes and successfull transfers
"""
oAccounting = DataOperation()
oAccounting.setEndTime( transEndTime )
oAccounting.setStartTime( self.submitTime )
accountingDict = {}
accountingDict['OperationType'] = 'replicateAndRegister'
result = getProxyInfo()
if not result['OK']:
userName = '******'
else:
userName = result['Value'].get( 'username', 'unknown' )
accountingDict['User'] = userName
accountingDict['Protocol'] = 'FTS' if 'fts3' not in self.ftsServer else 'FTS3'
accountingDict['RegistrationTime'] = regTime
accountingDict['RegistrationOK'] = regSuc
accountingDict['RegistrationTotal'] = regTotal
accountingDict['TransferOK'] = transDict['transOK']
accountingDict['TransferTotal'] = transDict['transTotal']
accountingDict['TransferSize'] = transDict['transSize']
accountingDict['FinalStatus'] = self.requestStatus
accountingDict['Source'] = self.sourceSE
accountingDict['Destination'] = self.targetSE
accountingDict['TransferTime'] = self.transferTime
oAccounting.setValuesFromDict( accountingDict )
self.log.verbose( "Attempting to commit accounting message..." )
oAccounting.commit()
self.log.verbose( "...committed." )
return S_OK()
def _filterReplicas(self, opFile):
""" filter out banned/invalid source SEs """
from DIRAC.Core.Utilities.Adler import compareAdler
ret = {"Valid": [], "Banned": [], "Bad": []}
replicas = self.rm.getActiveReplicas(opFile.LFN)
if not replicas["OK"]:
self.log.error(replicas["Message"])
reNotExists = re.compile("not such file or directory")
replicas = replicas["Value"]
failed = replicas["Failed"].get(opFile.LFN, "")
if reNotExists.match(failed.lower()):
opFile.Status = "Failed"
opFile.Error = failed
return S_ERROR(failed)
replicas = replicas["Successful"][
opFile.LFN] if opFile.LFN in replicas["Successful"] else {}
for repSEName in replicas:
seRead = self.rssSEStatus(repSEName, "ReadAccess")
if not seRead["OK"]:
self.log.info(seRead["Message"])
ret["Banned"].append(repSEName)
continue
if not seRead["Value"]:
self.log.info("StorageElement '%s' is banned for reading" %
(repSEName))
repSE = self.seCache.get(repSEName, None)
if not repSE:
repSE = StorageElement(repSEName, "SRM2")
self.seCache[repSE] = repSE
pfn = repSE.getPfnForLfn(opFile.LFN)
if not pfn["OK"]:
self.log.warn("unable to create pfn for %s lfn: %s" %
(opFile.LFN, pfn["Message"]))
ret["Banned"].append(repSEName)
continue
pfn = pfn["Value"]
repSEMetadata = repSE.getFileMetadata(pfn, singleFile=True)
if not repSEMetadata["OK"]:
self.log.warn(repSEMetadata["Message"])
ret["Banned"].append(repSEName)
continue
repSEMetadata = repSEMetadata["Value"]
seChecksum = repSEMetadata.get("Checksum")
if opFile.Checksum and seChecksum and not compareAdler(
seChecksum, opFile.Checksum):
self.log.warn(" %s checksum mismatch: %s %s:%s" %
(opFile.LFN, opFile.Checksum, repSE, seChecksum))
ret["Bad"].append(repSEName)
continue
# # if we're here repSE is OK
ret["Valid"].append(repSEName)
return S_OK(ret)
def _checkFilesToStage(seToLFNs, onlineLFNs, offlineLFNs, absentLFNs):
"""
Checks on SEs whether the file is NEARLINE or ONLINE
onlineLFNs is modified to contain the files found online
"""
# Only check on storage if it is a tape SE
failed = {}
for se, lfnsInSEList in seToLFNs.iteritems():
seObj = StorageElement(se)
status = seObj.getStatus()
if not status['OK']:
gLogger.error("Could not get SE status",
"%s - %s" % (se, status['Message']))
return status
tapeSE = status['Value']['TapeSE']
# File is at a disk SE, no need to stage
fileMetadata = seObj.getFileMetadata(lfnsInSEList)
if not fileMetadata['OK']:
failed[se] = dict.fromkeys(lfnsInSEList, fileMetadata['Message'])
else:
if fileMetadata['Value']['Failed']:
failed[se] = fileMetadata['Value']['Failed']
# is there at least one replica online?
for lfn, mDict in fileMetadata['Value']['Successful'].iteritems():
# SRM returns Cached, but others may only return Accessible
if mDict.get('Cached', mDict['Accessible']):
onlineLFNs.add(lfn)
elif tapeSE:
# A file can be staged only at Tape SE
offlineLFNs.setdefault(lfn, []).append(se)
else:
# File not available at a diskSE... we shall retry later
pass
# Doesn't matter if some files are Offline if they are also online
for lfn in set(offlineLFNs) & onlineLFNs:
offlineLFNs.pop(lfn)
# If the file was found staged, ignore possible errors, but print out errors
for se, failedLfns in failed.items():
gLogger.error("Errors when getting files metadata", 'at %s' % se)
for lfn, reason in failedLfns.items():
if lfn in onlineLFNs:
gLogger.info('%s: %s, but there is an online replica' %
(lfn, reason))
failed[se].pop(lfn)
else:
gLogger.error('%s: %s, no online replicas' % (lfn, reason))
if cmpError(reason, errno.ENOENT):
absentLFNs.setdefault(lfn, []).append(se)
failed[se].pop(lfn)
if not failed[se]:
failed.pop(se)
# Find the files that do not exist at SE
if failed:
gLogger.error(
"Error getting metadata", "for %d files" %
len(set(lfn for lfnList in failed.itervalues()
for lfn in lfnList)))
return S_OK()
class FTSRequest(object):
"""
.. class:: FTSRequest
Helper class for FTS job submission and monitoring.
"""
## default checksum type
__defaultCksmType = "ADLER32"
## flag to disablr/enable checksum test, default: disabled
__cksmTest = False
def __init__(self):
"""c'tor
:param self: self reference
"""
self.log = gLogger.getSubLogger(self.__class__.__name__, True)
## final states tuple
self.finalStates = ('Canceled', 'Failed', 'Hold', 'Finished',
'FinishedDirty')
## failed states tuple
self.failedStates = ('Canceled', 'Failed', 'Hold', 'FinishedDirty')
## successful states tuple
self.successfulStates = ('Finished', 'Done')
## all file states tuple
self.fileStates = ('Done', 'Active', 'Pending', 'Ready', 'Canceled',
'Failed', 'Finishing', 'Finished', 'Submitted',
'Hold', 'Waiting')
self.newlyCompletedFiles = []
self.newlyFailedFiles = []
self.statusSummary = {}
## request status
self.requestStatus = 'Unknown'
## dict for FTS job files
self.fileDict = {}
## dict for replicas information
self.catalogReplicas = {}
## dict for metadata information
self.catalogMetadata = {}
## dict for files that failed to register
self.failedRegistrations = {}
## placehoder for CatalogInterface reference
self.oCatalog = None
## submit timestamp
self.submitTime = ''
## placeholder FTS job GUID
self.ftsGUID = ''
## placeholder for FTS server URL
self.ftsServer = ''
## not used
self.priority = 3
## flag marking FTS job completness
self.isTerminal = False
## completness percentage
self.percentageComplete = 0.0
## source SE name
self.sourceSE = ''
## flag marking source SE validity
self.sourceValid = False
## source space token
self.sourceToken = ''
## target SE name
self.targetSE = ''
## flag marking target SE validity
self.targetValid = False
## target space token
self.targetToken = ''
## whatever
self.dumpStr = ''
## placeholder for surl file
self.surlFile = None
## placeholder for target StorageElement
self.oTargetSE = None
## placeholder for source StorageElement
self.oSourceSE = None
## checksum type, set it to default
self.__cksmType = self.__defaultCksmType
## disable checksum test by default
self.__cksmTest = False
## replica manager handler
self.replicaManager = ReplicaManager()
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE(self, se):
""" set SE for source
:param self: self reference
:param str se: source SE name
"""
if se == self.targetSE:
return S_ERROR("SourceSE is TargetSE")
self.sourceSE = se
self.oSourceSE = StorageElement(self.sourceSE)
return self.__checkSourceSE()
def getSourceSE(self):
""" source SE getter
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR("Source SE not defined")
return S_OK(self.sourceSE)
def setSourceToken(self, token):
""" set source space token
:param self: self reference
:param str token: source space token
"""
self.sourceToken = token
return S_OK()
def getSourceToken(self):
""" source space token getter
:param self: self reference
"""
if not self.sourceToken:
return S_ERROR("Source token not defined")
return S_OK(self.sourceToken)
def __checkSourceSE(self):
""" check source SE availability
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR("SourceSE not set")
res = self.oSourceSE.isValid('Read')
if not res['OK']:
return S_ERROR("SourceSE not available for reading")
res = self.__getSESpaceToken(self.oSourceSE)
if not res['OK']:
self.log.error(
"FTSRequest failed to get SRM Space Token for SourceSE",
res['Message'])
return S_ERROR("SourceSE does not support FTS transfers")
if self.__cksmTest:
res = self.oSourceSE.getChecksumType()
if not res["OK"]:
self.log.error(
"Unable to get checksum type for SourceSE %s: %s" %
(self.sourceSE, res["Message"]))
cksmType = res["Value"]
if cksmType in ("NONE", "NULL"):
self.log.warn(
"Checksum type set to %s at SourceSE %s, disabling checksum test"
% (cksmType, self.sourceSE))
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn(
"Checksum type mismatch, disabling checksum test")
self.__cksmTest = False
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE(self, se):
""" set target SE
:param self: self reference
:param str se: target SE name
"""
if se == self.sourceSE:
return S_ERROR("TargetSE is SourceSE")
self.targetSE = se
self.oTargetSE = StorageElement(self.targetSE)
return self.__checkTargetSE()
def getTargetSE(self):
""" target SE getter
:param self: self reference
"""
if not self.targetSE:
return S_ERROR("Target SE not defined")
return S_OK(self.targetSE)
def setTargetToken(self, token):
""" target space token setter
:param self: self reference
:param str token: target space token
"""
self.targetToken = token
return S_OK()
def getTargetToken(self):
""" target space token getter
:param self: self reference
"""
if not self.targetToken:
return S_ERROR("Target token not defined")
return S_OK(self.targetToken)
def __checkTargetSE(self):
""" check target SE availability
:param self: self reference
"""
if not self.targetSE:
return S_ERROR("TargetSE not set")
res = self.oTargetSE.isValid('Write')
if not res['OK']:
return S_ERROR("TargetSE not available for writing")
res = self.__getSESpaceToken(self.oTargetSE)
if not res['OK']:
self.log.error(
"FTSRequest failed to get SRM Space Token for TargetSE",
res['Message'])
return S_ERROR("TargetSE does not support FTS transfers")
## check checksum types
if self.__cksmTest:
res = self.oTargetSE.getChecksumType()
if not res["OK"]:
self.log.error(
"Unable to get checksum type for TargetSE %s: %s" %
(self.targetSE, res["Message"]))
cksmType = res["Value"]
if cksmType in ("NONE", "NULL"):
self.log.warn(
"Checksum type set to %s at TargetSE %s, disabling checksum test"
% (cksmType, self.targetSE))
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn(
"Checksum type mismatch, disabling checksum test")
self.__cksmTest = False
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
@staticmethod
def __getSESpaceToken(oSE):
""" get space token from StorageElement instance
:param self: self reference
:param StorageElement oSE: StorageElement instance
"""
res = oSE.getStorageParameters("SRM2")
if not res['OK']:
return res
return S_OK(res['Value'].get('SpaceToken'))
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID(self, guid):
""" FTS job GUID setter
:param self: self reference
:param str guid: string containg GUID
"""
if not checkGuid(guid):
return S_ERROR("Incorrect GUID format")
self.ftsGUID = guid
return S_OK()
def getFTSGUID(self):
""" FTS job GUID getter
:param self: self refenece
"""
if not self.ftsGUID:
return S_ERROR("FTSGUID not set")
return S_OK(self.ftsGUID)
def setFTSServer(self, server):
""" FTS server setter
:param self: self reference
:param str server: FTS server URL
"""
self.ftsServer = server
return S_OK()
def getFTSServer(self):
""" FTS server getter
:param self: self reference
"""
if not self.ftsServer:
return S_ERROR("FTSServer not set")
return S_OK(self.ftsServer)
def setPriority(self, priority):
""" set priority for FTS job
:param self: self reference
:param int priority: a new priority
"""
if not type(priority) in (IntType, LongType):
return S_ERROR("Priority must be integer")
if priority < 0:
priority = 0
elif priority > 5:
priority = 5
self.priority = priority
return S_OK(self.priority)
def getPriority(self):
""" FTS job priority getter
:param self: self reference
"""
return S_OK(self.priority)
def getPercentageComplete(self):
""" get completness percentage
:param self: self reference
"""
completedFiles = 0
totalFiles = 0
for state in (self.statusSummary.keys()):
if state in self.successfulStates:
completedFiles += self.statusSummary[state]
totalFiles += self.statusSummary[state]
self.percentageComplete = (float(completedFiles) *
100.0) / float(totalFiles)
return S_OK(self.percentageComplete)
def isRequestTerminal(self):
""" check if FTS job has terminated
:param self: self reference
"""
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK(self.isTerminal)
def getStatus(self):
""" get FTS job status
:param self: self reference
"""
return S_OK(self.requestStatus)
def setCksmType(self, cksm=None):
""" set checksum type to use
:param self: self reference
:param mixed cksm: checksum type, should be one of 'Adler32', 'md5', 'sha1', None
"""
if str(cksm).upper() not in ("ADLER32", "MD5", "SHA1", "NONE"):
return S_ERROR("Not supported checksum type: %s" % str(cksm))
if not cksm:
self.__cksmType = None
return S_OK(False)
self.__cksmType = str(cksm).upper()
return S_OK(True)
def getCksmType(self):
""" get checksum type
:param self: self reference
"""
return S_OK(self.__cksmType)
def setCksmTest(self, cksmTest=False):
""" set cksm test
:param self: self reference
:param bool cksmTest: flag to enable/disable checksum test
"""
self.__cksmTest = bool(cksmTest)
return S_OK(self.__cksmTest)
def getCksmTest(self):
""" get cksm test flag
:param self: self reference
"""
return S_OK(self.__cksmTest)
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN(self, lfn):
""" add LFN :lfn: to :fileDict:
:param self: self reference
:param str lfn: LFN to add to
"""
if lfn not in self.fileDict:
self.fileDict[lfn] = {}
return S_OK()
def setSourceSURL(self, lfn, surl):
""" source SURL setter
:param self: self reference
:param str lfn: LFN
:param str surl: source SURL
"""
target = self.fileDict[lfn].get('Target')
if target == surl:
return S_ERROR("Source and target the same")
self.__setFileParameter(lfn, 'Source', surl)
return S_OK()
def getSourceSURL(self, lfn):
""" get source SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter(lfn, 'Source')
def setTargetSURL(self, lfn, surl):
""" set target SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
:param str surl: target SURL
"""
source = self.fileDict[lfn].get('Source')
if source == surl:
return S_ERROR("Source and target the same")
self.__setFileParameter(lfn, 'Target', surl)
return S_OK()
def getTargetSURL(self, lfn):
""" target SURL getter
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter(lfn, 'Target')
def getFailReason(self, lfn):
""" get fail reason for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter(lfn, 'Reason')
def getRetries(self, lfn):
""" get number of attepmts made to transfer file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter(lfn, 'Retries')
def getTransferTime(self, lfn):
""" get duration of transfer for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter(lfn, 'Duration')
def getFailed(self):
""" get list of wrongly transferred LFNs
:param self: self reference
"""
return S_OK([
lfn for lfn in self.fileDict
if self.fileDict[lfn].get('Status', '') in self.failedStates
])
def getDone(self):
""" get list of succesfully transferred LFNs
:param self: self reference
"""
return S_OK([
lfn for lfn in self.fileDict
if self.fileDict[lfn].get('Status', '') in self.successfulStates
])
def __setFileParameter(self, lfn, paramName, paramValue):
""" set :paramName: to :paramValue: for :lfn: file
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
:param mixed paramValue: a new parameter value
"""
self.setLFN(lfn)
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter(self, lfn, paramName):
""" get value of :paramName: for file :lfn:
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
"""
if lfn not in self.fileDict:
return S_ERROR("Supplied file not set")
if paramName not in self.fileDict[lfn]:
return S_ERROR("%s not set for file" % paramName)
return S_OK(self.fileDict[lfn][paramName])
####################################################################
#
# Methods for submission
#
def submit(self, monitor=False, printOutput=True):
""" submit FTS job
:param self: self reference
:param bool monitor: flag to monitor progress of FTS job
:param bool printOutput: flag to print output of execution to stdout
"""
res = self.__isSubmissionValid()
if not res['OK']:
return res
res = self.__createSURLPairFile()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = {'ftsGUID': self.ftsGUID, 'ftsServer': self.ftsServer}
print "Submitted %s @ %s" % (self.ftsGUID, self.ftsServer)
if monitor:
self.monitor(untilTerminal=True, printOutput=printOutput)
return S_OK(resDict)
def __isSubmissionValid(self):
""" check validity of job before submission
:param self: self reference
"""
if not self.fileDict:
return S_ERROR("No files set")
if not self.sourceValid:
return S_ERROR("SourceSE not valid")
if not self.targetValid:
return S_ERROR("TargetSE not valid")
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR("FTSServer not valid")
self.resolveSource()
self.resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR("No files to submit")
return S_OK()
def __getCatalogObject(self):
""" CatalogInterface instance facade
:param self: self reference
"""
try:
if not self.oCatalog:
self.oCatalog = CatalogInterface()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache(self, lfns=None, overwrite=False):
""" update replica cache for list of :lfns:
:param self: self reference
:param mixed lfns: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [
lfn for lfn in lfns
if (lfn not in self.catalogReplicas) or overwrite
]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogReplicas(toUpdate)
if not res['OK']:
return S_ERROR("Failed to update replica cache: %s" %
res['Message'])
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache(self, lfns=None, overwrite=False):
""" update metadata cache for list of LFNs
:param self: self reference
:param list lnfs: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [
lfn for lfn in lfns
if (lfn not in self.catalogMetadata) or overwrite
]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogFileMetadata(toUpdate)
if not res['OK']:
return S_ERROR("Failed to get source catalog metadata: %s" %
res['Message'])
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def resolveSource(self):
""" resolve source SE eligible for submission
:param self: self reference
"""
toResolve = [lfn for lfn in self.fileDict]
if not toResolve:
return S_OK()
res = self.__updateMetadataCache(toResolve)
if not res['OK']:
return res
res = self.__updateReplicaCache(toResolve)
if not res['OK']:
return res
for lfn in toResolve:
if self.fileDict[lfn].get("Status", "") == "Failed":
continue
replicas = self.catalogReplicas.get(lfn, {})
if self.sourceSE not in replicas:
gLogger.warn(
"resolveSource: skipping %s - not replicas at SourceSE %s"
% (lfn, self.sourceSE))
self.__setFileParameter(lfn, 'Reason',
"No replica at SourceSE")
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.oSourceSE.getPfnForProtocol(replicas[self.sourceSE],
'SRM2',
withPort=True)
if not res['OK']:
gLogger.warn("resolveSource: skipping %s - %s" %
(lfn, res["Message"]))
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.setSourceSURL(lfn, res['Value'])
if not res['OK']:
gLogger.warn("resolveSource: skipping %s - %s" %
(lfn, res["Message"]))
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
toResolve = {}
for lfn in self.fileDict:
if "Source" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Source']] = lfn
if not toResolve:
return S_ERROR("No eligible Source files")
res = self.oSourceSE.getFileMetadata(toResolve.keys())
if not res['OK']:
return S_ERROR("Failed to check source file metadata")
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
if re.search('File does not exist', error):
gLogger.warn(
"resolveSource: skipping %s - source file does not exists"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Source file does not exist")
self.__setFileParameter(lfn, 'Status', 'Failed')
else:
gLogger.warn(
"resolveSource: skipping %s - failed to get source metadata"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Failed to get Source metadata")
self.__setFileParameter(lfn, 'Status', 'Failed')
for pfn, metadata in res['Value']['Successful'].items():
lfn = toResolve[pfn]
if metadata['Unavailable']:
gLogger.warn(
"resolveSource: skipping %s - source file unavailable" %
lfn)
self.__setFileParameter(lfn, 'Reason',
"Source file Unavailable")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif metadata['Lost']:
gLogger.warn("resolveSource: skipping %s - source file lost" %
lfn)
self.__setFileParameter(lfn, 'Reason', "Source file Lost")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif not metadata['Cached']:
gLogger.warn(
"resolveSource: source file %s not cached, prestaging..." %
lfn)
stage = self.replicaManager.prestageStorageFile(
pfn, self.sourceSE, singleFile=True)
if not stage["OK"]:
gLogger.warn("resolveSource: skipping %s - %s" %
(lfn, stage["Message"]))
self.__setFileParameter(lfn, 'Reason', stage["Message"])
self.__setFileParameter(lfn, 'Status', 'Failed')
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
gLogger.warn(
"resolveSource: skipping %s - source file size mismatch" %
lfn)
self.__setFileParameter(lfn, 'Reason', "Source size mismatch")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \
not ( compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ) ):
gLogger.warn(
"resolveSource: skipping %s - source file checksum mismatch"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Source checksum mismatch")
self.__setFileParameter(lfn, 'Status', 'Failed')
return S_OK()
def resolveTarget(self):
""" find target SE eligible for submission
:param self: self reference
"""
toResolve = [lfn for lfn in self.fileDict]
if not toResolve:
return S_OK()
res = self.__updateReplicaCache(toResolve)
if not res['OK']:
return res
for lfn in toResolve:
if (self.fileDict[lfn].get('Status') == 'Failed'):
continue
res = self.oTargetSE.getPfnForLfn(lfn)
if not res['OK']:
gLogger.warn(
"resolveTarget: skipping %s - failed to create target pfn"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Failed to create Target")
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.oTargetSE.getPfnForProtocol(res['Value'],
'SRM2',
withPort=True)
if not res['OK']:
gLogger.warn("resolveTarget: skipping %s - %s" %
(lfn, res["Message"]))
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.setTargetSURL(lfn, res['Value'])
if not res['OK']:
gLogger.warn("resolveTarget: skipping %s - %s" %
(lfn, res["Message"]))
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
toResolve = {}
for lfn in self.fileDict:
if "Target" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Target']] = lfn
if not toResolve:
return S_ERROR("No eligible Target files")
res = self.oTargetSE.exists(toResolve.keys())
if not res['OK']:
return S_ERROR("Failed to check target existence")
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
toRemove = []
for pfn, exists in res['Value']['Successful'].items():
if exists:
lfn = toResolve[pfn]
res = self.getSourceSURL(lfn)
if not res['OK']:
gLogger.warn("resolveTarget: skipping %s - target exists" %
lfn)
self.__setFileParameter(lfn, 'Reason', "Target exists")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif res['Value'] == pfn:
gLogger.warn(
"resolveTarget: skipping %s - source and target pfns are teh same"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Source and Target the same")
self.__setFileParameter(lfn, 'Status', 'Failed')
else:
toRemove.append(pfn)
if toRemove:
self.oTargetSE.removeFile(toRemove)
return S_OK()
def __filesToSubmit(self):
"""
check if there is at least one file to submit
:return: S_OK if at least one file is present, S_ERROR otherwise
"""
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get('Status')
source = self.fileDict[lfn].get('Source')
target = self.fileDict[lfn].get('Target')
if (lfnStatus != 'Failed') and (lfnStatus !=
'Done') and source and target:
return S_OK()
return S_ERROR()
def __createSURLPairFile(self):
""" create LFNs file for glite-transfer-submit command
This file consists one line for each fiel to be transferred:
sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM]
:param self: self reference
"""
fd, fileName = tempfile.mkstemp()
surlFile = os.fdopen(fd, 'w')
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get('Status')
source = self.fileDict[lfn].get('Source')
target = self.fileDict[lfn].get('Target')
if (lfnStatus not in ('Failed', 'Done')) and source and target:
cksmStr = ""
## add chsmType:cksm only if cksmType is specified, else let FTS decide by itself
if self.__cksmTest and self.__cksmType:
if lfn in self.catalogMetadata and "Checksum" in self.catalogMetadata[
lfn]:
cksmStr = " %s:%s" % (self.__cksmType, self.
catalogMetadata[lfn]["Checksum"])
surlFile.write("%s %s%s\n" % (source, target, cksmStr))
surlFile.close()
self.surlFile = fileName
return S_OK()
def __submitFTSTransfer(self):
""" create and execute glite-transfer-submit CLI command
:param self: self reference
"""
comm = [
'glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile,
'-o'
]
if self.targetToken:
comm.append('-t')
comm.append(self.targetToken)
if self.sourceToken:
comm.append('-S')
comm.append(self.sourceToken)
if self.__cksmTest:
comm.append("--compare-checksums")
res = executeGridCommand('', comm)
os.remove(self.surlFile)
if not res['OK']:
return res
returnCode, output, errStr = res['Value']
if not returnCode == 0:
return S_ERROR(errStr)
guid = output.replace('\n', '')
if not checkGuid(guid):
return S_ERROR('Wrong GUID format returned')
self.ftsGUID = guid
#if self.priority != 3:
# comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)]
# executeGridCommand('',comm)
return res
def __resolveFTSServer(self):
"""
resolve FTS server to use, it should be the closest one from target SE
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR("Source SE not set")
if not self.targetSE:
return S_ERROR("Target SE not set")
res = getSitesForSE(self.sourceSE, 'LCG')
if not res['OK'] or not res['Value']:
return S_ERROR("Could not determine source site")
sourceSite = res['Value'][0]
res = getSitesForSE(self.targetSE, 'LCG')
if not res['OK'] or not res['Value']:
return S_ERROR("Could not determine target site")
targetSite = res['Value'][0]
if (sourceSite == 'LCG.CERN.ch') or (targetSite == 'LCG.CERN.ch'):
ep = 'LCG.CERN.ch'
else:
# Target site FTS server should be used
ep = targetSite
try:
configPath = '/Resources/FTSEndpoints/%s' % ep
endpointURL = gConfig.getValue(configPath)
if not endpointURL:
errStr = "FTSRequest.__resolveFTSServer: Failed to find FTS endpoint, check CS entry for '%s'." % ep
return S_ERROR(errStr)
self.ftsServer = endpointURL
return S_OK(endpointURL)
except Exception, x:
return S_ERROR(
'FTSRequest.__resolveFTSServer: Failed to obtain endpoint details from CS'
)
def resolvePFNNotRegistered( self, problematicDict ):
""" This takes the problematic dictionary returned by the integrity DB and resolved the PFNNotRegistered prognosis
"""
lfn = problematicDict['LFN']
pfn = problematicDict['PFN']
seName = problematicDict['SE']
fileID = problematicDict['FileID']
se = StorageElement( seName )
res = Utils.executeSingleFileOrDirWrapper( self.fc.exists( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if not res['Value']:
# The file does not exist in the catalog
res = Utils.executeSingleFileOrDirWrapper( se.removeFile( pfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
return self.__updateCompletedFiles( 'PFNNotRegistered', fileID )
res = Utils.executeSingleFileOrDirWrapper( se.getFileMetadata( pfn ) )
if ( not res['OK'] ) and ( re.search( 'File does not exist', res['Message'] ) ):
gLogger.info( "PFNNotRegistered replica (%d) found to be missing." % fileID )
return self.__updateCompletedFiles( 'PFNNotRegistered', fileID )
elif not res['OK']:
return self.__returnProblematicError( fileID, res )
storageMetadata = res['Value']
if storageMetadata['Lost']:
gLogger.info( "PFNNotRegistered replica (%d) found to be Lost. Updating prognosis" % fileID )
return self.changeProblematicPrognosis( fileID, 'PFNLost' )
if storageMetadata['Unavailable']:
gLogger.info( "PFNNotRegistered replica (%d) found to be Unavailable. Updating retry count" % fileID )
return self.incrementProblematicRetry( fileID )
# HACK until we can obtain the space token descriptions through GFAL
site = seName.split( '_' )[0].split( '-' )[0]
if not storageMetadata['Cached']:
if lfn.endswith( '.raw' ):
seName = '%s-RAW' % site
else:
seName = '%s-RDST' % site
elif storageMetadata['Migrated']:
if lfn.startswith( '/lhcb/data' ):
seName = '%s_M-DST' % site
else:
seName = '%s_MC_M-DST' % site
else:
if lfn.startswith( '/lhcb/data' ):
seName = '%s-DST' % site
else:
seName = '%s_MC-DST' % site
problematicDict['SE'] = seName
res = se.getPfnForProtocol( pfn, withPort = False )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
for pfn, error in res['Value']['Failed'].items():
gLogger.error( 'Failed to obtain registered PFN for physical file', '%s %s' % ( pfn, error ) )
return S_ERROR( 'Failed to obtain registered PFNs from physical file' )
problematicDict['PFN'] = res['Value']['Successful'][pfn]
res = Utils.executeSingleFileOrDirWrapper( self.fc.addReplica( {lfn:problematicDict} ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
res = Utils.executeSingleFileOrDirWrapper( self.fc.getFileMetadata( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if res['Value']['Size'] != storageMetadata['Size']:
gLogger.info( "PFNNotRegistered replica (%d) found with catalog size mismatch. Updating prognosis" % fileID )
return self.changeProblematicPrognosis( fileID, 'CatalogPFNSizeMismatch' )
return self.__updateCompletedFiles( 'PFNNotRegistered', fileID )
class FTSRequest( object ):
"""
.. class:: FTSRequest
Helper class for FTS job submission and monitoring.
"""
# # default checksum type
__defaultCksmType = "ADLER32"
# # flag to disablr/enable checksum test, default: disabled
__cksmTest = False
def __init__( self ):
"""c'tor
:param self: self reference
"""
self.log = gLogger.getSubLogger( self.__class__.__name__, True )
# # final states tuple
self.finalStates = ( 'Canceled', 'Failed', 'Hold',
'Finished', 'FinishedDirty' )
# # failed states tuple
self.failedStates = ( 'Canceled', 'Failed',
'Hold', 'FinishedDirty' )
# # successful states tuple
self.successfulStates = ( 'Finished', 'Done' )
# # all file states tuple
self.fileStates = ( 'Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed',
'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting' )
self.statusSummary = {}
# # request status
self.requestStatus = 'Unknown'
# # dict for FTS job files
self.fileDict = {}
# # dict for replicas information
self.catalogReplicas = {}
# # dict for metadata information
self.catalogMetadata = {}
# # dict for files that failed to register
self.failedRegistrations = {}
# # placehoder for FileCatalog reference
self.oCatalog = None
# # submit timestamp
self.submitTime = ''
# # placeholder FTS job GUID
self.ftsGUID = ''
# # placeholder for FTS server URL
self.ftsServer = ''
# # flag marking FTS job completness
self.isTerminal = False
# # completness percentage
self.percentageComplete = 0.0
# # source SE name
self.sourceSE = ''
# # flag marking source SE validity
self.sourceValid = False
# # source space token
self.sourceToken = ''
# # target SE name
self.targetSE = ''
# # flag marking target SE validity
self.targetValid = False
# # target space token
self.targetToken = ''
# # placeholder for target StorageElement
self.oTargetSE = None
# # placeholder for source StorageElement
self.oSourceSE = None
# # checksum type, set it to default
self.__cksmType = self.__defaultCksmType
# # disable checksum test by default
self.__cksmTest = False
# # statuses that prevent submitting to FTS
self.noSubmitStatus = ( 'Failed', 'Done', 'Staging' )
# # were sources resolved?
self.sourceResolved = False
# # Number of file transfers actually submitted
self.submittedFiles = 0
self.transferTime = 0
self.submitCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/SubmitCommand', 'glite-transfer-submit' )
self.monitorCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/MonitorCommand', 'glite-transfer-status' )
self.ftsJob = None
self.ftsFiles = []
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE( self, se ):
""" set SE for source
:param self: self reference
:param str se: source SE name
"""
if se == self.targetSE:
return S_ERROR( "SourceSE is TargetSE" )
self.sourceSE = se
self.oSourceSE = StorageElement( self.sourceSE )
return self.__checkSourceSE()
def __checkSourceSE( self ):
""" check source SE availability
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR( "SourceSE not set" )
res = self.oSourceSE.isValid( 'Read' )
if not res['OK']:
return S_ERROR( "SourceSE not available for reading" )
res = self.__getSESpaceToken( self.oSourceSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message'] )
return S_ERROR( "SourceSE does not support FTS transfers" )
if self.__cksmTest:
res = self.oSourceSE.getChecksumType()
if not res["OK"]:
self.log.error( "Unable to get checksum type for SourceSE %s: %s" % ( self.sourceSE,
res["Message"] ) )
cksmType = res["Value"]
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at SourceSE %s, disabling checksum test" % ( cksmType,
self.sourceSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE( self, se ):
""" set target SE
:param self: self reference
:param str se: target SE name
"""
if se == self.sourceSE:
return S_ERROR( "TargetSE is SourceSE" )
self.targetSE = se
self.oTargetSE = StorageElement( self.targetSE )
return self.__checkTargetSE()
def setTargetToken( self, token ):
""" target space token setter
:param self: self reference
:param str token: target space token
"""
self.targetToken = token
return S_OK()
def __checkTargetSE( self ):
""" check target SE availability
:param self: self reference
"""
if not self.targetSE:
return S_ERROR( "TargetSE not set" )
res = self.oTargetSE.isValid( 'Write' )
if not res['OK']:
return S_ERROR( "TargetSE not available for writing" )
res = self.__getSESpaceToken( self.oTargetSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message'] )
return S_ERROR( "TargetSE does not support FTS transfers" )
# # check checksum types
if self.__cksmTest:
res = self.oTargetSE.getChecksumType()
if not res["OK"]:
self.log.error( "Unable to get checksum type for TargetSE %s: %s" % ( self.targetSE,
res["Message"] ) )
cksmType = res["Value"]
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at TargetSE %s, disabling checksum test" % ( cksmType,
self.targetSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
@staticmethod
def __getSESpaceToken( oSE ):
""" get space token from StorageElement instance
:param self: self reference
:param StorageElement oSE: StorageElement instance
"""
res = oSE.getStorageParameters( "SRM2" )
if not res['OK']:
return res
return S_OK( res['Value'].get( 'SpaceToken' ) )
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID( self, guid ):
""" FTS job GUID setter
:param self: self reference
:param str guid: string containg GUID
"""
if not checkGuid( guid ):
return S_ERROR( "Incorrect GUID format" )
self.ftsGUID = guid
return S_OK()
def setFTSServer( self, server ):
""" FTS server setter
:param self: self reference
:param str server: FTS server URL
"""
self.ftsServer = server
return S_OK()
def isRequestTerminal( self ):
""" check if FTS job has terminated
:param self: self reference
"""
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK( self.isTerminal )
def setCksmTest( self, cksmTest = False ):
""" set cksm test
:param self: self reference
:param bool cksmTest: flag to enable/disable checksum test
"""
self.__cksmTest = bool( cksmTest )
return S_OK( self.__cksmTest )
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN( self, lfn ):
""" add LFN :lfn: to :fileDict:
:param self: self reference
:param str lfn: LFN to add to
"""
self.fileDict.setdefault( lfn, {'Status':'Waiting'} )
return S_OK()
def setSourceSURL( self, lfn, surl ):
""" source SURL setter
:param self: self reference
:param str lfn: LFN
:param str surl: source SURL
"""
target = self.fileDict[lfn].get( 'Target' )
if target == surl:
return S_ERROR( "Source and target the same" )
return self.__setFileParameter( lfn, 'Source', surl )
def getSourceSURL( self, lfn ):
""" get source SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Source' )
def setTargetSURL( self, lfn, surl ):
""" set target SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
:param str surl: target SURL
"""
source = self.fileDict[lfn].get( 'Source' )
if source == surl:
return S_ERROR( "Source and target the same" )
return self.__setFileParameter( lfn, 'Target', surl )
def getFailReason( self, lfn ):
""" get fail reason for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Reason' )
def getRetries( self, lfn ):
""" get number of attepmts made to transfer file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Retries' )
def getTransferTime( self, lfn ):
""" get duration of transfer for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Duration' )
def getFailed( self ):
""" get list of wrongly transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.failedStates ] )
def getStaging( self ):
""" get files set for prestaging """
return S_OK( [lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) == 'Staging'] )
def getDone( self ):
""" get list of succesfully transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.successfulStates ] )
def __setFileParameter( self, lfn, paramName, paramValue ):
""" set :paramName: to :paramValue: for :lfn: file
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
:param mixed paramValue: a new parameter value
"""
self.setLFN( lfn )
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter( self, lfn, paramName ):
""" get value of :paramName: for file :lfn:
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
"""
if lfn not in self.fileDict:
return S_ERROR( "Supplied file not set" )
if paramName not in self.fileDict[lfn]:
return S_ERROR( "%s not set for file" % paramName )
return S_OK( self.fileDict[lfn][paramName] )
####################################################################
#
# Methods for submission
#
def submit( self, monitor = False, printOutput = True ):
""" submit FTS job
:param self: self reference
:param bool monitor: flag to monitor progress of FTS job
:param bool printOutput: flag to print output of execution to stdout
"""
res = self.__prepareForSubmission()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = { 'ftsGUID' : self.ftsGUID, 'ftsServer' : self.ftsServer, 'submittedFiles' : self.submittedFiles }
if monitor or printOutput:
gLogger.always( "Submitted %s@%s" % ( self.ftsGUID, self.ftsServer ) )
if monitor:
self.monitor( untilTerminal = True, printOutput = printOutput, full = False )
return S_OK( resDict )
def __prepareForSubmission( self ):
""" check validity of job before submission
:param self: self reference
"""
if not self.fileDict:
return S_ERROR( "No files set" )
if not self.sourceValid:
return S_ERROR( "SourceSE not valid" )
if not self.targetValid:
return S_ERROR( "TargetSE not valid" )
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR( "FTSServer not valid" )
self.resolveSource()
self.resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR( "No files to submit" )
return S_OK()
def __getCatalogObject( self ):
""" CatalogInterface instance facade
:param self: self reference
"""
try:
if not self.oCatalog:
self.oCatalog = FileCatalog()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache( self, lfns = None, overwrite = False ):
""" update replica cache for list of :lfns:
:param self: self reference
:param mixed lfns: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if ( lfn not in self.catalogReplicas ) or overwrite ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getReplicas( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to update replica cache: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache( self, lfns = None ):
""" update metadata cache for list of LFNs
:param self: self reference
:param list lnfs: list of LFNs
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if lfn not in self.catalogMetadata ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getFileMetadata( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to get source catalog metadata: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def resolveSource( self ):
""" resolve source SE eligible for submission
:param self: self reference
"""
# Avoid resolving sources twice
if self.sourceResolved:
return S_OK()
# Only resolve files that need a transfer
toResolve = [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( "Status", "" ) != "Failed" ]
if not toResolve:
return S_OK()
res = self.__updateMetadataCache( toResolve )
if not res['OK']:
return res
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
# Define the source URLs
for lfn in toResolve:
replicas = self.catalogReplicas.get( lfn, {} )
if self.sourceSE not in replicas:
gLogger.warn( "resolveSource: skipping %s - not replicas at SourceSE %s" % ( lfn, self.sourceSE ) )
self.__setFileParameter( lfn, 'Reason', "No replica at SourceSE" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
# Fix first the PFN
pfn = self.oSourceSE.getPfnForLfn( lfn ).get( 'Value', {} ).get( 'Successful', {} ).get( lfn, replicas[self.sourceSE] )
res = returnSingleResult( self.oSourceSE.getPfnForProtocol( pfn, protocol = 'SRM2', withPort = True ) )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setSourceSURL( lfn, res['Value'] )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict:
if "Source" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Source']] = lfn
if not toResolve:
return S_ERROR( "No eligible Source files" )
# Get metadata of the sources, to check for existance, availability and caching
res = self.oSourceSE.getFileMetadata( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check source file metadata" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
if re.search( 'File does not exist', error ):
gLogger.warn( "resolveSource: skipping %s - source file does not exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file does not exist" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
gLogger.warn( "resolveSource: skipping %s - failed to get source metadata" % lfn )
self.__setFileParameter( lfn, 'Reason', "Failed to get Source metadata" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toStage = []
nbStagedFiles = 0
for pfn, metadata in res['Value']['Successful'].items():
lfn = toResolve[pfn]
lfnStatus = self.fileDict.get( lfn, {} ).get( 'Status' )
if metadata['Unavailable']:
gLogger.warn( "resolveSource: skipping %s - source file unavailable" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Unavailable" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif metadata['Lost']:
gLogger.warn( "resolveSource: skipping %s - source file lost" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Lost" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif not metadata['Cached']:
if lfnStatus != 'Staging':
toStage.append( pfn )
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
gLogger.warn( "resolveSource: skipping %s - source file size mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source size mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \
not compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ):
gLogger.warn( "resolveSource: skipping %s - source file checksum mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source checksum mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif lfnStatus == 'Staging':
# file that was staging is now cached
self.__setFileParameter( lfn, 'Status', 'Waiting' )
nbStagedFiles += 1
# Some files were being staged
if nbStagedFiles:
self.log.info( 'resolveSource: %d files have been staged' % nbStagedFiles )
# Launching staging of files not in cache
if toStage:
gLogger.warn( "resolveSource: %s source files not cached, prestaging..." % len( toStage ) )
stage = self.oSourceSE.prestageFile( toStage )
if not stage["OK"]:
gLogger.error( "resolveSource: error is prestaging - %s" % stage["Message"] )
for pfn in toStage:
lfn = toResolve[pfn]
self.__setFileParameter( lfn, 'Reason', stage["Message"] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
for pfn in toStage:
lfn = toResolve[pfn]
if pfn in stage['Value']['Successful']:
self.__setFileParameter( lfn, 'Status', 'Staging' )
elif pfn in stage['Value']['Failed']:
self.__setFileParameter( lfn, 'Reason', stage['Value']['Failed'][pfn] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
self.sourceResolved = True
return S_OK()
def resolveTarget( self ):
""" find target SE eligible for submission
:param self: self reference
"""
toResolve = [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status' ) not in self.noSubmitStatus ]
if not toResolve:
return S_OK()
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
for lfn in toResolve:
res = self.oTargetSE.getPfnForLfn( lfn )
if not res['OK'] or lfn not in res['Value']['Successful']:
gLogger.warn( "resolveTarget: skipping %s - failed to create target pfn" % lfn )
self.__setFileParameter( lfn, 'Reason', "Failed to create Target" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
pfn = res['Value']['Successful'][lfn]
res = self.oTargetSE.getPfnForProtocol( pfn, protocol = 'SRM2', withPort = True )
if not res['OK'] or pfn not in res['Value']['Successful']:
reason = res.get( 'Message', res.get( 'Value', {} ).get( 'Failed', {} ).get( pfn ) )
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, reason ) )
self.__setFileParameter( lfn, 'Reason', reason )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
pfn = res['Value']['Successful'][pfn]
res = self.setTargetSURL( lfn, pfn )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict:
if "Target" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Target']] = lfn
if not toResolve:
return S_ERROR( "No eligible Target files" )
res = self.oTargetSE.exists( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check target existence" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toRemove = []
for pfn, exists in res['Value']['Successful'].items():
if exists:
lfn = toResolve[pfn]
res = self.getSourceSURL( lfn )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - target exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Target exists" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif res['Value'] == pfn:
gLogger.warn( "resolveTarget: skipping %s - source and target pfns are the same" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source and Target the same" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRemove.append( pfn )
if toRemove:
self.oTargetSE.removeFile( toRemove )
return S_OK()
def __filesToSubmit( self ):
"""
check if there is at least one file to submit
:return: S_OK if at least one file is present, S_ERROR otherwise
"""
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if lfnStatus not in self.noSubmitStatus and source and target:
return S_OK()
return S_ERROR()
def __createFTSFiles( self ):
""" create LFNs file for glite-transfer-submit command
This file consists one line for each fiel to be transferred:
sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM]
:param self: self reference
"""
self.__updateMetadataCache()
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
if lfnStatus not in self.noSubmitStatus:
cksmStr = ""
# # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself
if self.__cksmTest and self.__cksmType:
checkSum = self.catalogMetadata.get( lfn, {} ).get( 'Checksum' )
if checkSum:
cksmStr = " %s:%s" % ( self.__cksmType, intAdlerToHex( hexAdlerToInt( checkSum ) ) )
ftsFile = FTSFile()
ftsFile.LFN = lfn
ftsFile.SourceSURL = self.fileDict[lfn].get( 'Source' )
ftsFile.TargetSURL = self.fileDict[lfn].get( 'Target' )
ftsFile.SourceSE = self.sourceSE
ftsFile.TargetSE = self.targetSE
ftsFile.Status = self.fileDict[lfn].get( 'Status' )
ftsFile.Checksum = cksmStr
ftsFile.Size = self.catalogMetadata.get( lfn, {} ).get( 'Size' )
self.ftsFiles.append( ftsFile )
self.submittedFiles += 1
return S_OK()
def __createFTSJob( self, guid = None ):
self.__createFTSFiles()
ftsJob = FTSJob()
ftsJob.RequestID = 0
ftsJob.OperationID = 0
ftsJob.SourceSE = self.sourceSE
ftsJob.TargetSE = self.targetSE
ftsJob.SourceToken = self.sourceToken
ftsJob.TargetToken = self.targetToken
ftsJob.FTSServer = self.ftsServer
if guid:
ftsJob.FTSGUID = guid
for ftsFile in self.ftsFiles:
ftsFile.Attempt += 1
ftsFile.Error = ""
ftsJob.addFile( ftsFile )
self.ftsJob = ftsJob
def __submitFTSTransfer( self ):
""" create and execute glite-transfer-submit CLI command
:param self: self reference
"""
log = gLogger.getSubLogger( 'Submit' )
self.__createFTSJob()
submit = self.ftsJob.submitFTS2( command = self.submitCommand )
if not submit["OK"]:
log.error( "unable to submit FTSJob: %s" % submit["Message"] )
return submit
log.info( "FTSJob '%s'@'%s' has been submitted" % ( self.ftsJob.FTSGUID, self.ftsJob.FTSServer ) )
# # update statuses for job files
for ftsFile in self.ftsJob:
ftsFile.FTSGUID = self.ftsJob.FTSGUID
ftsFile.Status = "Submitted"
ftsFile.Attempt += 1
log.info( "FTSJob '%s'@'%s' has been submitted" % ( self.ftsJob.FTSGUID, self.ftsJob.FTSServer ) )
self.ftsGUID = self.ftsJob.FTSGUID
return S_OK()
def __resolveFTSServer( self ):
"""
resolve FTS server to use, it should be the closest one from target SE
:param self: self reference
"""
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTSServersForSites
if not self.targetSE:
return S_ERROR( "Target SE not set" )
res = getSitesForSE( self.targetSE )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine target site" )
targetSites = res['Value']
targetSite = ''
for targetSite in targetSites:
targetFTS = getFTSServersForSites( [targetSite] )
if targetFTS['OK']:
ftsTarget = targetFTS['Value'][targetSite]
if ftsTarget:
self.ftsServer = ftsTarget
return S_OK( self.ftsServer )
else:
return targetFTS
return S_ERROR( 'No FTS server found for %s' % targetSite )
####################################################################
#
# Methods for monitoring
#
def summary( self, untilTerminal = False, printOutput = False ):
""" summary of FTS job
:param self: self reference
:param bool untilTerminal: flag to monitor FTS job to its final state
:param bool printOutput: flag to print out monitoring information to the stdout
"""
res = self.__isSummaryValid()
if not res['OK']:
return res
while not self.isTerminal:
res = self.__parseOutput( full = True )
if not res['OK']:
return res
if untilTerminal:
self.__print()
self.isRequestTerminal()
if res['Value'] or ( not untilTerminal ):
break
time.sleep( 1 )
if untilTerminal:
print ""
if printOutput and ( not untilTerminal ):
return self.dumpSummary( printOutput = printOutput )
return S_OK()
def monitor( self, untilTerminal = False, printOutput = False, full = True ):
""" monitor FTS job
:param self: self reference
:param bool untilTerminal: flag to monitor FTS job to its final state
:param bool printOutput: flag to print out monitoring information to the stdout
"""
if not self.ftsJob:
self.resolveSource()
self.__createFTSJob( self.ftsGUID )
res = self.__isSummaryValid()
if not res['OK']:
return res
if untilTerminal:
res = self.summary( untilTerminal = untilTerminal, printOutput = printOutput )
if not res['OK']:
return res
res = self.__parseOutput( full = full )
if not res['OK']:
return res
if untilTerminal:
self.finalize()
if printOutput:
self.dump()
return res
def dumpSummary( self, printOutput = False ):
""" get FTS job summary as str
:param self: self reference
:param bool printOutput: print summary to stdout
"""
outStr = ''
for status in sorted( self.statusSummary ):
if self.statusSummary[status]:
outStr = '%s\t%-10s : %-10s\n' % ( outStr, status, str( self.statusSummary[status] ) )
outStr = outStr.rstrip( '\n' )
if printOutput:
print outStr
return S_OK( outStr )
def __print( self ):
""" print progress bar of FTS job completeness to stdout
:param self: self reference
"""
width = 100
bits = int( ( width * self.percentageComplete ) / 100 )
outStr = "|%s>%s| %.1f%s %s %s" % ( "="*bits, " "*( width - bits ),
self.percentageComplete, "%",
self.requestStatus, " "*10 )
sys.stdout.write( "%s\r" % ( outStr ) )
sys.stdout.flush()
def dump( self ):
""" print FTS job parameters and files to stdout
:param self: self reference
"""
print "%-10s : %-10s" % ( "Status", self.requestStatus )
print "%-10s : %-10s" % ( "Source", self.sourceSE )
print "%-10s : %-10s" % ( "Target", self.targetSE )
print "%-10s : %-128s" % ( "Server", self.ftsServer )
print "%-10s : %-128s" % ( "GUID", self.ftsGUID )
for lfn in sorted( self.fileDict ):
print "\n %-15s : %-128s" % ( 'LFN', lfn )
for key in ['Source', 'Target', 'Status', 'Reason', 'Duration']:
print " %-15s : %-128s" % ( key, str( self.fileDict[lfn].get( key ) ) )
return S_OK()
def __isSummaryValid( self ):
""" check validity of FTS job summary report
:param self: self reference
"""
if not self.ftsServer:
return S_ERROR( "FTSServer not set" )
if not self.ftsGUID:
return S_ERROR( "FTSGUID not set" )
return S_OK()
def __parseOutput( self, full = False ):
""" execute glite-transfer-status command and parse its output
:param self: self reference
:param bool full: glite-transfer-status verbosity level, when set, collect information of files as well
"""
monitor = self.ftsJob.monitorFTS2( command = self.monitorCommand, full = full )
if not monitor['OK']:
return monitor
self.percentageComplete = self.ftsJob.Completeness
self.requestStatus = self.ftsJob.Status
self.submitTime = self.ftsJob.SubmitTime
statusSummary = monitor['Value']
if statusSummary:
for state in statusSummary:
self.statusSummary[state] = statusSummary[state]
self.transferTime = 0
for ftsFile in self.ftsJob:
lfn = ftsFile.LFN
self.__setFileParameter( lfn, 'Status', ftsFile.Status )
self.__setFileParameter( lfn, 'Reason', ftsFile.Error )
self.__setFileParameter( lfn, 'Duration', ftsFile._duration )
targetURL = self.__getFileParameter( lfn, 'Target' )
if not targetURL['OK']:
self.__setFileParameter( lfn, 'Target', ftsFile.TargetSURL )
self.transferTime += int( ftsFile._duration )
return S_OK()
####################################################################
#
# Methods for finalization
#
def finalize( self ):
""" finalize FTS job
:param self: self reference
"""
self.__updateMetadataCache()
transEndTime = dateTime()
regStartTime = time.time()
res = self.getTransferStatistics()
transDict = res['Value']
res = self.__registerSuccessful( transDict['transLFNs'] )
regSuc, regTotal = res['Value']
regTime = time.time() - regStartTime
if self.sourceSE and self.targetSE:
self.__sendAccounting( regSuc, regTotal, regTime, transEndTime, transDict )
return S_OK()
def getTransferStatistics( self ):
""" collect information of Transfers that can be used by Accounting
:param self: self reference
"""
transDict = { 'transTotal': len( self.fileDict ),
'transLFNs': [],
'transOK': 0,
'transSize': 0 }
for lfn in self.fileDict:
if self.fileDict[lfn].get( 'Status' ) in self.successfulStates:
if self.fileDict[lfn].get( 'Duration', 0 ):
transDict['transLFNs'].append( lfn )
transDict['transOK'] += 1
if lfn in self.catalogMetadata:
transDict['transSize'] += self.catalogMetadata[lfn].get( 'Size', 0 )
return S_OK( transDict )
def getFailedRegistrations( self ):
""" get failed registrations dict
:param self: self reference
"""
return S_OK( self.failedRegistrations )
def __registerSuccessful( self, transLFNs ):
""" register successfully transferred files to the catalogs,
fill failedRegistrations dict for files that failed to register
:param self: self reference
:param list transLFNs: LFNs in FTS job
"""
self.failedRegistrations = {}
toRegister = {}
for lfn in transLFNs:
res = returnSingleResult( self.oTargetSE.getPfnForProtocol( self.fileDict[lfn].get( 'Target' ), protocol = 'SRM2', withPort = False ) )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRegister[lfn] = { 'PFN' : res['Value'], 'SE' : self.targetSE }
if not toRegister:
return S_OK( ( 0, 0 ) )
res = self.__getCatalogObject()
if not res['OK']:
for lfn in toRegister:
self.failedRegistrations = toRegister
self.log.error( 'Failed to get Catalog Object', res['Message'] )
return S_OK( ( 0, len( toRegister ) ) )
res = self.oCatalog.addReplica( toRegister )
if not res['OK']:
self.failedRegistrations = toRegister
self.log.error( 'Failed to get Catalog Object', res['Message'] )
return S_OK( ( 0, len( toRegister ) ) )
for lfn, error in res['Value']['Failed'].items():
self.failedRegistrations[lfn] = toRegister[lfn]
self.log.error( 'Registration of Replica failed', '%s : %s' % ( lfn, str( error ) ) )
return S_OK( ( len( res['Value']['Successful'] ), len( toRegister ) ) )
def __sendAccounting( self, regSuc, regTotal, regTime, transEndTime, transDict ):
""" send accounting record
:param self: self reference
:param regSuc: number of files successfully registered
:param regTotal: number of files attepted to register
:param regTime: time stamp at the end of registration
:param transEndTime: time stamp at the end of FTS job
:param dict transDict: dict holding couters for files being transerred, their sizes and successfull transfers
"""
oAccounting = DataOperation()
oAccounting.setEndTime( transEndTime )
oAccounting.setStartTime( self.submitTime )
accountingDict = {}
accountingDict['OperationType'] = 'replicateAndRegister'
result = getProxyInfo()
if not result['OK']:
userName = '******'
else:
userName = result['Value'].get( 'username', 'unknown' )
accountingDict['User'] = userName
accountingDict['Protocol'] = 'FTS' if 'fts3' not in self.ftsServer else 'FTS3'
accountingDict['RegistrationTime'] = regTime
accountingDict['RegistrationOK'] = regSuc
accountingDict['RegistrationTotal'] = regTotal
accountingDict['TransferOK'] = transDict['transOK']
accountingDict['TransferTotal'] = transDict['transTotal']
accountingDict['TransferSize'] = transDict['transSize']
accountingDict['FinalStatus'] = self.requestStatus
accountingDict['Source'] = self.sourceSE
accountingDict['Destination'] = self.targetSE
accountingDict['TransferTime'] = self.transferTime
oAccounting.setValuesFromDict( accountingDict )
self.log.verbose( "Attempting to commit accounting message..." )
oAccounting.commit()
self.log.verbose( "...committed." )
return S_OK()
def filterReplicas( opFile, logger = None, dataManager = None ):
""" filter out banned/invalid source SEs """
if logger is None:
logger = gLogger
if dataManager is None:
dataManager = DataManager()
log = logger.getSubLogger( "filterReplicas" )
ret = { "Valid" : [], "NoMetadata" : [], "Bad" : [], 'NoReplicas':[], 'NoPFN':[] }
replicas = dataManager.getActiveReplicas( opFile.LFN )
if not replicas["OK"]:
log.error( 'Failed to get active replicas', replicas["Message"] )
return replicas
reNotExists = re.compile( r".*such file.*" )
replicas = replicas["Value"]
failed = replicas["Failed"].get( opFile.LFN , "" )
if reNotExists.match( failed.lower() ):
opFile.Status = "Failed"
opFile.Error = failed
return S_ERROR( failed )
replicas = replicas["Successful"].get( opFile.LFN, {} )
if not opFile.Checksum:
# Set Checksum to FC checksum if not set in the request
fcMetadata = FileCatalog().getFileMetadata( opFile.LFN )
fcChecksum = fcMetadata.get( 'Value', {} ).get( 'Successful', {} ).get( opFile.LFN, {} ).get( 'Checksum', '' )
# Replace opFile.Checksum if it doesn't match a valid FC checksum
if fcChecksum:
opFile.Checksum = fcChecksum
opFile.ChecksumType = fcMetadata['Value']['Successful'][opFile.LFN].get( 'ChecksumType', 'Adler32' )
for repSEName in replicas:
repSE = StorageElement( repSEName )
repSEMetadata = repSE.getFileMetadata( opFile.LFN )
error = repSEMetadata.get( 'Message', repSEMetadata.get( 'Value', {} ).get( 'Failed', {} ).get( opFile.LFN ) )
if error:
log.warn( 'unable to get metadata at %s for %s' % ( repSEName, opFile.LFN ), error.replace( '\n', '' ) )
if 'File does not exist' in error:
ret['NoReplicas'].append( repSEName )
else:
ret["NoMetadata"].append( repSEName )
else:
repSEMetadata = repSEMetadata['Value']['Successful'][opFile.LFN]
seChecksum = repSEMetadata.get( "Checksum" )
if ( opFile.Checksum and seChecksum and compareAdler( seChecksum, opFile.Checksum ) ) or\
( not opFile.Checksum and not seChecksum ):
# # All checksums are OK
ret["Valid"].append( repSEName )
else:
log.warn( " %s checksum mismatch, FC: '%s' @%s: '%s'" % ( opFile.LFN,
opFile.Checksum,
repSEName,
seChecksum ) )
ret["Bad"].append( repSEName )
return S_OK( ret )
def compareChecksum(self, lfns):
"""compare the checksum of the file in the FC and the checksum of the physical replicas.
Returns a dictionary containing 3 sub-dictionaries: one with files with missing PFN, one with
files with all replicas corrupted, and one with files with some replicas corrupted and at least
one good replica
"""
retDict = {
"AllReplicasCorrupted": {},
"SomeReplicasCorrupted": {},
"MissingReplica": {},
"MissingAllReplicas": {},
"NoReplicas": {},
}
chunkSize = 100
replicas = {}
setLfns = set(lfns)
cachedLfns = setLfns & set(self.cachedReplicas)
for lfn in cachedLfns:
replicas[lfn] = self.cachedReplicas[lfn]
lfnsLeft = list(setLfns - cachedLfns)
if lfnsLeft:
self.__write("Get replicas for %d files (chunks of %d): " %
(len(lfnsLeft), chunkSize))
for lfnChunk in breakListIntoChunks(lfnsLeft, chunkSize):
self.__write(".")
replicasRes = self.fileCatalog.getReplicas(lfnChunk)
if not replicasRes["OK"]:
gLogger.error("error: %s" % replicasRes["Message"])
return S_ERROR(errno.ENOENT,
"error: %s" % replicasRes["Message"])
replicasRes = replicasRes["Value"]
if replicasRes["Failed"]:
retDict["NoReplicas"].update(replicasRes["Failed"])
replicas.update(replicasRes["Successful"])
self.__write("Get FC metadata for %d files to be checked: " %
len(lfns))
metadata = {}
for lfnChunk in breakListIntoChunks(replicas, chunkSize):
self.__write(".")
res = self.fileCatalog.getFileMetadata(lfnChunk)
if not res["OK"]:
return S_ERROR(errno.ENOENT, "error %s" % res["Message"])
metadata.update(res["Value"]["Successful"])
gLogger.notice("Check existence and compare checksum file by file...")
csDict = {}
seFiles = {}
# Reverse the LFN->SE dictionary
nReps = 0
for lfn in replicas:
csDict.setdefault(lfn, {})["FCChecksum"] = metadata.get(
lfn, {}).get("Checksum")
for se in replicas[lfn]:
seFiles.setdefault(se, []).append(lfn)
nReps += 1
gLogger.notice("Getting checksum of %d replicas in %d SEs" %
(nReps, len(seFiles)))
checkSum = {}
lfnNotExisting = {}
lfnNoInfo = {}
logLevel = gLogger.getLevel()
gLogger.setLevel("FATAL")
for num, se in enumerate(sorted(seFiles)):
self.__write("\n%d. At %s (%d files): " %
(num, se, len(seFiles[se])))
oSe = StorageElement(se)
notFound = 0
for surlChunk in breakListIntoChunks(seFiles[se], chunkSize):
self.__write(".")
metadata = oSe.getFileMetadata(surlChunk)
if not metadata["OK"]:
gLogger.error(
"Error: getFileMetadata returns %s. Ignore those replicas"
% (metadata["Message"]))
# Remove from list of replicas as we don't know whether it is OK or
# not
for lfn in seFiles[se]:
lfnNoInfo.setdefault(lfn, []).append(se)
else:
metadata = metadata["Value"]
notFound += len(metadata["Failed"])
for lfn in metadata["Failed"]:
lfnNotExisting.setdefault(lfn, []).append(se)
for lfn in metadata["Successful"]:
checkSum.setdefault(
lfn,
{})[se] = metadata["Successful"][lfn]["Checksum"]
if notFound:
gLogger.error("%d files not found" % notFound)
gLogger.setLevel(logLevel)
gLogger.notice("Verifying checksum of %d files" % len(replicas))
for lfn in replicas:
# get the lfn checksum from the FC
replicaDict = replicas[lfn]
oneGoodReplica = False
allGoodReplicas = True
fcChecksum = csDict[lfn].pop("FCChecksum")
for se in replicaDict:
# If replica doesn't exist skip check
if se in lfnNotExisting.get(lfn, []):
allGoodReplicas = False
continue
if se in lfnNoInfo.get(lfn, []):
# If there is no info, a priori it could be good
oneGoodReplica = True
continue
# get the surls metadata and compare the checksum
surlChecksum = checkSum.get(lfn, {}).get(se, "")
if not surlChecksum or not compareAdler(
fcChecksum, surlChecksum):
# if fcChecksum does not match surlChecksum
csDict[lfn][se] = {"PFNChecksum": surlChecksum}
gLogger.info(
"ERROR!! checksum mismatch at %s for LFN %s: FC checksum: %s , PFN checksum : %s "
% (se, lfn, fcChecksum, surlChecksum))
allGoodReplicas = False
else:
oneGoodReplica = True
if not oneGoodReplica:
if lfn in lfnNotExisting:
gLogger.info("=> All replicas are missing", lfn)
retDict["MissingAllReplicas"][lfn] = "All"
else:
gLogger.info("=> All replicas have bad checksum", lfn)
retDict["AllReplicasCorrupted"][lfn] = csDict[lfn]
elif not allGoodReplicas:
if lfn in lfnNotExisting:
gLogger.info("=> At least one replica missing", lfn)
retDict["MissingReplica"][lfn] = lfnNotExisting[lfn]
else:
gLogger.info("=> At least one replica with good Checksum",
lfn)
retDict["SomeReplicasCorrupted"][lfn] = csDict[lfn]
return S_OK(retDict)
class FTSRequest:
def __init__(self):
self.gridEnv = '/afs/cern.ch/project/gd/LCG-share/3.2.8-0/etc/profile.d/grid-env'
self.finalStates = [
'Canceled', 'Failed', 'Hold', 'Finished', 'FinishedDirty'
]
self.failedStates = ['Canceled', 'Failed', 'Hold', 'FinishedDirty']
self.successfulStates = ['Finished', 'Done']
self.fileStates = [
'Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed',
'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting'
]
self.newlyCompletedFiles = []
self.newlyFailedFiles = []
self.statusSummary = {}
self.requestStatus = 'Unknown'
self.fileDict = {}
self.catalogReplicas = {}
self.catalogMetadata = {}
self.oCatalog = None
self.submitTime = ''
self.ftsGUID = ''
self.ftsServer = ''
self.priority = 3
self.isTerminal = False
self.percentageComplete = 0.0
self.sourceSE = ''
self.sourceValid = False
self.sourceToken = ''
self.targetSE = ''
self.targetValid = False
self.targetToken = ''
self.dumpStr = ''
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE(self, se):
if se == self.targetSE:
return S_ERROR("SourceSE is TargetSE")
self.sourceSE = se
self.oSourceSE = StorageElement(self.sourceSE)
return self.__checkSourceSE()
def getSourceSE(self):
if not self.sourceSE:
return S_ERROR("Source SE not defined")
return S_OK(self.sourceSE)
def setSourceToken(self, token):
self.sourceToken = token
return S_OK()
def getSourceToken(self):
if not self.sourceToken:
return S_ERROR("Source token not defined")
return S_OK(self.sourceToken)
def __checkSourceSE(self):
if not self.sourceSE:
return S_ERROR("SourceSE not set")
res = self.oSourceSE.isValid('Read')
if not res['OK']:
return S_ERROR("SourceSE not available for reading")
res = self.__getSESpaceToken(self.oSourceSE)
if not res['OK']:
gLogger.error(
"FTSRequest failed to get SRM Space Token for SourceSE",
res['Message'])
return S_ERROR("SourceSE does not support FTS transfers")
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE(self, se):
if se == self.sourceSE:
return S_ERROR("TargetSE is SourceSE")
self.targetSE = se
self.oTargetSE = StorageElement(self.targetSE)
return self.__checkTargetSE()
def getTargetSE(self):
if not self.targetSE:
return S_ERROR("Target SE not defined")
return S_OK(self.targetSE)
def setTargetToken(self, token):
self.targetToken = token
return S_OK()
def getTargetToken(self):
if not self.targetToken:
return S_ERROR("Target token not defined")
return S_OK(self.targetToken)
def __checkTargetSE(self):
if not self.targetSE:
return S_ERROR("TargetSE not set")
res = self.oTargetSE.isValid('Write')
if not res['OK']:
return S_ERROR("TargetSE not available for writing")
res = self.__getSESpaceToken(self.oTargetSE)
if not res['OK']:
gLogger.error(
"FTSRequest failed to get SRM Space Token for TargetSE",
res['Message'])
return S_ERROR("SourceSE does not support FTS transfers")
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
def __getSESpaceToken(self, oSE):
res = oSE.getStorageParameters("SRM2")
if not res['OK']:
return res
return S_OK(res['Value'].get('SpaceToken'))
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID(self, guid):
if not checkGuid(guid):
return S_ERROR("Incorrect GUID format")
self.ftsGUID = guid
return S_OK()
def getFTSGUID(self):
if not self.ftsGUID:
return S_ERROR("FTSGUID not set")
return S_OK(self.ftsGUID)
def setFTSServer(self, server):
self.ftsServer = server
return S_OK()
def getFTSServer(self):
if not self.ftsServer:
return S_ERROR("FTSServer not set")
return S_OK(self.ftsServer)
def setPriority(self, priority):
if not type(priority) in [types.IntType, types.LongType]:
return S_ERROR("Priority must be integer")
if priority < 0:
priority = 0
elif priority > 5:
priority = 5
self.priority = priority
return S_OK(self.priority)
def getPriority(self):
return S_OK(self.priority)
def getPercentageComplete(self):
completedFiles = 0
totalFiles = 0
for state in (self.statusSummary.keys()):
if state in self.successfulStates:
completedFiles += self.statusSummary[state]
totalFiles += self.statusSummary[state]
self.percentageComplete = (float(completedFiles) *
100.0) / float(totalFiles)
return S_OK(self.percentageComplete)
def isRequestTerminal(self):
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK(self.isTerminal)
def getStatus(self):
return S_OK(self.requestStatus)
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN(self, lfn):
if not self.fileDict.has_key(lfn):
self.fileDict[lfn] = {}
return S_OK()
def setSourceSURL(self, lfn, surl):
target = self.fileDict[lfn].get('Target')
if target == surl:
return S_ERROR("Source and target the same")
self.__setFileParameter(lfn, 'Source', surl)
return S_OK()
def getSourceSURL(self, lfn):
return self.__getFileParameter(lfn, 'Source')
def setTargetSURL(self, lfn, surl):
source = self.fileDict[lfn].get('Source')
if source == surl:
return S_ERROR("Source and target the same")
self.__setFileParameter(lfn, 'Target', surl)
return S_OK()
def getTargetSURL(self, lfn):
return self.__getFileParameter(lfn, 'Target')
def getFailReason(self, lfn):
return self.__getFileParameter(lfn, 'Reason')
def getRetries(self, lfn):
return self.__getFileParameter(lfn, 'Retries')
def getTransferTime(self, lfn):
return self.__getFileParameter(lfn, 'Duration')
def getFailed(self):
failed = []
for lfn in self.fileDict.keys():
status = self.fileDict[lfn].get('Status', '')
if status in self.failedStates:
failed.append(lfn)
return S_OK(failed)
def getDone(self):
done = []
for lfn in self.fileDict.keys():
status = self.fileDict[lfn].get('Status', '')
if status in self.successfulStates:
done.append(lfn)
return S_OK(done)
def __setFileParameter(self, lfn, paramName, paramValue):
self.setLFN(lfn)
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter(self, lfn, paramName):
if not self.fileDict.has_key(lfn):
return S_ERROR("Supplied file not set")
if not self.fileDict[lfn].has_key(paramName):
return S_ERROR("%s not set for file" % paramName)
return S_OK(self.fileDict[lfn][paramName])
####################################################################
#
# Methods for submission
#
def submit(self, monitor=False, printOutput=True):
res = self.__isSubmissionValid()
if not res['OK']:
return res
res = self.__createSURLPairFile()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = {'ftsGUID': self.ftsGUID, 'ftsServer': self.ftsServer}
print "Submitted %s @ %s" % (self.ftsGUID, self.ftsServer)
if monitor:
self.monitor(untilTerminal=True, printOutput=printOutput)
return S_OK(resDict)
def __isSubmissionValid(self):
if not self.fileDict:
return S_ERROR("No files set")
if not self.sourceValid:
return S_ERROR("SourceSE not valid")
if not self.targetValid:
return S_ERROR("TargetSE not valid")
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR("FTSServer not valid")
self.__resolveSource()
self.__resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR("No files to submit")
return S_OK()
def __getCatalogObject(self):
try:
if not self.oCatalog:
self.oCatalog = CatalogInterface()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache(self, lfns=[], overwrite=False):
if not lfns:
lfns = self.fileDict.keys()
toUpdate = []
for lfn in lfns:
if (not lfn in self.catalogReplicas.keys()) or overwrite:
toUpdate.append(lfn)
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogReplicas(toUpdate)
if not res['OK']:
return S_ERROR("Failed to update replica cache", res['Message'])
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache(self, lfns=[], overwrite=False):
if not lfns:
lfns = self.fileDict.keys()
toUpdate = []
for lfn in lfns:
if (not lfn in self.catalogMetadata.keys()) or overwrite:
toUpdate.append(lfn)
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogFileMetadata(toUpdate)
if not res['OK']:
return S_ERROR("Failed to get source catalog metadata",
res['Message'])
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def __resolveSource(self):
toResolve = []
for lfn in self.fileDict.keys():
if (not self.fileDict[lfn].has_key('Source')) and (
self.fileDict[lfn].get('Status') != 'Failed'):
toResolve.append(lfn)
if not toResolve:
return S_OK()
res = self.__updateMetadataCache(toResolve)
if not res['OK']:
return res
res = self.__updateReplicaCache(toResolve)
if not res['OK']:
return res
for lfn in toResolve:
if self.fileDict[lfn].get('Status') == 'Failed':
continue
replicas = self.catalogReplicas.get(lfn, {})
if not replicas.has_key(self.sourceSE):
self.__setFileParameter(lfn, 'Reason',
"No replica at SourceSE")
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.oSourceSE.getPfnForProtocol(replicas[self.sourceSE],
'SRM2',
withPort=True)
if not res['OK']:
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.setSourceSURL(lfn, res['Value'])
if not res['OK']:
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
toResolve = {}
for lfn in self.fileDict.keys():
if self.fileDict[lfn].has_key('Source'):
toResolve[self.fileDict[lfn]['Source']] = lfn
if not toResolve:
return S_ERROR("No eligible Source files")
res = self.oSourceSE.getFileMetadata(toResolve.keys())
if not res['OK']:
return S_ERROR("Failed to check source file metadata")
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
if re.search('File does not exist', error):
self.__setFileParameter(lfn, 'Reason',
"Source file does not exist")
self.__setFileParameter(lfn, 'Status', 'Failed')
else:
self.__setFileParameter(lfn, 'Reason',
"Failed to get Source metadata")
self.__setFileParameter(lfn, 'Status', 'Failed')
for pfn, metadata in res['Value']['Successful'].items():
lfn = toResolve[pfn]
if metadata['Unavailable']:
self.__setFileParameter(lfn, 'Reason',
"Source file Unavailable")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif metadata['Lost']:
self.__setFileParameter(lfn, 'Reason', "Source file Lost")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif not metadata['Cached']:
self.__setFileParameter(lfn, 'Reason',
"Source file not Cached")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
self.__setFileParameter(lfn, 'Reason', "Source size mismatch")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif self.catalogMetadata[lfn]['Checksum'] and metadata[
'Checksum'] and not (compareAdler(
metadata['Checksum'],
self.catalogMetadata[lfn]['Checksum'])):
self.__setFileParameter(lfn, 'Reason',
"Source checksum mismatch")
self.__setFileParameter(lfn, 'Status', 'Failed')
return S_OK()
def __resolveTarget(self):
toResolve = []
for lfn in self.fileDict.keys():
if not self.fileDict[lfn].has_key('Target') and (
self.fileDict[lfn].get('Status') != 'Failed'):
toResolve.append(lfn)
if not toResolve:
return S_OK()
res = self.__updateReplicaCache(toResolve)
if not res['OK']:
return res
atTarget = []
for lfn in sortList(toResolve):
if self.fileDict[lfn].get('Status') == 'Failed':
continue
replicas = self.catalogReplicas.get(lfn, {})
if replicas.has_key(self.targetSE):
self.__setFileParameter(lfn, 'Reason',
"File already at Target")
self.__setFileParameter(lfn, 'Status', 'Done')
atTarget.append(lfn)
for lfn in toResolve:
if (self.fileDict[lfn].get('Status') == 'Failed') or (lfn
in atTarget):
continue
res = self.oTargetSE.getPfnForLfn(lfn)
if not res['OK']:
self.__setFileParameter(lfn, 'Reason',
"Failed to create Target")
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.oTargetSE.getPfnForProtocol(res['Value'],
'SRM2',
withPort=True)
if not res['OK']:
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.setTargetSURL(lfn, res['Value'])
if not res['OK']:
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
toResolve = {}
for lfn in self.fileDict.keys():
if self.fileDict[lfn].has_key('Target'):
toResolve[self.fileDict[lfn]['Target']] = lfn
if not toResolve:
return S_ERROR("No eligible Target files")
res = self.oTargetSE.exists(toResolve.keys())
if not res['OK']:
return S_ERROR("Failed to check target existence")
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
toRemove = []
for pfn, exists in res['Value']['Successful'].items():
if exists:
lfn = toResolve[pfn]
res = self.getSourceSURL(lfn)
if not res['OK']:
self.__setFileParameter(lfn, 'Reason', "Target exists")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif res['Value'] == pfn:
self.__setFileParameter(lfn, 'Reason',
"Source and Target the same")
self.__setFileParameter(lfn, 'Status', 'Failed')
else:
toRemove.append(pfn)
if toRemove:
self.oTargetSE.removeFile(toRemove)
return S_OK()
def __filesToSubmit(self):
for lfn in self.fileDict.keys():
lfnStatus = self.fileDict[lfn].get('Status')
source = self.fileDict[lfn].get('Source')
target = self.fileDict[lfn].get('Target')
if (lfnStatus != 'Failed') and (lfnStatus !=
'Done') and source and target:
return S_OK()
return S_ERROR()
def __createSURLPairFile(self):
fd, fileName = tempfile.mkstemp()
surlFile = os.fdopen(fd, 'w')
for lfn in self.fileDict.keys():
lfnStatus = self.fileDict[lfn].get('Status')
source = self.fileDict[lfn].get('Source')
target = self.fileDict[lfn].get('Target')
if (lfnStatus != 'Failed') and (lfnStatus !=
'Done') and source and target:
surlString = '%s %s\n' % (source, target)
surlFile.write(surlString)
surlFile.close()
self.surlFile = fileName
return S_OK()
def __submitFTSTransfer(self):
comm = [
'glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile,
'-o'
]
if self.targetToken:
comm.append('-t')
comm.append(self.targetToken)
if self.sourceToken:
comm.append('-S')
comm.append(self.sourceToken)
res = executeGridCommand('', comm, self.gridEnv)
os.remove(self.surlFile)
if not res['OK']:
return res
returnCode, output, errStr = res['Value']
if not returnCode == 0:
return S_ERROR(errStr)
guid = output.replace('\n', '')
if not checkGuid(guid):
return S_ERROR('Wrong GUID format returned')
self.ftsGUID = guid
#if self.priority != 3:
# comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)]
# executeGridCommand('',comm,self.gridEnv)
return res
def __resolveFTSServer(self):
if not self.sourceSE:
return S_ERROR("Source SE not set")
if not self.targetSE:
return S_ERROR("Target SE not set")
res = getSitesForSE(self.sourceSE, 'LCG')
if not res['OK'] or not res['Value']:
return S_ERROR("Could not determine source site")
sourceSite = res['Value'][0]
res = getSitesForSE(self.targetSE, 'LCG')
if not res['OK'] or not res['Value']:
return S_ERROR("Could not determine target site")
targetSite = res['Value'][0]
if (sourceSite == 'LCG.CERN.ch') or (targetSite == 'LCG.CERN.ch'):
ep = 'LCG.CERN.ch'
else:
# Target site FTS server should be used
ep = targetSite
try:
configPath = '/Resources/FTSEndpoints/%s' % ep
endpointURL = gConfig.getValue(configPath)
if not endpointURL:
errStr = "FTSRequest.__resolveFTSEndpoint: Failed to find FTS endpoint, check CS entry for '%s'." % ep
return S_ERROR(errStr)
self.ftsServer = endpointURL
return S_OK(endpointURL)
except Exception, x:
return S_ERROR(
'FTSRequest.__resolveFTSEndpoint: Failed to obtain endpoint details from CS'
)
