class FTSRequest( object ):
"""
.. class:: FTSRequest
Helper class for FTS job submission and monitoring.
"""
# # default checksum type
__defaultCksmType = "ADLER32"
# # flag to disablr/enable checksum test, default: disabled
__cksmTest = False
def __init__( self ):
"""c'tor
:param self: self reference
"""
self.log = gLogger.getSubLogger( self.__class__.__name__, True )
# # final states tuple
self.finalStates = ( 'Canceled', 'Failed', 'Hold',
'Finished', 'FinishedDirty' )
# # failed states tuple
self.failedStates = ( 'Canceled', 'Failed',
'Hold', 'FinishedDirty' )
# # successful states tuple
self.successfulStates = ( 'Finished', 'Done' )
# # all file states tuple
self.fileStates = ( 'Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed',
'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting' )
self.statusSummary = {}
# # request status
self.requestStatus = 'Unknown'
# # dict for FTS job files
self.fileDict = {}
# # dict for replicas information
self.catalogReplicas = {}
# # dict for metadata information
self.catalogMetadata = {}
# # dict for files that failed to register
self.failedRegistrations = {}
# # placehoder for FileCatalog reference
self.oCatalog = None
# # submit timestamp
self.submitTime = ''
# # placeholder FTS job GUID
self.ftsGUID = ''
# # placeholder for FTS server URL
self.ftsServer = ''
# # flag marking FTS job completness
self.isTerminal = False
# # completness percentage
self.percentageComplete = 0.0
# # source SE name
self.sourceSE = ''
# # flag marking source SE validity
self.sourceValid = False
# # source space token
self.sourceToken = ''
# # target SE name
self.targetSE = ''
# # flag marking target SE validity
self.targetValid = False
# # target space token
self.targetToken = ''
# # placeholder for target StorageElement
self.oTargetSE = None
# # placeholder for source StorageElement
self.oSourceSE = None
# # checksum type, set it to default
self.__cksmType = self.__defaultCksmType
# # disable checksum test by default
self.__cksmTest = False
# # statuses that prevent submitting to FTS
self.noSubmitStatus = ( 'Failed', 'Done', 'Staging' )
# # were sources resolved?
self.sourceResolved = False
# # Number of file transfers actually submitted
self.submittedFiles = 0
self.transferTime = 0
self.submitCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/SubmitCommand', 'glite-transfer-submit' )
self.monitorCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/MonitorCommand', 'glite-transfer-status' )
self.ftsJob = None
self.ftsFiles = []
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE( self, se ):
""" set SE for source
:param self: self reference
:param str se: source SE name
"""
if se == self.targetSE:
return S_ERROR( "SourceSE is TargetSE" )
self.sourceSE = se
self.oSourceSE = StorageElement( self.sourceSE )
return self.__checkSourceSE()
def __checkSourceSE( self ):
""" check source SE availability
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR( "SourceSE not set" )
res = self.oSourceSE.isValid( 'Read' )
if not res['OK']:
return S_ERROR( "SourceSE not available for reading" )
res = self.__getSESpaceToken( self.oSourceSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message'] )
return S_ERROR( "SourceSE does not support FTS transfers" )
if self.__cksmTest:
res = self.oSourceSE.getChecksumType()
if not res["OK"]:
self.log.error( "Unable to get checksum type for SourceSE %s: %s" % ( self.sourceSE,
res["Message"] ) )
cksmType = res["Value"]
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at SourceSE %s, disabling checksum test" % ( cksmType,
self.sourceSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE( self, se ):
""" set target SE
:param self: self reference
:param str se: target SE name
"""
if se == self.sourceSE:
return S_ERROR( "TargetSE is SourceSE" )
self.targetSE = se
self.oTargetSE = StorageElement( self.targetSE )
return self.__checkTargetSE()
def setTargetToken( self, token ):
""" target space token setter
:param self: self reference
:param str token: target space token
"""
self.targetToken = token
return S_OK()
def __checkTargetSE( self ):
""" check target SE availability
:param self: self reference
"""
if not self.targetSE:
return S_ERROR( "TargetSE not set" )
res = self.oTargetSE.isValid( 'Write' )
if not res['OK']:
return S_ERROR( "TargetSE not available for writing" )
res = self.__getSESpaceToken( self.oTargetSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message'] )
return S_ERROR( "TargetSE does not support FTS transfers" )
# # check checksum types
if self.__cksmTest:
res = self.oTargetSE.getChecksumType()
if not res["OK"]:
self.log.error( "Unable to get checksum type for TargetSE %s: %s" % ( self.targetSE,
res["Message"] ) )
cksmType = res["Value"]
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at TargetSE %s, disabling checksum test" % ( cksmType,
self.targetSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
@staticmethod
def __getSESpaceToken( oSE ):
""" get space token from StorageElement instance
:param self: self reference
:param StorageElement oSE: StorageElement instance
"""
res = oSE.getStorageParameters( "SRM2" )
if not res['OK']:
return res
return S_OK( res['Value'].get( 'SpaceToken' ) )
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID( self, guid ):
""" FTS job GUID setter
:param self: self reference
:param str guid: string containg GUID
"""
if not checkGuid( guid ):
return S_ERROR( "Incorrect GUID format" )
self.ftsGUID = guid
return S_OK()
def setFTSServer( self, server ):
""" FTS server setter
:param self: self reference
:param str server: FTS server URL
"""
self.ftsServer = server
return S_OK()
def isRequestTerminal( self ):
""" check if FTS job has terminated
:param self: self reference
"""
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK( self.isTerminal )
def setCksmTest( self, cksmTest = False ):
""" set cksm test
:param self: self reference
:param bool cksmTest: flag to enable/disable checksum test
"""
self.__cksmTest = bool( cksmTest )
return S_OK( self.__cksmTest )
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN( self, lfn ):
""" add LFN :lfn: to :fileDict:
:param self: self reference
:param str lfn: LFN to add to
"""
self.fileDict.setdefault( lfn, {'Status':'Waiting'} )
return S_OK()
def setSourceSURL( self, lfn, surl ):
""" source SURL setter
:param self: self reference
:param str lfn: LFN
:param str surl: source SURL
"""
target = self.fileDict[lfn].get( 'Target' )
if target == surl:
return S_ERROR( "Source and target the same" )
return self.__setFileParameter( lfn, 'Source', surl )
def getSourceSURL( self, lfn ):
""" get source SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Source' )
def setTargetSURL( self, lfn, surl ):
""" set target SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
:param str surl: target SURL
"""
source = self.fileDict[lfn].get( 'Source' )
if source == surl:
return S_ERROR( "Source and target the same" )
return self.__setFileParameter( lfn, 'Target', surl )
def getFailReason( self, lfn ):
""" get fail reason for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Reason' )
def getRetries( self, lfn ):
""" get number of attepmts made to transfer file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Retries' )
def getTransferTime( self, lfn ):
""" get duration of transfer for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Duration' )
def getFailed( self ):
""" get list of wrongly transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.failedStates ] )
def getStaging( self ):
""" get files set for prestaging """
return S_OK( [lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) == 'Staging'] )
def getDone( self ):
""" get list of succesfully transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.successfulStates ] )
def __setFileParameter( self, lfn, paramName, paramValue ):
""" set :paramName: to :paramValue: for :lfn: file
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
:param mixed paramValue: a new parameter value
"""
self.setLFN( lfn )
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter( self, lfn, paramName ):
""" get value of :paramName: for file :lfn:
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
"""
if lfn not in self.fileDict:
return S_ERROR( "Supplied file not set" )
if paramName not in self.fileDict[lfn]:
return S_ERROR( "%s not set for file" % paramName )
return S_OK( self.fileDict[lfn][paramName] )
####################################################################
#
# Methods for submission
#
def submit( self, monitor = False, printOutput = True ):
""" submit FTS job
:param self: self reference
:param bool monitor: flag to monitor progress of FTS job
:param bool printOutput: flag to print output of execution to stdout
"""
res = self.__prepareForSubmission()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = { 'ftsGUID' : self.ftsGUID, 'ftsServer' : self.ftsServer, 'submittedFiles' : self.submittedFiles }
if monitor or printOutput:
gLogger.always( "Submitted %s@%s" % ( self.ftsGUID, self.ftsServer ) )
if monitor:
self.monitor( untilTerminal = True, printOutput = printOutput, full = False )
return S_OK( resDict )
def __prepareForSubmission( self ):
""" check validity of job before submission
:param self: self reference
"""
if not self.fileDict:
return S_ERROR( "No files set" )
if not self.sourceValid:
return S_ERROR( "SourceSE not valid" )
if not self.targetValid:
return S_ERROR( "TargetSE not valid" )
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR( "FTSServer not valid" )
self.resolveSource()
self.resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR( "No files to submit" )
return S_OK()
def __getCatalogObject( self ):
""" CatalogInterface instance facade
:param self: self reference
"""
try:
if not self.oCatalog:
self.oCatalog = FileCatalog()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache( self, lfns = None, overwrite = False ):
""" update replica cache for list of :lfns:
:param self: self reference
:param mixed lfns: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if ( lfn not in self.catalogReplicas ) or overwrite ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getReplicas( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to update replica cache: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache( self, lfns = None ):
""" update metadata cache for list of LFNs
:param self: self reference
:param list lnfs: list of LFNs
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if lfn not in self.catalogMetadata ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getFileMetadata( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to get source catalog metadata: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def resolveSource( self ):
""" resolve source SE eligible for submission
:param self: self reference
"""
# Avoid resolving sources twice
if self.sourceResolved:
return S_OK()
# Only resolve files that need a transfer
toResolve = [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( "Status", "" ) != "Failed" ]
if not toResolve:
return S_OK()
res = self.__updateMetadataCache( toResolve )
if not res['OK']:
return res
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
# Define the source URLs
for lfn in toResolve:
replicas = self.catalogReplicas.get( lfn, {} )
if self.sourceSE not in replicas:
gLogger.warn( "resolveSource: skipping %s - not replicas at SourceSE %s" % ( lfn, self.sourceSE ) )
self.__setFileParameter( lfn, 'Reason', "No replica at SourceSE" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
# Fix first the PFN
pfn = self.oSourceSE.getPfnForLfn( lfn ).get( 'Value', {} ).get( 'Successful', {} ).get( lfn, replicas[self.sourceSE] )
res = returnSingleResult( self.oSourceSE.getPfnForProtocol( pfn, protocol = 'SRM2', withPort = True ) )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setSourceSURL( lfn, res['Value'] )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict:
if "Source" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Source']] = lfn
if not toResolve:
return S_ERROR( "No eligible Source files" )
# Get metadata of the sources, to check for existance, availability and caching
res = self.oSourceSE.getFileMetadata( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check source file metadata" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
if re.search( 'File does not exist', error ):
gLogger.warn( "resolveSource: skipping %s - source file does not exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file does not exist" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
gLogger.warn( "resolveSource: skipping %s - failed to get source metadata" % lfn )
self.__setFileParameter( lfn, 'Reason', "Failed to get Source metadata" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toStage = []
nbStagedFiles = 0
for pfn, metadata in res['Value']['Successful'].items():
lfn = toResolve[pfn]
lfnStatus = self.fileDict.get( lfn, {} ).get( 'Status' )
if metadata['Unavailable']:
gLogger.warn( "resolveSource: skipping %s - source file unavailable" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Unavailable" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif metadata['Lost']:
gLogger.warn( "resolveSource: skipping %s - source file lost" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Lost" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif not metadata['Cached']:
if lfnStatus != 'Staging':
toStage.append( pfn )
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
gLogger.warn( "resolveSource: skipping %s - source file size mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source size mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \
not compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ):
gLogger.warn( "resolveSource: skipping %s - source file checksum mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source checksum mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif lfnStatus == 'Staging':
# file that was staging is now cached
self.__setFileParameter( lfn, 'Status', 'Waiting' )
nbStagedFiles += 1
# Some files were being staged
if nbStagedFiles:
self.log.info( 'resolveSource: %d files have been staged' % nbStagedFiles )
# Launching staging of files not in cache
if toStage:
gLogger.warn( "resolveSource: %s source files not cached, prestaging..." % len( toStage ) )
stage = self.oSourceSE.prestageFile( toStage )
if not stage["OK"]:
gLogger.error( "resolveSource: error is prestaging - %s" % stage["Message"] )
for pfn in toStage:
lfn = toResolve[pfn]
self.__setFileParameter( lfn, 'Reason', stage["Message"] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
for pfn in toStage:
lfn = toResolve[pfn]
if pfn in stage['Value']['Successful']:
self.__setFileParameter( lfn, 'Status', 'Staging' )
elif pfn in stage['Value']['Failed']:
self.__setFileParameter( lfn, 'Reason', stage['Value']['Failed'][pfn] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
self.sourceResolved = True
return S_OK()
def resolveTarget( self ):
""" find target SE eligible for submission
:param self: self reference
"""
toResolve = [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status' ) not in self.noSubmitStatus ]
if not toResolve:
return S_OK()
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
for lfn in toResolve:
res = self.oTargetSE.getPfnForLfn( lfn )
if not res['OK'] or lfn not in res['Value']['Successful']:
gLogger.warn( "resolveTarget: skipping %s - failed to create target pfn" % lfn )
self.__setFileParameter( lfn, 'Reason', "Failed to create Target" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
pfn = res['Value']['Successful'][lfn]
res = self.oTargetSE.getPfnForProtocol( pfn, protocol = 'SRM2', withPort = True )
if not res['OK'] or pfn not in res['Value']['Successful']:
reason = res.get( 'Message', res.get( 'Value', {} ).get( 'Failed', {} ).get( pfn ) )
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, reason ) )
self.__setFileParameter( lfn, 'Reason', reason )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
pfn = res['Value']['Successful'][pfn]
res = self.setTargetSURL( lfn, pfn )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict:
if "Target" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Target']] = lfn
if not toResolve:
return S_ERROR( "No eligible Target files" )
res = self.oTargetSE.exists( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check target existence" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toRemove = []
for pfn, exists in res['Value']['Successful'].items():
if exists:
lfn = toResolve[pfn]
res = self.getSourceSURL( lfn )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - target exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Target exists" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif res['Value'] == pfn:
gLogger.warn( "resolveTarget: skipping %s - source and target pfns are the same" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source and Target the same" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRemove.append( pfn )
if toRemove:
self.oTargetSE.removeFile( toRemove )
return S_OK()
def __filesToSubmit( self ):
"""
check if there is at least one file to submit
:return: S_OK if at least one file is present, S_ERROR otherwise
"""
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if lfnStatus not in self.noSubmitStatus and source and target:
return S_OK()
return S_ERROR()
def __createFTSFiles( self ):
""" create LFNs file for glite-transfer-submit command
This file consists one line for each fiel to be transferred:
sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM]
:param self: self reference
"""
self.__updateMetadataCache()
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
if lfnStatus not in self.noSubmitStatus:
cksmStr = ""
# # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself
if self.__cksmTest and self.__cksmType:
checkSum = self.catalogMetadata.get( lfn, {} ).get( 'Checksum' )
if checkSum:
cksmStr = " %s:%s" % ( self.__cksmType, intAdlerToHex( hexAdlerToInt( checkSum ) ) )
ftsFile = FTSFile()
ftsFile.LFN = lfn
ftsFile.SourceSURL = self.fileDict[lfn].get( 'Source' )
ftsFile.TargetSURL = self.fileDict[lfn].get( 'Target' )
ftsFile.SourceSE = self.sourceSE
ftsFile.TargetSE = self.targetSE
ftsFile.Status = self.fileDict[lfn].get( 'Status' )
ftsFile.Checksum = cksmStr
ftsFile.Size = self.catalogMetadata.get( lfn, {} ).get( 'Size' )
self.ftsFiles.append( ftsFile )
self.submittedFiles += 1
return S_OK()
def __createFTSJob( self, guid = None ):
self.__createFTSFiles()
ftsJob = FTSJob()
ftsJob.RequestID = 0
ftsJob.OperationID = 0
ftsJob.SourceSE = self.sourceSE
ftsJob.TargetSE = self.targetSE
ftsJob.SourceToken = self.sourceToken
ftsJob.TargetToken = self.targetToken
ftsJob.FTSServer = self.ftsServer
if guid:
ftsJob.FTSGUID = guid
for ftsFile in self.ftsFiles:
ftsFile.Attempt += 1
ftsFile.Error = ""
ftsJob.addFile( ftsFile )
self.ftsJob = ftsJob
def __submitFTSTransfer( self ):
""" create and execute glite-transfer-submit CLI command
:param self: self reference
"""
log = gLogger.getSubLogger( 'Submit' )
self.__createFTSJob()
submit = self.ftsJob.submitFTS2( command = self.submitCommand )
if not submit["OK"]:
log.error( "unable to submit FTSJob: %s" % submit["Message"] )
return submit
log.info( "FTSJob '%s'@'%s' has been submitted" % ( self.ftsJob.FTSGUID, self.ftsJob.FTSServer ) )
# # update statuses for job files
for ftsFile in self.ftsJob:
ftsFile.FTSGUID = self.ftsJob.FTSGUID
ftsFile.Status = "Submitted"
ftsFile.Attempt += 1
log.info( "FTSJob '%s'@'%s' has been submitted" % ( self.ftsJob.FTSGUID, self.ftsJob.FTSServer ) )
self.ftsGUID = self.ftsJob.FTSGUID
return S_OK()
def __resolveFTSServer( self ):
"""
resolve FTS server to use, it should be the closest one from target SE
:param self: self reference
"""
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTSServersForSites
if not self.targetSE:
return S_ERROR( "Target SE not set" )
res = getSitesForSE( self.targetSE )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine target site" )
targetSites = res['Value']
targetSite = ''
for targetSite in targetSites:
targetFTS = getFTSServersForSites( [targetSite] )
if targetFTS['OK']:
ftsTarget = targetFTS['Value'][targetSite]
if ftsTarget:
self.ftsServer = ftsTarget
return S_OK( self.ftsServer )
else:
return targetFTS
return S_ERROR( 'No FTS server found for %s' % targetSite )
####################################################################
#
# Methods for monitoring
#
def summary( self, untilTerminal = False, printOutput = False ):
""" summary of FTS job
:param self: self reference
:param bool untilTerminal: flag to monitor FTS job to its final state
:param bool printOutput: flag to print out monitoring information to the stdout
"""
res = self.__isSummaryValid()
if not res['OK']:
return res
while not self.isTerminal:
res = self.__parseOutput( full = True )
if not res['OK']:
return res
if untilTerminal:
self.__print()
self.isRequestTerminal()
if res['Value'] or ( not untilTerminal ):
break
time.sleep( 1 )
if untilTerminal:
print ""
if printOutput and ( not untilTerminal ):
return self.dumpSummary( printOutput = printOutput )
return S_OK()
def monitor( self, untilTerminal = False, printOutput = False, full = True ):
""" monitor FTS job
:param self: self reference
:param bool untilTerminal: flag to monitor FTS job to its final state
:param bool printOutput: flag to print out monitoring information to the stdout
"""
if not self.ftsJob:
self.resolveSource()
self.__createFTSJob( self.ftsGUID )
res = self.__isSummaryValid()
if not res['OK']:
return res
if untilTerminal:
res = self.summary( untilTerminal = untilTerminal, printOutput = printOutput )
if not res['OK']:
return res
res = self.__parseOutput( full = full )
if not res['OK']:
return res
if untilTerminal:
self.finalize()
if printOutput:
self.dump()
return res
def dumpSummary( self, printOutput = False ):
""" get FTS job summary as str
:param self: self reference
:param bool printOutput: print summary to stdout
"""
outStr = ''
for status in sorted( self.statusSummary ):
if self.statusSummary[status]:
outStr = '%s\t%-10s : %-10s\n' % ( outStr, status, str( self.statusSummary[status] ) )
outStr = outStr.rstrip( '\n' )
if printOutput:
print outStr
return S_OK( outStr )
def __print( self ):
""" print progress bar of FTS job completeness to stdout
:param self: self reference
"""
width = 100
bits = int( ( width * self.percentageComplete ) / 100 )
outStr = "|%s>%s| %.1f%s %s %s" % ( "="*bits, " "*( width - bits ),
self.percentageComplete, "%",
self.requestStatus, " "*10 )
sys.stdout.write( "%s\r" % ( outStr ) )
sys.stdout.flush()
def dump( self ):
""" print FTS job parameters and files to stdout
:param self: self reference
"""
print "%-10s : %-10s" % ( "Status", self.requestStatus )
print "%-10s : %-10s" % ( "Source", self.sourceSE )
print "%-10s : %-10s" % ( "Target", self.targetSE )
print "%-10s : %-128s" % ( "Server", self.ftsServer )
print "%-10s : %-128s" % ( "GUID", self.ftsGUID )
for lfn in sorted( self.fileDict ):
print "\n %-15s : %-128s" % ( 'LFN', lfn )
for key in ['Source', 'Target', 'Status', 'Reason', 'Duration']:
print " %-15s : %-128s" % ( key, str( self.fileDict[lfn].get( key ) ) )
return S_OK()
def __isSummaryValid( self ):
""" check validity of FTS job summary report
:param self: self reference
"""
if not self.ftsServer:
return S_ERROR( "FTSServer not set" )
if not self.ftsGUID:
return S_ERROR( "FTSGUID not set" )
return S_OK()
def __parseOutput( self, full = False ):
""" execute glite-transfer-status command and parse its output
:param self: self reference
:param bool full: glite-transfer-status verbosity level, when set, collect information of files as well
"""
monitor = self.ftsJob.monitorFTS2( command = self.monitorCommand, full = full )
if not monitor['OK']:
return monitor
self.percentageComplete = self.ftsJob.Completeness
self.requestStatus = self.ftsJob.Status
self.submitTime = self.ftsJob.SubmitTime
statusSummary = monitor['Value']
if statusSummary:
for state in statusSummary:
self.statusSummary[state] = statusSummary[state]
self.transferTime = 0
for ftsFile in self.ftsJob:
lfn = ftsFile.LFN
self.__setFileParameter( lfn, 'Status', ftsFile.Status )
self.__setFileParameter( lfn, 'Reason', ftsFile.Error )
self.__setFileParameter( lfn, 'Duration', ftsFile._duration )
targetURL = self.__getFileParameter( lfn, 'Target' )
if not targetURL['OK']:
self.__setFileParameter( lfn, 'Target', ftsFile.TargetSURL )
self.transferTime += int( ftsFile._duration )
return S_OK()
####################################################################
#
# Methods for finalization
#
def finalize( self ):
""" finalize FTS job
:param self: self reference
"""
self.__updateMetadataCache()
transEndTime = dateTime()
regStartTime = time.time()
res = self.getTransferStatistics()
transDict = res['Value']
res = self.__registerSuccessful( transDict['transLFNs'] )
regSuc, regTotal = res['Value']
regTime = time.time() - regStartTime
if self.sourceSE and self.targetSE:
self.__sendAccounting( regSuc, regTotal, regTime, transEndTime, transDict )
return S_OK()
def getTransferStatistics( self ):
""" collect information of Transfers that can be used by Accounting
:param self: self reference
"""
transDict = { 'transTotal': len( self.fileDict ),
'transLFNs': [],
'transOK': 0,
'transSize': 0 }
for lfn in self.fileDict:
if self.fileDict[lfn].get( 'Status' ) in self.successfulStates:
if self.fileDict[lfn].get( 'Duration', 0 ):
transDict['transLFNs'].append( lfn )
transDict['transOK'] += 1
if lfn in self.catalogMetadata:
transDict['transSize'] += self.catalogMetadata[lfn].get( 'Size', 0 )
return S_OK( transDict )
def getFailedRegistrations( self ):
""" get failed registrations dict
:param self: self reference
"""
return S_OK( self.failedRegistrations )
def __registerSuccessful( self, transLFNs ):
""" register successfully transferred files to the catalogs,
fill failedRegistrations dict for files that failed to register
:param self: self reference
:param list transLFNs: LFNs in FTS job
"""
self.failedRegistrations = {}
toRegister = {}
for lfn in transLFNs:
res = returnSingleResult( self.oTargetSE.getPfnForProtocol( self.fileDict[lfn].get( 'Target' ), protocol = 'SRM2', withPort = False ) )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRegister[lfn] = { 'PFN' : res['Value'], 'SE' : self.targetSE }
if not toRegister:
return S_OK( ( 0, 0 ) )
res = self.__getCatalogObject()
if not res['OK']:
for lfn in toRegister:
self.failedRegistrations = toRegister
self.log.error( 'Failed to get Catalog Object', res['Message'] )
return S_OK( ( 0, len( toRegister ) ) )
res = self.oCatalog.addReplica( toRegister )
if not res['OK']:
self.failedRegistrations = toRegister
self.log.error( 'Failed to get Catalog Object', res['Message'] )
return S_OK( ( 0, len( toRegister ) ) )
for lfn, error in res['Value']['Failed'].items():
self.failedRegistrations[lfn] = toRegister[lfn]
self.log.error( 'Registration of Replica failed', '%s : %s' % ( lfn, str( error ) ) )
return S_OK( ( len( res['Value']['Successful'] ), len( toRegister ) ) )
def __sendAccounting( self, regSuc, regTotal, regTime, transEndTime, transDict ):
""" send accounting record
:param self: self reference
:param regSuc: number of files successfully registered
:param regTotal: number of files attepted to register
:param regTime: time stamp at the end of registration
:param transEndTime: time stamp at the end of FTS job
:param dict transDict: dict holding couters for files being transerred, their sizes and successfull transfers
"""
oAccounting = DataOperation()
oAccounting.setEndTime( transEndTime )
oAccounting.setStartTime( self.submitTime )
accountingDict = {}
accountingDict['OperationType'] = 'replicateAndRegister'
result = getProxyInfo()
if not result['OK']:
userName = '******'
else:
userName = result['Value'].get( 'username', 'unknown' )
accountingDict['User'] = userName
accountingDict['Protocol'] = 'FTS' if 'fts3' not in self.ftsServer else 'FTS3'
accountingDict['RegistrationTime'] = regTime
accountingDict['RegistrationOK'] = regSuc
accountingDict['RegistrationTotal'] = regTotal
accountingDict['TransferOK'] = transDict['transOK']
accountingDict['TransferTotal'] = transDict['transTotal']
accountingDict['TransferSize'] = transDict['transSize']
accountingDict['FinalStatus'] = self.requestStatus
accountingDict['Source'] = self.sourceSE
accountingDict['Destination'] = self.targetSE
accountingDict['TransferTime'] = self.transferTime
oAccounting.setValuesFromDict( accountingDict )
self.log.verbose( "Attempting to commit accounting message..." )
oAccounting.commit()
self.log.verbose( "...committed." )
return S_OK()
class FTSJob( object ):
""" Class describing one FTS job
"""
# # initial states
INITSTATES = ( "Submitted", "Ready", "Staging" )
# # ongoing transfer states
TRANSSTATES = ( "Active", "Hold" )
# # failed states
FAILEDSTATES = ( "Canceled", "Failed" )
# # finished (careful, must be capitalized)
FINALSTATES = ( "Finished", "Finisheddirty", "FinishedDirty", "Failed", "Canceled" )
# # missing source regexp patterns
missingSourceErrors = [
re.compile( r".*INVALID_PATH\] Failed" ),
re.compile( r".*INVALID_PATH\] No such file or directory" ),
re.compile( r".*INVALID_PATH\] The requested file either does not exist" ),
re.compile( r".*INVALID_PATH\] the server sent an error response: 500 500"\
" Command failed. : open error: No such file or directory" ),
re.compile( r"SOURCE error during TRANSFER_PREPARATION phase: \[USER_ERROR\] source file doesnt exist" ) ]
def __init__( self, fromDict = None ):
"""c'tor
:param self: self reference
:param dict fromDict: data dict
"""
self.__data__ = dict.fromkeys( self.tableDesc()["Fields"].keys(), None )
now = datetime.datetime.utcnow().replace( microsecond = 0 )
self.__data__["CreationTime"] = now
self.__data__["SubmitTime"] = now
self.__data__["LastUpdate"] = now
self.__data__["Status"] = "Submitted"
self.__data__["Completeness"] = 0
self.__data__["FTSJobID"] = 0
self._regTime = 0.
self._regSuccess = 0
self._regTotal = 0
self.__files__ = TypedList( allowedTypes = FTSFile )
self._fc = FileCatalog()
self._fts3context = None
self._states = tuple( set( self.INITSTATES + self.TRANSSTATES + self.FAILEDSTATES + self.FINALSTATES ) )
fromDict = fromDict if fromDict else {}
for ftsFileDict in fromDict.get( "FTSFiles", [] ):
self +=FTSFile( ftsFileDict )
if "FTSFiles" in fromDict:
del fromDict["FTSFiles"]
for key, value in fromDict.items():
if key not in self.__data__:
raise AttributeError( "Unknown FTSJob attribute '%s'" % key )
if value:
setattr( self, key, value )
self._log = gLogger.getSubLogger( "req_%s/FTSJob-%s" % ( self.RequestID, self.FTSGUID ) , True )
@staticmethod
def tableDesc():
""" get table desc """
return { "Fields" :
{ "FTSJobID" : "INTEGER NOT NULL AUTO_INCREMENT",
"FTSGUID" : "VARCHAR(64) NOT NULL",
"OperationID": "INTEGER NOT NULL",
"RequestID": "INTEGER NOT NULL",
"SourceSE" : "VARCHAR(128) NOT NULL",
"TargetSE" : "VARCHAR(128) NOT NULL",
"FTSServer" : "VARCHAR(255) NOT NULL",
"TargetToken": "VARCHAR(255)",
"SourceToken": "VARCHAR(255)",
"Size": "BIGINT NOT NULL",
"Files": "INTEGER NOT NULL",
"Completeness": "INTEGER NOT NULL DEFAULT 0",
"FailedFiles": "INTEGER DEFAULT 0",
"FailedSize": "INTEGER DEFAULT 0",
"Status" : "ENUM( 'Submitted', 'Ready', 'Staging', 'Canceled', 'Active', 'Hold', "\
"'Failed', 'Finished', 'FinishedDirty', 'Assigned' ) DEFAULT 'Submitted'",
"Error" : "VARCHAR(255)",
"CreationTime" : "DATETIME",
"SubmitTime" : "DATETIME",
"LastUpdate" : "DATETIME" },
"PrimaryKey" : [ "FTSJobID" ],
"Indexes" : { "FTSJobID" : [ "FTSJobID" ], "FTSGUID": [ "FTSGUID" ] } }
@property
def FTSJobID( self ):
""" FTSJobID getter """
return self.__data__["FTSJobID"]
@FTSJobID.setter
def FTSJobID( self, value ):
""" FTSJobID setter """
self.__data__["FTSJobID"] = long( value ) if value else 0
@property
def RequestID( self ):
""" RequestID getter """
return self.__data__["RequestID"]
@RequestID.setter
def RequestID( self, value ):
""" RequestID setter """
self.__data__["RequestID"] = long( value ) if value else 0
@property
def OperationID( self ):
""" OperationID getter """
return self.__data__["OperationID"]
@OperationID.setter
def OperationID( self, value ):
""" OperationID setter """
self.__data__["OperationID"] = long( value ) if value else 0
@property
def FTSGUID( self ):
""" FTSGUID prop """
return self.__data__["FTSGUID"]
@FTSGUID.setter
def FTSGUID( self, value ):
""" FTSGUID setter """
if value:
if type( value ) not in ( str, unicode ):
raise TypeError( "FTSGUID should be a string!" )
if not checkGuid( value ):
raise ValueError( "'%s' is not a valid GUID!" % str( value ) )
self.__data__["FTSGUID"] = value
@property
def FTSServer( self ):
""" FTSServer getter """
return self.__data__["FTSServer"]
@FTSServer.setter
def FTSServer( self, url ):
""" FTSServer getter """
self.__data__["FTSServer"] = url
# I REALLY don't see that happening
# but in case we change the server after the
# context was created, I reset it
# (I don't initialize because maybe we are in FTS2 mode...)
self._fts3context = None
@property
def Completeness( self ):
""" completeness getter """
return self.__data__["Completeness"]
@Completeness.setter
def Completeness( self, value ):
""" completeness setter """
self.__data__["Completeness"] = int( value ) if value else 0
@property
def Error( self ):
""" error getter """
return self.__data__["Error"]
@Error.setter
def Error( self, error ):
""" error setter """
self.__data__["Error"] = str( error )[255:]
@property
def Files( self ):
""" nb files getter """
self.__data__["Files"] = len( self )
return self.__data__["Files"]
@Files.setter
def Files( self, value ):
""" nb files setter """
self.__data__["Files"] = len( self )
@property
def Status( self ):
""" status prop """
if not self.__data__["Status"]:
self.__data__["Status"] = "Waiting"
return self.__data__["Status"]
@Status.setter
def Status( self, value ):
""" status setter """
value = self._normalizedStatus( value.strip() )
if value not in self._states:
raise ValueError( "Unknown FTSJob Status: '%s'" % str( value ) )
self.__data__["Status"] = value
@property
def FailedFiles( self ):
""" nb failed files getter """
self.__data__["FailedFiles"] = len( [ ftsFile for ftsFile in self
if ftsFile.Status in FTSFile.FAILED_STATES ] )
return self.__data__["FailedFiles"]
@FailedFiles.setter
def FailedFiles( self, value ):
""" nb failed files setter """
if value:
self.__data__["FailedFiles"] = value
else:
self.__data__["FailedFiles"] = len( [ftsFile for ftsFile in self if ftsFile.Status in FTSFile.FAILED_STATES] )
@property
def Size( self ):
""" size getter """
# if not self.__data__["Size"]:
self.__data__["Size"] = sum( ftsFile.Size for ftsFile in self )
return self.__data__["Size"]
@Size.setter
def Size( self, value ):
""" size setter """
if value:
self.__data__["Size"] = value
else:
self.__data__["Size"] = sum( ftsFile.Size for ftsFile in self )
@property
def FailedSize( self ):
""" size getter """
if not self.__data__["FailedSize"]:
self.__data__["FailedSize"] = sum( ftsFile.Size for ftsFile in self if ftsFile.Status in FTSFile.FAILED_STATES )
return self.__data__["FailedSize"]
@FailedSize.setter
def FailedSize( self, value ):
""" size setter """
if value:
self.__data__["FailedSize"] = value
else:
self.__data__["FailedSize"] = sum( ftsFile.Size for ftsFile in self if ftsFile.Status in FTSFile.FAILED_STATES )
@property
def CreationTime( self ):
""" creation time getter """
return self.__data__["CreationTime"]
@CreationTime.setter
def CreationTime( self, value = None ):
""" creation time setter """
if type( value ) not in ( datetime.datetime, str ) :
raise TypeError( "CreationTime should be a datetime.datetime!" )
if type( value ) == str:
value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' )
self.__data__["CreationTime"] = value
@property
def SubmitTime( self ):
""" request's submission time getter """
return self.__data__["SubmitTime"]
@SubmitTime.setter
def SubmitTime( self, value = None ):
""" submission time setter """
if type( value ) not in ( datetime.datetime, str ):
raise TypeError( "SubmitTime should be a datetime.datetime!" )
if type( value ) == str:
value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' )
self.__data__["SubmitTime"] = value
@property
def LastUpdate( self ):
""" last update getter """
return self.__data__["LastUpdate"]
@LastUpdate.setter
def LastUpdate( self, value = None ):
""" last update setter """
if type( value ) not in ( datetime.datetime, str ):
raise TypeError( "LastUpdate should be a datetime.datetime!" )
if type( value ) == str:
value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' )
self.__data__["LastUpdate"] = value
@property
def TargetSE( self ):
""" target SE getter """
return self.__data__["TargetSE"]
@TargetSE.setter
def TargetSE( self, targetSE ):
""" target SE setter """
self.__data__["TargetSE"] = targetSE
@property
def SourceSE( self ):
""" source SE getter """
return self.__data__["SourceSE"]
@SourceSE.setter
def SourceSE( self, sourceSE ):
""" source SE setter """
self.__data__["SourceSE"] = sourceSE
@property
def SourceToken( self ):
""" source token getter """
return self.__data__["SourceToken"]
@SourceToken.setter
def SourceToken( self, sourceToken ):
""" source SE setter """
self.__data__["SourceToken"] = sourceToken
@property
def TargetToken( self ):
""" target token getter """
return self.__data__["TargetToken"]
@TargetToken.setter
def TargetToken( self, targetToken ):
""" target SE setter """
self.__data__["TargetToken"] = targetToken
# # FTSJobFiles arithmetics
def __contains__( self, subFile ):
""" in operator """
return subFile in self.__files__
def __iadd__( self, ftsFile ):
""" += operator """
if ftsFile not in self:
self.__files__.append( ftsFile )
ftsFile._parent = self
self.Files
self.Size
return self
def __add__( self, ftsFile ):
""" + operator """
self +=ftsFile
def addFile( self, ftsFile ):
""" add :ftsFile: to FTS job """
self +=ftsFile
def subFile( self, ftsFile ):
""" remove ftsFile from this job """
if ftsFile in self:
ftsFile._parent = None
self.__files__.remove( ftsFile )
# # helpers for looping
def __iter__( self ):
""" files iterator """
return self.__files__.__iter__()
def __getitem__( self, i ):
""" [] op for files """
return self.__files__.__getitem__( i )
def __delitem__( self, i ):
""" del ftsJob[i] """
self.__files__.__delitem__( i )
def __setitem__( self, i, ftsFile ):
""" ftsJob[i] = ftsFile """
self.__files__.__setitem__( i, ftsFile )
def fileStatusList( self ):
""" get list of files statuses """
return [ ftsFile.Status for ftsFile in self ]
def __nonzero__( self ):
""" for comparisons
"""
return True
def __len__( self ):
""" nb of subFiles """
return len( self.__files__ )
def _surlPairs( self ):
""" create and return SURL pair file """
surls = []
for ftsFile in self:
checksum = "%s:%s" % ( ftsFile.ChecksumType, ftsFile.Checksum ) if ftsFile.ChecksumType and ftsFile.Checksum else ""
surls.append( "%s %s %s" % ( ftsFile.SourceSURL, ftsFile.TargetSURL, checksum ) )
return "\n".join( surls )
def submitFTS2( self, command = 'glite-transfer-submit', pinTime = False ):
""" submit fts job using FTS2 client """
if self.FTSGUID:
return S_ERROR( "FTSJob has already been submitted" )
surls = self._surlPairs()
if not surls:
return S_ERROR( "No files to submit" )
fd, fileName = tempfile.mkstemp()
surlFile = os.fdopen( fd, 'w' )
surlFile.write( surls )
surlFile.close()
submitCommand = command.split() + \
[ "-s",
self.FTSServer,
"-f",
fileName,
"-o",
"-K" ]
if self.TargetToken:
submitCommand += [ "-t", self.TargetToken]
if self.SourceToken:
submitCommand += [ "-S", self.SourceToken ]
if pinTime:
submitCommand += [ "--copy-pin-lifetime", "%d" % pinTime, "--bring-online", '86400' ]
submit = executeGridCommand( "", submitCommand )
os.remove( fileName )
if not submit["OK"]:
return submit
returnCode, output, errStr = submit["Value"]
if returnCode != 0:
return S_ERROR( errStr if errStr else output )
self.FTSGUID = output.replace( "\n", "" )
self.Status = "Submitted"
for ftsFile in self:
ftsFile.FTSGUID = self.FTSGUID
ftsFile.Status = "Submitted"
return S_OK()
def _normalizedStatus( self, status ):
for st in self._states:
if status.lower() == st.lower():
return st
return status
def monitorFTS2( self, command = "glite-transfer-status", full = False ):
""" monitor fts job """
if not self.FTSGUID:
return S_ERROR( "FTSGUID not set, FTS job not submitted?" )
monitorCommand = command.split() + \
["--verbose",
"-s",
self.FTSServer,
self.FTSGUID ]
if full:
monitorCommand.append( "-l" )
monitor = executeGridCommand( "", monitorCommand )
if not monitor["OK"]:
return monitor
returnCode, outputStr, errStr = monitor["Value"]
# Returns a non zero status if error
if returnCode != 0:
if 'was not found' in outputStr and not errStr:
errStr = 'Job was not found'
return S_ERROR( errStr )
outputStr = outputStr.replace( "'" , "" ).replace( "<", "" ).replace( ">", "" )
# # set FTS job status
regExp = re.compile( "Status:\\s+(\\S+)" )
# with FTS3 this can be uppercase
self.Status = re.search( regExp, outputStr ).group( 1 )
statusSummary = {}
# This is capitalized, even in FTS3!
for state in FTSFile.ALL_STATES:
regExp = re.compile( "\\s+%s:\\s+(\\d+)" % state )
if regExp.search( outputStr ):
statusSummary[state] = int( re.search( regExp, outputStr ).group( 1 ) )
total = sum( statusSummary.values() )
completed = sum( statusSummary.get( state, 0 ) for state in FTSFile.FINAL_STATES )
self.Completeness = 100 * completed / total if total else 0
if not full:
return S_OK( statusSummary )
# The order of informations is not the same for glite- and fts- !!!
# In order: new fts-, old fts-, glite-
realJob = len( self ) != 0
iExptr = None
for iExptr, exptr in enumerate( (
'[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Staging:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)',
'[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)',
'[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Retries:[ ]+(\\d+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)'
) ):
regExp = re.compile( exptr, re.S )
fileInfo = re.findall( regExp, outputStr )
if fileInfo:
break
if not fileInfo:
return S_ERROR( "Error monitoring job (no regexp match)" )
for info in fileInfo:
if iExptr == 0:
# version >= 3.2.30
sourceURL, targetURL, fileStatus, reason, duration, _retries, _staging = info
elif iExptr == 1:
# version FTS3 < 3.2.30
sourceURL, targetURL, fileStatus, reason, duration, _retries = info
elif iExptr == 2:
# version FTS2
sourceURL, targetURL, fileStatus, _retries, reason, duration = info
else:
return S_ERROR( 'Error monitoring job (implement match %d)' % iExptr )
candidateFile = None
if not realJob:
# This is used by the CLI monitoring of jobs in case no file was specified
candidateFile = FTSFile()
candidateFile.LFN = overlap( sourceURL, targetURL )
candidateFile.SourceSURL = sourceURL
candidateFile.Size = 0
self +=candidateFile
else:
for ftsFile in self:
if ftsFile.SourceSURL == sourceURL:
candidateFile = ftsFile
break
if not candidateFile:
continue
# Can be uppercase for FTS3
if not candidateFile.TargetSURL:
candidateFile.TargetSURL = targetURL
candidateFile.Status = fileStatus
candidateFile.Error = reason
candidateFile._duration = duration
if candidateFile.Status == "Failed":
for missingSource in self.missingSourceErrors:
if missingSource.match( reason ):
candidateFile.Error = "MissingSource"
# If the staging info was present, record it
if len( info ) > 6:
candidateFile._staging = info[6]
# # register successful files
if self.Status in FTSJob.FINALSTATES:
return self.finalize()
return S_OK()
def submitFTS3( self, pinTime = False ):
""" submit fts job using FTS3 rest API """
if self.FTSGUID:
return S_ERROR( "FTSJob already has been submitted" )
transfers = []
for ftsFile in self:
trans = fts3.new_transfer( ftsFile.SourceSURL,
ftsFile.TargetSURL,
checksum = 'ADLER32:%s'%ftsFile.Checksum,
filesize = ftsFile.Size )
transfers.append( trans )
source_spacetoken = self.SourceToken if self.SourceToken else None
dest_spacetoken = self.TargetToken if self.TargetToken else None
copy_pin_lifetime = pinTime if pinTime else None
bring_online = 86400 if pinTime else None
job = fts3.new_job( transfers = transfers, overwrite = True,
source_spacetoken = source_spacetoken, spacetoken = dest_spacetoken,
bring_online = bring_online, copy_pin_lifetime = copy_pin_lifetime, retry = 3 )
try:
if not self._fts3context:
self._fts3context = fts3.Context( endpoint = self.FTSServer, request_class = ftsSSLRequest, verify = False )
context = self._fts3context
self.FTSGUID = fts3.submit( context, job )
except Exception as e:
return S_ERROR( "Error at submission: %s" % e )
self.Status = "Submitted"
self._log = gLogger.getSubLogger( "req_%s/FTSJob-%s" % ( self.RequestID, self.FTSGUID ) , True )
for ftsFile in self:
ftsFile.FTSGUID = self.FTSGUID
ftsFile.Status = "Submitted"
return S_OK()
def monitorFTS3( self, full = False ):
if not self.FTSGUID:
return S_ERROR( "FTSGUID not set, FTS job not submitted?" )
jobStatusDict = None
try:
if not self._fts3context:
self._fts3context = fts3.Context( endpoint = self.FTSServer, request_class = ftsSSLRequest, verify = False )
context = self._fts3context
jobStatusDict = fts3.get_job_status( context, self.FTSGUID, list_files = True )
except Exception as e:
return S_ERROR( "Error getting the job status %s" % e )
self.Status = jobStatusDict['job_state'].capitalize()
filesInfoList = jobStatusDict['files']
statusSummary = {}
for fileDict in filesInfoList:
file_state = fileDict['file_state'].capitalize()
statusSummary[file_state] = statusSummary.get( file_state, 0 ) + 1
total = len( filesInfoList )
completed = sum( [ statusSummary.get( state, 0 ) for state in FTSFile.FINAL_STATES ] )
self.Completeness = 100 * completed / total
if not full:
return S_OK( statusSummary )
ftsFilesPrinted = False
for fileDict in filesInfoList:
sourceURL = fileDict['source_surl']
targetURL = fileDict['dest_surl']
fileStatus = fileDict['file_state'].capitalize()
reason = fileDict['reason']
duration = fileDict['tx_duration']
candidateFile = None
for ftsFile in self:
if ftsFile.SourceSURL == sourceURL and ftsFile.TargetSURL == targetURL :
candidateFile = ftsFile
break
if candidateFile is None:
self._log.warn( 'FTSFile not found', 'Source: %s, Target: %s' % ( sourceURL, targetURL ) )
if not ftsFilesPrinted:
ftsFilesPrinted = True
if not len( self ):
self._log.warn( 'Monitored FTS job is empty!' )
else:
self._log.warn( 'All FTS files are:', '\n' + '\n'.join( ['Source: %s, Target: %s' % ( ftsFile.SourceSURL, ftsFile.TargetSURL ) for ftsFile in self] ) )
else:
candidateFile.Status = fileStatus
candidateFile.Error = reason
candidateFile._duration = duration
if candidateFile.Status == "Failed":
for missingSource in self.missingSourceErrors:
if missingSource.match( reason ):
candidateFile.Error = "MissingSource"
# # register successful files
if self.Status in FTSJob.FINALSTATES:
return self.finalize()
return S_OK()
def monitorFTS( self, ftsVersion, command = "glite-transfer-status", full = False ):
""" Wrapper calling the proper method for a given version of FTS"""
if ftsVersion == "FTS2":
return self.monitorFTS2( command = command, full = full )
elif ftsVersion == "FTS3":
return self.monitorFTS3( full = full )
else:
return S_ERROR( "monitorFTS: unknown FTS version %s" % ftsVersion )
def submitFTS( self, ftsVersion, command = 'glite-transfer-submit', pinTime = False ):
""" Wrapper calling the proper method for a given version of FTS"""
if ftsVersion == "FTS2":
return self.submitFTS2( command = command, pinTime = pinTime )
elif ftsVersion == "FTS3":
return self.submitFTS3( pinTime = pinTime )
else:
return S_ERROR( "submitFTS: unknown FTS version %s" % ftsVersion )
def finalize( self ):
""" register successfully transferred files """
if self.Status not in FTSJob.FINALSTATES:
return S_OK()
if not len( self ):
return S_ERROR( "Empty job in finalize" )
startTime = time.time()
targetSE = StorageElement( self.TargetSE )
toRegister = [ ftsFile for ftsFile in self if ftsFile.Status == "Finished" ]
toRegisterDict = {}
for ftsFile in toRegister:
pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = 'srm' ) )
if pfn["OK"]:
pfn = pfn["Value"]
toRegisterDict[ ftsFile.LFN ] = { "PFN": pfn, "SE": self.TargetSE }
else:
self._log.error( "Error getting SRM URL", pfn['Message'] )
if toRegisterDict:
self._regTotal += len( toRegisterDict )
register = self._fc.addReplica( toRegisterDict )
self._regTime += time.time() - startTime
if not register["OK"]:
self._log.error( 'Error registering replica', register['Message'] )
for ftsFile in toRegister:
ftsFile.Error = "AddCatalogReplicaFailed"
return register
register = register["Value"]
self._regSuccess += len( register.get( 'Successful', {} ) )
if self._regSuccess:
self._log.info( 'Successfully registered %d replicas' % self._regSuccess )
failedFiles = register.get( "Failed", {} )
errorReason = {}
for lfn, reason in failedFiles.items():
errorReason.setdefault( str( reason ), [] ).append( lfn )
for reason in errorReason:
self._log.error( 'Error registering %d replicas' % len( errorReason[reason] ), reason )
for ftsFile in toRegister:
if ftsFile.LFN in failedFiles:
ftsFile.Error = "AddCatalogReplicaFailed"
else:
statuses = set( [ftsFile.Status for ftsFile in self] )
self._log.warn( "No replicas to register for FTSJob (%s) - Files status: '%s'" % \
( self.Status, ','.join( sorted( statuses ) ) ) )
return S_OK()
def toSQL( self ):
""" prepare SQL INSERT or UPDATE statement
:return: str with SQL fragment
"""
colVals = []
for column, value in self.__data__.items():
if value and column not in ( "FTSJobID", "LastUpdate" ):
colStr = "`%s`" % column
if isinstance( value, datetime.datetime ) or isinstance( value, basestring ):
valStr = "'%s'" % value
else:
valStr = str( value )
colVals.append( ( colStr, valStr ) )
colVals.append( ( "`LastUpdate`", "UTC_TIMESTAMP()" ) )
query = []
if self.FTSJobID:
query.append( "UPDATE `FTSJob` SET " )
query.append( ",".join( [ "%s=%s" % item for item in colVals ] ) )
query.append( " WHERE `FTSJobID`=%d;\n" % self.FTSJobID )
else:
query.append( "INSERT INTO `FTSJob` " )
columns = "(%s)" % ",".join( [ column for column, value in colVals ] )
values = "(%s)" % ",".join( [ value for column, value in colVals ] )
query.append( columns )
query.append( " VALUES %s;" % values )
return S_OK( "".join( query ) )
def toJSON( self ):
""" dump to JSON format """
digest = dict( zip( self.__data__.keys(),
[ str( val ) if val else "" for val in self.__data__.values() ] ) )
digest["FTSFiles"] = []
for ftsFile in self:
fileJSON = ftsFile.toJSON()
if not fileJSON["OK"]:
return fileJSON
digest["FTSFiles"].append( fileJSON["Value"] )
return S_OK( digest )
class DataIntegrityClient(Client):
"""
The following methods are supported in the service but are not mentioned explicitly here:
getProblematic()
Obtains a problematic file from the IntegrityDB based on the LastUpdate time
getPrognosisProblematics(prognosis)
Obtains all the problematics of a particular prognosis from the integrityDB
getProblematicsSummary()
Obtains a count of the number of problematics for each prognosis found
getDistinctPrognosis()
Obtains the distinct prognosis found in the integrityDB
getTransformationProblematics(prodID)
Obtains the problematics for a given production
incrementProblematicRetry(fileID)
Increments the retry count for the supplied file ID
changeProblematicPrognosis(fileID,newPrognosis)
Changes the prognosis of the supplied file to the new prognosis
setProblematicStatus(fileID,status)
Updates the status of a problematic in the integrityDB
removeProblematic(self,fileID)
This removes the specified file ID from the integrity DB
insertProblematic(sourceComponent,fileMetadata)
Inserts file with supplied metadata into the integrity DB
"""
def __init__(self, **kwargs):
Client.__init__(self, **kwargs)
self.setServer('DataManagement/DataIntegrity')
self.dm = DataManager()
self.fc = FileCatalog()
##########################################################################
#
# This section contains the specific methods for LFC->SE checks
#
def catalogDirectoryToSE(self, lfnDir):
""" This obtains the replica and metadata information from the catalog for the supplied directory and checks against the storage elements.
"""
gLogger.info("-" * 40)
gLogger.info("Performing the LFC->SE check")
gLogger.info("-" * 40)
if type(lfnDir) in types.StringTypes:
lfnDir = [lfnDir]
res = self.__getCatalogDirectoryContents(lfnDir)
if not res['OK']:
return res
replicas = res['Value']['Replicas']
catalogMetadata = res['Value']['Metadata']
res = self.__checkPhysicalFiles(replicas, catalogMetadata)
if not res['OK']:
return res
resDict = {
'CatalogMetadata': catalogMetadata,
'CatalogReplicas': replicas
}
return S_OK(resDict)
def catalogFileToSE(self, lfns):
""" This obtains the replica and metadata information from the catalog and checks against the storage elements.
"""
gLogger.info("-" * 40)
gLogger.info("Performing the LFC->SE check")
gLogger.info("-" * 40)
if type(lfns) in types.StringTypes:
lfns = [lfns]
res = self.__getCatalogMetadata(lfns)
if not res['OK']:
return res
catalogMetadata = res['Value']
res = self.__getCatalogReplicas(catalogMetadata.keys())
if not res['OK']:
return res
replicas = res['Value']
res = self.__checkPhysicalFiles(replicas, catalogMetadata)
if not res['OK']:
return res
resDict = {
'CatalogMetadata': catalogMetadata,
'CatalogReplicas': replicas
}
return S_OK(resDict)
def checkPhysicalFiles(self, replicas, catalogMetadata, ses=[]):
""" This obtains takes the supplied replica and metadata information obtained from the catalog and checks against the storage elements.
"""
gLogger.info("-" * 40)
gLogger.info("Performing the LFC->SE check")
gLogger.info("-" * 40)
return self.__checkPhysicalFiles(replicas, catalogMetadata, ses=ses)
def __checkPhysicalFiles(self, replicas, catalogMetadata, ses=[]):
""" This obtains the physical file metadata and checks the metadata against the catalog entries
"""
seLfns = {}
for lfn, replicaDict in replicas.items():
for se, _url in replicaDict.items():
if (ses) and (se not in ses):
continue
seLfns.setdefault(se, []).append(lfn)
gLogger.info('%s %s' %
('Storage Element'.ljust(20), 'Replicas'.rjust(20)))
for se in sortList(seLfns):
files = len(seLfns[se])
gLogger.info('%s %s' % (se.ljust(20), str(files).rjust(20)))
lfns = seLfns[se]
sizeMismatch = []
res = self.__checkPhysicalFileMetadata(lfns, se)
if not res['OK']:
gLogger.error('Failed to get physical file metadata.',
res['Message'])
return res
for lfn, metadata in res['Value'].items():
if lfn in catalogMetadata:
if (metadata['Size'] != catalogMetadata[lfn]['Size']) and (
metadata['Size'] != 0):
sizeMismatch.append((lfn, 'deprecatedUrl', se,
'CatalogPFNSizeMismatch'))
if sizeMismatch:
self.__reportProblematicReplicas(sizeMismatch, se,
'CatalogPFNSizeMismatch')
return S_OK()
def __checkPhysicalFileMetadata(self, lfns, se):
""" Check obtain the physical file metadata and check the files are available
"""
gLogger.info('Checking the integrity of %s physical files at %s' %
(len(lfns), se))
res = StorageElement(se).getFileMetadata(lfns)
if not res['OK']:
gLogger.error('Failed to get metadata for lfns.', res['Message'])
return res
lfnMetadataDict = res['Value']['Successful']
# If the replicas are completely missing
missingReplicas = []
for lfn, reason in res['Value']['Failed'].items():
if re.search('File does not exist', reason):
missingReplicas.append(
(lfn, 'deprecatedUrl', se, 'PFNMissing'))
if missingReplicas:
self.__reportProblematicReplicas(missingReplicas, se, 'PFNMissing')
lostReplicas = []
unavailableReplicas = []
zeroSizeReplicas = []
# If the files are not accessible
for lfn, lfnMetadata in lfnMetadataDict.items():
if lfnMetadata['Lost']:
lostReplicas.append((lfn, 'deprecatedUrl', se, 'PFNLost'))
if lfnMetadata['Unavailable']:
unavailableReplicas.append(
(lfn, 'deprecatedUrl', se, 'PFNUnavailable'))
if lfnMetadata['Size'] == 0:
zeroSizeReplicas.append(
(lfn, 'deprecatedUrl', se, 'PFNZeroSize'))
if lostReplicas:
self.__reportProblematicReplicas(lostReplicas, se, 'PFNLost')
if unavailableReplicas:
self.__reportProblematicReplicas(unavailableReplicas, se,
'PFNUnavailable')
if zeroSizeReplicas:
self.__reportProblematicReplicas(zeroSizeReplicas, se,
'PFNZeroSize')
gLogger.info(
'Checking the integrity of physical files at %s complete' % se)
return S_OK(lfnMetadataDict)
##########################################################################
#
# This section contains the specific methods for SE->LFC checks
#
def storageDirectoryToCatalog(self, lfnDir, storageElement):
""" This obtains the file found on the storage element in the supplied directories and determines whether they exist in the catalog and checks their metadata elements
"""
gLogger.info("-" * 40)
gLogger.info("Performing the SE->LFC check at %s" % storageElement)
gLogger.info("-" * 40)
if type(lfnDir) in types.StringTypes:
lfnDir = [lfnDir]
res = self.__getStorageDirectoryContents(lfnDir, storageElement)
if not res['OK']:
return res
storageFileMetadata = res['Value']
if storageFileMetadata:
return self.__checkCatalogForSEFiles(storageFileMetadata,
storageElement)
return S_OK({'CatalogMetadata': {}, 'StorageMetadata': {}})
def __checkCatalogForSEFiles(self, storageMetadata, storageElement):
gLogger.info('Checking %s storage files exist in the catalog' %
len(storageMetadata))
res = self.fc.getReplicas(storageMetadata)
if not res['OK']:
gLogger.error("Failed to get replicas for LFN", res['Message'])
return res
failedLfns = res['Value']['Failed']
successfulLfns = res['Value']['Successful']
notRegisteredLfns = []
for lfn in storageMetadata:
if lfn in failedLfns:
if 'No such file or directory' in failedLfns[lfn]:
notRegisteredLfns.append(
(lfn, 'deprecatedUrl', storageElement,
'LFNNotRegistered'))
failedLfns.pop(lfn)
elif storageElement not in successfulLfns[lfn]:
notRegisteredLfns.append(
(lfn, 'deprecatedUrl', storageElement, 'LFNNotRegistered'))
if notRegisteredLfns:
self.__reportProblematicReplicas(notRegisteredLfns, storageElement,
'LFNNotRegistered')
if failedLfns:
return S_ERROR('Failed to obtain replicas')
# For the LFNs found to be registered obtain the file metadata from the catalog and verify against the storage metadata
res = self.__getCatalogMetadata(storageMetadata)
if not res['OK']:
return res
catalogMetadata = res['Value']
sizeMismatch = []
for lfn, lfnCatalogMetadata in catalogMetadata.items():
lfnStorageMetadata = storageMetadata[lfn]
if (lfnStorageMetadata['Size'] != lfnCatalogMetadata['Size']) and (
lfnStorageMetadata['Size'] != 0):
sizeMismatch.append((lfn, 'deprecatedUrl', storageElement,
'CatalogPFNSizeMismatch'))
if sizeMismatch:
self.__reportProblematicReplicas(sizeMismatch, storageElement,
'CatalogPFNSizeMismatch')
gLogger.info('Checking storage files exist in the catalog complete')
resDict = {
'CatalogMetadata': catalogMetadata,
'StorageMetadata': storageMetadata
}
return S_OK(resDict)
def getStorageDirectoryContents(self, lfnDir, storageElement):
""" This obtains takes the supplied lfn directories and recursively obtains the files in the supplied storage element
"""
return self.__getStorageDirectoryContents(lfnDir, storageElement)
def __getStorageDirectoryContents(self, lfnDir, storageElement):
""" Obtians the contents of the supplied directory on the storage
"""
gLogger.info('Obtaining the contents for %s directories at %s' %
(len(lfnDir), storageElement))
se = StorageElement(storageElement)
res = se.exists(lfnDir)
if not res['OK']:
gLogger.error("Failed to obtain existance of directories",
res['Message'])
return res
for directory, error in res['Value']['Failed'].items():
gLogger.error('Failed to determine existance of directory',
'%s %s' % (directory, error))
if res['Value']['Failed']:
return S_ERROR('Failed to determine existance of directory')
directoryExists = res['Value']['Successful']
activeDirs = []
for directory in sorted(directoryExists):
exists = directoryExists[directory]
if exists:
activeDirs.append(directory)
allFiles = {}
while len(activeDirs) > 0:
currentDir = activeDirs[0]
res = se.listDirectory(currentDir)
activeDirs.remove(currentDir)
if not res['OK']:
gLogger.error('Failed to get directory contents',
res['Message'])
return res
elif currentDir in res['Value']['Failed']:
gLogger.error(
'Failed to get directory contents',
'%s %s' % (currentDir, res['Value']['Failed'][currentDir]))
return S_ERROR(res['Value']['Failed'][currentDir])
else:
dirContents = res['Value']['Successful'][currentDir]
activeDirs.extend(
se.getLFNFromURL(dirContents['SubDirs']).get(
'Value', {}).get('Successful', []))
fileURLMetadata = dirContents['Files']
fileMetadata = {}
res = se.getLFNFromURL(fileURLMetadata)
if not res['OK']:
gLogger.error('Failed to get directory content LFNs',
res['Message'])
return res
for url, error in res['Value']['Failed'].items():
gLogger.error("Failed to get LFN for URL",
"%s %s" % (url, error))
if res['Value']['Failed']:
return S_ERROR("Failed to get LFNs for PFNs")
urlLfns = res['Value']['Successful']
for urlLfn, lfn in urlLfns.items():
fileMetadata[lfn] = fileURLMetadata[urlLfn]
allFiles.update(fileMetadata)
zeroSizeFiles = []
for lfn in sorted(allFiles):
if os.path.basename(lfn) == 'dirac_directory':
allFiles.pop(lfn)
else:
metadata = allFiles[lfn]
if metadata['Size'] == 0:
zeroSizeFiles.append(
(lfn, 'deprecatedUrl', storageElement, 'PFNZeroSize'))
if zeroSizeFiles:
self.__reportProblematicReplicas(zeroSizeFiles, storageElement,
'PFNZeroSize')
gLogger.info('Obtained at total of %s files for directories at %s' %
(len(allFiles), storageElement))
return S_OK(allFiles)
def __getStoragePathExists(self, lfnPaths, storageElement):
gLogger.info('Determining the existance of %d files at %s' %
(len(lfnPaths), storageElement))
se = StorageElement(storageElement)
res = se.exists(lfnPaths)
if not res['OK']:
gLogger.error("Failed to obtain existance of paths",
res['Message'])
return res
for lfnPath, error in res['Value']['Failed'].items():
gLogger.error('Failed to determine existance of path',
'%s %s' % (lfnPath, error))
if res['Value']['Failed']:
return S_ERROR('Failed to determine existance of paths')
pathExists = res['Value']['Successful']
resDict = {}
for lfn, exists in pathExists.items():
if exists:
resDict[lfn] = True
return S_OK(resDict)
##########################################################################
#
# This section contains the specific methods for obtaining replica and metadata information from the catalog
#
def __getCatalogDirectoryContents(self, lfnDir):
""" Obtain the contents of the supplied directory
"""
gLogger.info('Obtaining the catalog contents for %s directories' %
len(lfnDir))
activeDirs = lfnDir
allFiles = {}
while len(activeDirs) > 0:
currentDir = activeDirs[0]
res = self.fc.listDirectory(currentDir)
activeDirs.remove(currentDir)
if not res['OK']:
gLogger.error('Failed to get directory contents',
res['Message'])
return res
elif res['Value']['Failed'].has_key(currentDir):
gLogger.error(
'Failed to get directory contents',
'%s %s' % (currentDir, res['Value']['Failed'][currentDir]))
else:
dirContents = res['Value']['Successful'][currentDir]
activeDirs.extend(dirContents['SubDirs'])
allFiles.update(dirContents['Files'])
zeroReplicaFiles = []
zeroSizeFiles = []
allReplicaDict = {}
allMetadataDict = {}
for lfn, lfnDict in allFiles.items():
lfnReplicas = {}
for se, replicaDict in lfnDict['Replicas'].items():
lfnReplicas[se] = replicaDict['PFN']
if not lfnReplicas:
zeroReplicaFiles.append(lfn)
allReplicaDict[lfn] = lfnReplicas
allMetadataDict[lfn] = lfnDict['MetaData']
if lfnDict['MetaData']['Size'] == 0:
zeroSizeFiles.append(lfn)
if zeroReplicaFiles:
self.__reportProblematicFiles(zeroReplicaFiles, 'LFNZeroReplicas')
if zeroSizeFiles:
self.__reportProblematicFiles(zeroSizeFiles, 'LFNZeroSize')
gLogger.info(
'Obtained at total of %s files for the supplied directories' %
len(allMetadataDict))
resDict = {'Metadata': allMetadataDict, 'Replicas': allReplicaDict}
return S_OK(resDict)
def __getCatalogReplicas(self, lfns):
""" Obtain the file replicas from the catalog while checking that there are replicas
"""
gLogger.info('Obtaining the replicas for %s files' % len(lfns))
zeroReplicaFiles = []
res = self.fc.getReplicas(lfns, allStatus=True)
if not res['OK']:
gLogger.error('Failed to get catalog replicas', res['Message'])
return res
allReplicas = res['Value']['Successful']
for lfn, error in res['Value']['Failed'].items():
if re.search('File has zero replicas', error):
zeroReplicaFiles.append(lfn)
if zeroReplicaFiles:
self.__reportProblematicFiles(zeroReplicaFiles, 'LFNZeroReplicas')
gLogger.info('Obtaining the replicas for files complete')
return S_OK(allReplicas)
def __getCatalogMetadata(self, lfns):
""" Obtain the file metadata from the catalog while checking they exist
"""
if not lfns:
return S_OK({})
gLogger.info('Obtaining the catalog metadata for %s files' % len(lfns))
missingCatalogFiles = []
zeroSizeFiles = []
res = self.fc.getFileMetadata(lfns)
if not res['OK']:
gLogger.error('Failed to get catalog metadata', res['Message'])
return res
allMetadata = res['Value']['Successful']
for lfn, error in res['Value']['Failed'].items():
if re.search('No such file or directory', error):
missingCatalogFiles.append(lfn)
if missingCatalogFiles:
self.__reportProblematicFiles(missingCatalogFiles,
'LFNCatalogMissing')
for lfn, metadata in allMetadata.items():
if metadata['Size'] == 0:
zeroSizeFiles.append(lfn)
if zeroSizeFiles:
self.__reportProblematicFiles(zeroSizeFiles, 'LFNZeroSize')
gLogger.info('Obtaining the catalog metadata complete')
return S_OK(allMetadata)
##########################################################################
#
# This section contains the methods for inserting problematic files into the integrity DB
#
def __reportProblematicFiles(self, lfns, reason):
""" Simple wrapper function around setFileProblematic """
gLogger.info('The following %s files were found with %s' %
(len(lfns), reason))
for lfn in sortList(lfns):
gLogger.info(lfn)
res = self.setFileProblematic(lfns,
reason,
sourceComponent='DataIntegrityClient')
if not res['OK']:
gLogger.info('Failed to update integrity DB with files',
res['Message'])
else:
gLogger.info('Successfully updated integrity DB with files')
def setFileProblematic(self, lfn, reason, sourceComponent=''):
""" This method updates the status of the file in the FileCatalog and the IntegrityDB
lfn - the lfn of the file
reason - this is given to the integrity DB and should reflect the problem observed with the file
sourceComponent is the component issuing the request.
"""
if type(lfn) == types.ListType:
lfns = lfn
elif type(lfn) == types.StringType:
lfns = [lfn]
else:
errStr = "DataIntegrityClient.setFileProblematic: Supplied file info must be list or a single LFN."
gLogger.error(errStr)
return S_ERROR(errStr)
gLogger.info(
"DataIntegrityClient.setFileProblematic: Attempting to update %s files."
% len(lfns))
fileMetadata = {}
for lfn in lfns:
fileMetadata[lfn] = {
'Prognosis': reason,
'LFN': lfn,
'PFN': '',
'SE': ''
}
res = self.insertProblematic(sourceComponent, fileMetadata)
if not res['OK']:
gLogger.error(
"DataIntegrityClient.setReplicaProblematic: Failed to insert problematics to integrity DB"
)
return res
def __reportProblematicReplicas(self, replicaTuple, se, reason):
""" Simple wrapper function around setReplicaProblematic """
gLogger.info('The following %s files had %s at %s' %
(len(replicaTuple), reason, se))
for lfn, _pfn, se, reason in sortList(replicaTuple):
if lfn:
gLogger.info(lfn)
res = self.setReplicaProblematic(replicaTuple,
sourceComponent='DataIntegrityClient')
if not res['OK']:
gLogger.info('Failed to update integrity DB with replicas',
res['Message'])
else:
gLogger.info('Successfully updated integrity DB with replicas')
def setReplicaProblematic(self, replicaTuple, sourceComponent=''):
""" This method updates the status of the replica in the FileCatalog and the IntegrityDB
The supplied replicaDict should be of the form {lfn :{'PFN':pfn,'SE':se,'Prognosis':prognosis}
lfn - the lfn of the file
pfn - the pfn if available (otherwise '')
se - the storage element of the problematic replica (otherwise '')
prognosis - this is given to the integrity DB and should reflect the problem observed with the file
sourceComponent is the component issuing the request.
"""
if type(replicaTuple) == types.TupleType:
replicaTuple = [replicaTuple]
elif type(replicaTuple) == types.ListType:
pass
else:
errStr = "DataIntegrityClient.setReplicaProblematic: Supplied replica info must be a tuple or list of tuples."
gLogger.error(errStr)
return S_ERROR(errStr)
gLogger.info(
"DataIntegrityClient.setReplicaProblematic: Attempting to update %s replicas."
% len(replicaTuple))
replicaDict = {}
for lfn, pfn, se, reason in replicaTuple:
replicaDict[lfn] = {
'Prognosis': reason,
'LFN': lfn,
'PFN': pfn,
'SE': se
}
res = self.insertProblematic(sourceComponent, replicaDict)
if not res['OK']:
gLogger.error(
"DataIntegrityClient.setReplicaProblematic: Failed to insert problematic to integrity DB"
)
return res
for lfn in replicaDict.keys():
replicaDict[lfn]['Status'] = 'Problematic'
res = self.fc.setReplicaStatus(replicaDict)
if not res['OK']:
errStr = "DataIntegrityClient.setReplicaProblematic: Completely failed to update replicas."
gLogger.error(errStr, res['Message'])
return res
failed = res['Value']['Failed']
successful = res['Value']['Successful']
resDict = {'Successful': successful, 'Failed': failed}
return S_OK(resDict)
##########################################################################
#
# This section contains the resolution methods for various prognoses
#
def __updateCompletedFiles(self, prognosis, fileID):
gLogger.info("%s file (%d) is resolved" % (prognosis, fileID))
return self.setProblematicStatus(fileID, 'Resolved')
def __returnProblematicError(self, fileID, res):
self.incrementProblematicRetry(fileID)
gLogger.error('DataIntegrityClient failure', res['Message'])
return res
# def __getRegisteredPFNLFN( self, pfn, storageElement ):
#
# res = StorageElement( storageElement ).getURL( pfn )
# if not res['OK']:
# gLogger.error( "Failed to get registered PFN for physical files", res['Message'] )
# return res
# for pfn, error in res['Value']['Failed'].items():
# gLogger.error( 'Failed to obtain registered PFN for physical file', '%s %s' % ( pfn, error ) )
# return S_ERROR( 'Failed to obtain registered PFNs from physical file' )
# registeredPFN = res['Value']['Successful'][pfn]
# res = returnSingleResult( self.fc.getLFNForPFN( registeredPFN ) )
# if ( not res['OK'] ) and re.search( 'No such file or directory', res['Message'] ):
# return S_OK( False )
# return S_OK( res['Value'] )
def __updateReplicaToChecked(self, problematicDict):
lfn = problematicDict['LFN']
fileID = problematicDict['FileID']
prognosis = problematicDict['Prognosis']
problematicDict['Status'] = 'Checked'
res = returnSingleResult(
self.fc.setReplicaStatus({lfn: problematicDict}))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
gLogger.info("%s replica (%d) is updated to Checked status" %
(prognosis, fileID))
return self.__updateCompletedFiles(prognosis, fileID)
def resolveCatalogPFNSizeMismatch(self, problematicDict):
""" This takes the problematic dictionary returned by the integrity DB and resolved the CatalogPFNSizeMismatch prognosis
"""
lfn = problematicDict['LFN']
se = problematicDict['SE']
fileID = problematicDict['FileID']
res = returnSingleResult(self.fc.getFileSize(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
catalogSize = res['Value']
res = returnSingleResult(StorageElement(se).getFileSize(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
storageSize = res['Value']
bkKCatalog = FileCatalog(['BookkeepingDB'])
res = returnSingleResult(bkKCatalog.getFileSize(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
bookkeepingSize = res['Value']
if bookkeepingSize == catalogSize == storageSize:
gLogger.info(
"CatalogPFNSizeMismatch replica (%d) matched all registered sizes."
% fileID)
return self.__updateReplicaToChecked(problematicDict)
if (catalogSize == bookkeepingSize):
gLogger.info(
"CatalogPFNSizeMismatch replica (%d) found to mismatch the bookkeeping also"
% fileID)
res = returnSingleResult(self.fc.getReplicas(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if len(res['Value']) <= 1:
gLogger.info(
"CatalogPFNSizeMismatch replica (%d) has no other replicas."
% fileID)
return S_ERROR(
"Not removing catalog file mismatch since the only replica"
)
else:
gLogger.info(
"CatalogPFNSizeMismatch replica (%d) has other replicas. Removing..."
% fileID)
res = self.dm.removeReplica(se, lfn)
if not res['OK']:
return self.__returnProblematicError(fileID, res)
return self.__updateCompletedFiles('CatalogPFNSizeMismatch',
fileID)
if (catalogSize != bookkeepingSize) and (bookkeepingSize
== storageSize):
gLogger.info(
"CatalogPFNSizeMismatch replica (%d) found to match the bookkeeping size"
% fileID)
res = self.__updateReplicaToChecked(problematicDict)
if not res['OK']:
return self.__returnProblematicError(fileID, res)
return self.changeProblematicPrognosis(fileID,
'BKCatalogSizeMismatch')
gLogger.info(
"CatalogPFNSizeMismatch replica (%d) all sizes found mismatch. Updating retry count"
% fileID)
return self.incrementProblematicRetry(fileID)
def resolvePFNNotRegistered(self, problematicDict):
""" This takes the problematic dictionary returned by the integrity DB and resolved the PFNNotRegistered prognosis
"""
lfn = problematicDict['LFN']
seName = problematicDict['SE']
fileID = problematicDict['FileID']
se = StorageElement(seName)
res = returnSingleResult(self.fc.exists(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if not res['Value']:
# The file does not exist in the catalog
res = returnSingleResult(se.removeFile(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
return self.__updateCompletedFiles('PFNNotRegistered', fileID)
res = returnSingleResult(se.getFileMetadata(lfn))
if (not res['OK']) and (re.search('File does not exist',
res['Message'])):
gLogger.info("PFNNotRegistered replica (%d) found to be missing." %
fileID)
return self.__updateCompletedFiles('PFNNotRegistered', fileID)
elif not res['OK']:
return self.__returnProblematicError(fileID, res)
storageMetadata = res['Value']
if storageMetadata['Lost']:
gLogger.info(
"PFNNotRegistered replica (%d) found to be Lost. Updating prognosis"
% fileID)
return self.changeProblematicPrognosis(fileID, 'PFNLost')
if storageMetadata['Unavailable']:
gLogger.info(
"PFNNotRegistered replica (%d) found to be Unavailable. Updating retry count"
% fileID)
return self.incrementProblematicRetry(fileID)
# HACK until we can obtain the space token descriptions through GFAL
site = seName.split('_')[0].split('-')[0]
if not storageMetadata['Cached']:
if lfn.endswith('.raw'):
seName = '%s-RAW' % site
else:
seName = '%s-RDST' % site
elif storageMetadata['Migrated']:
if lfn.startswith('/lhcb/data'):
seName = '%s_M-DST' % site
else:
seName = '%s_MC_M-DST' % site
else:
if lfn.startswith('/lhcb/data'):
seName = '%s-DST' % site
else:
seName = '%s_MC-DST' % site
problematicDict['SE'] = seName
res = returnSingleResult(se.getURL(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
problematicDict['PFN'] = res['Value']
res = returnSingleResult(self.fc.addReplica({lfn: problematicDict}))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
res = returnSingleResult(self.fc.getFileMetadata(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if res['Value']['Size'] != storageMetadata['Size']:
gLogger.info(
"PFNNotRegistered replica (%d) found with catalog size mismatch. Updating prognosis"
% fileID)
return self.changeProblematicPrognosis(fileID,
'CatalogPFNSizeMismatch')
return self.__updateCompletedFiles('PFNNotRegistered', fileID)
def resolveLFNCatalogMissing(self, problematicDict):
""" This takes the problematic dictionary returned by the integrity DB and resolved the LFNCatalogMissing prognosis
"""
lfn = problematicDict['LFN']
fileID = problematicDict['FileID']
res = returnSingleResult(self.fc.exists(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if res['Value']:
return self.__updateCompletedFiles('LFNCatalogMissing', fileID)
# Remove the file from all catalogs
# RF_NOTE : here I can do it because it's a single file, but otherwise I would need to sort the path
res = returnSingleResult(self.fc.removeFile(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
return self.__updateCompletedFiles('LFNCatalogMissing', fileID)
def resolvePFNMissing(self, problematicDict):
""" This takes the problematic dictionary returned by the integrity DB and resolved the PFNMissing prognosis
"""
se = problematicDict['SE']
lfn = problematicDict['LFN']
fileID = problematicDict['FileID']
res = returnSingleResult(self.fc.exists(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if not res['Value']:
gLogger.info("PFNMissing file (%d) no longer exists in catalog" %
fileID)
return self.__updateCompletedFiles('PFNMissing', fileID)
res = returnSingleResult(StorageElement(se).exists(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if res['Value']:
gLogger.info("PFNMissing replica (%d) is no longer missing" %
fileID)
return self.__updateReplicaToChecked(problematicDict)
gLogger.info("PFNMissing replica (%d) does not exist" % fileID)
res = returnSingleResult(self.fc.getReplicas(lfn, allStatus=True))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
replicas = res['Value']
seSite = se.split('_')[0].split('-')[0]
found = False
print replicas
for replicaSE in replicas.keys():
if re.search(seSite, replicaSE):
found = True
problematicDict['SE'] = replicaSE
se = replicaSE
if not found:
gLogger.info(
"PFNMissing replica (%d) is no longer registered at SE. Resolved."
% fileID)
return self.__updateCompletedFiles('PFNMissing', fileID)
gLogger.info(
"PFNMissing replica (%d) does not exist. Removing from catalog..."
% fileID)
res = returnSingleResult(self.fc.removeReplica({lfn: problematicDict}))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if len(replicas) == 1:
gLogger.info(
"PFNMissing replica (%d) had a single replica. Updating prognosis"
% fileID)
return self.changeProblematicPrognosis(fileID, 'LFNZeroReplicas')
res = self.dm.replicateAndRegister(problematicDict['LFN'], se)
if not res['OK']:
return self.__returnProblematicError(fileID, res)
# If we get here the problem is solved so we can update the integrityDB
return self.__updateCompletedFiles('PFNMissing', fileID)
def resolvePFNUnavailable(self, problematicDict):
""" This takes the problematic dictionary returned by the integrity DB and resolved the PFNUnavailable prognosis
"""
lfn = problematicDict['LFN']
se = problematicDict['SE']
fileID = problematicDict['FileID']
res = returnSingleResult(StorageElement(se).getFileMetadata(lfn))
if (not res['OK']) and (re.search('File does not exist',
res['Message'])):
# The file is no longer Unavailable but has now dissapeared completely
gLogger.info(
"PFNUnavailable replica (%d) found to be missing. Updating prognosis"
% fileID)
return self.changeProblematicPrognosis(fileID, 'PFNMissing')
if (not res['OK']) or res['Value']['Unavailable']:
gLogger.info(
"PFNUnavailable replica (%d) found to still be Unavailable" %
fileID)
return self.incrementProblematicRetry(fileID)
if res['Value']['Lost']:
gLogger.info(
"PFNUnavailable replica (%d) is now found to be Lost. Updating prognosis"
% fileID)
return self.changeProblematicPrognosis(fileID, 'PFNLost')
gLogger.info("PFNUnavailable replica (%d) is no longer Unavailable" %
fileID)
# Need to make the replica okay in the Catalog
return self.__updateReplicaToChecked(problematicDict)
def resolvePFNZeroSize(self, problematicDict):
""" This takes the problematic dictionary returned by the integrity DB and resolves the PFNZeroSize prognosis
"""
lfn = problematicDict['LFN']
seName = problematicDict['SE']
fileID = problematicDict['FileID']
se = StorageElement(seName)
res = returnSingleResult(se.getFileSize(lfn))
if (not res['OK']) and (re.search('File does not exist',
res['Message'])):
gLogger.info(
"PFNZeroSize replica (%d) found to be missing. Updating prognosis"
% problematicDict['FileID'])
return self.changeProblematicPrognosis(fileID, 'PFNMissing')
storageSize = res['Value']
if storageSize == 0:
res = returnSingleResult(se.removeFile(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
gLogger.info(
"PFNZeroSize replica (%d) removed. Updating prognosis" %
problematicDict['FileID'])
return self.changeProblematicPrognosis(fileID, 'PFNMissing')
res = returnSingleResult(self.fc.getReplicas(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if seName not in res['Value']:
gLogger.info(
"PFNZeroSize replica (%d) not registered in catalog. Updating prognosis"
% problematicDict['FileID'])
return self.changeProblematicPrognosis(fileID, 'PFNNotRegistered')
res = returnSingleResult(self.fc.getFileMetadata(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
catalogSize = res['Value']['Size']
if catalogSize != storageSize:
gLogger.info(
"PFNZeroSize replica (%d) size found to differ from registered metadata. Updating prognosis"
% problematicDict['FileID'])
return self.changeProblematicPrognosis(fileID,
'CatalogPFNSizeMismatch')
return self.__updateCompletedFiles('PFNZeroSize', fileID)
############################################################################################
def resolveLFNZeroReplicas(self, problematicDict):
""" This takes the problematic dictionary returned by the integrity DB and resolves the LFNZeroReplicas prognosis
"""
lfn = problematicDict['LFN']
fileID = problematicDict['FileID']
res = returnSingleResult(self.fc.getReplicas(lfn, allStatus=True))
if res['OK'] and res['Value']:
gLogger.info("LFNZeroReplicas file (%d) found to have replicas" %
fileID)
else:
gLogger.info(
"LFNZeroReplicas file (%d) does not have replicas. Checking storage..."
% fileID)
pfnsFound = False
for storageElementName in sorted(
gConfig.getValue(
'Resources/StorageElementGroups/Tier1_MC_M-DST', [])):
res = self.__getStoragePathExists([lfn], storageElementName)
if lfn in res['Value']:
gLogger.info(
"LFNZeroReplicas file (%d) found storage file at %s" %
(fileID, storageElementName))
self.__reportProblematicReplicas(
[(lfn, 'deprecatedUrl', storageElementName,
'PFNNotRegistered')], storageElementName,
'PFNNotRegistered')
pfnsFound = True
if not pfnsFound:
gLogger.info(
"LFNZeroReplicas file (%d) did not have storage files. Removing..."
% fileID)
res = returnSingleResult(self.fc.removeFile(lfn))
if not res['OK']:
gLogger.error('DataIntegrityClient: failed to remove file',
res['Message'])
# Increment the number of retries for this file
self.server.incrementProblematicRetry(fileID)
return res
gLogger.info("LFNZeroReplicas file (%d) removed from catalog" %
fileID)
# If we get here the problem is solved so we can update the integrityDB
return self.__updateCompletedFiles('LFNZeroReplicas', fileID)
class FTSJob(Record):
"""
.. class:: FTSJob
class describing one FTS job
"""
# # initial states
INITSTATES = ("Submitted", "Ready", "Staging")
# # ongoing transfer states
TRANSSTATES = ("Active", "Hold")
# # failed states
FAILEDSTATES = ("Canceled", "Failed")
# # finished
FINALSTATES = ("Finished", "FinishedDirty", "Failed", "Canceled")
# # missing source regexp patterns
missingSourceErrors = [
re.compile( r"SOURCE error during TRANSFER_PREPARATION phase: \[INVALID_PATH\] Failed" ),
re.compile( r"SOURCE error during TRANSFER_PREPARATION phase: \[INVALID_PATH\] No such file or directory" ),
re.compile( r"SOURCE error during PREPARATION phase: \[INVALID_PATH\] Failed" ),
re.compile( r"SOURCE error during PREPARATION phase: \[INVALID_PATH\] The requested file either does not exist" ),
re.compile( r"TRANSFER error during TRANSFER phase: \[INVALID_PATH\] the server sent an error response: 500 500"\
" Command failed. : open error: No such file or directory" ),
re.compile( r"SOURCE error during TRANSFER_PREPARATION phase: \[USER_ERROR\] source file doesnt exist" ) ]
def __init__(self, fromDict=None):
"""c'tor
:param self: self reference
:param dict fromDict: data dict
"""
Record.__init__(self)
now = datetime.datetime.utcnow().replace(microsecond=0)
self.__data__["CreationTime"] = now
self.__data__["SubmitTime"] = now
self.__data__["LastUpdate"] = now
self.__data__["Status"] = "Submitted"
self.__data__["Completeness"] = 0
self.__data__["FTSJobID"] = 0
self._regTime = 0.
self._regSuccess = 0
self._regTotal = 0
self.__files__ = TypedList(allowedTypes=FTSFile)
self._fc = FileCatalog()
self._log = gLogger.getSubLogger("FTSJob-%s" % self.FTSJobID, True)
self._states = tuple(
set(self.INITSTATES + self.TRANSSTATES + self.FAILEDSTATES +
self.FINALSTATES))
fromDict = fromDict if fromDict else {}
for ftsFileDict in fromDict.get("FTSFiles", []):
self += FTSFile(ftsFileDict)
if "FTSFiles" in fromDict: del fromDict["FTSFiles"]
for key, value in fromDict.items():
if key not in self.__data__:
raise AttributeError("Unknown FTSJob attribute '%s'" % key)
if value:
setattr(self, key, value)
@staticmethod
def tableDesc():
""" get table desc """
return { "Fields" :
{ "FTSJobID" : "INTEGER NOT NULL AUTO_INCREMENT",
"FTSGUID" : "VARCHAR(64) NOT NULL",
"OperationID": "INTEGER NOT NULL",
"RequestID": "INTEGER NOT NULL",
"SourceSE" : "VARCHAR(128) NOT NULL",
"TargetSE" : "VARCHAR(128) NOT NULL",
"FTSServer" : "VARCHAR(255) NOT NULL",
"TargetToken": "VARCHAR(255)",
"SourceToken": "VARCHAR(255)",
"Size": "BIGINT NOT NULL",
"Files": "INTEGER NOT NULL",
"Completeness": "INTEGER NOT NULL DEFAULT 0",
"FailedFiles": "INTEGER DEFAULT 0",
"FailedSize": "INTEGER DEFAULT 0",
"Status" : "ENUM( 'Submitted', 'Ready', 'Staging', 'Canceled', 'Active', 'Hold', "\
"'Failed', 'Finished', 'FinishedDirty', 'Assigned' ) DEFAULT 'Submitted'",
"Error" : "VARCHAR(255)",
"CreationTime" : "DATETIME",
"SubmitTime" : "DATETIME",
"LastUpdate" : "DATETIME" },
"PrimaryKey" : [ "FTSJobID" ],
"Indexes" : { "FTSJobID" : [ "FTSJobID" ], "FTSGUID": [ "FTSGUID" ] } }
@property
def FTSJobID(self):
""" FTSJobID getter """
return self.__data__["FTSJobID"]
@FTSJobID.setter
def FTSJobID(self, value):
""" FTSJobID setter """
self.__data__["FTSJobID"] = long(value) if value else 0
@property
def RequestID(self):
""" RequestID getter """
return self.__data__["RequestID"]
@RequestID.setter
def RequestID(self, value):
""" RequestID setter """
self.__data__["RequestID"] = long(value) if value else 0
@property
def OperationID(self):
""" OperationID getter """
return self.__data__["OperationID"]
@OperationID.setter
def OperationID(self, value):
""" OperationID setter """
self.__data__["OperationID"] = long(value) if value else 0
@property
def FTSGUID(self):
""" FTSGUID prop """
return self.__data__["FTSGUID"]
@FTSGUID.setter
def FTSGUID(self, value):
""" FTSGUID setter """
if value:
if type(value) not in (str, unicode):
raise TypeError("FTSGUID should be a string!")
if not checkGuid(value):
raise ValueError("'%s' is not a valid GUID!" % str(value))
self.__data__["FTSGUID"] = value
@property
def FTSServer(self):
""" FTSServer getter """
return self.__data__["FTSServer"]
@FTSServer.setter
def FTSServer(self, url):
""" FTSServer getter """
self.__data__["FTSServer"] = url
@property
def Completeness(self):
""" completeness getter """
return self.__data__["Completeness"]
@Completeness.setter
def Completeness(self, value):
""" completeness setter """
self.__data__["Completeness"] = int(value) if value else 0
@property
def Error(self):
""" error getter """
return self.__data__["Error"]
@Error.setter
def Error(self, error):
""" error setter """
self.__data__["Error"] = str(error)[255:]
@property
def Files(self):
""" nb files getter """
self.__data__["Files"] = len(self)
return self.__data__["Files"]
@Files.setter
def Files(self, value):
""" nb files setter """
self.__data__["Files"] = len(self)
@property
def Status(self):
""" status prop """
if not self.__data__["Status"]:
self.__data__["Status"] = "Waiting"
return self.__data__["Status"]
@Status.setter
def Status(self, value):
""" status setter """
value = self._normalizedStatus(value.strip())
if value not in self._states:
raise ValueError("Unknown FTSJob Status: '%s'" % str(value))
self.__data__["Status"] = value
@property
def FailedFiles(self):
""" nb failed files getter """
self.__data__["FailedFiles"] = len([
ftsFile for ftsFile in self
if ftsFile.Status in FTSFile.FAILED_STATES
])
return self.__data__["FailedFiles"]
@FailedFiles.setter
def FailedFiles(self, value):
""" nb failed files setter """
if value:
self.__data__["FailedFiles"] = value
else:
self.__data__["FailedFiles"] = sum([
ftsFile for ftsFile in self
if ftsFile.Status in FTSFile.FAILED_STATES
])
@property
def Size(self):
""" size getter """
# if not self.__data__["Size"]:
self.__data__["Size"] = sum([ftsFile.Size for ftsFile in self])
return self.__data__["Size"]
@Size.setter
def Size(self, value):
""" size setter """
if value:
self.__data__["Size"] = value
else:
self.__data__["Size"] = sum([ftsFile.Size for ftsFile in self])
@property
def FailedSize(self):
""" size getter """
if not self.__data__["FailedSize"]:
self.__data__["FailedSize"] = sum([
ftsFile.Size for ftsFile in self
if ftsFile.Status in FTSFile.FAILED_STATES
])
return self.__data__["FailedSize"]
@FailedSize.setter
def FailedSize(self, value):
""" size setter """
if value:
self.__data__["FailedSize"] = value
else:
self.__data__["FailedSize"] = sum([
ftsFile.Size for ftsFile in self
if ftsFile.Status in FTSFile.FAILED_STATES
])
@property
def CreationTime(self):
""" creation time getter """
return self.__data__["CreationTime"]
@CreationTime.setter
def CreationTime(self, value=None):
""" creation time setter """
if type(value) not in (datetime.datetime, str):
raise TypeError("CreationTime should be a datetime.datetime!")
if type(value) == str:
value = datetime.datetime.strptime(
value.split(".")[0], '%Y-%m-%d %H:%M:%S')
self.__data__["CreationTime"] = value
@property
def SubmitTime(self):
""" request's submission time getter """
return self.__data__["SubmitTime"]
@SubmitTime.setter
def SubmitTime(self, value=None):
""" submission time setter """
if type(value) not in (datetime.datetime, str):
raise TypeError("SubmitTime should be a datetime.datetime!")
if type(value) == str:
value = datetime.datetime.strptime(
value.split(".")[0], '%Y-%m-%d %H:%M:%S')
self.__data__["SubmitTime"] = value
@property
def LastUpdate(self):
""" last update getter """
return self.__data__["LastUpdate"]
@LastUpdate.setter
def LastUpdate(self, value=None):
""" last update setter """
if type(value) not in (datetime.datetime, str):
raise TypeError("LastUpdate should be a datetime.datetime!")
if type(value) == str:
value = datetime.datetime.strptime(
value.split(".")[0], '%Y-%m-%d %H:%M:%S')
self.__data__["LastUpdate"] = value
@property
def TargetSE(self):
""" target SE getter """
return self.__data__["TargetSE"]
@TargetSE.setter
def TargetSE(self, targetSE):
""" target SE setter """
self.__data__["TargetSE"] = targetSE
@property
def SourceSE(self):
""" source SE getter """
return self.__data__["SourceSE"]
@SourceSE.setter
def SourceSE(self, sourceSE):
""" source SE setter """
self.__data__["SourceSE"] = sourceSE
@property
def SourceToken(self):
""" source token getter """
return self.__data__["SourceToken"]
@SourceToken.setter
def SourceToken(self, sourceToken):
""" source SE setter """
self.__data__["SourceToken"] = sourceToken
@property
def TargetToken(self):
""" target token getter """
return self.__data__["TargetToken"]
@TargetToken.setter
def TargetToken(self, targetToken):
""" target SE setter """
self.__data__["TargetToken"] = targetToken
# # FTSJobFiles arithmetics
def __contains__(self, subFile):
""" in operator """
return subFile in self.__files__
def __iadd__(self, ftsFile):
""" += operator """
if ftsFile not in self:
self.__files__.append(ftsFile)
ftsFile._parent = self
self.Files
self.Size
return self
def __add__(self, ftsFile):
""" + operator """
self += ftsFile
def addFile(self, ftsFile):
""" add :ftsFile: to FTS job """
self += ftsFile
def subFile(self, ftsFile):
""" remove ftsFile from this job """
if ftsFile in self:
ftsFile._parent = None
self.__files__.remove(ftsFile)
# # helpers for looping
def __iter__(self):
""" files iterator """
return self.__files__.__iter__()
def __getitem__(self, i):
""" [] op for files """
return self.__files__.__getitem__(i)
def __delitem__(self, i):
""" del ftsJob[i] """
self.__files__.__delitem__(i)
def __setitem__(self, i, ftsFile):
""" ftsJob[i] = ftsFile """
self.__files__.__setitem__(i, ftsFile)
def fileStatusList(self):
""" get list of files statuses """
return [ftsFile.Status for ftsFile in self]
def __nonzero__(self):
""" for comparisons
"""
return True
def __len__(self):
""" nb of subFiles """
return len(self.__files__)
def _surlPairs(self):
""" create and return SURL pair file """
surls = []
for ftsFile in self:
checksum = "%s:%s" % (
ftsFile.ChecksumType, ftsFile.Checksum
) if ftsFile.ChecksumType and ftsFile.Checksum else ""
surls.append("%s %s %s" %
(ftsFile.SourceSURL, ftsFile.TargetSURL, checksum))
return "\n".join(surls)
def submitFTS2(self, command='glite-transfer-submit', pinTime=False):
""" submit fts job using FTS2 client """
if self.FTSGUID:
return S_ERROR("FTSJob has already been submitted")
surls = self._surlPairs()
if not surls:
return S_ERROR("No files to submit")
fd, fileName = tempfile.mkstemp()
surlFile = os.fdopen(fd, 'w')
surlFile.write(surls)
surlFile.close()
submitCommand = command.split() + \
[ "-s",
self.FTSServer,
"-f",
fileName,
"-o",
"-K" ]
if self.TargetToken:
submitCommand += ["-t", self.TargetToken]
if self.SourceToken:
submitCommand += ["-S", self.SourceToken]
if pinTime:
submitCommand += [
"--copy-pin-lifetime",
"%d" % pinTime, "--bring-online", '86400'
]
submit = executeGridCommand("", submitCommand)
os.remove(fileName)
if not submit["OK"]:
return submit
returnCode, output, errStr = submit["Value"]
if returnCode != 0:
return S_ERROR(errStr if errStr else output)
self.FTSGUID = output.replace("\n", "")
self.Status = "Submitted"
for ftsFile in self:
ftsFile.FTSGUID = self.FTSGUID
ftsFile.Status = "Submitted"
return S_OK()
def _normalizedStatus(self, status):
for st in self._states:
if status.lower() == st.lower():
return st
return status
def monitorFTS2(self, command="glite-transfer-status", full=False):
""" monitor fts job """
if not self.FTSGUID:
return S_ERROR("FTSGUID not set, FTS job not submitted?")
monitorCommand = command.split() + \
["--verbose",
"-s",
self.FTSServer,
self.FTSGUID ]
if full:
monitorCommand.append("-l")
monitor = executeGridCommand("", monitorCommand)
if not monitor["OK"]:
return monitor
returnCode, outputStr, errStr = monitor["Value"]
# Returns a non zero status if error
if returnCode != 0:
if 'was not found' in outputStr and not errStr:
errStr = 'Job was not found'
return S_ERROR(errStr)
outputStr = outputStr.replace("'", "").replace("<",
"").replace(">", "")
# # set FTS job status
regExp = re.compile("Status:\\s+(\\S+)")
# with FTS3 this can be uppercase
self.Status = re.search(regExp, outputStr).group(1)
statusSummary = {}
# This is capitalized, even in FTS3!
for state in FTSFile.ALL_STATES:
regExp = re.compile("\\s+%s:\\s+(\\d+)" % state)
if regExp.search(outputStr):
statusSummary[state] = int(
re.search(regExp, outputStr).group(1))
total = sum(statusSummary.values())
completed = sum(
[statusSummary.get(state, 0) for state in FTSFile.FINAL_STATES])
self.Completeness = 100 * completed / total if total else 0
if not full:
return S_OK(statusSummary)
# The order of informations is not the same for glite- and fts- !!!
# In order: new fts-, old fts-, glite-
iExptr = None
for iExptr, exptr in enumerate(
('[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Staging:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)',
'[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)',
'[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Retries:[ ]+(\\d+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)'
)):
regExp = re.compile(exptr, re.S)
fileInfo = re.findall(regExp, outputStr)
if fileInfo:
break
if not fileInfo:
return S_ERROR("Error monitoring job (no regexp match)")
for info in fileInfo:
if iExptr == 0:
# version >= 3.2.30
sourceURL, targetURL, fileStatus, reason, duration, _retries, _staging = info
elif iExptr == 1:
# version FTS3 < 3.2.30
sourceURL, targetURL, fileStatus, reason, duration, _retries = info
elif iExptr == 2:
# version FTS2
sourceURL, targetURL, fileStatus, _retries, reason, duration = info
else:
return S_ERROR('Error monitoring job (implement match %d)' %
iExptr)
candidateFile = None
for ftsFile in self:
if ftsFile.SourceSURL == sourceURL:
candidateFile = ftsFile
break
if not candidateFile:
continue
# Can be uppercase for FTS3
if not candidateFile.TargetSURL:
candidateFile.TargetSURL = targetURL
candidateFile.Status = fileStatus
candidateFile.Error = reason
candidateFile._duration = duration
if candidateFile.Status == "Failed":
for missingSource in self.missingSourceErrors:
if missingSource.match(reason):
candidateFile.Error = "MissingSource"
# If the staging info was present, record it
if len(info) > 6:
candidateFile._staging = info[6]
# # register successful files
if self.Status in FTSJob.FINALSTATES:
return self.finalize()
return S_OK()
def finalize(self):
""" register successfully transferred files """
if self.Status not in FTSJob.FINALSTATES:
return S_OK()
startTime = time.time()
targetSE = StorageElement(self.TargetSE)
toRegister = [
ftsFile for ftsFile in self if ftsFile.Status == "Finished"
]
toRegisterDict = {}
for ftsFile in toRegister:
pfn = returnSingleResult(
targetSE.getPfnForProtocol(ftsFile.TargetSURL,
protocol="SRM2",
withPort=False))
if not pfn["OK"]:
continue
pfn = pfn["Value"]
toRegisterDict[ftsFile.LFN] = {"PFN": pfn, "SE": self.TargetSE}
if toRegisterDict:
self._regTotal += len(toRegisterDict)
register = self._fc.addReplica(toRegisterDict)
self._regTime += time.time() - startTime
if not register["OK"]:
for ftsFile in toRegister:
ftsFile.Error = "AddCatalogReplicaFailed"
return register
register = register["Value"]
self._regSuccess += len(register.get('Successful', {}))
failedFiles = register.get("Failed", {})
for ftsFile in toRegister:
if ftsFile.LFN in failedFiles:
ftsFile.Error = "AddCatalogReplicaFailed"
return S_OK()
def toSQL(self):
""" prepare SQL INSERT or UPDATE statement
:return: str with SQL fragment
"""
colVals = [
("`%s`" % column, "'%s'" %
value if type(value) in (str, datetime.datetime) else str(value))
for column, value in self.__data__.items()
if value and column not in ("FTSJobID", "LastUpdate")
]
colVals.append(("`LastUpdate`", "UTC_TIMESTAMP()"))
query = []
if self.FTSJobID:
query.append("UPDATE `FTSJob` SET ")
query.append(",".join(["%s=%s" % item for item in colVals]))
query.append(" WHERE `FTSJobID`=%d;\n" % self.FTSJobID)
else:
query.append("INSERT INTO `FTSJob` ")
columns = "(%s)" % ",".join([column for column, value in colVals])
values = "(%s)" % ",".join([value for column, value in colVals])
query.append(columns)
query.append(" VALUES %s;" % values)
return S_OK("".join(query))
def toJSON(self):
""" dump to JSON format """
digest = dict(
zip(self.__data__.keys(),
[str(val) if val else "" for val in self.__data__.values()]))
digest["FTSFiles"] = []
for ftsFile in self:
fileJSON = ftsFile.toJSON()
if not fileJSON["OK"]:
return fileJSON
digest["FTSFiles"].append(fileJSON["Value"])
return S_OK(digest)
class FTSRequest( object ):
"""
.. class:: FTSRequest
Helper class for FTS job submission and monitoring.
"""
# # default checksum type
__defaultCksmType = "ADLER32"
# # flag to disablr/enable checksum test, default: disabled
__cksmTest = False
def __init__( self ):
"""c'tor
:param self: self reference
"""
self.log = gLogger.getSubLogger( self.__class__.__name__, True )
# # final states tuple
self.finalStates = ( 'Canceled', 'Failed', 'Hold',
'Finished', 'FinishedDirty' )
# # failed states tuple
self.failedStates = ( 'Canceled', 'Failed',
'Hold', 'FinishedDirty' )
# # successful states tuple
self.successfulStates = ( 'Finished', 'Done' )
# # all file states tuple
self.fileStates = ( 'Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed',
'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting' )
self.statusSummary = {}
# # request status
self.requestStatus = 'Unknown'
# # dict for FTS job files
self.fileDict = {}
# # dict for replicas information
self.catalogReplicas = {}
# # dict for metadata information
self.catalogMetadata = {}
# # dict for files that failed to register
self.failedRegistrations = {}
# # placehoder for FileCatalog reference
self.oCatalog = None
# # submit timestamp
self.submitTime = ''
# # placeholder FTS job GUID
self.ftsGUID = ''
# # placeholder for FTS server URL
self.ftsServer = ''
# # flag marking FTS job completness
self.isTerminal = False
# # completness percentage
self.percentageComplete = 0.0
# # source SE name
self.sourceSE = ''
# # flag marking source SE validity
self.sourceValid = False
# # source space token
self.sourceToken = ''
# # target SE name
self.targetSE = ''
# # flag marking target SE validity
self.targetValid = False
# # target space token
self.targetToken = ''
# # placeholder for target StorageElement
self.oTargetSE = None
# # placeholder for source StorageElement
self.oSourceSE = None
# # checksum type, set it to default
self.__cksmType = self.__defaultCksmType
# # disable checksum test by default
self.__cksmTest = False
# # statuses that prevent submitting to FTS
self.noSubmitStatus = ( 'Failed', 'Done', 'Staging' )
# # were sources resolved?
self.sourceResolved = False
# # Number of file transfers actually submitted
self.submittedFiles = 0
self.transferTime = 0
self.submitCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/SubmitCommand', 'glite-transfer-submit' )
self.monitorCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/MonitorCommand', 'glite-transfer-status' )
self.ftsVersion = Operations().getValue( 'DataManagement/FTSVersion', 'FTS2' )
self.ftsJob = None
self.ftsFiles = []
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE( self, se ):
""" set SE for source
:param self: self reference
:param str se: source SE name
"""
if se == self.targetSE:
return S_ERROR( "SourceSE is TargetSE" )
self.sourceSE = se
self.oSourceSE = StorageElement( self.sourceSE )
return self.__checkSourceSE()
def __checkSourceSE( self ):
""" check source SE availability
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR( "SourceSE not set" )
res = self.oSourceSE.isValid( 'Read' )
if not res['OK']:
return S_ERROR( "SourceSE not available for reading" )
res = self.__getSESpaceToken( self.oSourceSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message'] )
return S_ERROR( "SourceSE does not support FTS transfers" )
if self.__cksmTest:
cksmType = self.oSourceSE.checksumType()
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at SourceSE %s, disabling checksum test" % ( cksmType,
self.sourceSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE( self, se ):
""" set target SE
:param self: self reference
:param str se: target SE name
"""
if se == self.sourceSE:
return S_ERROR( "TargetSE is SourceSE" )
self.targetSE = se
self.oTargetSE = StorageElement( self.targetSE )
return self.__checkTargetSE()
def setTargetToken( self, token ):
""" target space token setter
:param self: self reference
:param str token: target space token
"""
self.targetToken = token
return S_OK()
def __checkTargetSE( self ):
""" check target SE availability
:param self: self reference
"""
if not self.targetSE:
return S_ERROR( "TargetSE not set" )
res = self.oTargetSE.isValid( 'Write' )
if not res['OK']:
return S_ERROR( "TargetSE not available for writing" )
res = self.__getSESpaceToken( self.oTargetSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message'] )
return S_ERROR( "TargetSE does not support FTS transfers" )
# # check checksum types
if self.__cksmTest:
cksmType = self.oTargetSE.checksumType()
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at TargetSE %s, disabling checksum test" % ( cksmType,
self.targetSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
@staticmethod
def __getSESpaceToken( oSE ):
""" get space token from StorageElement instance
:param self: self reference
:param StorageElement oSE: StorageElement instance
"""
res = oSE.getStorageParameters( protocol = 'srm' )
if not res['OK']:
return res
return S_OK( res['Value'].get( 'SpaceToken' ) )
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID( self, guid ):
""" FTS job GUID setter
:param self: self reference
:param str guid: string containg GUID
"""
if not checkGuid( guid ):
return S_ERROR( "Incorrect GUID format" )
self.ftsGUID = guid
return S_OK()
def setFTSServer( self, server ):
""" FTS server setter
:param self: self reference
:param str server: FTS server URL
"""
self.ftsServer = server
return S_OK()
def isRequestTerminal( self ):
""" check if FTS job has terminated
:param self: self reference
"""
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK( self.isTerminal )
def setCksmTest( self, cksmTest = False ):
""" set cksm test
:param self: self reference
:param bool cksmTest: flag to enable/disable checksum test
"""
self.__cksmTest = bool( cksmTest )
return S_OK( self.__cksmTest )
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN( self, lfn ):
""" add LFN :lfn: to :fileDict:
:param self: self reference
:param str lfn: LFN to add to
"""
self.fileDict.setdefault( lfn, {'Status':'Waiting'} )
return S_OK()
def setSourceSURL( self, lfn, surl ):
""" source SURL setter
:param self: self reference
:param str lfn: LFN
:param str surl: source SURL
"""
target = self.fileDict[lfn].get( 'Target' )
if target == surl:
return S_ERROR( "Source and target the same" )
return self.__setFileParameter( lfn, 'Source', surl )
def getSourceSURL( self, lfn ):
""" get source SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Source' )
def setTargetSURL( self, lfn, surl ):
""" set target SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
:param str surl: target SURL
"""
source = self.fileDict[lfn].get( 'Source' )
if source == surl:
return S_ERROR( "Source and target the same" )
return self.__setFileParameter( lfn, 'Target', surl )
def getFailReason( self, lfn ):
""" get fail reason for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Reason' )
def getRetries( self, lfn ):
""" get number of attepmts made to transfer file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Retries' )
def getTransferTime( self, lfn ):
""" get duration of transfer for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Duration' )
def getFailed( self ):
""" get list of wrongly transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.failedStates ] )
def getStaging( self ):
""" get files set for prestaging """
return S_OK( [lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) == 'Staging'] )
def getDone( self ):
""" get list of succesfully transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.successfulStates ] )
def __setFileParameter( self, lfn, paramName, paramValue ):
""" set :paramName: to :paramValue: for :lfn: file
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
:param mixed paramValue: a new parameter value
"""
self.setLFN( lfn )
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter( self, lfn, paramName ):
""" get value of :paramName: for file :lfn:
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
"""
if lfn not in self.fileDict:
return S_ERROR( "Supplied file not set" )
if paramName not in self.fileDict[lfn]:
return S_ERROR( "%s not set for file" % paramName )
return S_OK( self.fileDict[lfn][paramName] )
####################################################################
#
# Methods for submission
#
def submit( self, monitor = False, printOutput = True ):
""" submit FTS job
:param self: self reference
:param bool monitor: flag to monitor progress of FTS job
:param bool printOutput: flag to print output of execution to stdout
"""
res = self.__prepareForSubmission()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = { 'ftsGUID' : self.ftsGUID, 'ftsServer' : self.ftsServer, 'submittedFiles' : self.submittedFiles }
if monitor or printOutput:
gLogger.always( "Submitted %s@%s" % ( self.ftsGUID, self.ftsServer ) )
if monitor:
self.monitor( untilTerminal = True, printOutput = printOutput, full = False )
return S_OK( resDict )
def __prepareForSubmission( self ):
""" check validity of job before submission
:param self: self reference
"""
if not self.fileDict:
return S_ERROR( "No files set" )
if not self.sourceValid:
return S_ERROR( "SourceSE not valid" )
if not self.targetValid:
return S_ERROR( "TargetSE not valid" )
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR( "FTSServer not valid" )
self.resolveSource()
self.resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR( "No files to submit" )
return S_OK()
def __getCatalogObject( self ):
""" CatalogInterface instance facade
:param self: self reference
"""
try:
if not self.oCatalog:
self.oCatalog = FileCatalog()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache( self, lfns = None, overwrite = False ):
""" update replica cache for list of :lfns:
:param self: self reference
:param mixed lfns: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if ( lfn not in self.catalogReplicas ) or overwrite ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getReplicas( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to update replica cache: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache( self, lfns = None ):
""" update metadata cache for list of LFNs
:param self: self reference
:param list lnfs: list of LFNs
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if lfn not in self.catalogMetadata ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getFileMetadata( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to get source catalog metadata: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def resolveSource( self ):
""" resolve source SE eligible for submission
:param self: self reference
"""
# Avoid resolving sources twice
if self.sourceResolved:
return S_OK()
# Only resolve files that need a transfer
toResolve = [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( "Status", "" ) != "Failed" ]
if not toResolve:
return S_OK()
res = self.__updateMetadataCache( toResolve )
if not res['OK']:
return res
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
# Define the source URLs
for lfn in toResolve:
replicas = self.catalogReplicas.get( lfn, {} )
if self.sourceSE not in replicas:
gLogger.warn( "resolveSource: skipping %s - not replicas at SourceSE %s" % ( lfn, self.sourceSE ) )
self.__setFileParameter( lfn, 'Reason', "No replica at SourceSE" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = returnSingleResult( self.oSourceSE.getURL( lfn, protocol = 'srm' ) )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setSourceSURL( lfn, res['Value'] )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = []
for lfn in self.fileDict:
if "Source" in self.fileDict[lfn]:
toResolve.append( lfn )
if not toResolve:
return S_ERROR( "No eligible Source files" )
# Get metadata of the sources, to check for existance, availability and caching
res = self.oSourceSE.getFileMetadata( toResolve )
if not res['OK']:
return S_ERROR( "Failed to check source file metadata" )
for lfn, error in res['Value']['Failed'].items():
if re.search( 'File does not exist', error ):
gLogger.warn( "resolveSource: skipping %s - source file does not exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file does not exist" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
gLogger.warn( "resolveSource: skipping %s - failed to get source metadata" % lfn )
self.__setFileParameter( lfn, 'Reason', "Failed to get Source metadata" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toStage = []
nbStagedFiles = 0
for lfn, metadata in res['Value']['Successful'].items():
lfnStatus = self.fileDict.get( lfn, {} ).get( 'Status' )
if metadata.get( 'Unavailable', False ):
gLogger.warn( "resolveSource: skipping %s - source file unavailable" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Unavailable" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif metadata.get( 'Lost', False ):
gLogger.warn( "resolveSource: skipping %s - source file lost" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Lost" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif not metadata.get( 'Cached', metadata['Accessible'] ):
if lfnStatus != 'Staging':
toStage.append( lfn )
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
gLogger.warn( "resolveSource: skipping %s - source file size mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source size mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \
not compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ):
gLogger.warn( "resolveSource: skipping %s - source file checksum mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source checksum mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif lfnStatus == 'Staging':
# file that was staging is now cached
self.__setFileParameter( lfn, 'Status', 'Waiting' )
nbStagedFiles += 1
# Some files were being staged
if nbStagedFiles:
self.log.info( 'resolveSource: %d files have been staged' % nbStagedFiles )
# Launching staging of files not in cache
if toStage:
gLogger.warn( "resolveSource: %s source files not cached, prestaging..." % len( toStage ) )
stage = self.oSourceSE.prestageFile( toStage )
if not stage["OK"]:
gLogger.error( "resolveSource: error is prestaging", stage["Message"] )
for lfn in toStage:
self.__setFileParameter( lfn, 'Reason', stage["Message"] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
for lfn in toStage:
if lfn in stage['Value']['Successful']:
self.__setFileParameter( lfn, 'Status', 'Staging' )
elif lfn in stage['Value']['Failed']:
self.__setFileParameter( lfn, 'Reason', stage['Value']['Failed'][lfn] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
self.sourceResolved = True
return S_OK()
def resolveTarget( self ):
""" find target SE eligible for submission
:param self: self reference
"""
toResolve = [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status' ) not in self.noSubmitStatus ]
if not toResolve:
return S_OK()
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
for lfn in toResolve:
res = returnSingleResult( self.oTargetSE.getURL( lfn, protocol = 'srm' ) )
if not res['OK']:
reason = res.get( 'Message', res['Message'] )
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, reason ) )
self.__setFileParameter( lfn, 'Reason', reason )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setTargetSURL( lfn, res['Value'] )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = []
for lfn in self.fileDict:
if "Target" in self.fileDict[lfn]:
toResolve.append( lfn )
if not toResolve:
return S_ERROR( "No eligible Target files" )
res = self.oTargetSE.exists( toResolve )
if not res['OK']:
return S_ERROR( "Failed to check target existence" )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toRemove = []
for lfn, exists in res['Value']['Successful'].items():
if exists:
res = self.getSourceSURL( lfn )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - target exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Target exists" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif res['Value'] == self.fileDict[lfn]['Target']:
gLogger.warn( "resolveTarget: skipping %s - source and target pfns are the same" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source and Target the same" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRemove.append( lfn )
if toRemove:
self.oTargetSE.removeFile( toRemove )
return S_OK()
def __filesToSubmit( self ):
"""
check if there is at least one file to submit
:return: S_OK if at least one file is present, S_ERROR otherwise
"""
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if lfnStatus not in self.noSubmitStatus and source and target:
return S_OK()
return S_ERROR()
def __createFTSFiles( self ):
""" create LFNs file for glite-transfer-submit command
This file consists one line for each fiel to be transferred:
sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM]
:param self: self reference
"""
self.__updateMetadataCache()
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
if lfnStatus not in self.noSubmitStatus:
cksmStr = ""
# # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself
if self.__cksmTest and self.__cksmType:
checkSum = self.catalogMetadata.get( lfn, {} ).get( 'Checksum' )
if checkSum:
cksmStr = " %s:%s" % ( self.__cksmType, intAdlerToHex( hexAdlerToInt( checkSum ) ) )
ftsFile = FTSFile()
ftsFile.LFN = lfn
ftsFile.SourceSURL = self.fileDict[lfn].get( 'Source' )
ftsFile.TargetSURL = self.fileDict[lfn].get( 'Target' )
ftsFile.SourceSE = self.sourceSE
ftsFile.TargetSE = self.targetSE
ftsFile.Status = self.fileDict[lfn].get( 'Status' )
ftsFile.Checksum = cksmStr
ftsFile.Size = self.catalogMetadata.get( lfn, {} ).get( 'Size' )
self.ftsFiles.append( ftsFile )
self.submittedFiles += 1
return S_OK()
def __createFTSJob( self, guid = None ):
self.__createFTSFiles()
ftsJob = FTSJob()
ftsJob.RequestID = 0
ftsJob.OperationID = 0
ftsJob.SourceSE = self.sourceSE
ftsJob.TargetSE = self.targetSE
ftsJob.SourceToken = self.sourceToken
ftsJob.TargetToken = self.targetToken
ftsJob.FTSServer = self.ftsServer
if guid:
ftsJob.FTSGUID = guid
for ftsFile in self.ftsFiles:
ftsFile.Attempt += 1
ftsFile.Error = ""
ftsJob.addFile( ftsFile )
self.ftsJob = ftsJob
def __submitFTSTransfer( self ):
""" create and execute glite-transfer-submit CLI command
:param self: self reference
"""
log = gLogger.getSubLogger( 'Submit' )
self.__createFTSJob()
submit = self.ftsJob.submitFTS( self.ftsVersion, command = self.submitCommand )
if not submit["OK"]:
log.error( "unable to submit FTSJob: %s" % submit["Message"] )
return submit
log.info( "FTSJob '%s'@'%s' has been submitted" % ( self.ftsJob.FTSGUID, self.ftsJob.FTSServer ) )
# # update statuses for job files
for ftsFile in self.ftsJob:
ftsFile.FTSGUID = self.ftsJob.FTSGUID
ftsFile.Status = "Submitted"
ftsFile.Attempt += 1
log.info( "FTSJob '%s'@'%s' has been submitted" % ( self.ftsJob.FTSGUID, self.ftsJob.FTSServer ) )
self.ftsGUID = self.ftsJob.FTSGUID
return S_OK()
def __resolveFTSServer( self ):
"""
resolve FTS server to use, it should be the closest one from target SE
:param self: self reference
"""
if self.ftsVersion.upper() == 'FTS2':
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTS2ServersForSites
if not self.targetSE:
return S_ERROR( "Target SE not set" )
res = getSitesForSE( self.targetSE )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine target site" )
targetSites = res['Value']
targetSite = ''
for targetSite in targetSites:
targetFTS = getFTS2ServersForSites( [targetSite] )
if targetFTS['OK']:
ftsTarget = targetFTS['Value'][targetSite]
if ftsTarget:
self.ftsServer = ftsTarget
return S_OK( self.ftsServer )
else:
return targetFTS
elif self.ftsVersion.upper() == 'FTS3':
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTS3Servers
res = getFTS3Servers()
if not res['OK']:
return res
ftsServerList = res['Value']
if ftsServerList:
# Here we take the first one, regardless of the policy...
# Unclean but all this will disapear after refactoring the fts code
self.ftsServer = ftsServerList[0]
return S_OK( self.ftsServer )
else:
return S_ERROR( 'Unknown FTS version %s' % self.ftsVersion )
return S_ERROR( 'No FTS server found for %s' % targetSite )
####################################################################
#
# Methods for monitoring
#
def summary( self, untilTerminal = False, printOutput = False ):
""" summary of FTS job
:param self: self reference
:param bool untilTerminal: flag to monitor FTS job to its final state
:param bool printOutput: flag to print out monitoring information to the stdout
"""
res = self.__isSummaryValid()
if not res['OK']:
return res
while not self.isTerminal:
res = self.__parseOutput( full = True )
if not res['OK']:
return res
if untilTerminal:
self.__print()
self.isRequestTerminal()
if res['Value'] or ( not untilTerminal ):
break
time.sleep( 1 )
if untilTerminal:
print ""
if printOutput and ( not untilTerminal ):
return self.dumpSummary( printOutput = printOutput )
return S_OK()
def monitor( self, untilTerminal = False, printOutput = False, full = True ):
""" monitor FTS job
:param self: self reference
:param bool untilTerminal: flag to monitor FTS job to its final state
:param bool printOutput: flag to print out monitoring information to the stdout
"""
if not self.ftsJob:
self.resolveSource()
self.__createFTSJob( self.ftsGUID )
res = self.__isSummaryValid()
if not res['OK']:
return res
if untilTerminal:
res = self.summary( untilTerminal = untilTerminal, printOutput = printOutput )
if not res['OK']:
return res
res = self.__parseOutput( full = full )
if not res['OK']:
return res
if untilTerminal:
self.finalize()
if printOutput:
self.dump()
return res
def dumpSummary( self, printOutput = False ):
""" get FTS job summary as str
:param self: self reference
:param bool printOutput: print summary to stdout
"""
outStr = ''
for status in sorted( self.statusSummary ):
if self.statusSummary[status]:
outStr = '%s\t%-10s : %-10s\n' % ( outStr, status, str( self.statusSummary[status] ) )
outStr = outStr.rstrip( '\n' )
if printOutput:
print outStr
return S_OK( outStr )
def __print( self ):
""" print progress bar of FTS job completeness to stdout
:param self: self reference
"""
width = 100
bits = int( ( width * self.percentageComplete ) / 100 )
outStr = "|%s>%s| %.1f%s %s %s" % ( "="*bits, " "*( width - bits ),
self.percentageComplete, "%",
self.requestStatus, " "*10 )
sys.stdout.write( "%s\r" % ( outStr ) )
sys.stdout.flush()
def dump( self ):
""" print FTS job parameters and files to stdout
:param self: self reference
"""
print "%-10s : %-10s" % ( "Status", self.requestStatus )
print "%-10s : %-10s" % ( "Source", self.sourceSE )
print "%-10s : %-10s" % ( "Target", self.targetSE )
print "%-10s : %-128s" % ( "Server", self.ftsServer )
print "%-10s : %-128s" % ( "GUID", self.ftsGUID )
for lfn in sorted( self.fileDict ):
print "\n %-15s : %-128s" % ( 'LFN', lfn )
for key in ['Source', 'Target', 'Status', 'Reason', 'Duration']:
print " %-15s : %-128s" % ( key, str( self.fileDict[lfn].get( key ) ) )
return S_OK()
def __isSummaryValid( self ):
""" check validity of FTS job summary report
:param self: self reference
"""
if not self.ftsServer:
return S_ERROR( "FTSServer not set" )
if not self.ftsGUID:
return S_ERROR( "FTSGUID not set" )
return S_OK()
def __parseOutput( self, full = False ):
""" execute glite-transfer-status command and parse its output
:param self: self reference
:param bool full: glite-transfer-status verbosity level, when set, collect information of files as well
"""
monitor = self.ftsJob.monitorFTS( self.ftsVersion, command = self.monitorCommand, full = full )
if not monitor['OK']:
return monitor
self.percentageComplete = self.ftsJob.Completeness
self.requestStatus = self.ftsJob.Status
self.submitTime = self.ftsJob.SubmitTime
statusSummary = monitor['Value']
if statusSummary:
for state in statusSummary:
self.statusSummary[state] = statusSummary[state]
self.transferTime = 0
for ftsFile in self.ftsJob:
lfn = ftsFile.LFN
self.__setFileParameter( lfn, 'Status', ftsFile.Status )
self.__setFileParameter( lfn, 'Reason', ftsFile.Error )
self.__setFileParameter( lfn, 'Duration', ftsFile._duration )
targetURL = self.__getFileParameter( lfn, 'Target' )
if not targetURL['OK']:
self.__setFileParameter( lfn, 'Target', ftsFile.TargetSURL )
sourceURL = self.__getFileParameter( lfn, 'Source' )
if not sourceURL['OK']:
self.__setFileParameter( lfn, 'Source', ftsFile.SourceSURL )
self.transferTime += int( ftsFile._duration )
return S_OK()
####################################################################
#
# Methods for finalization
#
def finalize( self ):
""" finalize FTS job
:param self: self reference
"""
self.__updateMetadataCache()
transEndTime = dateTime()
regStartTime = time.time()
res = self.getTransferStatistics()
transDict = res['Value']
res = self.__registerSuccessful( transDict['transLFNs'] )
regSuc, regTotal = res['Value']
regTime = time.time() - regStartTime
if self.sourceSE and self.targetSE:
self.__sendAccounting( regSuc, regTotal, regTime, transEndTime, transDict )
return S_OK()
def getTransferStatistics( self ):
""" collect information of Transfers that can be used by Accounting
:param self: self reference
"""
transDict = { 'transTotal': len( self.fileDict ),
'transLFNs': [],
'transOK': 0,
'transSize': 0 }
for lfn in self.fileDict:
if self.fileDict[lfn].get( 'Status' ) in self.successfulStates:
if self.fileDict[lfn].get( 'Duration', 0 ):
transDict['transLFNs'].append( lfn )
transDict['transOK'] += 1
if lfn in self.catalogMetadata:
transDict['transSize'] += self.catalogMetadata[lfn].get( 'Size', 0 )
return S_OK( transDict )
def getFailedRegistrations( self ):
""" get failed registrations dict
:param self: self reference
"""
return S_OK( self.failedRegistrations )
def __registerSuccessful( self, transLFNs ):
""" register successfully transferred files to the catalogs,
fill failedRegistrations dict for files that failed to register
:param self: self reference
:param list transLFNs: LFNs in FTS job
"""
self.failedRegistrations = {}
toRegister = {}
for lfn in transLFNs:
res = returnSingleResult( self.oTargetSE.getURL( self.fileDict[lfn].get( 'Target' ), protocol = 'srm' ) )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRegister[lfn] = { 'PFN' : res['Value'], 'SE' : self.targetSE }
if not toRegister:
return S_OK( ( 0, 0 ) )
res = self.__getCatalogObject()
if not res['OK']:
for lfn in toRegister:
self.failedRegistrations = toRegister
self.log.error( 'Failed to get Catalog Object', res['Message'] )
return S_OK( ( 0, len( toRegister ) ) )
res = self.oCatalog.addReplica( toRegister )
if not res['OK']:
self.failedRegistrations = toRegister
self.log.error( 'Failed to get Catalog Object', res['Message'] )
return S_OK( ( 0, len( toRegister ) ) )
for lfn, error in res['Value']['Failed'].items():
self.failedRegistrations[lfn] = toRegister[lfn]
self.log.error( 'Registration of Replica failed', '%s : %s' % ( lfn, str( error ) ) )
return S_OK( ( len( res['Value']['Successful'] ), len( toRegister ) ) )
def __sendAccounting( self, regSuc, regTotal, regTime, transEndTime, transDict ):
""" send accounting record
:param self: self reference
:param regSuc: number of files successfully registered
:param regTotal: number of files attepted to register
:param regTime: time stamp at the end of registration
:param transEndTime: time stamp at the end of FTS job
:param dict transDict: dict holding couters for files being transerred, their sizes and successfull transfers
"""
oAccounting = DataOperation()
oAccounting.setEndTime( transEndTime )
oAccounting.setStartTime( self.submitTime )
accountingDict = {}
accountingDict['OperationType'] = 'replicateAndRegister'
result = getProxyInfo()
if not result['OK']:
userName = '******'
else:
userName = result['Value'].get( 'username', 'unknown' )
accountingDict['User'] = userName
accountingDict['Protocol'] = 'FTS' if 'fts3' not in self.ftsServer else 'FTS3'
accountingDict['RegistrationTime'] = regTime
accountingDict['RegistrationOK'] = regSuc
accountingDict['RegistrationTotal'] = regTotal
accountingDict['TransferOK'] = transDict['transOK']
accountingDict['TransferTotal'] = transDict['transTotal']
accountingDict['TransferSize'] = transDict['transSize']
accountingDict['FinalStatus'] = self.requestStatus
accountingDict['Source'] = self.sourceSE
accountingDict['Destination'] = self.targetSE
accountingDict['TransferTime'] = self.transferTime
oAccounting.setValuesFromDict( accountingDict )
self.log.verbose( "Attempting to commit accounting message..." )
oAccounting.commit()
self.log.verbose( "...committed." )
return S_OK()
class DataIntegrityClient(Client):
"""
The following methods are supported in the service but are not mentioned explicitly here:
getProblematic()
Obtains a problematic file from the IntegrityDB based on the LastUpdate time
getPrognosisProblematics(prognosis)
Obtains all the problematics of a particular prognosis from the integrityDB
getProblematicsSummary()
Obtains a count of the number of problematics for each prognosis found
getDistinctPrognosis()
Obtains the distinct prognosis found in the integrityDB
getTransformationProblematics(prodID)
Obtains the problematics for a given production
incrementProblematicRetry(fileID)
Increments the retry count for the supplied file ID
changeProblematicPrognosis(fileID,newPrognosis)
Changes the prognosis of the supplied file to the new prognosis
setProblematicStatus(fileID,status)
Updates the status of a problematic in the integrityDB
removeProblematic(self,fileID)
This removes the specified file ID from the integrity DB
insertProblematic(sourceComponent,fileMetadata)
Inserts file with supplied metadata into the integrity DB
"""
def __init__(self, **kwargs):
super(DataIntegrityClient, self).__init__(**kwargs)
self.setServer('DataManagement/DataIntegrity')
self.dm = DataManager()
self.fc = FileCatalog()
def setFileProblematic(self, lfn, reason, sourceComponent=''):
""" This method updates the status of the file in the FileCatalog and the IntegrityDB
lfn - the lfn of the file
reason - this is given to the integrity DB and should reflect the problem observed with the file
sourceComponent is the component issuing the request.
"""
if isinstance(lfn, list):
lfns = lfn
elif isinstance(lfn, basestring):
lfns = [lfn]
else:
errStr = "DataIntegrityClient.setFileProblematic: Supplied file info must be list or a single LFN."
gLogger.error(errStr)
return S_ERROR(errStr)
gLogger.info(
"DataIntegrityClient.setFileProblematic: Attempting to update %s files."
% len(lfns))
fileMetadata = {}
for lfn in lfns:
fileMetadata[lfn] = {
'Prognosis': reason,
'LFN': lfn,
'PFN': '',
'SE': ''
}
res = self.insertProblematic(sourceComponent, fileMetadata)
if not res['OK']:
gLogger.error(
"DataIntegrityClient.setReplicaProblematic: Failed to insert problematics to integrity DB"
)
return res
def reportProblematicReplicas(self, replicaTuple, se, reason):
""" Simple wrapper function around setReplicaProblematic """
gLogger.info('The following %s files had %s at %s' %
(len(replicaTuple), reason, se))
for lfn, _pfn, se, reason in sorted(replicaTuple):
if lfn:
gLogger.info(lfn)
res = self.setReplicaProblematic(replicaTuple,
sourceComponent='DataIntegrityClient')
if not res['OK']:
gLogger.info('Failed to update integrity DB with replicas',
res['Message'])
else:
gLogger.info('Successfully updated integrity DB with replicas')
def setReplicaProblematic(self, replicaTuple, sourceComponent=''):
""" This method updates the status of the replica in the FileCatalog and the IntegrityDB
The supplied replicaDict should be of the form {lfn :{'PFN':pfn,'SE':se,'Prognosis':prognosis}
lfn - the lfn of the file
pfn - the pfn if available (otherwise '')
se - the storage element of the problematic replica (otherwise '')
prognosis - this is given to the integrity DB and should reflect the problem observed with the file
sourceComponent is the component issuing the request.
"""
if isinstance(replicaTuple, tuple):
replicaTuple = [replicaTuple]
elif isinstance(replicaTuple, list):
pass
else:
errStr = "DataIntegrityClient.setReplicaProblematic: Supplied replica info must be a tuple or list of tuples."
gLogger.error(errStr)
return S_ERROR(errStr)
gLogger.info(
"DataIntegrityClient.setReplicaProblematic: Attempting to update %s replicas."
% len(replicaTuple))
replicaDict = {}
for lfn, pfn, se, reason in replicaTuple:
replicaDict[lfn] = {
'Prognosis': reason,
'LFN': lfn,
'PFN': pfn,
'SE': se
}
res = self.insertProblematic(sourceComponent, replicaDict)
if not res['OK']:
gLogger.error(
"DataIntegrityClient.setReplicaProblematic: Failed to insert problematic to integrity DB"
)
return res
for lfn in replicaDict.keys():
replicaDict[lfn]['Status'] = 'Problematic'
res = self.fc.setReplicaStatus(replicaDict)
if not res['OK']:
errStr = "DataIntegrityClient.setReplicaProblematic: Completely failed to update replicas."
gLogger.error(errStr, res['Message'])
return res
failed = res['Value']['Failed']
successful = res['Value']['Successful']
resDict = {'Successful': successful, 'Failed': failed}
return S_OK(resDict)
##########################################################################
#
# This section contains the resolution methods for various prognoses
#
def __updateCompletedFiles(self, prognosis, fileID):
gLogger.info("%s file (%d) is resolved" % (prognosis, fileID))
return self.setProblematicStatus(fileID, 'Resolved')
def __returnProblematicError(self, fileID, res):
self.incrementProblematicRetry(fileID)
gLogger.error('DataIntegrityClient failure', res['Message'])
return res
def __updateReplicaToChecked(self, problematicDict):
lfn = problematicDict['LFN']
fileID = problematicDict['FileID']
prognosis = problematicDict['Prognosis']
problematicDict['Status'] = 'Checked'
res = returnSingleResult(
self.fc.setReplicaStatus({lfn: problematicDict}))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
gLogger.info("%s replica (%d) is updated to Checked status" %
(prognosis, fileID))
return self.__updateCompletedFiles(prognosis, fileID)
def resolveCatalogPFNSizeMismatch(self, problematicDict):
""" This takes the problematic dictionary returned by the integrity DB and resolved the CatalogPFNSizeMismatch prognosis
"""
lfn = problematicDict['LFN']
se = problematicDict['SE']
fileID = problematicDict['FileID']
res = returnSingleResult(self.fc.getFileSize(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
catalogSize = res['Value']
res = returnSingleResult(StorageElement(se).getFileSize(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
storageSize = res['Value']
bkKCatalog = FileCatalog(['BookkeepingDB'])
res = returnSingleResult(bkKCatalog.getFileSize(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
bookkeepingSize = res['Value']
if bookkeepingSize == catalogSize == storageSize:
gLogger.info(
"CatalogPFNSizeMismatch replica (%d) matched all registered sizes."
% fileID)
return self.__updateReplicaToChecked(problematicDict)
if catalogSize == bookkeepingSize:
gLogger.info(
"CatalogPFNSizeMismatch replica (%d) found to mismatch the bookkeeping also"
% fileID)
res = returnSingleResult(self.fc.getReplicas(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if len(res['Value']) <= 1:
gLogger.info(
"CatalogPFNSizeMismatch replica (%d) has no other replicas."
% fileID)
return S_ERROR(
"Not removing catalog file mismatch since the only replica"
)
else:
gLogger.info(
"CatalogPFNSizeMismatch replica (%d) has other replicas. Removing..."
% fileID)
res = self.dm.removeReplica(se, lfn)
if not res['OK']:
return self.__returnProblematicError(fileID, res)
return self.__updateCompletedFiles('CatalogPFNSizeMismatch',
fileID)
if (catalogSize != bookkeepingSize) and (bookkeepingSize
== storageSize):
gLogger.info(
"CatalogPFNSizeMismatch replica (%d) found to match the bookkeeping size"
% fileID)
res = self.__updateReplicaToChecked(problematicDict)
if not res['OK']:
return self.__returnProblematicError(fileID, res)
return self.changeProblematicPrognosis(fileID,
'BKCatalogSizeMismatch')
gLogger.info(
"CatalogPFNSizeMismatch replica (%d) all sizes found mismatch. Updating retry count"
% fileID)
return self.incrementProblematicRetry(fileID)
#FIXME: Unused?
def resolvePFNNotRegistered(self, problematicDict):
""" This takes the problematic dictionary returned by the integrity DB and resolved the PFNNotRegistered prognosis
"""
lfn = problematicDict['LFN']
seName = problematicDict['SE']
fileID = problematicDict['FileID']
se = StorageElement(seName)
res = returnSingleResult(self.fc.exists(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if not res['Value']:
# The file does not exist in the catalog
res = returnSingleResult(se.removeFile(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
return self.__updateCompletedFiles('PFNNotRegistered', fileID)
res = returnSingleResult(se.getFileMetadata(lfn))
if (not res['OK']) and (re.search('File does not exist',
res['Message'])):
gLogger.info("PFNNotRegistered replica (%d) found to be missing." %
fileID)
return self.__updateCompletedFiles('PFNNotRegistered', fileID)
elif not res['OK']:
return self.__returnProblematicError(fileID, res)
storageMetadata = res['Value']
if storageMetadata['Lost']:
gLogger.info(
"PFNNotRegistered replica (%d) found to be Lost. Updating prognosis"
% fileID)
return self.changeProblematicPrognosis(fileID, 'PFNLost')
if storageMetadata['Unavailable']:
gLogger.info(
"PFNNotRegistered replica (%d) found to be Unavailable. Updating retry count"
% fileID)
return self.incrementProblematicRetry(fileID)
# HACK until we can obtain the space token descriptions through GFAL
site = seName.split('_')[0].split('-')[0]
if not storageMetadata['Cached']:
if lfn.endswith('.raw'):
seName = '%s-RAW' % site
else:
seName = '%s-RDST' % site
elif storageMetadata['Migrated']:
if lfn.startswith('/lhcb/data'):
seName = '%s_M-DST' % site
else:
seName = '%s_MC_M-DST' % site
else:
if lfn.startswith('/lhcb/data'):
seName = '%s-DST' % site
else:
seName = '%s_MC-DST' % site
problematicDict['SE'] = seName
res = returnSingleResult(se.getURL(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
problematicDict['PFN'] = res['Value']
res = returnSingleResult(self.fc.addReplica({lfn: problematicDict}))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
res = returnSingleResult(self.fc.getFileMetadata(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if res['Value']['Size'] != storageMetadata['Size']:
gLogger.info(
"PFNNotRegistered replica (%d) found with catalog size mismatch. Updating prognosis"
% fileID)
return self.changeProblematicPrognosis(fileID,
'CatalogPFNSizeMismatch')
return self.__updateCompletedFiles('PFNNotRegistered', fileID)
#FIXME: Unused?
def resolveLFNCatalogMissing(self, problematicDict):
""" This takes the problematic dictionary returned by the integrity DB and resolved the LFNCatalogMissing prognosis
"""
lfn = problematicDict['LFN']
fileID = problematicDict['FileID']
res = returnSingleResult(self.fc.exists(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if res['Value']:
return self.__updateCompletedFiles('LFNCatalogMissing', fileID)
# Remove the file from all catalogs
# RF_NOTE : here I can do it because it's a single file, but otherwise I would need to sort the path
res = returnSingleResult(self.fc.removeFile(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
return self.__updateCompletedFiles('LFNCatalogMissing', fileID)
#FIXME: Unused?
def resolvePFNMissing(self, problematicDict):
""" This takes the problematic dictionary returned by the integrity DB and resolved the PFNMissing prognosis
"""
se = problematicDict['SE']
lfn = problematicDict['LFN']
fileID = problematicDict['FileID']
res = returnSingleResult(self.fc.exists(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if not res['Value']:
gLogger.info("PFNMissing file (%d) no longer exists in catalog" %
fileID)
return self.__updateCompletedFiles('PFNMissing', fileID)
res = returnSingleResult(StorageElement(se).exists(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if res['Value']:
gLogger.info("PFNMissing replica (%d) is no longer missing" %
fileID)
return self.__updateReplicaToChecked(problematicDict)
gLogger.info("PFNMissing replica (%d) does not exist" % fileID)
res = returnSingleResult(self.fc.getReplicas(lfn, allStatus=True))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
replicas = res['Value']
seSite = se.split('_')[0].split('-')[0]
found = False
print replicas
for replicaSE in replicas.keys():
if re.search(seSite, replicaSE):
found = True
problematicDict['SE'] = replicaSE
se = replicaSE
if not found:
gLogger.info(
"PFNMissing replica (%d) is no longer registered at SE. Resolved."
% fileID)
return self.__updateCompletedFiles('PFNMissing', fileID)
gLogger.info(
"PFNMissing replica (%d) does not exist. Removing from catalog..."
% fileID)
res = returnSingleResult(self.fc.removeReplica({lfn: problematicDict}))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if len(replicas) == 1:
gLogger.info(
"PFNMissing replica (%d) had a single replica. Updating prognosis"
% fileID)
return self.changeProblematicPrognosis(fileID, 'LFNZeroReplicas')
res = self.dm.replicateAndRegister(problematicDict['LFN'], se)
if not res['OK']:
return self.__returnProblematicError(fileID, res)
# If we get here the problem is solved so we can update the integrityDB
return self.__updateCompletedFiles('PFNMissing', fileID)
#FIXME: Unused?
def resolvePFNUnavailable(self, problematicDict):
""" This takes the problematic dictionary returned by the integrity DB and resolved the PFNUnavailable prognosis
"""
lfn = problematicDict['LFN']
se = problematicDict['SE']
fileID = problematicDict['FileID']
res = returnSingleResult(StorageElement(se).getFileMetadata(lfn))
if (not res['OK']) and (re.search('File does not exist',
res['Message'])):
# The file is no longer Unavailable but has now dissapeared completely
gLogger.info(
"PFNUnavailable replica (%d) found to be missing. Updating prognosis"
% fileID)
return self.changeProblematicPrognosis(fileID, 'PFNMissing')
if (not res['OK']) or res['Value']['Unavailable']:
gLogger.info(
"PFNUnavailable replica (%d) found to still be Unavailable" %
fileID)
return self.incrementProblematicRetry(fileID)
if res['Value']['Lost']:
gLogger.info(
"PFNUnavailable replica (%d) is now found to be Lost. Updating prognosis"
% fileID)
return self.changeProblematicPrognosis(fileID, 'PFNLost')
gLogger.info("PFNUnavailable replica (%d) is no longer Unavailable" %
fileID)
# Need to make the replica okay in the Catalog
return self.__updateReplicaToChecked(problematicDict)
#FIXME: Unused?
def resolvePFNZeroSize(self, problematicDict):
""" This takes the problematic dictionary returned by the integrity DB and resolves the PFNZeroSize prognosis
"""
lfn = problematicDict['LFN']
seName = problematicDict['SE']
fileID = problematicDict['FileID']
se = StorageElement(seName)
res = returnSingleResult(se.getFileSize(lfn))
if (not res['OK']) and (re.search('File does not exist',
res['Message'])):
gLogger.info(
"PFNZeroSize replica (%d) found to be missing. Updating prognosis"
% problematicDict['FileID'])
return self.changeProblematicPrognosis(fileID, 'PFNMissing')
storageSize = res['Value']
if storageSize == 0:
res = returnSingleResult(se.removeFile(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
gLogger.info(
"PFNZeroSize replica (%d) removed. Updating prognosis" %
problematicDict['FileID'])
return self.changeProblematicPrognosis(fileID, 'PFNMissing')
res = returnSingleResult(self.fc.getReplicas(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if seName not in res['Value']:
gLogger.info(
"PFNZeroSize replica (%d) not registered in catalog. Updating prognosis"
% problematicDict['FileID'])
return self.changeProblematicPrognosis(fileID, 'PFNNotRegistered')
res = returnSingleResult(self.fc.getFileMetadata(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
catalogSize = res['Value']['Size']
if catalogSize != storageSize:
gLogger.info(
"PFNZeroSize replica (%d) size found to differ from registered metadata. Updating prognosis"
% problematicDict['FileID'])
return self.changeProblematicPrognosis(fileID,
'CatalogPFNSizeMismatch')
return self.__updateCompletedFiles('PFNZeroSize', fileID)
############################################################################################
#FIXME: Unused?
def resolveLFNZeroReplicas(self, problematicDict):
""" This takes the problematic dictionary returned by the integrity DB and resolves the LFNZeroReplicas prognosis
"""
lfn = problematicDict['LFN']
fileID = problematicDict['FileID']
res = returnSingleResult(self.fc.getReplicas(lfn, allStatus=True))
if res['OK'] and res['Value']:
gLogger.info("LFNZeroReplicas file (%d) found to have replicas" %
fileID)
else:
gLogger.info(
"LFNZeroReplicas file (%d) does not have replicas. Checking storage..."
% fileID)
pfnsFound = False
for storageElementName in sorted(
gConfig.getValue(
'Resources/StorageElementGroups/Tier1_MC_M-DST', [])):
res = self.__getStoragePathExists([lfn], storageElementName)
if lfn in res['Value']:
gLogger.info(
"LFNZeroReplicas file (%d) found storage file at %s" %
(fileID, storageElementName))
self.reportProblematicReplicas(
[(lfn, 'deprecatedUrl', storageElementName,
'PFNNotRegistered')], storageElementName,
'PFNNotRegistered')
pfnsFound = True
if not pfnsFound:
gLogger.info(
"LFNZeroReplicas file (%d) did not have storage files. Removing..."
% fileID)
res = returnSingleResult(self.fc.removeFile(lfn))
if not res['OK']:
gLogger.error('DataIntegrityClient: failed to remove file',
res['Message'])
# Increment the number of retries for this file
self.server.incrementProblematicRetry(fileID)
return res
gLogger.info("LFNZeroReplicas file (%d) removed from catalog" %
fileID)
# If we get here the problem is solved so we can update the integrityDB
return self.__updateCompletedFiles('LFNZeroReplicas', fileID)
def _reportProblematicFiles(self, lfns, reason):
""" Simple wrapper function around setFileProblematic
"""
gLogger.info('The following %s files were found with %s' %
(len(lfns), reason))
for lfn in sorted(lfns):
gLogger.info(lfn)
res = self.setFileProblematic(lfns,
reason,
sourceComponent='DataIntegrityClient')
if not res['OK']:
gLogger.info('Failed to update integrity DB with files',
res['Message'])
else:
gLogger.info('Successfully updated integrity DB with files')
class DataIntegrityClient( Client ):
"""
The following methods are supported in the service but are not mentioned explicitly here:
getProblematic()
Obtains a problematic file from the IntegrityDB based on the LastUpdate time
getPrognosisProblematics(prognosis)
Obtains all the problematics of a particular prognosis from the integrityDB
getProblematicsSummary()
Obtains a count of the number of problematics for each prognosis found
getDistinctPrognosis()
Obtains the distinct prognosis found in the integrityDB
getTransformationProblematics(prodID)
Obtains the problematics for a given production
incrementProblematicRetry(fileID)
Increments the retry count for the supplied file ID
changeProblematicPrognosis(fileID,newPrognosis)
Changes the prognosis of the supplied file to the new prognosis
setProblematicStatus(fileID,status)
Updates the status of a problematic in the integrityDB
removeProblematic(self,fileID)
This removes the specified file ID from the integrity DB
insertProblematic(sourceComponent,fileMetadata)
Inserts file with supplied metadata into the integrity DB
"""
def __init__( self, **kwargs ):
Client.__init__( self, **kwargs )
self.setServer( 'DataManagement/DataIntegrity' )
self.dm = DataManager()
self.fc = FileCatalog()
##########################################################################
#
# This section contains the specific methods for LFC->SE checks
#
def catalogDirectoryToSE( self, lfnDir ):
""" This obtains the replica and metadata information from the catalog for the supplied directory and checks against the storage elements.
"""
gLogger.info( "-" * 40 )
gLogger.info( "Performing the LFC->SE check" )
gLogger.info( "-" * 40 )
if type( lfnDir ) in types.StringTypes:
lfnDir = [lfnDir]
res = self.__getCatalogDirectoryContents( lfnDir )
if not res['OK']:
return res
replicas = res['Value']['Replicas']
catalogMetadata = res['Value']['Metadata']
res = self.__checkPhysicalFiles( replicas, catalogMetadata )
if not res['OK']:
return res
resDict = {'CatalogMetadata':catalogMetadata, 'CatalogReplicas':replicas}
return S_OK( resDict )
def catalogFileToSE( self, lfns ):
""" This obtains the replica and metadata information from the catalog and checks against the storage elements.
"""
gLogger.info( "-" * 40 )
gLogger.info( "Performing the LFC->SE check" )
gLogger.info( "-" * 40 )
if type( lfns ) in types.StringTypes:
lfns = [lfns]
res = self.__getCatalogMetadata( lfns )
if not res['OK']:
return res
catalogMetadata = res['Value']
res = self.__getCatalogReplicas( catalogMetadata.keys() )
if not res['OK']:
return res
replicas = res['Value']
res = self.__checkPhysicalFiles( replicas, catalogMetadata )
if not res['OK']:
return res
resDict = {'CatalogMetadata':catalogMetadata, 'CatalogReplicas':replicas}
return S_OK( resDict )
def checkPhysicalFiles( self, replicas, catalogMetadata, ses = [] ):
""" This obtains takes the supplied replica and metadata information obtained from the catalog and checks against the storage elements.
"""
gLogger.info( "-" * 40 )
gLogger.info( "Performing the LFC->SE check" )
gLogger.info( "-" * 40 )
return self.__checkPhysicalFiles( replicas, catalogMetadata, ses = ses )
def __checkPhysicalFiles( self, replicas, catalogMetadata, ses = [] ):
""" This obtains the physical file metadata and checks the metadata against the catalog entries
"""
sePfns = {}
pfnLfns = {}
for lfn, replicaDict in replicas.items():
for se, pfn in replicaDict.items():
if ( ses ) and ( se not in ses ):
continue
if not sePfns.has_key( se ):
sePfns[se] = []
sePfns[se].append( pfn )
pfnLfns[pfn] = lfn
gLogger.info( '%s %s' % ( 'Storage Element'.ljust( 20 ), 'Replicas'.rjust( 20 ) ) )
for site in sortList( sePfns.keys() ):
files = len( sePfns[site] )
gLogger.info( '%s %s' % ( site.ljust( 20 ), str( files ).rjust( 20 ) ) )
for se in sortList( sePfns.keys() ):
pfns = sePfns[se]
pfnDict = {}
for pfn in pfns:
pfnDict[pfn] = pfnLfns[pfn]
sizeMismatch = []
res = self.__checkPhysicalFileMetadata( pfnDict, se )
if not res['OK']:
gLogger.error( 'Failed to get physical file metadata.', res['Message'] )
return res
for pfn, metadata in res['Value'].items():
if catalogMetadata.has_key( pfnLfns[pfn] ):
if ( metadata['Size'] != catalogMetadata[pfnLfns[pfn]]['Size'] ) and ( metadata['Size'] != 0 ):
sizeMismatch.append( ( pfnLfns[pfn], pfn, se, 'CatalogPFNSizeMismatch' ) )
if sizeMismatch:
self.__reportProblematicReplicas( sizeMismatch, se, 'CatalogPFNSizeMismatch' )
return S_OK()
def __checkPhysicalFileMetadata( self, pfnLfns, se ):
""" Check obtain the physical file metadata and check the files are available
"""
gLogger.info( 'Checking the integrity of %s physical files at %s' % ( len( pfnLfns ), se ) )
res = StorageElement( se ).getFileMetadata( pfnLfns.keys() )
if not res['OK']:
gLogger.error( 'Failed to get metadata for pfns.', res['Message'] )
return res
pfnMetadataDict = res['Value']['Successful']
# If the replicas are completely missing
missingReplicas = []
for pfn, reason in res['Value']['Failed'].items():
if re.search( 'File does not exist', reason ):
missingReplicas.append( ( pfnLfns[pfn], pfn, se, 'PFNMissing' ) )
if missingReplicas:
self.__reportProblematicReplicas( missingReplicas, se, 'PFNMissing' )
lostReplicas = []
unavailableReplicas = []
zeroSizeReplicas = []
# If the files are not accessible
for pfn, pfnMetadata in pfnMetadataDict.items():
if pfnMetadata['Lost']:
lostReplicas.append( ( pfnLfns[pfn], pfn, se, 'PFNLost' ) )
if pfnMetadata['Unavailable']:
unavailableReplicas.append( ( pfnLfns[pfn], pfn, se, 'PFNUnavailable' ) )
if pfnMetadata['Size'] == 0:
zeroSizeReplicas.append( ( pfnLfns[pfn], pfn, se, 'PFNZeroSize' ) )
if lostReplicas:
self.__reportProblematicReplicas( lostReplicas, se, 'PFNLost' )
if unavailableReplicas:
self.__reportProblematicReplicas( unavailableReplicas, se, 'PFNUnavailable' )
if zeroSizeReplicas:
self.__reportProblematicReplicas( zeroSizeReplicas, se, 'PFNZeroSize' )
gLogger.info( 'Checking the integrity of physical files at %s complete' % se )
return S_OK( pfnMetadataDict )
##########################################################################
#
# This section contains the specific methods for SE->LFC checks
#
def storageDirectoryToCatalog( self, lfnDir, storageElement ):
""" This obtains the file found on the storage element in the supplied directories and determines whether they exist in the catalog and checks their metadata elements
"""
gLogger.info( "-" * 40 )
gLogger.info( "Performing the SE->LFC check at %s" % storageElement )
gLogger.info( "-" * 40 )
if type( lfnDir ) in types.StringTypes:
lfnDir = [lfnDir]
res = self.__getStorageDirectoryContents( lfnDir, storageElement )
if not res['OK']:
return res
storageFileMetadata = res['Value']
if storageFileMetadata:
return self.__checkCatalogForSEFiles( storageFileMetadata, storageElement )
return S_OK( {'CatalogMetadata':{}, 'StorageMetadata':{}} )
def __checkCatalogForSEFiles( self, storageMetadata, storageElement ):
gLogger.info( 'Checking %s storage files exist in the catalog' % len( storageMetadata ) )
# RF_NOTE : this comment is completely wrong
# First get all the PFNs as they should be registered in the catalog
res = StorageElement( storageElement ).getPfnForProtocol( storageMetadata.keys(), withPort = False )
if not res['OK']:
gLogger.error( "Failed to get registered PFNs for physical files", res['Message'] )
return res
for pfn, error in res['Value']['Failed'].items():
gLogger.error( 'Failed to obtain registered PFN for physical file', '%s %s' % ( pfn, error ) )
if res['Value']['Failed']:
return S_ERROR( 'Failed to obtain registered PFNs from physical file' )
for original, registered in res['Value']['Successful'].items():
storageMetadata[registered] = storageMetadata.pop( original )
# Determine whether these PFNs are registered and if so obtain the LFN
res = self.fc.getLFNForPFN( storageMetadata.keys() )
if not res['OK']:
gLogger.error( "Failed to get registered LFNs for PFNs", res['Message'] )
return res
failedPfns = res['Value']['Failed']
notRegisteredPfns = []
for pfn, error in failedPfns.items():
if re.search( 'No such file or directory', error ):
notRegisteredPfns.append( ( storageMetadata[pfn]['LFN'], pfn, storageElement, 'PFNNotRegistered' ) )
failedPfns.pop( pfn )
if notRegisteredPfns:
self.__reportProblematicReplicas( notRegisteredPfns, storageElement, 'PFNNotRegistered' )
if failedPfns:
return S_ERROR( 'Failed to obtain LFNs for PFNs' )
pfnLfns = res['Value']['Successful']
for pfn in storageMetadata.keys():
pfnMetadata = storageMetadata.pop( pfn )
if pfn in pfnLfns.keys():
lfn = pfnLfns[pfn]
storageMetadata[lfn] = pfnMetadata
storageMetadata[lfn]['PFN'] = pfn
# For the LFNs found to be registered obtain the file metadata from the catalog and verify against the storage metadata
res = self.__getCatalogMetadata( storageMetadata.keys() )
if not res['OK']:
return res
catalogMetadata = res['Value']
sizeMismatch = []
for lfn, lfnCatalogMetadata in catalogMetadata.items():
lfnStorageMetadata = storageMetadata[lfn]
if ( lfnStorageMetadata['Size'] != lfnCatalogMetadata['Size'] ) and ( lfnStorageMetadata['Size'] != 0 ):
sizeMismatch.append( ( lfn, storageMetadata[lfn]['PFN'], storageElement, 'CatalogPFNSizeMismatch' ) )
if sizeMismatch:
self.__reportProblematicReplicas( sizeMismatch, storageElement, 'CatalogPFNSizeMismatch' )
gLogger.info( 'Checking storage files exist in the catalog complete' )
resDict = {'CatalogMetadata':catalogMetadata, 'StorageMetadata':storageMetadata}
return S_OK( resDict )
def getStorageDirectoryContents( self, lfnDir, storageElement ):
""" This obtains takes the supplied lfn directories and recursively obtains the files in the supplied storage element
"""
return self.__getStorageDirectoryContents( lfnDir, storageElement )
def __getStorageDirectoryContents( self, lfnDir, storageElement ):
""" Obtians the contents of the supplied directory on the storage
"""
gLogger.info( 'Obtaining the contents for %s directories at %s' % ( len( lfnDir ), storageElement ) )
se = StorageElement( storageElement )
res = se.getPfnForLfn( lfnDir )
if not res['OK']:
gLogger.error( "Failed to get PFNs for directories", res['Message'] )
return res
for directory, error in res['Value']['Failed'].items():
gLogger.error( 'Failed to obtain directory PFN from LFNs', '%s %s' % ( directory, error ) )
if res['Value']['Failed']:
return S_ERROR( 'Failed to obtain directory PFN from LFNs' )
storageDirectories = res['Value']['Successful'].values()
res = se.exists( storageDirectories )
if not res['OK']:
gLogger.error( "Failed to obtain existance of directories", res['Message'] )
return res
for directory, error in res['Value']['Failed'].items():
gLogger.error( 'Failed to determine existance of directory', '%s %s' % ( directory, error ) )
if res['Value']['Failed']:
return S_ERROR( 'Failed to determine existance of directory' )
directoryExists = res['Value']['Successful']
activeDirs = []
for directory in sortList( directoryExists.keys() ):
exists = directoryExists[directory]
if exists:
activeDirs.append( directory )
allFiles = {}
while len( activeDirs ) > 0:
currentDir = activeDirs[0]
res = se.listDirectory( currentDir )
activeDirs.remove( currentDir )
if not res['OK']:
gLogger.error( 'Failed to get directory contents', res['Message'] )
return res
elif res['Value']['Failed'].has_key( currentDir ):
gLogger.error( 'Failed to get directory contents', '%s %s' % ( currentDir, res['Value']['Failed'][currentDir] ) )
return S_ERROR( res['Value']['Failed'][currentDir] )
else:
dirContents = res['Value']['Successful'][currentDir]
activeDirs.extend( dirContents['SubDirs'] )
fileMetadata = dirContents['Files']
# RF_NOTE This ugly trick is needed because se.getPfnPath does not follow the Successful/Failed convention
# res = { "Successful" : {}, "Failed" : {} }
# for pfn in fileMetadata:
# inRes = se.getPfnPath( pfn )
# if inRes["OK"]:
# res["Successful"][pfn] = inRes["Value"]
# else:
# res["Failed"][pfn] = inRes["Message"]
res = se.getLfnForPfn( fileMetadata.keys() )
if not res['OK']:
gLogger.error( 'Failed to get directory content LFNs', res['Message'] )
return res
for pfn, error in res['Value']['Failed'].items():
gLogger.error( "Failed to get LFN for PFN", "%s %s" % ( pfn, error ) )
if res['Value']['Failed']:
return S_ERROR( "Failed to get LFNs for PFNs" )
pfnLfns = res['Value']['Successful']
for pfn, lfn in pfnLfns.items():
fileMetadata[pfn]['LFN'] = lfn
allFiles.update( fileMetadata )
zeroSizeFiles = []
lostFiles = []
unavailableFiles = []
for pfn in sortList( allFiles.keys() ):
if os.path.basename( pfn ) == 'dirac_directory':
allFiles.pop( pfn )
else:
metadata = allFiles[pfn]
if metadata['Size'] == 0:
zeroSizeFiles.append( ( metadata['LFN'], pfn, storageElement, 'PFNZeroSize' ) )
# if metadata['Lost']:
# lostFiles.append((metadata['LFN'],pfn,storageElement,'PFNLost'))
# if metadata['Unavailable']:
# unavailableFiles.append((metadata['LFN'],pfn,storageElement,'PFNUnavailable'))
if zeroSizeFiles:
self.__reportProblematicReplicas( zeroSizeFiles, storageElement, 'PFNZeroSize' )
if lostFiles:
self.__reportProblematicReplicas( lostFiles, storageElement, 'PFNLost' )
if unavailableFiles:
self.__reportProblematicReplicas( unavailableFiles, storageElement, 'PFNUnavailable' )
gLogger.info( 'Obtained at total of %s files for directories at %s' % ( len( allFiles ), storageElement ) )
return S_OK( allFiles )
def __getStoragePathExists( self, lfnPaths, storageElement ):
gLogger.info( 'Determining the existance of %d files at %s' % ( len( lfnPaths ), storageElement ) )
se = StorageElement( storageElement )
res = se.getPfnForLfn( lfnPaths )
if not res['OK']:
gLogger.error( "Failed to get PFNs for LFNs", res['Message'] )
return res
for lfnPath, error in res['Value']['Failed'].items():
gLogger.error( 'Failed to obtain PFN from LFN', '%s %s' % ( lfnPath, error ) )
if res['Value']['Failed']:
return S_ERROR( 'Failed to obtain PFNs from LFNs' )
lfnPfns = res['Value']['Successful']
pfnLfns = {}
for lfn, pfn in lfnPfns.items():
pfnLfns[pfn] = lfn
res = se.exists( pfnLfns )
if not res['OK']:
gLogger.error( "Failed to obtain existance of paths", res['Message'] )
return res
for lfnPath, error in res['Value']['Failed'].items():
gLogger.error( 'Failed to determine existance of path', '%s %s' % ( lfnPath, error ) )
if res['Value']['Failed']:
return S_ERROR( 'Failed to determine existance of paths' )
pathExists = res['Value']['Successful']
resDict = {}
for pfn, exists in pathExists.items():
if exists:
resDict[pfnLfns[pfn]] = pfn
return S_OK( resDict )
##########################################################################
#
# This section contains the specific methods for obtaining replica and metadata information from the catalog
#
def __getCatalogDirectoryContents( self, lfnDir ):
""" Obtain the contents of the supplied directory
"""
gLogger.info( 'Obtaining the catalog contents for %s directories' % len( lfnDir ) )
activeDirs = lfnDir
allFiles = {}
while len( activeDirs ) > 0:
currentDir = activeDirs[0]
res = self.fc.listDirectory( currentDir )
activeDirs.remove( currentDir )
if not res['OK']:
gLogger.error( 'Failed to get directory contents', res['Message'] )
return res
elif res['Value']['Failed'].has_key( currentDir ):
gLogger.error( 'Failed to get directory contents', '%s %s' % ( currentDir, res['Value']['Failed'][currentDir] ) )
else:
dirContents = res['Value']['Successful'][currentDir]
activeDirs.extend( dirContents['SubDirs'] )
allFiles.update( dirContents['Files'] )
zeroReplicaFiles = []
zeroSizeFiles = []
allReplicaDict = {}
allMetadataDict = {}
for lfn, lfnDict in allFiles.items():
lfnReplicas = {}
for se, replicaDict in lfnDict['Replicas'].items():
lfnReplicas[se] = replicaDict['PFN']
if not lfnReplicas:
zeroReplicaFiles.append( lfn )
allReplicaDict[lfn] = lfnReplicas
allMetadataDict[lfn] = lfnDict['MetaData']
if lfnDict['MetaData']['Size'] == 0:
zeroSizeFiles.append( lfn )
if zeroReplicaFiles:
self.__reportProblematicFiles( zeroReplicaFiles, 'LFNZeroReplicas' )
if zeroSizeFiles:
self.__reportProblematicFiles( zeroSizeFiles, 'LFNZeroSize' )
gLogger.info( 'Obtained at total of %s files for the supplied directories' % len( allMetadataDict ) )
resDict = {'Metadata':allMetadataDict, 'Replicas':allReplicaDict}
return S_OK( resDict )
def __getCatalogReplicas( self, lfns ):
""" Obtain the file replicas from the catalog while checking that there are replicas
"""
gLogger.info( 'Obtaining the replicas for %s files' % len( lfns ) )
zeroReplicaFiles = []
res = self.fc.getReplicas( lfns, allStatus = True )
if not res['OK']:
gLogger.error( 'Failed to get catalog replicas', res['Message'] )
return res
allReplicas = res['Value']['Successful']
for lfn, error in res['Value']['Failed'].items():
if re.search( 'File has zero replicas', error ):
zeroReplicaFiles.append( lfn )
if zeroReplicaFiles:
self.__reportProblematicFiles( zeroReplicaFiles, 'LFNZeroReplicas' )
gLogger.info( 'Obtaining the replicas for files complete' )
return S_OK( allReplicas )
def __getCatalogMetadata( self, lfns ):
""" Obtain the file metadata from the catalog while checking they exist
"""
if not lfns:
return S_OK( {} )
gLogger.info( 'Obtaining the catalog metadata for %s files' % len( lfns ) )
missingCatalogFiles = []
zeroSizeFiles = []
res = self.fc.getFileMetadata( lfns )
if not res['OK']:
gLogger.error( 'Failed to get catalog metadata', res['Message'] )
return res
allMetadata = res['Value']['Successful']
for lfn, error in res['Value']['Failed'].items():
if re.search( 'No such file or directory', error ):
missingCatalogFiles.append( lfn )
if missingCatalogFiles:
self.__reportProblematicFiles( missingCatalogFiles, 'LFNCatalogMissing' )
for lfn, metadata in allMetadata.items():
if metadata['Size'] == 0:
zeroSizeFiles.append( lfn )
if zeroSizeFiles:
self.__reportProblematicFiles( zeroSizeFiles, 'LFNZeroSize' )
gLogger.info( 'Obtaining the catalog metadata complete' )
return S_OK( allMetadata )
##########################################################################
#
# This section contains the methods for inserting problematic files into the integrity DB
#
def __reportProblematicFiles( self, lfns, reason ):
""" Simple wrapper function around setFileProblematic """
gLogger.info( 'The following %s files were found with %s' % ( len( lfns ), reason ) )
for lfn in sortList( lfns ):
gLogger.info( lfn )
res = self.setFileProblematic( lfns, reason, sourceComponent = 'DataIntegrityClient' )
if not res['OK']:
gLogger.info( 'Failed to update integrity DB with files', res['Message'] )
else:
gLogger.info( 'Successfully updated integrity DB with files' )
def setFileProblematic( self, lfn, reason, sourceComponent = '' ):
""" This method updates the status of the file in the FileCatalog and the IntegrityDB
lfn - the lfn of the file
reason - this is given to the integrity DB and should reflect the problem observed with the file
sourceComponent is the component issuing the request.
"""
if type( lfn ) == types.ListType:
lfns = lfn
elif type( lfn ) == types.StringType:
lfns = [lfn]
else:
errStr = "DataIntegrityClient.setFileProblematic: Supplied file info must be list or a single LFN."
gLogger.error( errStr )
return S_ERROR( errStr )
gLogger.info( "DataIntegrityClient.setFileProblematic: Attempting to update %s files." % len( lfns ) )
fileMetadata = {}
for lfn in lfns:
fileMetadata[lfn] = {'Prognosis':reason, 'LFN':lfn, 'PFN':'', 'SE':''}
res = self.insertProblematic( sourceComponent, fileMetadata )
if not res['OK']:
gLogger.error( "DataIntegrityClient.setReplicaProblematic: Failed to insert problematics to integrity DB" )
return res
def __reportProblematicReplicas( self, replicaTuple, se, reason ):
""" Simple wrapper function around setReplicaProblematic """
gLogger.info( 'The following %s files had %s at %s' % ( len( replicaTuple ), reason, se ) )
for lfn, pfn, se, reason in sortList( replicaTuple ):
if lfn:
gLogger.info( lfn )
else:
gLogger.info( pfn )
res = self.setReplicaProblematic( replicaTuple, sourceComponent = 'DataIntegrityClient' )
if not res['OK']:
gLogger.info( 'Failed to update integrity DB with replicas', res['Message'] )
else:
gLogger.info( 'Successfully updated integrity DB with replicas' )
def setReplicaProblematic( self, replicaTuple, sourceComponent = '' ):
""" This method updates the status of the replica in the FileCatalog and the IntegrityDB
The supplied replicaDict should be of the form {lfn :{'PFN':pfn,'SE':se,'Prognosis':prognosis}
lfn - the lfn of the file
pfn - the pfn if available (otherwise '')
se - the storage element of the problematic replica (otherwise '')
prognosis - this is given to the integrity DB and should reflect the problem observed with the file
sourceComponent is the component issuing the request.
"""
if type( replicaTuple ) == types.TupleType:
replicaTuple = [replicaTuple]
elif type( replicaTuple ) == types.ListType:
pass
else:
errStr = "DataIntegrityClient.setReplicaProblematic: Supplied replica info must be a tuple or list of tuples."
gLogger.error( errStr )
return S_ERROR( errStr )
gLogger.info( "DataIntegrityClient.setReplicaProblematic: Attempting to update %s replicas." % len( replicaTuple ) )
replicaDict = {}
for lfn, pfn, se, reason in replicaTuple:
replicaDict[lfn] = {'Prognosis':reason, 'LFN':lfn, 'PFN':pfn, 'SE':se}
res = self.insertProblematic( sourceComponent, replicaDict )
if not res['OK']:
gLogger.error( "DataIntegrityClient.setReplicaProblematic: Failed to insert problematic to integrity DB" )
return res
for lfn in replicaDict.keys():
replicaDict[lfn]['Status'] = 'Problematic'
res = self.fc.setReplicaStatus( replicaDict )
if not res['OK']:
errStr = "DataIntegrityClient.setReplicaProblematic: Completely failed to update replicas."
gLogger.error( errStr, res['Message'] )
return res
failed = res['Value']['Failed']
successful = res['Value']['Successful']
resDict = {'Successful':successful, 'Failed':failed}
return S_OK( resDict )
##########################################################################
#
# This section contains the resolution methods for various prognoses
#
def __updateCompletedFiles( self, prognosis, fileID ):
gLogger.info( "%s file (%d) is resolved" % ( prognosis, fileID ) )
return self.setProblematicStatus( fileID, 'Resolved' )
def __returnProblematicError( self, fileID, res ):
self.incrementProblematicRetry( fileID )
gLogger.error( res['Message'] )
return res
def __getRegisteredPFNLFN( self, pfn, storageElement ):
res = StorageElement( storageElement ).getPfnForProtocol( pfn, withPort = False )
if not res['OK']:
gLogger.error( "Failed to get registered PFN for physical files", res['Message'] )
return res
for pfn, error in res['Value']['Failed'].items():
gLogger.error( 'Failed to obtain registered PFN for physical file', '%s %s' % ( pfn, error ) )
return S_ERROR( 'Failed to obtain registered PFNs from physical file' )
registeredPFN = res['Value']['Successful'][pfn]
res = Utils.executeSingleFileOrDirWrapper( self.fc.getLFNForPFN( registeredPFN ) )
if ( not res['OK'] ) and re.search( 'No such file or directory', res['Message'] ):
return S_OK( False )
return S_OK( res['Value'] )
def __updateReplicaToChecked( self, problematicDict ):
lfn = problematicDict['LFN']
fileID = problematicDict['FileID']
prognosis = problematicDict['Prognosis']
problematicDict['Status'] = 'Checked'
res = Utils.executeSingleFileOrDirWrapper( self.fc.setReplicaStatus( {lfn:problematicDict} ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
gLogger.info( "%s replica (%d) is updated to Checked status" % ( prognosis, fileID ) )
return self.__updateCompletedFiles( prognosis, fileID )
def resolveCatalogPFNSizeMismatch( self, problematicDict ):
""" This takes the problematic dictionary returned by the integrity DB and resolved the CatalogPFNSizeMismatch prognosis
"""
lfn = problematicDict['LFN']
pfn = problematicDict['PFN']
se = problematicDict['SE']
fileID = problematicDict['FileID']
res = Utils.executeSingleFileOrDirWrapper( self.fc.getFileSize( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
catalogSize = res['Value']
res = Utils.executeSingleFileOrDirWrapper( StorageElement( se ).getFileSize( pfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
storageSize = res['Value']
bkKCatalog = FileCatalog( ['BookkeepingDB'] )
res = Utils.executeSingleFileOrDirWrapper( bkKCatalog.getFileSize( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
bookkeepingSize = res['Value']
if bookkeepingSize == catalogSize == storageSize:
gLogger.info( "CatalogPFNSizeMismatch replica (%d) matched all registered sizes." % fileID )
return self.__updateReplicaToChecked( problematicDict )
if ( catalogSize == bookkeepingSize ):
gLogger.info( "CatalogPFNSizeMismatch replica (%d) found to mismatch the bookkeeping also" % fileID )
res = Utils.executeSingleFileOrDirWrapper( self.fc.getReplicas( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if len( res['Value'] ) <= 1:
gLogger.info( "CatalogPFNSizeMismatch replica (%d) has no other replicas." % fileID )
return S_ERROR( "Not removing catalog file mismatch since the only replica" )
else:
gLogger.info( "CatalogPFNSizeMismatch replica (%d) has other replicas. Removing..." % fileID )
res = self.dm.removeReplica( se, lfn )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
return self.__updateCompletedFiles( 'CatalogPFNSizeMismatch', fileID )
if ( catalogSize != bookkeepingSize ) and ( bookkeepingSize == storageSize ):
gLogger.info( "CatalogPFNSizeMismatch replica (%d) found to match the bookkeeping size" % fileID )
res = self.__updateReplicaToChecked( problematicDict )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
return self.changeProblematicPrognosis( fileID, 'BKCatalogSizeMismatch' )
gLogger.info( "CatalogPFNSizeMismatch replica (%d) all sizes found mismatch. Updating retry count" % fileID )
return self.incrementProblematicRetry( fileID )
def resolvePFNNotRegistered( self, problematicDict ):
""" This takes the problematic dictionary returned by the integrity DB and resolved the PFNNotRegistered prognosis
"""
lfn = problematicDict['LFN']
pfn = problematicDict['PFN']
seName = problematicDict['SE']
fileID = problematicDict['FileID']
se = StorageElement( seName )
res = Utils.executeSingleFileOrDirWrapper( self.fc.exists( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if not res['Value']:
# The file does not exist in the catalog
res = Utils.executeSingleFileOrDirWrapper( se.removeFile( pfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
return self.__updateCompletedFiles( 'PFNNotRegistered', fileID )
res = Utils.executeSingleFileOrDirWrapper( se.getFileMetadata( pfn ) )
if ( not res['OK'] ) and ( re.search( 'File does not exist', res['Message'] ) ):
gLogger.info( "PFNNotRegistered replica (%d) found to be missing." % fileID )
return self.__updateCompletedFiles( 'PFNNotRegistered', fileID )
elif not res['OK']:
return self.__returnProblematicError( fileID, res )
storageMetadata = res['Value']
if storageMetadata['Lost']:
gLogger.info( "PFNNotRegistered replica (%d) found to be Lost. Updating prognosis" % fileID )
return self.changeProblematicPrognosis( fileID, 'PFNLost' )
if storageMetadata['Unavailable']:
gLogger.info( "PFNNotRegistered replica (%d) found to be Unavailable. Updating retry count" % fileID )
return self.incrementProblematicRetry( fileID )
# HACK until we can obtain the space token descriptions through GFAL
site = seName.split( '_' )[0].split( '-' )[0]
if not storageMetadata['Cached']:
if lfn.endswith( '.raw' ):
seName = '%s-RAW' % site
else:
seName = '%s-RDST' % site
elif storageMetadata['Migrated']:
if lfn.startswith( '/lhcb/data' ):
seName = '%s_M-DST' % site
else:
seName = '%s_MC_M-DST' % site
else:
if lfn.startswith( '/lhcb/data' ):
seName = '%s-DST' % site
else:
seName = '%s_MC-DST' % site
problematicDict['SE'] = seName
res = se.getPfnForProtocol( pfn, withPort = False )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
for pfn, error in res['Value']['Failed'].items():
gLogger.error( 'Failed to obtain registered PFN for physical file', '%s %s' % ( pfn, error ) )
return S_ERROR( 'Failed to obtain registered PFNs from physical file' )
problematicDict['PFN'] = res['Value']['Successful'][pfn]
res = Utils.executeSingleFileOrDirWrapper( self.fc.addReplica( {lfn:problematicDict} ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
res = Utils.executeSingleFileOrDirWrapper( self.fc.getFileMetadata( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if res['Value']['Size'] != storageMetadata['Size']:
gLogger.info( "PFNNotRegistered replica (%d) found with catalog size mismatch. Updating prognosis" % fileID )
return self.changeProblematicPrognosis( fileID, 'CatalogPFNSizeMismatch' )
return self.__updateCompletedFiles( 'PFNNotRegistered', fileID )
def resolveLFNCatalogMissing( self, problematicDict ):
""" This takes the problematic dictionary returned by the integrity DB and resolved the LFNCatalogMissing prognosis
"""
lfn = problematicDict['LFN']
fileID = problematicDict['FileID']
res = Utils.executeSingleFileOrDirWrapper( self.fc.exists( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if res['Value']:
return self.__updateCompletedFiles( 'LFNCatalogMissing', fileID )
# Remove the file from all catalogs
# RF_NOTE : here I can do it because it's a single file, but otherwise I would need to sort the path
res = Utils.executeSingleFileOrDirWrapper( self.fc.removeFile( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
return self.__updateCompletedFiles( 'LFNCatalogMissing', fileID )
def resolvePFNMissing( self, problematicDict ):
""" This takes the problematic dictionary returned by the integrity DB and resolved the PFNMissing prognosis
"""
pfn = problematicDict['PFN']
se = problematicDict['SE']
lfn = problematicDict['LFN']
fileID = problematicDict['FileID']
res = Utils.executeSingleFileOrDirWrapper( self.fc.exists( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if not res['Value']:
gLogger.info( "PFNMissing file (%d) no longer exists in catalog" % fileID )
return self.__updateCompletedFiles( 'PFNMissing', fileID )
res = Utils.executeSingleFileOrDirWrapper( StorageElement( se ).exists( pfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if res['Value']:
gLogger.info( "PFNMissing replica (%d) is no longer missing" % fileID )
return self.__updateReplicaToChecked( problematicDict )
gLogger.info( "PFNMissing replica (%d) does not exist" % fileID )
res = Utils.executeSingleFileOrDirWrapper( self.fc.getReplicas( lfn, allStatus = True ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
replicas = res['Value']
seSite = se.split( '_' )[0].split( '-' )[0]
found = False
print replicas
for replicaSE in replicas.keys():
if re.search( seSite, replicaSE ):
found = True
problematicDict['SE'] = replicaSE
se = replicaSE
if not found:
gLogger.info( "PFNMissing replica (%d) is no longer registered at SE. Resolved." % fileID )
return self.__updateCompletedFiles( 'PFNMissing', fileID )
gLogger.info( "PFNMissing replica (%d) does not exist. Removing from catalog..." % fileID )
res = Utils.executeSingleFileOrDirWrapper( self.fc.removeReplica( {lfn:problematicDict} ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if len( replicas ) == 1:
gLogger.info( "PFNMissing replica (%d) had a single replica. Updating prognosis" % fileID )
return self.changeProblematicPrognosis( fileID, 'LFNZeroReplicas' )
res = self.dm.replicateAndRegister( problematicDict['LFN'], se )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
# If we get here the problem is solved so we can update the integrityDB
return self.__updateCompletedFiles( 'PFNMissing', fileID )
def resolvePFNUnavailable( self, problematicDict ):
""" This takes the problematic dictionary returned by the integrity DB and resolved the PFNUnavailable prognosis
"""
pfn = problematicDict['PFN']
se = problematicDict['SE']
fileID = problematicDict['FileID']
res = Utils.executeSingleFileOrDirWrapper( StorageElement( se ).getFileMetadata( pfn ) )
if ( not res['OK'] ) and ( re.search( 'File does not exist', res['Message'] ) ):
# The file is no longer Unavailable but has now dissapeared completely
gLogger.info( "PFNUnavailable replica (%d) found to be missing. Updating prognosis" % fileID )
return self.changeProblematicPrognosis( fileID, 'PFNMissing' )
if ( not res['OK'] ) or res['Value']['Unavailable']:
gLogger.info( "PFNUnavailable replica (%d) found to still be Unavailable" % fileID )
return self.incrementProblematicRetry( fileID )
if res['Value']['Lost']:
gLogger.info( "PFNUnavailable replica (%d) is now found to be Lost. Updating prognosis" % fileID )
return self.changeProblematicPrognosis( fileID, 'PFNLost' )
gLogger.info( "PFNUnavailable replica (%d) is no longer Unavailable" % fileID )
# Need to make the replica okay in the Catalog
return self.__updateReplicaToChecked( problematicDict )
def resolvePFNZeroSize( self, problematicDict ):
""" This takes the problematic dictionary returned by the integrity DB and resolves the PFNZeroSize prognosis
"""
pfn = problematicDict['PFN']
seName = problematicDict['SE']
fileID = problematicDict['FileID']
se = StorageElement( seName )
res = Utils.executeSingleFileOrDirWrapper( se.getFileSize( pfn ) )
if ( not res['OK'] ) and ( re.search( 'File does not exist', res['Message'] ) ):
gLogger.info( "PFNZeroSize replica (%d) found to be missing. Updating prognosis" % problematicDict['FileID'] )
return self.changeProblematicPrognosis( fileID, 'PFNMissing' )
storageSize = res['Value']
if storageSize == 0:
res = Utils.executeSingleFileOrDirWrapper( se.removeFile( pfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
gLogger.info( "PFNZeroSize replica (%d) removed. Updating prognosis" % problematicDict['FileID'] )
return self.changeProblematicPrognosis( fileID, 'PFNMissing' )
res = self.__getRegisteredPFNLFN( pfn, seName )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
lfn = res['Value']
if not lfn:
gLogger.info( "PFNZeroSize replica (%d) not registered in catalog. Updating prognosis" % problematicDict['FileID'] )
return self.changeProblematicPrognosis( fileID, 'PFNNotRegistered' )
res = Utils.executeSingleFileOrDirWrapper( self.fc.getFileMetadata( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
catalogSize = res['Value']['Size']
if catalogSize != storageSize:
gLogger.info( "PFNZeroSize replica (%d) size found to differ from registered metadata. Updating prognosis" % problematicDict['FileID'] )
return self.changeProblematicPrognosis( fileID, 'CatalogPFNSizeMismatch' )
return self.__updateCompletedFiles( 'PFNZeroSize', fileID )
############################################################################################
def resolveLFNZeroReplicas( self, problematicDict ):
""" This takes the problematic dictionary returned by the integrity DB and resolves the LFNZeroReplicas prognosis
"""
lfn = problematicDict['LFN']
fileID = problematicDict['FileID']
res = Utils.executeSingleFileOrDirWrapper( self.fc.getReplicas( lfn, allStatus = True ) )
if res['OK'] and res['Value']:
gLogger.info( "LFNZeroReplicas file (%d) found to have replicas" % fileID )
else:
gLogger.info( "LFNZeroReplicas file (%d) does not have replicas. Checking storage..." % fileID )
pfnsFound = False
for storageElementName in sortList( gConfig.getValue( 'Resources/StorageElementGroups/Tier1_MC_M-DST', [] ) ):
res = self.__getStoragePathExists( [lfn], storageElementName )
if res['Value'].has_key( lfn ):
gLogger.info( "LFNZeroReplicas file (%d) found storage file at %s" % ( fileID, storageElementName ) )
pfn = res['Value'][lfn]
self.__reportProblematicReplicas( [( lfn, pfn, storageElementName, 'PFNNotRegistered' )], storageElementName, 'PFNNotRegistered' )
pfnsFound = True
if not pfnsFound:
gLogger.info( "LFNZeroReplicas file (%d) did not have storage files. Removing..." % fileID )
res = Utils.executeSingleFileOrDirWrapper( self.fc.removeFile( lfn ) )
if not res['OK']:
gLogger.error( res['Message'] )
# Increment the number of retries for this file
self.server.incrementProblematicRetry( fileID )
return res
gLogger.info( "LFNZeroReplicas file (%d) removed from catalog" % fileID )
# If we get here the problem is solved so we can update the integrityDB
return self.__updateCompletedFiles( 'LFNZeroReplicas', fileID )
class FTSJob( Record ):
"""
.. class:: FTSJob
class describing one FTS job
"""
# # initial states
INITSTATES = ( "Submitted", "Ready", "Staging" )
# # ongoing transfer states
TRANSSTATES = ( "Active", "Hold" )
# # failed states
FAILEDSTATES = ( "Canceled", "Failed" )
# # finished
FINALSTATES = ( "Finished", "FinishedDirty", "Failed", "Canceled" )
# # missing source regexp patterns
missingSourceErrors = [
re.compile( r"SOURCE error during TRANSFER_PREPARATION phase: \[INVALID_PATH\] Failed" ),
re.compile( r"SOURCE error during TRANSFER_PREPARATION phase: \[INVALID_PATH\] No such file or directory" ),
re.compile( r"SOURCE error during PREPARATION phase: \[INVALID_PATH\] Failed" ),
re.compile( r"SOURCE error during PREPARATION phase: \[INVALID_PATH\] The requested file either does not exist" ),
re.compile( r"TRANSFER error during TRANSFER phase: \[INVALID_PATH\] the server sent an error response: 500 500"\
" Command failed. : open error: No such file or directory" ),
re.compile( r"SOURCE error during TRANSFER_PREPARATION phase: \[USER_ERROR\] source file doesnt exist" ) ]
def __init__( self, fromDict = None ):
"""c'tor
:param self: self reference
:param dict fromDict: data dict
"""
Record.__init__( self )
now = datetime.datetime.utcnow().replace( microsecond = 0 )
self.__data__["CreationTime"] = now
self.__data__["SubmitTime"] = now
self.__data__["LastUpdate"] = now
self.__data__["Status"] = "Submitted"
self.__data__["Completeness"] = 0
self.__data__["FTSJobID"] = 0
self._regTime = 0.
self._regSuccess = 0
self._regTotal = 0
self.__files__ = TypedList( allowedTypes = FTSFile )
self._fc = FileCatalog()
self._log = gLogger.getSubLogger( "FTSJob-%s" % self.FTSJobID , True )
self._states = tuple( set( self.INITSTATES + self.TRANSSTATES + self.FAILEDSTATES + self.FINALSTATES ) )
fromDict = fromDict if fromDict else {}
for ftsFileDict in fromDict.get( "FTSFiles", [] ):
self +=FTSFile( ftsFileDict )
if "FTSFiles" in fromDict: del fromDict["FTSFiles"]
for key, value in fromDict.items():
if key not in self.__data__:
raise AttributeError( "Unknown FTSJob attribute '%s'" % key )
if value:
setattr( self, key, value )
@staticmethod
def tableDesc():
""" get table desc """
return { "Fields" :
{ "FTSJobID" : "INTEGER NOT NULL AUTO_INCREMENT",
"FTSGUID" : "VARCHAR(64) NOT NULL",
"OperationID": "INTEGER NOT NULL",
"RequestID": "INTEGER NOT NULL",
"SourceSE" : "VARCHAR(128) NOT NULL",
"TargetSE" : "VARCHAR(128) NOT NULL",
"FTSServer" : "VARCHAR(255) NOT NULL",
"TargetToken": "VARCHAR(255)",
"SourceToken": "VARCHAR(255)",
"Size": "BIGINT NOT NULL",
"Files": "INTEGER NOT NULL",
"Completeness": "INTEGER NOT NULL DEFAULT 0",
"FailedFiles": "INTEGER DEFAULT 0",
"FailedSize": "INTEGER DEFAULT 0",
"Status" : "ENUM( 'Submitted', 'Ready', 'Staging', 'Canceled', 'Active', 'Hold', "\
"'Failed', 'Finished', 'FinishedDirty', 'Assigned' ) DEFAULT 'Submitted'",
"Error" : "VARCHAR(255)",
"CreationTime" : "DATETIME",
"SubmitTime" : "DATETIME",
"LastUpdate" : "DATETIME" },
"PrimaryKey" : [ "FTSJobID" ],
"Indexes" : { "FTSJobID" : [ "FTSJobID" ], "FTSGUID": [ "FTSGUID" ] } }
@property
def FTSJobID( self ):
""" FTSJobID getter """
return self.__data__["FTSJobID"]
@FTSJobID.setter
def FTSJobID( self, value ):
""" FTSJobID setter """
self.__data__["FTSJobID"] = long( value ) if value else 0
@property
def RequestID( self ):
""" RequestID getter """
return self.__data__["RequestID"]
@RequestID.setter
def RequestID( self, value ):
""" RequestID setter """
self.__data__["RequestID"] = long( value ) if value else 0
@property
def OperationID( self ):
""" OperationID getter """
return self.__data__["OperationID"]
@OperationID.setter
def OperationID( self, value ):
""" OperationID setter """
self.__data__["OperationID"] = long( value ) if value else 0
@property
def FTSGUID( self ):
""" FTSGUID prop """
return self.__data__["FTSGUID"]
@FTSGUID.setter
def FTSGUID( self, value ):
""" FTSGUID setter """
if value:
if type( value ) not in ( str, unicode ):
raise TypeError( "FTSGUID should be a string!" )
if not checkGuid( value ):
raise ValueError( "'%s' is not a valid GUID!" % str( value ) )
self.__data__["FTSGUID"] = value
@property
def FTSServer( self ):
""" FTSServer getter """
return self.__data__["FTSServer"]
@FTSServer.setter
def FTSServer( self, url ):
""" FTSServer getter """
self.__data__["FTSServer"] = url
@property
def Completeness( self ):
""" completeness getter """
return self.__data__["Completeness"]
@Completeness.setter
def Completeness( self, value ):
""" completeness setter """
self.__data__["Completeness"] = int( value ) if value else 0
@property
def Error( self ):
""" error getter """
return self.__data__["Error"]
@Error.setter
def Error( self, error ):
""" error setter """
self.__data__["Error"] = str( error )[255:]
@property
def Files( self ):
""" nb files getter """
self.__data__["Files"] = len( self )
return self.__data__["Files"]
@Files.setter
def Files( self, value ):
""" nb files setter """
self.__data__["Files"] = len( self )
@property
def Status( self ):
""" status prop """
if not self.__data__["Status"]:
self.__data__["Status"] = "Waiting"
return self.__data__["Status"]
@Status.setter
def Status( self, value ):
""" status setter """
value = self._normalizedStatus( value.strip() )
if value not in self._states:
raise ValueError( "Unknown FTSJob Status: '%s'" % str( value ) )
self.__data__["Status"] = value
@property
def FailedFiles( self ):
""" nb failed files getter """
self.__data__["FailedFiles"] = len( [ ftsFile for ftsFile in self
if ftsFile.Status in FTSFile.FAILED_STATES ] )
return self.__data__["FailedFiles"]
@FailedFiles.setter
def FailedFiles( self, value ):
""" nb failed files setter """
if value:
self.__data__["FailedFiles"] = value
else:
self.__data__["FailedFiles"] = sum( [ ftsFile for ftsFile in self
if ftsFile.Status in FTSFile.FAILED_STATES ] )
@property
def Size( self ):
""" size getter """
# if not self.__data__["Size"]:
self.__data__["Size"] = sum( [ ftsFile.Size for ftsFile in self ] )
return self.__data__["Size"]
@Size.setter
def Size( self, value ):
""" size setter """
if value:
self.__data__["Size"] = value
else:
self.__data__["Size"] = sum( [ ftsFile.Size for ftsFile in self ] )
@property
def FailedSize( self ):
""" size getter """
if not self.__data__["FailedSize"]:
self.__data__["FailedSize"] = sum( [ ftsFile.Size for ftsFile in self
if ftsFile.Status in FTSFile.FAILED_STATES ] )
return self.__data__["FailedSize"]
@FailedSize.setter
def FailedSize( self, value ):
""" size setter """
if value:
self.__data__["FailedSize"] = value
else:
self.__data__["FailedSize"] = sum( [ ftsFile.Size for ftsFile in self
if ftsFile.Status in FTSFile.FAILED_STATES ] )
@property
def CreationTime( self ):
""" creation time getter """
return self.__data__["CreationTime"]
@CreationTime.setter
def CreationTime( self, value = None ):
""" creation time setter """
if type( value ) not in ( datetime.datetime, str ) :
raise TypeError( "CreationTime should be a datetime.datetime!" )
if type( value ) == str:
value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' )
self.__data__["CreationTime"] = value
@property
def SubmitTime( self ):
""" request's submission time getter """
return self.__data__["SubmitTime"]
@SubmitTime.setter
def SubmitTime( self, value = None ):
""" submission time setter """
if type( value ) not in ( datetime.datetime, str ):
raise TypeError( "SubmitTime should be a datetime.datetime!" )
if type( value ) == str:
value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' )
self.__data__["SubmitTime"] = value
@property
def LastUpdate( self ):
""" last update getter """
return self.__data__["LastUpdate"]
@LastUpdate.setter
def LastUpdate( self, value = None ):
""" last update setter """
if type( value ) not in ( datetime.datetime, str ):
raise TypeError( "LastUpdate should be a datetime.datetime!" )
if type( value ) == str:
value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' )
self.__data__["LastUpdate"] = value
@property
def TargetSE( self ):
""" target SE getter """
return self.__data__["TargetSE"]
@TargetSE.setter
def TargetSE( self, targetSE ):
""" target SE setter """
self.__data__["TargetSE"] = targetSE
@property
def SourceSE( self ):
""" source SE getter """
return self.__data__["SourceSE"]
@SourceSE.setter
def SourceSE( self, sourceSE ):
""" source SE setter """
self.__data__["SourceSE"] = sourceSE
@property
def SourceToken( self ):
""" source token getter """
return self.__data__["SourceToken"]
@SourceToken.setter
def SourceToken( self, sourceToken ):
""" source SE setter """
self.__data__["SourceToken"] = sourceToken
@property
def TargetToken( self ):
""" target token getter """
return self.__data__["TargetToken"]
@TargetToken.setter
def TargetToken( self, targetToken ):
""" target SE setter """
self.__data__["TargetToken"] = targetToken
# # FTSJobFiles arithmetics
def __contains__( self, subFile ):
""" in operator """
return subFile in self.__files__
def __iadd__( self, ftsFile ):
""" += operator """
if ftsFile not in self:
self.__files__.append( ftsFile )
ftsFile._parent = self
self.Files
self.Size
return self
def __add__( self, ftsFile ):
""" + operator """
self +=ftsFile
def addFile( self, ftsFile ):
""" add :ftsFile: to FTS job """
self +=ftsFile
def subFile( self, ftsFile ):
""" remove ftsFile from this job """
if ftsFile in self:
ftsFile._parent = None
self.__files__.remove( ftsFile )
# # helpers for looping
def __iter__( self ):
""" files iterator """
return self.__files__.__iter__()
def __getitem__( self, i ):
""" [] op for files """
return self.__files__.__getitem__( i )
def __delitem__( self, i ):
""" del ftsJob[i] """
self.__files__.__delitem__( i )
def __setitem__( self, i, ftsFile ):
""" ftsJob[i] = ftsFile """
self.__files__.__setitem__( i, ftsFile )
def fileStatusList( self ):
""" get list of files statuses """
return [ ftsFile.Status for ftsFile in self ]
def __nonzero__( self ):
""" for comparisons
"""
return True
def __len__( self ):
""" nb of subFiles """
return len( self.__files__ )
def _surlPairs( self ):
""" create and return SURL pair file """
surls = []
for ftsFile in self:
checksum = "%s:%s" % ( ftsFile.ChecksumType, ftsFile.Checksum ) if ftsFile.ChecksumType and ftsFile.Checksum else ""
surls.append( "%s %s %s" % ( ftsFile.SourceSURL, ftsFile.TargetSURL, checksum ) )
return "\n".join( surls )
def submitFTS2( self, command = 'glite-transfer-submit', pinTime = False ):
""" submit fts job using FTS2 client """
if self.FTSGUID:
return S_ERROR( "FTSJob has already been submitted" )
surls = self._surlPairs()
if not surls:
return S_ERROR( "No files to submit" )
fd, fileName = tempfile.mkstemp()
surlFile = os.fdopen( fd, 'w' )
surlFile.write( surls )
surlFile.close()
submitCommand = command.split() + \
[ "-s",
self.FTSServer,
"-f",
fileName,
"-o",
"-K" ]
if self.TargetToken:
submitCommand += [ "-t", self.TargetToken]
if self.SourceToken:
submitCommand += [ "-S", self.SourceToken ]
if pinTime:
submitCommand += [ "--copy-pin-lifetime", "%d" % pinTime, "--bring-online", '86400' ]
submit = executeGridCommand( "", submitCommand )
os.remove( fileName )
if not submit["OK"]:
return submit
returnCode, output, errStr = submit["Value"]
if returnCode != 0:
return S_ERROR( errStr if errStr else output )
self.FTSGUID = output.replace( "\n", "" )
self.Status = "Submitted"
for ftsFile in self:
ftsFile.FTSGUID = self.FTSGUID
ftsFile.Status = "Submitted"
return S_OK()
def _normalizedStatus( self, status ):
for st in self._states:
if status.lower() == st.lower():
return st
return status
def monitorFTS2( self, command = "glite-transfer-status", full = False ):
""" monitor fts job """
if not self.FTSGUID:
return S_ERROR( "FTSGUID not set, FTS job not submitted?" )
monitorCommand = command.split() + \
["--verbose",
"-s",
self.FTSServer,
self.FTSGUID ]
if full:
monitorCommand.append( "-l" )
monitor = executeGridCommand( "", monitorCommand )
if not monitor["OK"]:
return monitor
returnCode, outputStr, errStr = monitor["Value"]
# Returns a non zero status if error
if returnCode != 0:
return S_ERROR( errStr )
outputStr = outputStr.replace( "'" , "" ).replace( "<", "" ).replace( ">", "" )
# # set FTS job status
regExp = re.compile( "Status:\\s+(\\S+)" )
# with FTS3 this can be uppercase
self.Status = re.search( regExp, outputStr ).group( 1 )
statusSummary = {}
# This is capitalized, even in FTS3!
for state in FTSFile.ALL_STATES:
regExp = re.compile( "\\s+%s:\\s+(\\d+)" % state )
if regExp.search( outputStr ):
statusSummary[state] = int( re.search( regExp, outputStr ).group( 1 ) )
total = sum( statusSummary.values() )
completed = sum( [ statusSummary.get( state, 0 ) for state in FTSFile.FINAL_STATES ] )
self.Completeness = 100 * completed / total if total else 0
if not full:
return S_OK( statusSummary )
# The order of informations is not the same for glite- and fts- !!!
# In order: new fts-, old fts-, glite-
iExptr = None
for iExptr, exptr in enumerate( (
'[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Staging:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)',
'[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)',
'[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Retries:[ ]+(\\d+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)'
) ):
regExp = re.compile( exptr, re.S )
fileInfo = re.findall( regExp, outputStr )
if fileInfo:
break
if not fileInfo:
return S_ERROR( "Error monitoring job (no regexp match)" )
for info in fileInfo:
if iExptr == 0:
# version >= 3.2.30
sourceURL, targetURL, fileStatus, reason, duration, _retries, _staging = info
elif iExptr == 1:
# version FTS3 < 3.2.30
sourceURL, targetURL, fileStatus, reason, duration, _retries = info
elif iExptr == 2:
# version FTS2
sourceURL, targetURL, fileStatus, _retries, reason, duration = info
else:
return S_ERROR( 'Error monitoring job (implement match %d)' % iExptr )
candidateFile = None
for ftsFile in self:
if ftsFile.SourceSURL == sourceURL:
candidateFile = ftsFile
break
if not candidateFile:
continue
# Can be uppercase for FTS3
if not candidateFile.TargetSURL:
candidateFile.TargetSURL = targetURL
candidateFile.Status = fileStatus
candidateFile.Error = reason
candidateFile._duration = duration
if candidateFile.Status == "Failed":
for missingSource in self.missingSourceErrors:
if missingSource.match( reason ):
candidateFile.Error = "MissingSource"
# If the staging info was present, record it
if len( info ) > 6:
candidateFile._staging = info[6]
# # register successful files
if self.Status in FTSJob.FINALSTATES:
return self.finalize()
return S_OK()
def finalize( self ):
""" register successfully transferred files """
if self.Status not in FTSJob.FINALSTATES:
return S_OK()
startTime = time.time()
targetSE = StorageElement( self.TargetSE )
toRegister = [ ftsFile for ftsFile in self if ftsFile.Status == "Finished" ]
toRegisterDict = {}
for ftsFile in toRegister:
pfn = returnSingleResult( targetSE.getPfnForProtocol( ftsFile.TargetSURL, protocol = "SRM2", withPort = False ) )
if not pfn["OK"]:
continue
pfn = pfn["Value"]
toRegisterDict[ ftsFile.LFN ] = { "PFN": pfn, "SE": self.TargetSE }
if toRegisterDict:
self._regTotal += len( toRegisterDict )
register = self._fc.addReplica( toRegisterDict )
self._regTime += time.time() - startTime
if not register["OK"]:
# FIXME: shouldn't be a print!
for ftsFile in toRegister:
ftsFile.Error = "AddCatalogReplicaFailed"
print ftsFile.Error
return register
register = register["Value"]
self._regSuccess += len( register.get( 'Successful', {} ) )
failedFiles = register.get( "Failed", {} )
# FIXME
for ftsFile in toRegister:
if ftsFile.LFN in failedFiles:
ftsFile.Error = "AddCatalogReplicaFailed"
print ftsFile.Error
return S_OK()
def toSQL( self ):
""" prepare SQL INSERT or UPDATE statement
:return: str with SQL fragment
"""
colVals = [ ( "`%s`" % column, "'%s'" % value if type( value ) in ( str, datetime.datetime ) else str( value ) )
for column, value in self.__data__.items()
if value and column not in ( "FTSJobID", "LastUpdate" ) ]
colVals.append( ( "`LastUpdate`", "UTC_TIMESTAMP()" ) )
query = []
if self.FTSJobID:
query.append( "UPDATE `FTSJob` SET " )
query.append( ",".join( [ "%s=%s" % item for item in colVals ] ) )
query.append( " WHERE `FTSJobID`=%d;\n" % self.FTSJobID )
else:
query.append( "INSERT INTO `FTSJob` " )
columns = "(%s)" % ",".join( [ column for column, value in colVals ] )
values = "(%s)" % ",".join( [ value for column, value in colVals ] )
query.append( columns )
query.append( " VALUES %s;" % values )
return S_OK( "".join( query ) )
def toJSON( self ):
""" dump to JSON format """
digest = dict( zip( self.__data__.keys(),
[ str( val ) if val else "" for val in self.__data__.values() ] ) )
digest["FTSFiles"] = []
for ftsFile in self:
fileJSON = ftsFile.toJSON()
if not fileJSON["OK"]:
return fileJSON
digest["FTSFiles"].append( fileJSON["Value"] )
return S_OK( digest )
class DataIntegrityClient( Client ):
"""
The following methods are supported in the service but are not mentioned explicitly here:
getProblematic()
Obtains a problematic file from the IntegrityDB based on the LastUpdate time
getPrognosisProblematics(prognosis)
Obtains all the problematics of a particular prognosis from the integrityDB
getProblematicsSummary()
Obtains a count of the number of problematics for each prognosis found
getDistinctPrognosis()
Obtains the distinct prognosis found in the integrityDB
getTransformationProblematics(prodID)
Obtains the problematics for a given production
incrementProblematicRetry(fileID)
Increments the retry count for the supplied file ID
changeProblematicPrognosis(fileID,newPrognosis)
Changes the prognosis of the supplied file to the new prognosis
setProblematicStatus(fileID,status)
Updates the status of a problematic in the integrityDB
removeProblematic(self,fileID)
This removes the specified file ID from the integrity DB
insertProblematic(sourceComponent,fileMetadata)
Inserts file with supplied metadata into the integrity DB
"""
def __init__( self, **kwargs ):
super(DataIntegrityClient, self).__init__( **kwargs )
self.setServer( 'DataManagement/DataIntegrity' )
self.dm = DataManager()
self.fc = FileCatalog()
def setFileProblematic( self, lfn, reason, sourceComponent = '' ):
""" This method updates the status of the file in the FileCatalog and the IntegrityDB
lfn - the lfn of the file
reason - this is given to the integrity DB and should reflect the problem observed with the file
sourceComponent is the component issuing the request.
"""
if isinstance( lfn, list ):
lfns = lfn
elif isinstance( lfn, basestring ):
lfns = [lfn]
else:
errStr = "DataIntegrityClient.setFileProblematic: Supplied file info must be list or a single LFN."
gLogger.error( errStr )
return S_ERROR( errStr )
gLogger.info( "DataIntegrityClient.setFileProblematic: Attempting to update %s files." % len( lfns ) )
fileMetadata = {}
for lfn in lfns:
fileMetadata[lfn] = {'Prognosis':reason, 'LFN':lfn, 'PFN':'', 'SE':''}
res = self.insertProblematic( sourceComponent, fileMetadata )
if not res['OK']:
gLogger.error( "DataIntegrityClient.setReplicaProblematic: Failed to insert problematics to integrity DB" )
return res
def reportProblematicReplicas( self, replicaTuple, se, reason ):
""" Simple wrapper function around setReplicaProblematic """
gLogger.info( 'The following %s files had %s at %s' % ( len( replicaTuple ), reason, se ) )
for lfn, _pfn, se, reason in sorted( replicaTuple ):
if lfn:
gLogger.info( lfn )
res = self.setReplicaProblematic( replicaTuple, sourceComponent = 'DataIntegrityClient' )
if not res['OK']:
gLogger.info( 'Failed to update integrity DB with replicas', res['Message'] )
else:
gLogger.info( 'Successfully updated integrity DB with replicas' )
def setReplicaProblematic( self, replicaTuple, sourceComponent = '' ):
""" This method updates the status of the replica in the FileCatalog and the IntegrityDB
The supplied replicaDict should be of the form {lfn :{'PFN':pfn,'SE':se,'Prognosis':prognosis}
lfn - the lfn of the file
pfn - the pfn if available (otherwise '')
se - the storage element of the problematic replica (otherwise '')
prognosis - this is given to the integrity DB and should reflect the problem observed with the file
sourceComponent is the component issuing the request.
"""
if isinstance( replicaTuple, tuple ):
replicaTuple = [replicaTuple]
elif isinstance( replicaTuple, list ):
pass
else:
errStr = "DataIntegrityClient.setReplicaProblematic: Supplied replica info must be a tuple or list of tuples."
gLogger.error( errStr )
return S_ERROR( errStr )
gLogger.info( "DataIntegrityClient.setReplicaProblematic: Attempting to update %s replicas." % len( replicaTuple ) )
replicaDict = {}
for lfn, pfn, se, reason in replicaTuple:
replicaDict[lfn] = {'Prognosis':reason, 'LFN':lfn, 'PFN':pfn, 'SE':se}
res = self.insertProblematic( sourceComponent, replicaDict )
if not res['OK']:
gLogger.error( "DataIntegrityClient.setReplicaProblematic: Failed to insert problematic to integrity DB" )
return res
for lfn in replicaDict.keys():
replicaDict[lfn]['Status'] = 'Problematic'
res = self.fc.setReplicaStatus( replicaDict )
if not res['OK']:
errStr = "DataIntegrityClient.setReplicaProblematic: Completely failed to update replicas."
gLogger.error( errStr, res['Message'] )
return res
failed = res['Value']['Failed']
successful = res['Value']['Successful']
resDict = {'Successful':successful, 'Failed':failed}
return S_OK( resDict )
##########################################################################
#
# This section contains the resolution methods for various prognoses
#
def __updateCompletedFiles( self, prognosis, fileID ):
gLogger.info( "%s file (%d) is resolved" % ( prognosis, fileID ) )
return self.setProblematicStatus( fileID, 'Resolved' )
def __returnProblematicError( self, fileID, res ):
self.incrementProblematicRetry( fileID )
gLogger.error( 'DataIntegrityClient failure', res['Message'] )
return res
def __updateReplicaToChecked( self, problematicDict ):
lfn = problematicDict['LFN']
fileID = problematicDict['FileID']
prognosis = problematicDict['Prognosis']
problematicDict['Status'] = 'Checked'
res = returnSingleResult( self.fc.setReplicaStatus( {lfn:problematicDict} ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
gLogger.info( "%s replica (%d) is updated to Checked status" % ( prognosis, fileID ) )
return self.__updateCompletedFiles( prognosis, fileID )
def resolveCatalogPFNSizeMismatch( self, problematicDict ):
""" This takes the problematic dictionary returned by the integrity DB and resolved the CatalogPFNSizeMismatch prognosis
"""
lfn = problematicDict['LFN']
se = problematicDict['SE']
fileID = problematicDict['FileID']
res = returnSingleResult( self.fc.getFileSize( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
catalogSize = res['Value']
res = returnSingleResult( StorageElement( se ).getFileSize( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
storageSize = res['Value']
bkKCatalog = FileCatalog( ['BookkeepingDB'] )
res = returnSingleResult( bkKCatalog.getFileSize( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
bookkeepingSize = res['Value']
if bookkeepingSize == catalogSize == storageSize:
gLogger.info( "CatalogPFNSizeMismatch replica (%d) matched all registered sizes." % fileID )
return self.__updateReplicaToChecked( problematicDict )
if catalogSize == bookkeepingSize:
gLogger.info( "CatalogPFNSizeMismatch replica (%d) found to mismatch the bookkeeping also" % fileID )
res = returnSingleResult( self.fc.getReplicas( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if len( res['Value'] ) <= 1:
gLogger.info( "CatalogPFNSizeMismatch replica (%d) has no other replicas." % fileID )
return S_ERROR( "Not removing catalog file mismatch since the only replica" )
else:
gLogger.info( "CatalogPFNSizeMismatch replica (%d) has other replicas. Removing..." % fileID )
res = self.dm.removeReplica( se, lfn )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
return self.__updateCompletedFiles( 'CatalogPFNSizeMismatch', fileID )
if ( catalogSize != bookkeepingSize ) and ( bookkeepingSize == storageSize ):
gLogger.info( "CatalogPFNSizeMismatch replica (%d) found to match the bookkeeping size" % fileID )
res = self.__updateReplicaToChecked( problematicDict )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
return self.changeProblematicPrognosis( fileID, 'BKCatalogSizeMismatch' )
gLogger.info( "CatalogPFNSizeMismatch replica (%d) all sizes found mismatch. Updating retry count" % fileID )
return self.incrementProblematicRetry( fileID )
#FIXME: Unused?
def resolvePFNNotRegistered( self, problematicDict ):
""" This takes the problematic dictionary returned by the integrity DB and resolved the PFNNotRegistered prognosis
"""
lfn = problematicDict['LFN']
seName = problematicDict['SE']
fileID = problematicDict['FileID']
se = StorageElement( seName )
res = returnSingleResult( self.fc.exists( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if not res['Value']:
# The file does not exist in the catalog
res = returnSingleResult( se.removeFile( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
return self.__updateCompletedFiles( 'PFNNotRegistered', fileID )
res = returnSingleResult( se.getFileMetadata( lfn ) )
if ( not res['OK'] ) and ( re.search( 'File does not exist', res['Message'] ) ):
gLogger.info( "PFNNotRegistered replica (%d) found to be missing." % fileID )
return self.__updateCompletedFiles( 'PFNNotRegistered', fileID )
elif not res['OK']:
return self.__returnProblematicError( fileID, res )
storageMetadata = res['Value']
if storageMetadata['Lost']:
gLogger.info( "PFNNotRegistered replica (%d) found to be Lost. Updating prognosis" % fileID )
return self.changeProblematicPrognosis( fileID, 'PFNLost' )
if storageMetadata['Unavailable']:
gLogger.info( "PFNNotRegistered replica (%d) found to be Unavailable. Updating retry count" % fileID )
return self.incrementProblematicRetry( fileID )
# HACK until we can obtain the space token descriptions through GFAL
site = seName.split( '_' )[0].split( '-' )[0]
if not storageMetadata['Cached']:
if lfn.endswith( '.raw' ):
seName = '%s-RAW' % site
else:
seName = '%s-RDST' % site
elif storageMetadata['Migrated']:
if lfn.startswith( '/lhcb/data' ):
seName = '%s_M-DST' % site
else:
seName = '%s_MC_M-DST' % site
else:
if lfn.startswith( '/lhcb/data' ):
seName = '%s-DST' % site
else:
seName = '%s_MC-DST' % site
problematicDict['SE'] = seName
res = returnSingleResult( se.getURL( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
problematicDict['PFN'] = res['Value']
res = returnSingleResult( self.fc.addReplica( {lfn:problematicDict} ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
res = returnSingleResult( self.fc.getFileMetadata( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if res['Value']['Size'] != storageMetadata['Size']:
gLogger.info( "PFNNotRegistered replica (%d) found with catalog size mismatch. Updating prognosis" % fileID )
return self.changeProblematicPrognosis( fileID, 'CatalogPFNSizeMismatch' )
return self.__updateCompletedFiles( 'PFNNotRegistered', fileID )
#FIXME: Unused?
def resolveLFNCatalogMissing( self, problematicDict ):
""" This takes the problematic dictionary returned by the integrity DB and resolved the LFNCatalogMissing prognosis
"""
lfn = problematicDict['LFN']
fileID = problematicDict['FileID']
res = returnSingleResult( self.fc.exists( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if res['Value']:
return self.__updateCompletedFiles( 'LFNCatalogMissing', fileID )
# Remove the file from all catalogs
# RF_NOTE : here I can do it because it's a single file, but otherwise I would need to sort the path
res = returnSingleResult( self.fc.removeFile( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
return self.__updateCompletedFiles( 'LFNCatalogMissing', fileID )
#FIXME: Unused?
def resolvePFNMissing( self, problematicDict ):
""" This takes the problematic dictionary returned by the integrity DB and resolved the PFNMissing prognosis
"""
se = problematicDict['SE']
lfn = problematicDict['LFN']
fileID = problematicDict['FileID']
res = returnSingleResult( self.fc.exists( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if not res['Value']:
gLogger.info( "PFNMissing file (%d) no longer exists in catalog" % fileID )
return self.__updateCompletedFiles( 'PFNMissing', fileID )
res = returnSingleResult( StorageElement( se ).exists( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if res['Value']:
gLogger.info( "PFNMissing replica (%d) is no longer missing" % fileID )
return self.__updateReplicaToChecked( problematicDict )
gLogger.info( "PFNMissing replica (%d) does not exist" % fileID )
res = returnSingleResult( self.fc.getReplicas( lfn, allStatus = True ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
replicas = res['Value']
seSite = se.split( '_' )[0].split( '-' )[0]
found = False
print replicas
for replicaSE in replicas.keys():
if re.search( seSite, replicaSE ):
found = True
problematicDict['SE'] = replicaSE
se = replicaSE
if not found:
gLogger.info( "PFNMissing replica (%d) is no longer registered at SE. Resolved." % fileID )
return self.__updateCompletedFiles( 'PFNMissing', fileID )
gLogger.info( "PFNMissing replica (%d) does not exist. Removing from catalog..." % fileID )
res = returnSingleResult( self.fc.removeReplica( {lfn:problematicDict} ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if len( replicas ) == 1:
gLogger.info( "PFNMissing replica (%d) had a single replica. Updating prognosis" % fileID )
return self.changeProblematicPrognosis( fileID, 'LFNZeroReplicas' )
res = self.dm.replicateAndRegister( problematicDict['LFN'], se )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
# If we get here the problem is solved so we can update the integrityDB
return self.__updateCompletedFiles( 'PFNMissing', fileID )
#FIXME: Unused?
def resolvePFNUnavailable( self, problematicDict ):
""" This takes the problematic dictionary returned by the integrity DB and resolved the PFNUnavailable prognosis
"""
lfn = problematicDict['LFN']
se = problematicDict['SE']
fileID = problematicDict['FileID']
res = returnSingleResult( StorageElement( se ).getFileMetadata( lfn ) )
if ( not res['OK'] ) and ( re.search( 'File does not exist', res['Message'] ) ):
# The file is no longer Unavailable but has now dissapeared completely
gLogger.info( "PFNUnavailable replica (%d) found to be missing. Updating prognosis" % fileID )
return self.changeProblematicPrognosis( fileID, 'PFNMissing' )
if ( not res['OK'] ) or res['Value']['Unavailable']:
gLogger.info( "PFNUnavailable replica (%d) found to still be Unavailable" % fileID )
return self.incrementProblematicRetry( fileID )
if res['Value']['Lost']:
gLogger.info( "PFNUnavailable replica (%d) is now found to be Lost. Updating prognosis" % fileID )
return self.changeProblematicPrognosis( fileID, 'PFNLost' )
gLogger.info( "PFNUnavailable replica (%d) is no longer Unavailable" % fileID )
# Need to make the replica okay in the Catalog
return self.__updateReplicaToChecked( problematicDict )
#FIXME: Unused?
def resolvePFNZeroSize( self, problematicDict ):
""" This takes the problematic dictionary returned by the integrity DB and resolves the PFNZeroSize prognosis
"""
lfn = problematicDict['LFN']
seName = problematicDict['SE']
fileID = problematicDict['FileID']
se = StorageElement( seName )
res = returnSingleResult( se.getFileSize( lfn ) )
if ( not res['OK'] ) and ( re.search( 'File does not exist', res['Message'] ) ):
gLogger.info( "PFNZeroSize replica (%d) found to be missing. Updating prognosis" % problematicDict['FileID'] )
return self.changeProblematicPrognosis( fileID, 'PFNMissing' )
storageSize = res['Value']
if storageSize == 0:
res = returnSingleResult( se.removeFile( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
gLogger.info( "PFNZeroSize replica (%d) removed. Updating prognosis" % problematicDict['FileID'] )
return self.changeProblematicPrognosis( fileID, 'PFNMissing' )
res = returnSingleResult( self.fc.getReplicas( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if seName not in res['Value']:
gLogger.info( "PFNZeroSize replica (%d) not registered in catalog. Updating prognosis" % problematicDict['FileID'] )
return self.changeProblematicPrognosis( fileID, 'PFNNotRegistered' )
res = returnSingleResult( self.fc.getFileMetadata( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
catalogSize = res['Value']['Size']
if catalogSize != storageSize:
gLogger.info( "PFNZeroSize replica (%d) size found to differ from registered metadata. Updating prognosis" % problematicDict['FileID'] )
return self.changeProblematicPrognosis( fileID, 'CatalogPFNSizeMismatch' )
return self.__updateCompletedFiles( 'PFNZeroSize', fileID )
############################################################################################
#FIXME: Unused?
def resolveLFNZeroReplicas( self, problematicDict ):
""" This takes the problematic dictionary returned by the integrity DB and resolves the LFNZeroReplicas prognosis
"""
lfn = problematicDict['LFN']
fileID = problematicDict['FileID']
res = returnSingleResult( self.fc.getReplicas( lfn, allStatus = True ) )
if res['OK'] and res['Value']:
gLogger.info( "LFNZeroReplicas file (%d) found to have replicas" % fileID )
else:
gLogger.info( "LFNZeroReplicas file (%d) does not have replicas. Checking storage..." % fileID )
pfnsFound = False
for storageElementName in sorted( gConfig.getValue( 'Resources/StorageElementGroups/Tier1_MC_M-DST', [] ) ):
res = self.__getStoragePathExists( [lfn], storageElementName )
if lfn in res['Value']:
gLogger.info( "LFNZeroReplicas file (%d) found storage file at %s" % ( fileID, storageElementName ) )
self.reportProblematicReplicas( [( lfn, 'deprecatedUrl', storageElementName, 'PFNNotRegistered' )], storageElementName, 'PFNNotRegistered' )
pfnsFound = True
if not pfnsFound:
gLogger.info( "LFNZeroReplicas file (%d) did not have storage files. Removing..." % fileID )
res = returnSingleResult( self.fc.removeFile( lfn ) )
if not res['OK']:
gLogger.error( 'DataIntegrityClient: failed to remove file', res['Message'] )
# Increment the number of retries for this file
self.server.incrementProblematicRetry( fileID )
return res
gLogger.info( "LFNZeroReplicas file (%d) removed from catalog" % fileID )
# If we get here the problem is solved so we can update the integrityDB
return self.__updateCompletedFiles( 'LFNZeroReplicas', fileID )
def _reportProblematicFiles( self, lfns, reason ):
""" Simple wrapper function around setFileProblematic
"""
gLogger.info( 'The following %s files were found with %s' % ( len( lfns ), reason ) )
for lfn in sorted( lfns ):
gLogger.info( lfn )
res = self.setFileProblematic( lfns, reason, sourceComponent = 'DataIntegrityClient' )
if not res['OK']:
gLogger.info( 'Failed to update integrity DB with files', res['Message'] )
else:
gLogger.info( 'Successfully updated integrity DB with files' )
