def __getStoragePathExists(self, lfnPaths, storageElement):
gLogger.info('Determining the existance of %d files at %s' %
(len(lfnPaths), storageElement))
se = StorageElement(storageElement)
res = se.getPfnForLfn(lfnPaths)
if not res['OK']:
gLogger.error("Failed to get PFNs for LFNs", res['Message'])
return res
for lfnPath, error in res['Value']['Failed'].items():
gLogger.error('Failed to obtain PFN from LFN',
'%s %s' % (lfnPath, error))
if res['Value']['Failed']:
return S_ERROR('Failed to obtain PFNs from LFNs')
lfnPfns = res['Value']['Successful']
pfnLfns = {}
for lfn, pfn in lfnPfns.items():
pfnLfns[pfn] = lfn
res = se.exists(pfnLfns)
if not res['OK']:
gLogger.error("Failed to obtain existance of paths",
res['Message'])
return res
for lfnPath, error in res['Value']['Failed'].items():
gLogger.error('Failed to determine existance of path',
'%s %s' % (lfnPath, error))
if res['Value']['Failed']:
return S_ERROR('Failed to determine existance of paths')
pathExists = res['Value']['Successful']
resDict = {}
for pfn, exists in pathExists.items():
if exists:
resDict[pfnLfns[pfn]] = pfn
return S_OK(resDict)
def checkSourceSE( self, sourceSE, lfn, catalogMetadata ):
""" filter out SourceSE where PFN is not existing
:param self: self reference
:param str lfn: logical file name
:param dict catalogMetadata: catalog metadata
"""
se = self.seCache.get( sourceSE, None )
if not se:
se = StorageElement( sourceSE, "SRM2" )
self.seCache[sourceSE] = se
pfn = se.getPfnForLfn( lfn )
if not pfn["OK"]:
self.log.warn("checkSourceSE: unable to create pfn for %s lfn: %s" % ( lfn, pfn["Message"] ) )
return pfn
pfn = pfn["Value"]
seMetadata = se.getFileMetadata( pfn, singleFile=True )
if not seMetadata["OK"]:
self.log.warn("checkSourceSE: %s" % seMetadata["Message"] )
return S_ERROR("checkSourceSE: failed to get metadata")
seMetadata = seMetadata["Value"]
catalogChecksum = catalogMetadata["Checksum"].replace("x", "0" ).zfill(8) if "Checksum" in catalogMetadata else None
storageChecksum = seMetadata["Checksum"].replace("x", "0").zfill(8) if "Checksum" in seMetadata else None
if catalogChecksum != storageChecksum:
self.log.warn( "checkSourceSE: %s checksum mismatch catalogue:%s %s:%s" % ( lfn,
catalogChecksum,
sourceSE,
storageChecksum ) )
return S_ERROR("checkSourceSE: checksum mismatch")
## if we're here everything is OK
return S_OK()
def __getStoragePathExists( self, lfnPaths, storageElement ):
gLogger.info( 'Determining the existance of %d files at %s' % ( len( lfnPaths ), storageElement ) )
se = StorageElement( storageElement )
res = se.getPfnForLfn( lfnPaths )
if not res['OK']:
gLogger.error( "Failed to get PFNs for LFNs", res['Message'] )
return res
for lfnPath, error in res['Value']['Failed'].items():
gLogger.error( 'Failed to obtain PFN from LFN', '%s %s' % ( lfnPath, error ) )
if res['Value']['Failed']:
return S_ERROR( 'Failed to obtain PFNs from LFNs' )
lfnPfns = res['Value']['Successful']
pfnLfns = {}
for lfn, pfn in lfnPfns.items():
pfnLfns[pfn] = lfn
res = se.exists( pfnLfns )
if not res['OK']:
gLogger.error( "Failed to obtain existance of paths", res['Message'] )
return res
for lfnPath, error in res['Value']['Failed'].items():
gLogger.error( 'Failed to determine existance of path', '%s %s' % ( lfnPath, error ) )
if res['Value']['Failed']:
return S_ERROR( 'Failed to determine existance of paths' )
pathExists = res['Value']['Successful']
resDict = {}
for pfn, exists in pathExists.items():
if exists:
resDict[pfnLfns[pfn]] = pfn
return S_OK( resDict )
def __setRegistrationRequest( self, lfn, targetSE, fileDict, catalog ):
""" Sets a registration request
:param str lfn: LFN
:param list se: list of SE
:param list catalog: list of catalogs to use
:param dict fileDict: file metadata
"""
self.log.info( 'Setting registration request for %s at %s.' % ( lfn, targetSE ) )
for cat in catalog:
register = Operation()
register.Type = "RegisterFile"
register.Catalog = cat
register.TargetSE = targetSE
regFile = File()
regFile.LFN = lfn
regFile.Checksum = fileDict.get( "Checksum", "" )
regFile.ChecksumType = fileDict.get( "ChecksumType", "" )
regFile.Size = fileDict.get( "Size", 0 )
regFile.GUID = fileDict.get( "GUID", "" )
se = StorageElement( targetSE )
pfn = se.getPfnForLfn( lfn )
if not pfn["OK"]:
self.log.error( "unable to get PFN for LFN: %s" % pfn["Message"] )
return pfn
regFile.PFN = pfn["Value"]
register.addFile( regFile )
self.request.addOperation( register )
return S_OK()
def _filterReplicas( self, opFile ):
""" filter out banned/invalid source SEs """
ret = { "Valid" : [], "Banned" : [], "Bad" : [] }
replicas = self.replicaManager().getActiveReplicas( opFile.LFN )
if not replicas["OK"]:
self.log.error( replicas["Message"] )
reNotExists = re.compile( "not such file or directory" )
replicas = replicas["Value"]
failed = replicas["Failed"].get( opFile.LFN , "" )
if reNotExists.match( failed.lower() ):
opFile.Status = "Failed"
opFile.Error = failed
return S_ERROR( failed )
replicas = replicas["Successful"][opFile.LFN] if opFile.LFN in replicas["Successful"] else {}
for repSEName in replicas:
seRead = self.rssSEStatus( repSEName, "ReadAccess" )
if not seRead["OK"]:
self.log.error( seRead["Message"] )
ret["Banned"].append( repSEName )
continue
if not seRead["Value"]:
self.log.error( "StorageElement '%s' is banned for reading" % ( repSEName ) )
repSE = self.seCache.get( repSEName, None )
if not repSE:
repSE = StorageElement( repSEName, "SRM2" )
self.seCache[repSE] = repSE
pfn = repSE.getPfnForLfn( opFile.LFN )
if not pfn["OK"]:
self.log.warn( "unable to create pfn for %s lfn: %s" % ( opFile.LFN, pfn["Message"] ) )
ret["Banned"].append( repSEName )
continue
pfn = pfn["Value"]
repSEMetadata = repSE.getFileMetadata( pfn, singleFile = True )
if not repSEMetadata["OK"]:
self.log.warn( repSEMetadata["Message"] )
ret["Banned"].append( repSEName )
continue
repSEMetadata = repSEMetadata["Value"]
seChecksum = repSEMetadata["Checksum"].replace( "x", "0" ).zfill( 8 ) if "Checksum" in repSEMetadata else None
if opFile.Checksum and opFile.Checksum != seChecksum:
self.log.warn( " %s checksum mismatch: %s %s:%s" % ( opFile.LFN,
opFile.Checksum,
repSE,
seChecksum ) )
ret["Bad"].append( repSEName )
continue
# # if we're here repSE is OK
ret["Valid"].append( repSEName )
return S_OK( ret )
class StorageElementTestCase( unittest.TestCase ):
""" Base class for the StorageElement test cases
"""
def setUp( self ):
self.numberOfFiles = 1
self.storageElement = StorageElement( storageElementToTest )
self.localSourceFile = "/etc/group"
self.localFileSize = getSize( self.localSourceFile )
self.destDirectory = "/lhcb/test/unit-test/TestStorageElement"
destinationDir = self.storageElement.getPfnForLfn( self.destDirectory )['Value']
res = self.storageElement.createDirectory( destinationDir, singleDirectory = True )
self.assert_( res['OK'] )
def tearDown( self ):
destinationDir = self.storageElement.getPfnForLfn( self.destDirectory )['Value']
res = self.storageElement.removeDirectory( destinationDir, recursive = True, singleDirectory = True )
self.assert_( res['OK'] )
class StorageElementTestCase(unittest.TestCase):
""" Base class for the StorageElement test cases
"""
def setUp(self):
self.numberOfFiles = 1
self.storageElement = StorageElement(storageElementToTest)
self.localSourceFile = "/etc/group"
self.localFileSize = getSize(self.localSourceFile)
self.destDirectory = "/lhcb/test/unit-test/TestStorageElement"
destinationDir = returnSingleResult(
self.storageElement.getPfnForLfn(self.destDirectory))
res = self.storageElement.createDirectory(destinationDir,
singleDirectory=True)
self.assert_(res['OK'])
def tearDown(self):
destinationDir = returnSingleResult(
self.storageElement.getPfnForLfn(self.destDirectory))
res = self.storageElement.removeDirectory(destinationDir,
recursive=True,
singleDirectory=True)
self.assert_(res['OK'])
def uploadAndRegisterFiles(self, localDir, SE='IHEP-USER', guid=None):
"""upload a set of files to SE and register it in DFC.
user input the directory of localfile.
we can treat localDir as a kind of datasetName.
"""
result_OK = 1
errorList = []
fileList = self.__getFilenamesByLocaldir(localDir)
for fullpath in fileList[:50]:
#get the attributes of the file
print fullpath
fileAttr = self.__getFileAttributes(fullpath)
#create dir and set dirMetadata to associated dir
metaDict = {}
metaDict['dataType'] = fileAttr['dataType']
metaDict['eventType'] = fileAttr['eventType']
metaDict['streamId'] = fileAttr['streamId']
metaDict['resonance'] = fileAttr['resonance']
metaDict['round'] = fileAttr['round']
metaDict['bossVer'] = fileAttr['bossVer']
lastDir = self.registerHierarchicalDir(metaDict,
rootDir='/zhanggang_test')
lfn = lastDir + os.sep + fileAttr['LFN']
fileAttr['LFN'] = lfn
#upload and register file.
dirac = Dirac()
result = dirac.addFile(lfn, fullpath, SE, guid, printOutput=True)
#register file metadata
if not result['OK']:
print 'ERROR %s' % (result['Message'])
#return S_ERROR(result['Message'])
errorList.append(fullpath)
result_OK = 0
else:
#get the truely PFN
storageElement = StorageElement(SE)
res = storageElement.getPfnForLfn(lfn)
destPfn = res['Value']
fileAttr['PFN'] = destPfn
result = self.__registerFileMetadata(lfn, fileAttr)
if not result['OK']:
result_OK = 0
print "failed to register file metadata"
if result_OK:
return S_OK()
else:
return S_ERROR(errorList)
def _setRegistrationRequest(self, lfn, targetSE, fileDict, catalog):
""" Sets a registration request
:param str lfn: LFN
:param list se: list of SE (or just string)
:param list catalog: list (or string) of catalogs to use
:param dict fileDict: file metadata
"""
self.log.info('Setting registration request for %s at %s.' %
(lfn, targetSE))
if not type(catalog) == type([]):
catalog = [catalog]
for cat in catalog:
register = Operation()
register.Type = "RegisterFile"
register.Catalog = cat
register.TargetSE = targetSE
regFile = File()
regFile.LFN = lfn
regFile.Checksum = fileDict.get("Checksum", "")
regFile.ChecksumType = fileDict.get("ChecksumType",
self.defaultChecksumType)
regFile.Size = fileDict.get("Size", 0)
regFile.GUID = fileDict.get("GUID", "")
se = StorageElement(targetSE)
pfn = se.getPfnForLfn(lfn)
if not pfn["OK"] or lfn not in pfn["Value"]['Successful']:
self.log.error(
"unable to get PFN for LFN: %s" %
pfn.get('Message',
pfn.get('Value', {}).get('Failed', {}).get(lfn)))
return pfn
regFile.PFN = pfn["Value"]['Successful'][lfn]
register.addFile(regFile)
self.request.addOperation(register)
return S_OK()
def __removeStorageDirectory(self, directory, storageElement):
""" wipe out all contents from :directory: at :storageElement:
:param self: self reference
:param str directory: path
:param str storageElement: SE name
"""
self.log.info('Removing the contents of %s at %s' %
(directory, storageElement))
se = StorageElement(storageElement)
res = se.getPfnForLfn([directory])
if not res['OK']:
self.log.error("Failed to get PFN for directory", res['Message'])
return res
if directory in res['Value']['Failed']:
self.log.verbose(
'Failed to obtain directory PFN from LFN',
'%s %s' % (directory, res['Value']['Failed'][directory]))
return S_ERROR('Failed to obtain directory PFN from LFNs')
storageDirectory = res['Value']['Successful'][directory]
res = returnSingleResult(se.exists(storageDirectory))
if not res['OK']:
self.log.error("Failed to obtain existance of directory",
res['Message'])
return res
exists = res['Value']
if not exists:
self.log.info("The directory %s does not exist at %s " %
(directory, storageElement))
return S_OK()
res = returnSingleResult(
se.removeDirectory(storageDirectory, recursive=True))
if not res['OK']:
self.log.error("Failed to remove storage directory",
res['Message'])
return res
self.log.info(
"Successfully removed %d files from %s at %s" %
(res['Value']['FilesRemoved'], directory, storageElement))
return S_OK()
def __generateLocation(self, sbPath):
"""
Generate the location string
"""
if self.__useLocalStorage:
return S_OK((self.__localSEName, sbPath))
#It's external storage
storageElement = StorageElement(self.__externalSEName)
res = storageElement.isValid()
if not res['OK']:
errStr = "Failed to instantiate destination StorageElement"
gLogger.error(errStr, self.__externalSEName)
return S_ERROR(errStr)
result = storageElement.getPfnForLfn(sbPath)
if not result['OK']:
errStr = "Failed to generate PFN"
gLogger.error(errStr, self.__externalSEName)
return S_ERROR(errStr)
destPfn = result['Value']
return S_OK((self.__externalSEName, destPfn))
def __generateLocation( self, sbPath ):
"""
Generate the location string
"""
if self.__useLocalStorage:
return S_OK( ( self.__localSEName, sbPath ) )
#It's external storage
storageElement = StorageElement( self.__externalSEName )
res = storageElement.isValid()
if not res['OK']:
errStr = "Failed to instantiate destination StorageElement"
gLogger.error( errStr, self.__externalSEName )
return S_ERROR( errStr )
result = storageElement.getPfnForLfn( sbPath )
if not result['OK']:
errStr = "Failed to generate PFN"
gLogger.error( errStr, self.__externalSEName )
return S_ERROR( errStr )
destPfn = result['Value']
return S_OK( ( self.__externalSEName, destPfn ) )
def _setRegistrationRequest( self, lfn, targetSE, fileDict, catalog ):
""" Sets a registration request
:param str lfn: LFN
:param list se: list of SE (or just string)
:param list catalog: list (or string) of catalogs to use
:param dict fileDict: file metadata
"""
self.log.info( 'Setting registration request for %s at %s.' % ( lfn, targetSE ) )
if not type( catalog ) == type( [] ):
catalog = [catalog]
for cat in catalog:
register = Operation()
register.Type = "RegisterFile"
register.Catalog = cat
register.TargetSE = targetSE
regFile = File()
regFile.LFN = lfn
regFile.Checksum = fileDict.get( "Checksum", "" )
regFile.ChecksumType = fileDict.get( "ChecksumType", self.defaultChecksumType )
regFile.Size = fileDict.get( "Size", 0 )
regFile.GUID = fileDict.get( "GUID", "" )
se = StorageElement( targetSE )
pfn = se.getPfnForLfn( lfn )
if not pfn["OK"] or lfn not in pfn["Value"]['Successful']:
self.log.error( "unable to get PFN for LFN: %s" % pfn.get( 'Message', pfn.get( 'Value', {} ).get( 'Failed', {} ).get( lfn ) ) )
return pfn
regFile.PFN = pfn["Value"]['Successful'][lfn]
register.addFile( regFile )
self.request.addOperation( register )
return S_OK()
def __removeStorageDirectory( self, directory, storageElement ):
""" wipe out all contents from :directory: at :storageElement:
:param self: self reference
:param str directory: path
:param str storageElement: SE name
"""
self.log.info( 'Removing the contents of %s at %s' % ( directory, storageElement ) )
se = StorageElement( storageElement )
res = se.getPfnForLfn( [directory] )
if not res['OK']:
self.log.error( "Failed to get PFN for directory", res['Message'] )
return res
if directory in res['Value']['Failed']:
self.log.verbose( 'Failed to obtain directory PFN from LFN', '%s %s' % ( directory, res['Value']['Failed'][directory] ) )
return S_ERROR( 'Failed to obtain directory PFN from LFNs' )
storageDirectory = res['Value']['Successful'][directory]
res = returnSingleResult( se.exists( storageDirectory ) )
if not res['OK']:
self.log.error( "Failed to obtain existance of directory", res['Message'] )
return res
exists = res['Value']
if not exists:
self.log.info( "The directory %s does not exist at %s " % ( directory, storageElement ) )
return S_OK()
res = returnSingleResult( se.removeDirectory( storageDirectory, recursive = True ) )
if not res['OK']:
self.log.error( "Failed to remove storage directory", res['Message'] )
return res
self.log.info( "Successfully removed %d files from %s at %s" % ( res['Value']['FilesRemoved'],
directory,
storageElement ) )
return S_OK()
class FTSRequest( object ):
"""
.. class:: FTSRequest
Helper class for FTS job submission and monitoring.
"""
# # default checksum type
__defaultCksmType = "ADLER32"
# # flag to disablr/enable checksum test, default: disabled
__cksmTest = False
def __init__( self ):
"""c'tor
:param self: self reference
"""
self.log = gLogger.getSubLogger( self.__class__.__name__, True )
# # final states tuple
self.finalStates = ( 'Canceled', 'Failed', 'Hold',
'Finished', 'FinishedDirty' )
# # failed states tuple
self.failedStates = ( 'Canceled', 'Failed',
'Hold', 'FinishedDirty' )
# # successful states tuple
self.successfulStates = ( 'Finished', 'Done' )
# # all file states tuple
self.fileStates = ( 'Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed',
'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting' )
self.statusSummary = {}
# # request status
self.requestStatus = 'Unknown'
# # dict for FTS job files
self.fileDict = {}
# # dict for replicas information
self.catalogReplicas = {}
# # dict for metadata information
self.catalogMetadata = {}
# # dict for files that failed to register
self.failedRegistrations = {}
# # placehoder for FileCatalog reference
self.oCatalog = None
# # submit timestamp
self.submitTime = ''
# # placeholder FTS job GUID
self.ftsGUID = ''
# # placeholder for FTS server URL
self.ftsServer = ''
# # flag marking FTS job completness
self.isTerminal = False
# # completness percentage
self.percentageComplete = 0.0
# # source SE name
self.sourceSE = ''
# # flag marking source SE validity
self.sourceValid = False
# # source space token
self.sourceToken = ''
# # target SE name
self.targetSE = ''
# # flag marking target SE validity
self.targetValid = False
# # target space token
self.targetToken = ''
# # placeholder for target StorageElement
self.oTargetSE = None
# # placeholder for source StorageElement
self.oSourceSE = None
# # checksum type, set it to default
self.__cksmType = self.__defaultCksmType
# # disable checksum test by default
self.__cksmTest = False
# # statuses that prevent submitting to FTS
self.noSubmitStatus = ( 'Failed', 'Done', 'Staging' )
# # were sources resolved?
self.sourceResolved = False
# # Number of file transfers actually submitted
self.submittedFiles = 0
self.transferTime = 0
self.submitCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/SubmitCommand', 'glite-transfer-submit' )
self.monitorCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/MonitorCommand', 'glite-transfer-status' )
self.ftsJob = None
self.ftsFiles = []
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE( self, se ):
""" set SE for source
:param self: self reference
:param str se: source SE name
"""
if se == self.targetSE:
return S_ERROR( "SourceSE is TargetSE" )
self.sourceSE = se
self.oSourceSE = StorageElement( self.sourceSE )
return self.__checkSourceSE()
def __checkSourceSE( self ):
""" check source SE availability
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR( "SourceSE not set" )
res = self.oSourceSE.isValid( 'Read' )
if not res['OK']:
return S_ERROR( "SourceSE not available for reading" )
res = self.__getSESpaceToken( self.oSourceSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message'] )
return S_ERROR( "SourceSE does not support FTS transfers" )
if self.__cksmTest:
res = self.oSourceSE.getChecksumType()
if not res["OK"]:
self.log.error( "Unable to get checksum type for SourceSE %s: %s" % ( self.sourceSE,
res["Message"] ) )
cksmType = res["Value"]
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at SourceSE %s, disabling checksum test" % ( cksmType,
self.sourceSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE( self, se ):
""" set target SE
:param self: self reference
:param str se: target SE name
"""
if se == self.sourceSE:
return S_ERROR( "TargetSE is SourceSE" )
self.targetSE = se
self.oTargetSE = StorageElement( self.targetSE )
return self.__checkTargetSE()
def setTargetToken( self, token ):
""" target space token setter
:param self: self reference
:param str token: target space token
"""
self.targetToken = token
return S_OK()
def __checkTargetSE( self ):
""" check target SE availability
:param self: self reference
"""
if not self.targetSE:
return S_ERROR( "TargetSE not set" )
res = self.oTargetSE.isValid( 'Write' )
if not res['OK']:
return S_ERROR( "TargetSE not available for writing" )
res = self.__getSESpaceToken( self.oTargetSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message'] )
return S_ERROR( "TargetSE does not support FTS transfers" )
# # check checksum types
if self.__cksmTest:
res = self.oTargetSE.getChecksumType()
if not res["OK"]:
self.log.error( "Unable to get checksum type for TargetSE %s: %s" % ( self.targetSE,
res["Message"] ) )
cksmType = res["Value"]
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at TargetSE %s, disabling checksum test" % ( cksmType,
self.targetSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
@staticmethod
def __getSESpaceToken( oSE ):
""" get space token from StorageElement instance
:param self: self reference
:param StorageElement oSE: StorageElement instance
"""
res = oSE.getStorageParameters( "SRM2" )
if not res['OK']:
return res
return S_OK( res['Value'].get( 'SpaceToken' ) )
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID( self, guid ):
""" FTS job GUID setter
:param self: self reference
:param str guid: string containg GUID
"""
if not checkGuid( guid ):
return S_ERROR( "Incorrect GUID format" )
self.ftsGUID = guid
return S_OK()
def setFTSServer( self, server ):
""" FTS server setter
:param self: self reference
:param str server: FTS server URL
"""
self.ftsServer = server
return S_OK()
def isRequestTerminal( self ):
""" check if FTS job has terminated
:param self: self reference
"""
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK( self.isTerminal )
def setCksmTest( self, cksmTest = False ):
""" set cksm test
:param self: self reference
:param bool cksmTest: flag to enable/disable checksum test
"""
self.__cksmTest = bool( cksmTest )
return S_OK( self.__cksmTest )
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN( self, lfn ):
""" add LFN :lfn: to :fileDict:
:param self: self reference
:param str lfn: LFN to add to
"""
self.fileDict.setdefault( lfn, {'Status':'Waiting'} )
return S_OK()
def setSourceSURL( self, lfn, surl ):
""" source SURL setter
:param self: self reference
:param str lfn: LFN
:param str surl: source SURL
"""
target = self.fileDict[lfn].get( 'Target' )
if target == surl:
return S_ERROR( "Source and target the same" )
return self.__setFileParameter( lfn, 'Source', surl )
def getSourceSURL( self, lfn ):
""" get source SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Source' )
def setTargetSURL( self, lfn, surl ):
""" set target SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
:param str surl: target SURL
"""
source = self.fileDict[lfn].get( 'Source' )
if source == surl:
return S_ERROR( "Source and target the same" )
return self.__setFileParameter( lfn, 'Target', surl )
def getFailReason( self, lfn ):
""" get fail reason for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Reason' )
def getRetries( self, lfn ):
""" get number of attepmts made to transfer file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Retries' )
def getTransferTime( self, lfn ):
""" get duration of transfer for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Duration' )
def getFailed( self ):
""" get list of wrongly transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.failedStates ] )
def getStaging( self ):
""" get files set for prestaging """
return S_OK( [lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) == 'Staging'] )
def getDone( self ):
""" get list of succesfully transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.successfulStates ] )
def __setFileParameter( self, lfn, paramName, paramValue ):
""" set :paramName: to :paramValue: for :lfn: file
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
:param mixed paramValue: a new parameter value
"""
self.setLFN( lfn )
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter( self, lfn, paramName ):
""" get value of :paramName: for file :lfn:
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
"""
if lfn not in self.fileDict:
return S_ERROR( "Supplied file not set" )
if paramName not in self.fileDict[lfn]:
return S_ERROR( "%s not set for file" % paramName )
return S_OK( self.fileDict[lfn][paramName] )
####################################################################
#
# Methods for submission
#
def submit( self, monitor = False, printOutput = True ):
""" submit FTS job
:param self: self reference
:param bool monitor: flag to monitor progress of FTS job
:param bool printOutput: flag to print output of execution to stdout
"""
res = self.__prepareForSubmission()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = { 'ftsGUID' : self.ftsGUID, 'ftsServer' : self.ftsServer, 'submittedFiles' : self.submittedFiles }
if monitor or printOutput:
gLogger.always( "Submitted %s@%s" % ( self.ftsGUID, self.ftsServer ) )
if monitor:
self.monitor( untilTerminal = True, printOutput = printOutput, full = False )
return S_OK( resDict )
def __prepareForSubmission( self ):
""" check validity of job before submission
:param self: self reference
"""
if not self.fileDict:
return S_ERROR( "No files set" )
if not self.sourceValid:
return S_ERROR( "SourceSE not valid" )
if not self.targetValid:
return S_ERROR( "TargetSE not valid" )
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR( "FTSServer not valid" )
self.resolveSource()
self.resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR( "No files to submit" )
return S_OK()
def __getCatalogObject( self ):
""" CatalogInterface instance facade
:param self: self reference
"""
try:
if not self.oCatalog:
self.oCatalog = FileCatalog()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache( self, lfns = None, overwrite = False ):
""" update replica cache for list of :lfns:
:param self: self reference
:param mixed lfns: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if ( lfn not in self.catalogReplicas ) or overwrite ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getReplicas( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to update replica cache: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache( self, lfns = None ):
""" update metadata cache for list of LFNs
:param self: self reference
:param list lnfs: list of LFNs
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if lfn not in self.catalogMetadata ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getFileMetadata( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to get source catalog metadata: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def resolveSource( self ):
""" resolve source SE eligible for submission
:param self: self reference
"""
# Avoid resolving sources twice
if self.sourceResolved:
return S_OK()
# Only resolve files that need a transfer
toResolve = [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( "Status", "" ) != "Failed" ]
if not toResolve:
return S_OK()
res = self.__updateMetadataCache( toResolve )
if not res['OK']:
return res
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
# Define the source URLs
for lfn in toResolve:
replicas = self.catalogReplicas.get( lfn, {} )
if self.sourceSE not in replicas:
gLogger.warn( "resolveSource: skipping %s - not replicas at SourceSE %s" % ( lfn, self.sourceSE ) )
self.__setFileParameter( lfn, 'Reason', "No replica at SourceSE" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
# Fix first the PFN
pfn = self.oSourceSE.getPfnForLfn( lfn ).get( 'Value', {} ).get( 'Successful', {} ).get( lfn, replicas[self.sourceSE] )
res = returnSingleResult( self.oSourceSE.getPfnForProtocol( pfn, protocol = 'SRM2', withPort = True ) )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setSourceSURL( lfn, res['Value'] )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict:
if "Source" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Source']] = lfn
if not toResolve:
return S_ERROR( "No eligible Source files" )
# Get metadata of the sources, to check for existance, availability and caching
res = self.oSourceSE.getFileMetadata( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check source file metadata" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
if re.search( 'File does not exist', error ):
gLogger.warn( "resolveSource: skipping %s - source file does not exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file does not exist" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
gLogger.warn( "resolveSource: skipping %s - failed to get source metadata" % lfn )
self.__setFileParameter( lfn, 'Reason', "Failed to get Source metadata" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toStage = []
nbStagedFiles = 0
for pfn, metadata in res['Value']['Successful'].items():
lfn = toResolve[pfn]
lfnStatus = self.fileDict.get( lfn, {} ).get( 'Status' )
if metadata['Unavailable']:
gLogger.warn( "resolveSource: skipping %s - source file unavailable" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Unavailable" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif metadata['Lost']:
gLogger.warn( "resolveSource: skipping %s - source file lost" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Lost" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif not metadata['Cached']:
if lfnStatus != 'Staging':
toStage.append( pfn )
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
gLogger.warn( "resolveSource: skipping %s - source file size mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source size mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \
not compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ):
gLogger.warn( "resolveSource: skipping %s - source file checksum mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source checksum mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif lfnStatus == 'Staging':
# file that was staging is now cached
self.__setFileParameter( lfn, 'Status', 'Waiting' )
nbStagedFiles += 1
# Some files were being staged
if nbStagedFiles:
self.log.info( 'resolveSource: %d files have been staged' % nbStagedFiles )
# Launching staging of files not in cache
if toStage:
gLogger.warn( "resolveSource: %s source files not cached, prestaging..." % len( toStage ) )
stage = self.oSourceSE.prestageFile( toStage )
if not stage["OK"]:
gLogger.error( "resolveSource: error is prestaging - %s" % stage["Message"] )
for pfn in toStage:
lfn = toResolve[pfn]
self.__setFileParameter( lfn, 'Reason', stage["Message"] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
for pfn in toStage:
lfn = toResolve[pfn]
if pfn in stage['Value']['Successful']:
self.__setFileParameter( lfn, 'Status', 'Staging' )
elif pfn in stage['Value']['Failed']:
self.__setFileParameter( lfn, 'Reason', stage['Value']['Failed'][pfn] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
self.sourceResolved = True
return S_OK()
def resolveTarget( self ):
""" find target SE eligible for submission
:param self: self reference
"""
toResolve = [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status' ) not in self.noSubmitStatus ]
if not toResolve:
return S_OK()
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
for lfn in toResolve:
res = self.oTargetSE.getPfnForLfn( lfn )
if not res['OK'] or lfn not in res['Value']['Successful']:
gLogger.warn( "resolveTarget: skipping %s - failed to create target pfn" % lfn )
self.__setFileParameter( lfn, 'Reason', "Failed to create Target" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
pfn = res['Value']['Successful'][lfn]
res = self.oTargetSE.getPfnForProtocol( pfn, protocol = 'SRM2', withPort = True )
if not res['OK'] or pfn not in res['Value']['Successful']:
reason = res.get( 'Message', res.get( 'Value', {} ).get( 'Failed', {} ).get( pfn ) )
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, reason ) )
self.__setFileParameter( lfn, 'Reason', reason )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
pfn = res['Value']['Successful'][pfn]
res = self.setTargetSURL( lfn, pfn )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict:
if "Target" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Target']] = lfn
if not toResolve:
return S_ERROR( "No eligible Target files" )
res = self.oTargetSE.exists( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check target existence" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toRemove = []
for pfn, exists in res['Value']['Successful'].items():
if exists:
lfn = toResolve[pfn]
res = self.getSourceSURL( lfn )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - target exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Target exists" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif res['Value'] == pfn:
gLogger.warn( "resolveTarget: skipping %s - source and target pfns are the same" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source and Target the same" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRemove.append( pfn )
if toRemove:
self.oTargetSE.removeFile( toRemove )
return S_OK()
def __filesToSubmit( self ):
"""
check if there is at least one file to submit
:return: S_OK if at least one file is present, S_ERROR otherwise
"""
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if lfnStatus not in self.noSubmitStatus and source and target:
return S_OK()
return S_ERROR()
def __createFTSFiles( self ):
""" create LFNs file for glite-transfer-submit command
This file consists one line for each fiel to be transferred:
sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM]
:param self: self reference
"""
self.__updateMetadataCache()
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
if lfnStatus not in self.noSubmitStatus:
cksmStr = ""
# # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself
if self.__cksmTest and self.__cksmType:
checkSum = self.catalogMetadata.get( lfn, {} ).get( 'Checksum' )
if checkSum:
cksmStr = " %s:%s" % ( self.__cksmType, intAdlerToHex( hexAdlerToInt( checkSum ) ) )
ftsFile = FTSFile()
ftsFile.LFN = lfn
ftsFile.SourceSURL = self.fileDict[lfn].get( 'Source' )
ftsFile.TargetSURL = self.fileDict[lfn].get( 'Target' )
ftsFile.SourceSE = self.sourceSE
ftsFile.TargetSE = self.targetSE
ftsFile.Status = self.fileDict[lfn].get( 'Status' )
ftsFile.Checksum = cksmStr
ftsFile.Size = self.catalogMetadata.get( lfn, {} ).get( 'Size' )
self.ftsFiles.append( ftsFile )
self.submittedFiles += 1
return S_OK()
def __createFTSJob( self, guid = None ):
self.__createFTSFiles()
ftsJob = FTSJob()
ftsJob.RequestID = 0
ftsJob.OperationID = 0
ftsJob.SourceSE = self.sourceSE
ftsJob.TargetSE = self.targetSE
ftsJob.SourceToken = self.sourceToken
ftsJob.TargetToken = self.targetToken
ftsJob.FTSServer = self.ftsServer
if guid:
ftsJob.FTSGUID = guid
for ftsFile in self.ftsFiles:
ftsFile.Attempt += 1
ftsFile.Error = ""
ftsJob.addFile( ftsFile )
self.ftsJob = ftsJob
def __submitFTSTransfer( self ):
""" create and execute glite-transfer-submit CLI command
:param self: self reference
"""
log = gLogger.getSubLogger( 'Submit' )
self.__createFTSJob()
submit = self.ftsJob.submitFTS2( command = self.submitCommand )
if not submit["OK"]:
log.error( "unable to submit FTSJob: %s" % submit["Message"] )
return submit
log.info( "FTSJob '%s'@'%s' has been submitted" % ( self.ftsJob.FTSGUID, self.ftsJob.FTSServer ) )
# # update statuses for job files
for ftsFile in self.ftsJob:
ftsFile.FTSGUID = self.ftsJob.FTSGUID
ftsFile.Status = "Submitted"
ftsFile.Attempt += 1
log.info( "FTSJob '%s'@'%s' has been submitted" % ( self.ftsJob.FTSGUID, self.ftsJob.FTSServer ) )
self.ftsGUID = self.ftsJob.FTSGUID
return S_OK()
def __resolveFTSServer( self ):
"""
resolve FTS server to use, it should be the closest one from target SE
:param self: self reference
"""
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTSServersForSites
if not self.targetSE:
return S_ERROR( "Target SE not set" )
res = getSitesForSE( self.targetSE )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine target site" )
targetSites = res['Value']
targetSite = ''
for targetSite in targetSites:
targetFTS = getFTSServersForSites( [targetSite] )
if targetFTS['OK']:
ftsTarget = targetFTS['Value'][targetSite]
if ftsTarget:
self.ftsServer = ftsTarget
return S_OK( self.ftsServer )
else:
return targetFTS
return S_ERROR( 'No FTS server found for %s' % targetSite )
####################################################################
#
# Methods for monitoring
#
def summary( self, untilTerminal = False, printOutput = False ):
""" summary of FTS job
:param self: self reference
:param bool untilTerminal: flag to monitor FTS job to its final state
:param bool printOutput: flag to print out monitoring information to the stdout
"""
res = self.__isSummaryValid()
if not res['OK']:
return res
while not self.isTerminal:
res = self.__parseOutput( full = True )
if not res['OK']:
return res
if untilTerminal:
self.__print()
self.isRequestTerminal()
if res['Value'] or ( not untilTerminal ):
break
time.sleep( 1 )
if untilTerminal:
print ""
if printOutput and ( not untilTerminal ):
return self.dumpSummary( printOutput = printOutput )
return S_OK()
def monitor( self, untilTerminal = False, printOutput = False, full = True ):
""" monitor FTS job
:param self: self reference
:param bool untilTerminal: flag to monitor FTS job to its final state
:param bool printOutput: flag to print out monitoring information to the stdout
"""
if not self.ftsJob:
self.resolveSource()
self.__createFTSJob( self.ftsGUID )
res = self.__isSummaryValid()
if not res['OK']:
return res
if untilTerminal:
res = self.summary( untilTerminal = untilTerminal, printOutput = printOutput )
if not res['OK']:
return res
res = self.__parseOutput( full = full )
if not res['OK']:
return res
if untilTerminal:
self.finalize()
if printOutput:
self.dump()
return res
def dumpSummary( self, printOutput = False ):
""" get FTS job summary as str
:param self: self reference
:param bool printOutput: print summary to stdout
"""
outStr = ''
for status in sorted( self.statusSummary ):
if self.statusSummary[status]:
outStr = '%s\t%-10s : %-10s\n' % ( outStr, status, str( self.statusSummary[status] ) )
outStr = outStr.rstrip( '\n' )
if printOutput:
print outStr
return S_OK( outStr )
def __print( self ):
""" print progress bar of FTS job completeness to stdout
:param self: self reference
"""
width = 100
bits = int( ( width * self.percentageComplete ) / 100 )
outStr = "|%s>%s| %.1f%s %s %s" % ( "="*bits, " "*( width - bits ),
self.percentageComplete, "%",
self.requestStatus, " "*10 )
sys.stdout.write( "%s\r" % ( outStr ) )
sys.stdout.flush()
def dump( self ):
""" print FTS job parameters and files to stdout
:param self: self reference
"""
print "%-10s : %-10s" % ( "Status", self.requestStatus )
print "%-10s : %-10s" % ( "Source", self.sourceSE )
print "%-10s : %-10s" % ( "Target", self.targetSE )
print "%-10s : %-128s" % ( "Server", self.ftsServer )
print "%-10s : %-128s" % ( "GUID", self.ftsGUID )
for lfn in sorted( self.fileDict ):
print "\n %-15s : %-128s" % ( 'LFN', lfn )
for key in ['Source', 'Target', 'Status', 'Reason', 'Duration']:
print " %-15s : %-128s" % ( key, str( self.fileDict[lfn].get( key ) ) )
return S_OK()
def __isSummaryValid( self ):
""" check validity of FTS job summary report
:param self: self reference
"""
if not self.ftsServer:
return S_ERROR( "FTSServer not set" )
if not self.ftsGUID:
return S_ERROR( "FTSGUID not set" )
return S_OK()
def __parseOutput( self, full = False ):
""" execute glite-transfer-status command and parse its output
:param self: self reference
:param bool full: glite-transfer-status verbosity level, when set, collect information of files as well
"""
monitor = self.ftsJob.monitorFTS2( command = self.monitorCommand, full = full )
if not monitor['OK']:
return monitor
self.percentageComplete = self.ftsJob.Completeness
self.requestStatus = self.ftsJob.Status
self.submitTime = self.ftsJob.SubmitTime
statusSummary = monitor['Value']
if statusSummary:
for state in statusSummary:
self.statusSummary[state] = statusSummary[state]
self.transferTime = 0
for ftsFile in self.ftsJob:
lfn = ftsFile.LFN
self.__setFileParameter( lfn, 'Status', ftsFile.Status )
self.__setFileParameter( lfn, 'Reason', ftsFile.Error )
self.__setFileParameter( lfn, 'Duration', ftsFile._duration )
targetURL = self.__getFileParameter( lfn, 'Target' )
if not targetURL['OK']:
self.__setFileParameter( lfn, 'Target', ftsFile.TargetSURL )
self.transferTime += int( ftsFile._duration )
return S_OK()
####################################################################
#
# Methods for finalization
#
def finalize( self ):
""" finalize FTS job
:param self: self reference
"""
self.__updateMetadataCache()
transEndTime = dateTime()
regStartTime = time.time()
res = self.getTransferStatistics()
transDict = res['Value']
res = self.__registerSuccessful( transDict['transLFNs'] )
regSuc, regTotal = res['Value']
regTime = time.time() - regStartTime
if self.sourceSE and self.targetSE:
self.__sendAccounting( regSuc, regTotal, regTime, transEndTime, transDict )
return S_OK()
def getTransferStatistics( self ):
""" collect information of Transfers that can be used by Accounting
:param self: self reference
"""
transDict = { 'transTotal': len( self.fileDict ),
'transLFNs': [],
'transOK': 0,
'transSize': 0 }
for lfn in self.fileDict:
if self.fileDict[lfn].get( 'Status' ) in self.successfulStates:
if self.fileDict[lfn].get( 'Duration', 0 ):
transDict['transLFNs'].append( lfn )
transDict['transOK'] += 1
if lfn in self.catalogMetadata:
transDict['transSize'] += self.catalogMetadata[lfn].get( 'Size', 0 )
return S_OK( transDict )
def getFailedRegistrations( self ):
""" get failed registrations dict
:param self: self reference
"""
return S_OK( self.failedRegistrations )
def __registerSuccessful( self, transLFNs ):
""" register successfully transferred files to the catalogs,
fill failedRegistrations dict for files that failed to register
:param self: self reference
:param list transLFNs: LFNs in FTS job
"""
self.failedRegistrations = {}
toRegister = {}
for lfn in transLFNs:
res = returnSingleResult( self.oTargetSE.getPfnForProtocol( self.fileDict[lfn].get( 'Target' ), protocol = 'SRM2', withPort = False ) )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRegister[lfn] = { 'PFN' : res['Value'], 'SE' : self.targetSE }
if not toRegister:
return S_OK( ( 0, 0 ) )
res = self.__getCatalogObject()
if not res['OK']:
for lfn in toRegister:
self.failedRegistrations = toRegister
self.log.error( 'Failed to get Catalog Object', res['Message'] )
return S_OK( ( 0, len( toRegister ) ) )
res = self.oCatalog.addReplica( toRegister )
if not res['OK']:
self.failedRegistrations = toRegister
self.log.error( 'Failed to get Catalog Object', res['Message'] )
return S_OK( ( 0, len( toRegister ) ) )
for lfn, error in res['Value']['Failed'].items():
self.failedRegistrations[lfn] = toRegister[lfn]
self.log.error( 'Registration of Replica failed', '%s : %s' % ( lfn, str( error ) ) )
return S_OK( ( len( res['Value']['Successful'] ), len( toRegister ) ) )
def __sendAccounting( self, regSuc, regTotal, regTime, transEndTime, transDict ):
""" send accounting record
:param self: self reference
:param regSuc: number of files successfully registered
:param regTotal: number of files attepted to register
:param regTime: time stamp at the end of registration
:param transEndTime: time stamp at the end of FTS job
:param dict transDict: dict holding couters for files being transerred, their sizes and successfull transfers
"""
oAccounting = DataOperation()
oAccounting.setEndTime( transEndTime )
oAccounting.setStartTime( self.submitTime )
accountingDict = {}
accountingDict['OperationType'] = 'replicateAndRegister'
result = getProxyInfo()
if not result['OK']:
userName = '******'
else:
userName = result['Value'].get( 'username', 'unknown' )
accountingDict['User'] = userName
accountingDict['Protocol'] = 'FTS' if 'fts3' not in self.ftsServer else 'FTS3'
accountingDict['RegistrationTime'] = regTime
accountingDict['RegistrationOK'] = regSuc
accountingDict['RegistrationTotal'] = regTotal
accountingDict['TransferOK'] = transDict['transOK']
accountingDict['TransferTotal'] = transDict['transTotal']
accountingDict['TransferSize'] = transDict['transSize']
accountingDict['FinalStatus'] = self.requestStatus
accountingDict['Source'] = self.sourceSE
accountingDict['Destination'] = self.targetSE
accountingDict['TransferTime'] = self.transferTime
oAccounting.setValuesFromDict( accountingDict )
self.log.verbose( "Attempting to commit accounting message..." )
oAccounting.commit()
self.log.verbose( "...committed." )
return S_OK()
class FTSRequest( object ):
"""
.. class:: FTSRequest
Helper class for FTS job submission and monitoring.
"""
# # default checksum type
__defaultCksmType = "ADLER32"
# # flag to disablr/enable checksum test, default: disabled
__cksmTest = False
def __init__( self ):
"""c'tor
:param self: self reference
"""
self.log = gLogger.getSubLogger( self.__class__.__name__, True )
# # final states tuple
self.finalStates = ( 'Canceled', 'Failed', 'Hold',
'Finished', 'FinishedDirty' )
# # failed states tuple
self.failedStates = ( 'Canceled', 'Failed',
'Hold', 'FinishedDirty' )
# # successful states tuple
self.successfulStates = ( 'Finished', 'Done' )
# # all file states tuple
self.fileStates = ( 'Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed',
'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting' )
self.newlyCompletedFiles = []
self.newlyFailedFiles = []
self.statusSummary = {}
# # request status
self.requestStatus = 'Unknown'
# # dict for FTS job files
self.fileDict = {}
# # dict for replicas information
self.catalogReplicas = {}
# # dict for metadata information
self.catalogMetadata = {}
# # dict for files that failed to register
self.failedRegistrations = {}
# # placehoder for CatalogInterface reference
self.oCatalog = None
# # submit timestamp
self.submitTime = ''
# # placeholder FTS job GUID
self.ftsGUID = ''
# # placeholder for FTS server URL
self.ftsServer = ''
# # not used
self.priority = 3
# # flag marking FTS job completness
self.isTerminal = False
# # completness percentage
self.percentageComplete = 0.0
# # source SE name
self.sourceSE = ''
# # flag marking source SE validity
self.sourceValid = False
# # source space token
self.sourceToken = ''
# # target SE name
self.targetSE = ''
# # flag marking target SE validity
self.targetValid = False
# # target space token
self.targetToken = ''
# # whatever
self.dumpStr = ''
# # placeholder for surl file
self.surlFile = None
# # placeholder for target StorageElement
self.oTargetSE = None
# # placeholder for source StorageElement
self.oSourceSE = None
# # checksum type, set it to default
self.__cksmType = self.__defaultCksmType
# # disable checksum test by default
self.__cksmTest = False
# # replica manager handler
self.replicaManager = ReplicaManager()
# # statuses that prevent submitting to FTS
self.noSubmitStatus = ( 'Failed', 'Done', 'Staging' )
# # were sources resolved?
self.sourceResolved = False
# # Number of file transfers actually submitted
self.submittedFiles = 0
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE( self, se ):
""" set SE for source
:param self: self reference
:param str se: source SE name
"""
if se == self.targetSE:
return S_ERROR( "SourceSE is TargetSE" )
self.sourceSE = se
self.oSourceSE = StorageElement( self.sourceSE )
return self.__checkSourceSE()
def getSourceSE( self ):
""" source SE getter
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR( "Source SE not defined" )
return S_OK( self.sourceSE )
def setSourceToken( self, token ):
""" set source space token
:param self: self reference
:param str token: source space token
"""
self.sourceToken = token
return S_OK()
def getSourceToken( self ):
""" source space token getter
:param self: self reference
"""
if not self.sourceToken:
return S_ERROR( "Source token not defined" )
return S_OK( self.sourceToken )
def __checkSourceSE( self ):
""" check source SE availability
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR( "SourceSE not set" )
res = self.oSourceSE.isValid( 'Read' )
if not res['OK']:
return S_ERROR( "SourceSE not available for reading" )
res = self.__getSESpaceToken( self.oSourceSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message'] )
return S_ERROR( "SourceSE does not support FTS transfers" )
if self.__cksmTest:
res = self.oSourceSE.getChecksumType()
if not res["OK"]:
self.log.error( "Unable to get checksum type for SourceSE %s: %s" % ( self.sourceSE,
res["Message"] ) )
cksmType = res["Value"]
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at SourceSE %s, disabling checksum test" % ( cksmType,
self.sourceSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE( self, se ):
""" set target SE
:param self: self reference
:param str se: target SE name
"""
if se == self.sourceSE:
return S_ERROR( "TargetSE is SourceSE" )
self.targetSE = se
self.oTargetSE = StorageElement( self.targetSE )
return self.__checkTargetSE()
def getTargetSE( self ):
""" target SE getter
:param self: self reference
"""
if not self.targetSE:
return S_ERROR( "Target SE not defined" )
return S_OK( self.targetSE )
def setTargetToken( self, token ):
""" target space token setter
:param self: self reference
:param str token: target space token
"""
self.targetToken = token
return S_OK()
def getTargetToken( self ):
""" target space token getter
:param self: self reference
"""
if not self.targetToken:
return S_ERROR( "Target token not defined" )
return S_OK( self.targetToken )
def __checkTargetSE( self ):
""" check target SE availability
:param self: self reference
"""
if not self.targetSE:
return S_ERROR( "TargetSE not set" )
res = self.oTargetSE.isValid( 'Write' )
if not res['OK']:
return S_ERROR( "TargetSE not available for writing" )
res = self.__getSESpaceToken( self.oTargetSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message'] )
return S_ERROR( "TargetSE does not support FTS transfers" )
# # check checksum types
if self.__cksmTest:
res = self.oTargetSE.getChecksumType()
if not res["OK"]:
self.log.error( "Unable to get checksum type for TargetSE %s: %s" % ( self.targetSE,
res["Message"] ) )
cksmType = res["Value"]
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at TargetSE %s, disabling checksum test" % ( cksmType,
self.targetSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
@staticmethod
def __getSESpaceToken( oSE ):
""" get space token from StorageElement instance
:param self: self reference
:param StorageElement oSE: StorageElement instance
"""
res = oSE.getStorageParameters( "SRM2" )
if not res['OK']:
return res
return S_OK( res['Value'].get( 'SpaceToken' ) )
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID( self, guid ):
""" FTS job GUID setter
:param self: self reference
:param str guid: string containg GUID
"""
if not checkGuid( guid ):
return S_ERROR( "Incorrect GUID format" )
self.ftsGUID = guid
return S_OK()
def getFTSGUID( self ):
""" FTS job GUID getter
:param self: self refenece
"""
if not self.ftsGUID:
return S_ERROR( "FTSGUID not set" )
return S_OK( self.ftsGUID )
def setFTSServer( self, server ):
""" FTS server setter
:param self: self reference
:param str server: FTS server URL
"""
self.ftsServer = server
return S_OK()
def getFTSServer( self ):
""" FTS server getter
:param self: self reference
"""
if not self.ftsServer:
return S_ERROR( "FTSServer not set" )
return S_OK( self.ftsServer )
def setPriority( self, priority ):
""" set priority for FTS job
:param self: self reference
:param int priority: a new priority
"""
if not type( priority ) in ( IntType, LongType ):
return S_ERROR( "Priority must be integer" )
if priority < 0:
priority = 0
elif priority > 5:
priority = 5
self.priority = priority
return S_OK( self.priority )
def getPriority( self ):
""" FTS job priority getter
:param self: self reference
"""
return S_OK( self.priority )
def getPercentageComplete( self ):
""" get completness percentage
:param self: self reference
"""
completedFiles = 0
totalFiles = 0
for state in self.statusSummary:
if state in self.successfulStates:
completedFiles += self.statusSummary[state]
totalFiles += self.statusSummary[state]
self.percentageComplete = ( float( completedFiles ) * 100.0 ) / float( totalFiles )
return S_OK( self.percentageComplete )
def isRequestTerminal( self ):
""" check if FTS job has terminated
:param self: self reference
"""
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK( self.isTerminal )
def getStatus( self ):
""" get FTS job status
:param self: self reference
"""
return S_OK( self.requestStatus )
def setCksmType( self, cksm = None ):
""" set checksum type to use
:param self: self reference
:param mixed cksm: checksum type, should be one of 'Adler32', 'md5', 'sha1', None
"""
if str( cksm ).upper() not in ( "ADLER32", "MD5", "SHA1", "NONE" ):
return S_ERROR( "Not supported checksum type: %s" % str( cksm ) )
if not cksm:
self.__cksmType = None
return S_OK( False )
self.__cksmType = str( cksm ).upper()
return S_OK( True )
def getCksmType( self ):
""" get checksum type
:param self: self reference
"""
return S_OK( self.__cksmType )
def setCksmTest( self, cksmTest = False ):
""" set cksm test
:param self: self reference
:param bool cksmTest: flag to enable/disable checksum test
"""
self.__cksmTest = bool( cksmTest )
return S_OK( self.__cksmTest )
def getCksmTest( self ):
""" get cksm test flag
:param self: self reference
"""
return S_OK( self.__cksmTest )
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN( self, lfn ):
""" add LFN :lfn: to :fileDict:
:param self: self reference
:param str lfn: LFN to add to
"""
self.fileDict.setdefault( lfn, {'Status':'Waiting'} )
return S_OK()
def setStatus( self, lfn, status ):
""" set status of a file """
return( self.__setFileParameter( lfn, 'Status', status ) )
def setSourceSURL( self, lfn, surl ):
""" source SURL setter
:param self: self reference
:param str lfn: LFN
:param str surl: source SURL
"""
target = self.fileDict[lfn].get( 'Target' )
if target == surl:
return S_ERROR( "Source and target the same" )
return( self.__setFileParameter( lfn, 'Source', surl ) )
def getSourceSURL( self, lfn ):
""" get source SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Source' )
def setTargetSURL( self, lfn, surl ):
""" set target SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
:param str surl: target SURL
"""
source = self.fileDict[lfn].get( 'Source' )
if source == surl:
return S_ERROR( "Source and target the same" )
return( self.__setFileParameter( lfn, 'Target', surl ) )
def getTargetSURL( self, lfn ):
""" target SURL getter
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Target' )
def getFailReason( self, lfn ):
""" get fail reason for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Reason' )
def getRetries( self, lfn ):
""" get number of attepmts made to transfer file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Retries' )
def getTransferTime( self, lfn ):
""" get duration of transfer for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Duration' )
def getFailed( self ):
""" get list of wrongly transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.failedStates ] )
def getStaging( self ):
""" get files set for prestaging """
return S_OK( [lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) == 'Staging'] )
def getDone( self ):
""" get list of succesfully transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.successfulStates ] )
def __setFileParameter( self, lfn, paramName, paramValue ):
""" set :paramName: to :paramValue: for :lfn: file
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
:param mixed paramValue: a new parameter value
"""
self.setLFN( lfn )
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter( self, lfn, paramName ):
""" get value of :paramName: for file :lfn:
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
"""
if lfn not in self.fileDict:
return S_ERROR( "Supplied file not set" )
if paramName not in self.fileDict[lfn]:
return S_ERROR( "%s not set for file" % paramName )
return S_OK( self.fileDict[lfn][paramName] )
####################################################################
#
# Methods for submission
#
def submit( self, monitor = False, printOutput = True ):
""" submit FTS job
:param self: self reference
:param bool monitor: flag to monitor progress of FTS job
:param bool printOutput: flag to print output of execution to stdout
"""
res = self.__isSubmissionValid()
if not res['OK']:
return res
res = self.__createSURLPairFile()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = { 'ftsGUID' : self.ftsGUID, 'ftsServer' : self.ftsServer, 'submittedFiles' : self.submittedFiles }
# print "Submitted %s @ %s" % ( self.ftsGUID, self.ftsServer )
if monitor:
self.monitor( untilTerminal = True, printOutput = printOutput )
return S_OK( resDict )
def __isSubmissionValid( self ):
""" check validity of job before submission
:param self: self reference
"""
if not self.fileDict:
return S_ERROR( "No files set" )
if not self.sourceValid:
return S_ERROR( "SourceSE not valid" )
if not self.targetValid:
return S_ERROR( "TargetSE not valid" )
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR( "FTSServer not valid" )
self.resolveSource()
self.resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR( "No files to submit" )
return S_OK()
def __getCatalogObject( self ):
""" CatalogInterface instance facade
:param self: self reference
"""
try:
if not self.oCatalog:
self.oCatalog = CatalogInterface()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache( self, lfns = None, overwrite = False ):
""" update replica cache for list of :lfns:
:param self: self reference
:param mixed lfns: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if ( lfn not in self.catalogReplicas ) or overwrite ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogReplicas( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to update replica cache: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache( self, lfns = None, overwrite = False ):
""" update metadata cache for list of LFNs
:param self: self reference
:param list lnfs: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if ( lfn not in self.catalogMetadata ) or overwrite ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogFileMetadata( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to get source catalog metadata: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def resolveSource( self ):
""" resolve source SE eligible for submission
:param self: self reference
"""
# Avoid resolving sources twice
if self.sourceResolved:
return S_OK()
# Only resolve files that need a transfer
toResolve = [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( "Status", "" ) != "Failed" ]
if not toResolve:
return S_OK()
res = self.__updateMetadataCache( toResolve )
if not res['OK']:
return res
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
# Define the source URLs
for lfn in toResolve:
replicas = self.catalogReplicas.get( lfn, {} )
if self.sourceSE not in replicas:
gLogger.warn( "resolveSource: skipping %s - not replicas at SourceSE %s" % ( lfn, self.sourceSE ) )
self.__setFileParameter( lfn, 'Reason', "No replica at SourceSE" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.oSourceSE.getPfnForProtocol( replicas[self.sourceSE], 'SRM2', withPort = True )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setSourceSURL( lfn, res['Value'] )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict:
if "Source" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Source']] = lfn
if not toResolve:
return S_ERROR( "No eligible Source files" )
# Get metadata of the sources, to check for existance, availability and caching
res = self.oSourceSE.getFileMetadata( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check source file metadata" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
if re.search( 'File does not exist', error ):
gLogger.warn( "resolveSource: skipping %s - source file does not exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file does not exist" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
gLogger.warn( "resolveSource: skipping %s - failed to get source metadata" % lfn )
self.__setFileParameter( lfn, 'Reason', "Failed to get Source metadata" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toStage = []
nbStagedFiles = 0
for pfn, metadata in res['Value']['Successful'].items():
lfn = toResolve[pfn]
lfnStatus = self.fileDict.get( lfn, {} ).get( 'Status' )
if metadata['Unavailable']:
gLogger.warn( "resolveSource: skipping %s - source file unavailable" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Unavailable" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif metadata['Lost']:
gLogger.warn( "resolveSource: skipping %s - source file lost" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Lost" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif not metadata['Cached']:
if lfnStatus != 'Staging':
toStage.append( pfn )
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
gLogger.warn( "resolveSource: skipping %s - source file size mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source size mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \
not ( compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ) ):
gLogger.warn( "resolveSource: skipping %s - source file checksum mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source checksum mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif lfnStatus == 'Staging':
# file that was staging is now cached
self.__setFileParameter( lfn, 'Status', 'Waiting' )
nbStagedFiles += 1
# Some files were being staged
if nbStagedFiles:
self.log.info( 'resolveSource: %d files have been staged' % nbStagedFiles )
# Launching staging of files not in cache
if toStage:
gLogger.warn( "resolveSource: %s source files not cached, prestaging..." % len( toStage ) )
stage = self.replicaManager.prestageStorageFile( toStage, self.sourceSE )
if not stage["OK"]:
gLogger.error( "resolveSource: error is prestaging - %s" % stage["Message"] )
for pfn in toStage:
lfn = toResolve[pfn]
self.__setFileParameter( lfn, 'Reason', stage["Message"] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
for pfn in toStage:
lfn = toResolve[pfn]
if pfn in stage['Value']['Successful']:
self.__setFileParameter( lfn, 'Status', 'Staging' )
elif pfn in stage['Value']['Failed']:
self.__setFileParameter( lfn, 'Reason', stage['Value']['Failed'][pfn] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
self.sourceResolved = True
return S_OK()
def resolveTarget( self ):
""" find target SE eligible for submission
:param self: self reference
"""
toResolve = [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status' ) not in self.noSubmitStatus ]
if not toResolve:
return S_OK()
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
for lfn in toResolve:
res = self.oTargetSE.getPfnForLfn( lfn )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - failed to create target pfn" % lfn )
self.__setFileParameter( lfn, 'Reason', "Failed to create Target" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.oTargetSE.getPfnForProtocol( res['Value'], 'SRM2', withPort = True )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setTargetSURL( lfn, res['Value'] )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict:
if "Target" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Target']] = lfn
if not toResolve:
return S_ERROR( "No eligible Target files" )
res = self.oTargetSE.exists( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check target existence" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toRemove = []
for pfn, exists in res['Value']['Successful'].items():
if exists:
lfn = toResolve[pfn]
res = self.getSourceSURL( lfn )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - target exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Target exists" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif res['Value'] == pfn:
gLogger.warn( "resolveTarget: skipping %s - source and target pfns are the same" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source and Target the same" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRemove.append( pfn )
if toRemove:
self.oTargetSE.removeFile( toRemove )
return S_OK()
def __filesToSubmit( self ):
"""
check if there is at least one file to submit
:return: S_OK if at least one file is present, S_ERROR otherwise
"""
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if lfnStatus not in self.noSubmitStatus and source and target:
return S_OK()
return S_ERROR()
def __createSURLPairFile( self ):
""" create LFNs file for glite-transfer-submit command
This file consists one line for each fiel to be transferred:
sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM]
:param self: self reference
"""
fd, fileName = tempfile.mkstemp()
surlFile = os.fdopen( fd, 'w' )
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if lfnStatus not in self.noSubmitStatus and source and target:
cksmStr = ""
# # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself
if self.__cksmTest and self.__cksmType:
checkSum = self.catalogMetadata.get( lfn, {} ).get( 'Checksum' )
if checkSum:
cksmStr = " %s:%s" % ( self.__cksmType, intAdlerToHex( hexAdlerToInt( checkSum ) ) )
surlFile.write( "%s %s%s\n" % ( source, target, cksmStr ) )
self.submittedFiles += 1
surlFile.close()
self.surlFile = fileName
return S_OK()
def __submitFTSTransfer( self ):
""" create and execute glite-transfer-submit CLI command
:param self: self reference
"""
comm = [ 'glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile, '-o' ]
if self.targetToken:
comm += [ '-t', self.targetToken ]
if self.sourceToken:
comm += [ '-S', self.sourceToken ]
if self.__cksmTest:
comm.append( "--compare-checksums" )
gLogger.verbose( 'Executing %s' % ' '.join( comm ) )
res = executeGridCommand( '', comm )
os.remove( self.surlFile )
if not res['OK']:
return res
returnCode, output, errStr = res['Value']
if not returnCode == 0:
return S_ERROR( errStr )
guid = output.replace( '\n', '' )
if not checkGuid( guid ):
return S_ERROR( 'Wrong GUID format returned' )
self.ftsGUID = guid
# if self.priority != 3:
# comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)]
# executeGridCommand('',comm)
return res
def __getFTSServer( self, site ):
try:
configPath = '/Resources/FTSEndpoints/%s' % site
endpointURL = gConfig.getValue( configPath )
if not endpointURL:
errStr = "FTSRequest.__getFTSServer: Failed to find FTS endpoint, check CS entry for '%s'." % site
return S_ERROR( errStr )
return S_OK( endpointURL )
except Exception, x:
return S_ERROR( 'FTSRequest.__getFTSServer: Failed to obtain endpoint details from CS' )
class FTSRequest:
def __init__( self ):
self.gridEnv = '/afs/cern.ch/project/gd/LCG-share/3.2.8-0/etc/profile.d/grid-env'
self.finalStates = ['Canceled', 'Failed', 'Hold', 'Finished', 'FinishedDirty']
self.failedStates = ['Canceled', 'Failed', 'Hold', 'FinishedDirty']
self.successfulStates = ['Finished', 'Done']
self.fileStates = ['Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed', 'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting']
self.newlyCompletedFiles = []
self.newlyFailedFiles = []
self.statusSummary = {}
self.requestStatus = 'Unknown'
self.fileDict = {}
self.catalogReplicas = {}
self.catalogMetadata = {}
self.oCatalog = None
self.submitTime = ''
self.ftsGUID = ''
self.ftsServer = ''
self.priority = 3
self.isTerminal = False
self.percentageComplete = 0.0
self.sourceSE = ''
self.sourceValid = False
self.sourceToken = ''
self.targetSE = ''
self.targetValid = False
self.targetToken = ''
self.dumpStr = ''
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE( self, se ):
if se == self.targetSE:
return S_ERROR( "SourceSE is TargetSE" )
self.sourceSE = se
self.oSourceSE = StorageElement( self.sourceSE )
return self.__checkSourceSE()
def getSourceSE( self ):
if not self.sourceSE:
return S_ERROR( "Source SE not defined" )
return S_OK( self.sourceSE )
def setSourceToken( self, token ):
self.sourceToken = token
return S_OK()
def getSourceToken( self ):
if not self.sourceToken:
return S_ERROR( "Source token not defined" )
return S_OK( self.sourceToken )
def __checkSourceSE( self ):
if not self.sourceSE:
return S_ERROR( "SourceSE not set" )
res = self.oSourceSE.isValid( 'Read' )
if not res['OK']:
return S_ERROR( "SourceSE not available for reading" )
res = self.__getSESpaceToken( self.oSourceSE )
if not res['OK']:
gLogger.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message'] )
return S_ERROR( "SourceSE does not support FTS transfers" )
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE( self, se ):
if se == self.sourceSE:
return S_ERROR( "TargetSE is SourceSE" )
self.targetSE = se
self.oTargetSE = StorageElement( self.targetSE )
return self.__checkTargetSE()
def getTargetSE( self ):
if not self.targetSE:
return S_ERROR( "Target SE not defined" )
return S_OK( self.targetSE )
def setTargetToken( self, token ):
self.targetToken = token
return S_OK()
def getTargetToken( self ):
if not self.targetToken:
return S_ERROR( "Target token not defined" )
return S_OK( self.targetToken )
def __checkTargetSE( self ):
if not self.targetSE:
return S_ERROR( "TargetSE not set" )
res = self.oTargetSE.isValid( 'Write' )
if not res['OK']:
return S_ERROR( "TargetSE not available for writing" )
res = self.__getSESpaceToken( self.oTargetSE )
if not res['OK']:
gLogger.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message'] )
return S_ERROR( "SourceSE does not support FTS transfers" )
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
def __getSESpaceToken( self, oSE ):
res = oSE.getStorageParameters( "SRM2" )
if not res['OK']:
return res
return S_OK( res['Value'].get( 'SpaceToken' ) )
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID( self, guid ):
if not checkGuid( guid ):
return S_ERROR( "Incorrect GUID format" )
self.ftsGUID = guid
return S_OK()
def getFTSGUID( self ):
if not self.ftsGUID:
return S_ERROR( "FTSGUID not set" )
return S_OK( self.ftsGUID )
def setFTSServer( self, server ):
self.ftsServer = server
return S_OK()
def getFTSServer( self ):
if not self.ftsServer:
return S_ERROR( "FTSServer not set" )
return S_OK( self.ftsServer )
def setPriority( self, priority ):
if not type( priority ) in [types.IntType, types.LongType]:
return S_ERROR( "Priority must be integer" )
if priority < 0:
priority = 0
elif priority > 5:
priority = 5
self.priority = priority
return S_OK( self.priority )
def getPriority( self ):
return S_OK( self.priority )
def getPercentageComplete( self ):
completedFiles = 0
totalFiles = 0
for state in ( self.statusSummary.keys() ):
if state in self.successfulStates:
completedFiles += self.statusSummary[state]
totalFiles += self.statusSummary[state]
self.percentageComplete = ( float( completedFiles ) * 100.0 ) / float( totalFiles )
return S_OK( self.percentageComplete )
def isRequestTerminal( self ):
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK( self.isTerminal )
def getStatus( self ):
return S_OK( self.requestStatus )
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN( self, lfn ):
if not self.fileDict.has_key( lfn ):
self.fileDict[lfn] = {}
return S_OK()
def setSourceSURL( self, lfn, surl ):
target = self.fileDict[lfn].get( 'Target' )
if target == surl:
return S_ERROR( "Source and target the same" )
self.__setFileParameter( lfn, 'Source', surl )
return S_OK()
def getSourceSURL( self, lfn ):
return self.__getFileParameter( lfn, 'Source' )
def setTargetSURL( self, lfn, surl ):
source = self.fileDict[lfn].get( 'Source' )
if source == surl:
return S_ERROR( "Source and target the same" )
self.__setFileParameter( lfn, 'Target', surl )
return S_OK()
def getTargetSURL( self, lfn ):
return self.__getFileParameter( lfn, 'Target' )
def getFailReason( self, lfn ):
return self.__getFileParameter( lfn, 'Reason' )
def getRetries( self, lfn ):
return self.__getFileParameter( lfn, 'Retries' )
def getTransferTime( self, lfn ):
return self.__getFileParameter( lfn, 'Duration' )
def getFailed( self ):
failed = []
for lfn in self.fileDict.keys():
status = self.fileDict[lfn].get( 'Status', '' )
if status in self.failedStates:
failed.append( lfn )
return S_OK( failed )
def getDone( self ):
done = []
for lfn in self.fileDict.keys():
status = self.fileDict[lfn].get( 'Status', '' )
if status in self.successfulStates:
done.append( lfn )
return S_OK( done )
def __setFileParameter( self, lfn, paramName, paramValue ):
self.setLFN( lfn )
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter( self, lfn, paramName ):
if not self.fileDict.has_key( lfn ):
return S_ERROR( "Supplied file not set" )
if not self.fileDict[lfn].has_key( paramName ):
return S_ERROR( "%s not set for file" % paramName )
return S_OK( self.fileDict[lfn][paramName] )
####################################################################
#
# Methods for submission
#
def submit( self, monitor = False, printOutput = True ):
res = self.__isSubmissionValid()
if not res['OK']:
return res
res = self.__createSURLPairFile()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = {'ftsGUID':self.ftsGUID, 'ftsServer':self.ftsServer}
print "Submitted %s @ %s" % ( self.ftsGUID, self.ftsServer )
if monitor:
self.monitor( untilTerminal = True, printOutput = printOutput )
return S_OK( resDict )
def __isSubmissionValid( self ):
if not self.fileDict:
return S_ERROR( "No files set" )
if not self.sourceValid:
return S_ERROR( "SourceSE not valid" )
if not self.targetValid:
return S_ERROR( "TargetSE not valid" )
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR( "FTSServer not valid" )
self.__resolveSource()
self.__resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR( "No files to submit" )
return S_OK()
def __getCatalogObject( self ):
try:
if not self.oCatalog:
self.oCatalog = CatalogInterface()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache( self, lfns = [], overwrite = False ):
if not lfns:
lfns = self.fileDict.keys()
toUpdate = []
for lfn in lfns:
if ( not lfn in self.catalogReplicas.keys() ) or overwrite:
toUpdate.append( lfn )
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogReplicas( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to update replica cache", res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache( self, lfns = [], overwrite = False ):
if not lfns:
lfns = self.fileDict.keys()
toUpdate = []
for lfn in lfns:
if ( not lfn in self.catalogMetadata.keys() ) or overwrite:
toUpdate.append( lfn )
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogFileMetadata( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to get source catalog metadata", res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def __resolveSource( self ):
toResolve = []
for lfn in self.fileDict.keys():
if ( not self.fileDict[lfn].has_key( 'Source' ) ) and ( self.fileDict[lfn].get( 'Status' ) != 'Failed' ):
toResolve.append( lfn )
if not toResolve:
return S_OK()
res = self.__updateMetadataCache( toResolve )
if not res['OK']:
return res
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
for lfn in toResolve:
if self.fileDict[lfn].get( 'Status' ) == 'Failed':
continue
replicas = self.catalogReplicas.get( lfn, {} )
if not replicas.has_key( self.sourceSE ):
self.__setFileParameter( lfn, 'Reason', "No replica at SourceSE" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.oSourceSE.getPfnForProtocol( replicas[self.sourceSE], 'SRM2', withPort = True )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setSourceSURL( lfn, res['Value'] )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict.keys():
if self.fileDict[lfn].has_key( 'Source' ):
toResolve[self.fileDict[lfn]['Source']] = lfn
if not toResolve:
return S_ERROR( "No eligible Source files" )
res = self.oSourceSE.getFileMetadata( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check source file metadata" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
if re.search( 'File does not exist', error ):
self.__setFileParameter( lfn, 'Reason', "Source file does not exist" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
self.__setFileParameter( lfn, 'Reason', "Failed to get Source metadata" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for pfn, metadata in res['Value']['Successful'].items():
lfn = toResolve[pfn]
if metadata['Unavailable']:
self.__setFileParameter( lfn, 'Reason', "Source file Unavailable" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif metadata['Lost']:
self.__setFileParameter( lfn, 'Reason', "Source file Lost" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif not metadata['Cached']:
self.__setFileParameter( lfn, 'Reason', "Source file not Cached" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
self.__setFileParameter( lfn, 'Reason', "Source size mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and not ( compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ) ):
self.__setFileParameter( lfn, 'Reason', "Source checksum mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
return S_OK()
def __resolveTarget( self ):
toResolve = []
for lfn in self.fileDict.keys():
if not self.fileDict[lfn].has_key( 'Target' ) and ( self.fileDict[lfn].get( 'Status' ) != 'Failed' ):
toResolve.append( lfn )
if not toResolve:
return S_OK()
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
atTarget = []
for lfn in sortList( toResolve ):
if self.fileDict[lfn].get( 'Status' ) == 'Failed':
continue
replicas = self.catalogReplicas.get( lfn, {} )
if replicas.has_key( self.targetSE ):
self.__setFileParameter( lfn, 'Reason', "File already at Target" )
self.__setFileParameter( lfn, 'Status', 'Done' )
atTarget.append( lfn )
for lfn in toResolve:
if ( self.fileDict[lfn].get( 'Status' ) == 'Failed' ) or ( lfn in atTarget ):
continue
res = self.oTargetSE.getPfnForLfn( lfn )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', "Failed to create Target" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.oTargetSE.getPfnForProtocol( res['Value'], 'SRM2', withPort = True )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setTargetSURL( lfn, res['Value'] )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict.keys():
if self.fileDict[lfn].has_key( 'Target' ):
toResolve[self.fileDict[lfn]['Target']] = lfn
if not toResolve:
return S_ERROR( "No eligible Target files" )
res = self.oTargetSE.exists( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check target existence" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toRemove = []
for pfn, exists in res['Value']['Successful'].items():
if exists:
lfn = toResolve[pfn]
res = self.getSourceSURL( lfn )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', "Target exists" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif res['Value'] == pfn:
self.__setFileParameter( lfn, 'Reason', "Source and Target the same" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRemove.append( pfn )
if toRemove:
self.oTargetSE.removeFile( toRemove )
return S_OK()
def __filesToSubmit( self ):
for lfn in self.fileDict.keys():
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if ( lfnStatus != 'Failed' ) and ( lfnStatus != 'Done' ) and source and target:
return S_OK()
return S_ERROR()
def __createSURLPairFile( self ):
fd, fileName = tempfile.mkstemp()
surlFile = os.fdopen( fd, 'w' )
for lfn in self.fileDict.keys():
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if ( lfnStatus != 'Failed' ) and ( lfnStatus != 'Done' ) and source and target:
surlString = '%s %s\n' % ( source, target )
surlFile.write( surlString )
surlFile.close()
self.surlFile = fileName
return S_OK()
def __submitFTSTransfer( self ):
comm = ['glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile, '-o']
if self.targetToken:
comm.append( '-t' )
comm.append( self.targetToken )
if self.sourceToken:
comm.append( '-S' )
comm.append( self.sourceToken )
res = executeGridCommand( '', comm, self.gridEnv )
os.remove( self.surlFile )
if not res['OK']:
return res
returnCode, output, errStr = res['Value']
if not returnCode == 0:
return S_ERROR( errStr )
guid = output.replace( '\n', '' )
if not checkGuid( guid ):
return S_ERROR( 'Wrong GUID format returned' )
self.ftsGUID = guid
#if self.priority != 3:
# comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)]
# executeGridCommand('',comm,self.gridEnv)
return res
def __resolveFTSServer( self ):
if not self.sourceSE:
return S_ERROR( "Source SE not set" )
if not self.targetSE:
return S_ERROR( "Target SE not set" )
res = getSitesForSE( self.sourceSE, 'LCG' )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine source site" )
sourceSite = res['Value'][0]
res = getSitesForSE( self.targetSE, 'LCG' )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine target site" )
targetSite = res['Value'][0]
if ( sourceSite == 'LCG.CERN.ch' ) or ( targetSite == 'LCG.CERN.ch' ):
ep = 'LCG.CERN.ch'
else:
# Target site FTS server should be used
ep = targetSite
try:
configPath = '/Resources/FTSEndpoints/%s' % ep
endpointURL = gConfig.getValue( configPath )
if not endpointURL:
errStr = "FTSRequest.__resolveFTSEndpoint: Failed to find FTS endpoint, check CS entry for '%s'." % ep
return S_ERROR( errStr )
self.ftsServer = endpointURL
return S_OK( endpointURL )
except Exception, x:
return S_ERROR( 'FTSRequest.__resolveFTSEndpoint: Failed to obtain endpoint details from CS' )
def _filterReplicas(self, opFile):
""" filter out banned/invalid source SEs """
from DIRAC.Core.Utilities.Adler import compareAdler
ret = {"Valid": [], "Banned": [], "Bad": []}
replicas = self.rm.getActiveReplicas(opFile.LFN)
if not replicas["OK"]:
self.log.error(replicas["Message"])
reNotExists = re.compile("not such file or directory")
replicas = replicas["Value"]
failed = replicas["Failed"].get(opFile.LFN, "")
if reNotExists.match(failed.lower()):
opFile.Status = "Failed"
opFile.Error = failed
return S_ERROR(failed)
replicas = replicas["Successful"][
opFile.LFN] if opFile.LFN in replicas["Successful"] else {}
for repSEName in replicas:
seRead = self.rssSEStatus(repSEName, "ReadAccess")
if not seRead["OK"]:
self.log.info(seRead["Message"])
ret["Banned"].append(repSEName)
continue
if not seRead["Value"]:
self.log.info("StorageElement '%s' is banned for reading" %
(repSEName))
repSE = self.seCache.get(repSEName, None)
if not repSE:
repSE = StorageElement(repSEName, "SRM2")
self.seCache[repSE] = repSE
pfn = repSE.getPfnForLfn(opFile.LFN)
if not pfn["OK"]:
self.log.warn("unable to create pfn for %s lfn: %s" %
(opFile.LFN, pfn["Message"]))
ret["Banned"].append(repSEName)
continue
pfn = pfn["Value"]
repSEMetadata = repSE.getFileMetadata(pfn, singleFile=True)
if not repSEMetadata["OK"]:
self.log.warn(repSEMetadata["Message"])
ret["Banned"].append(repSEName)
continue
repSEMetadata = repSEMetadata["Value"]
seChecksum = repSEMetadata.get("Checksum")
if opFile.Checksum and seChecksum and not compareAdler(
seChecksum, opFile.Checksum):
self.log.warn(" %s checksum mismatch: %s %s:%s" %
(opFile.LFN, opFile.Checksum, repSE, seChecksum))
ret["Bad"].append(repSEName)
continue
# # if we're here repSE is OK
ret["Valid"].append(repSEName)
return S_OK(ret)
def __getStorageDirectoryContents(self, lfnDir, storageElement):
""" Obtians the contents of the supplied directory on the storage
"""
gLogger.info('Obtaining the contents for %s directories at %s' %
(len(lfnDir), storageElement))
se = StorageElement(storageElement)
res = se.getPfnForLfn(lfnDir)
if not res['OK']:
gLogger.error("Failed to get PFNs for directories", res['Message'])
return res
for directory, error in res['Value']['Failed'].items():
gLogger.error('Failed to obtain directory PFN from LFNs',
'%s %s' % (directory, error))
if res['Value']['Failed']:
return S_ERROR('Failed to obtain directory PFN from LFNs')
storageDirectories = res['Value']['Successful'].values()
res = se.exists(storageDirectories)
if not res['OK']:
gLogger.error("Failed to obtain existance of directories",
res['Message'])
return res
for directory, error in res['Value']['Failed'].items():
gLogger.error('Failed to determine existance of directory',
'%s %s' % (directory, error))
if res['Value']['Failed']:
return S_ERROR('Failed to determine existance of directory')
directoryExists = res['Value']['Successful']
activeDirs = []
for directory in sortList(directoryExists.keys()):
exists = directoryExists[directory]
if exists:
activeDirs.append(directory)
allFiles = {}
while len(activeDirs) > 0:
currentDir = activeDirs[0]
res = se.listDirectory(currentDir)
activeDirs.remove(currentDir)
if not res['OK']:
gLogger.error('Failed to get directory contents',
res['Message'])
return res
elif res['Value']['Failed'].has_key(currentDir):
gLogger.error(
'Failed to get directory contents',
'%s %s' % (currentDir, res['Value']['Failed'][currentDir]))
return S_ERROR(res['Value']['Failed'][currentDir])
else:
dirContents = res['Value']['Successful'][currentDir]
activeDirs.extend(dirContents['SubDirs'])
fileMetadata = dirContents['Files']
# RF_NOTE This ugly trick is needed because se.getPfnPath does not follow the Successful/Failed convention
# res = { "Successful" : {}, "Failed" : {} }
# for pfn in fileMetadata:
# inRes = se.getPfnPath( pfn )
# if inRes["OK"]:
# res["Successful"][pfn] = inRes["Value"]
# else:
# res["Failed"][pfn] = inRes["Message"]
res = se.getLfnForPfn(fileMetadata.keys())
if not res['OK']:
gLogger.error('Failed to get directory content LFNs',
res['Message'])
return res
for pfn, error in res['Value']['Failed'].items():
gLogger.error("Failed to get LFN for PFN",
"%s %s" % (pfn, error))
if res['Value']['Failed']:
return S_ERROR("Failed to get LFNs for PFNs")
pfnLfns = res['Value']['Successful']
for pfn, lfn in pfnLfns.items():
fileMetadata[pfn]['LFN'] = lfn
allFiles.update(fileMetadata)
zeroSizeFiles = []
lostFiles = []
unavailableFiles = []
for pfn in sortList(allFiles.keys()):
if os.path.basename(pfn) == 'dirac_directory':
allFiles.pop(pfn)
else:
metadata = allFiles[pfn]
if metadata['Size'] == 0:
zeroSizeFiles.append(
(metadata['LFN'], pfn, storageElement, 'PFNZeroSize'))
# if metadata['Lost']:
# lostFiles.append((metadata['LFN'],pfn,storageElement,'PFNLost'))
# if metadata['Unavailable']:
# unavailableFiles.append((metadata['LFN'],pfn,storageElement,'PFNUnavailable'))
if zeroSizeFiles:
self.__reportProblematicReplicas(zeroSizeFiles, storageElement,
'PFNZeroSize')
if lostFiles:
self.__reportProblematicReplicas(lostFiles, storageElement,
'PFNLost')
if unavailableFiles:
self.__reportProblematicReplicas(unavailableFiles, storageElement,
'PFNUnavailable')
gLogger.info('Obtained at total of %s files for directories at %s' %
(len(allFiles), storageElement))
return S_OK(allFiles)
class FTSRequest(object):
"""
.. class:: FTSRequest
Helper class for FTS job submission and monitoring.
"""
## default checksum type
__defaultCksmType = "ADLER32"
## flag to disablr/enable checksum test, default: disabled
__cksmTest = False
def __init__(self):
"""c'tor
:param self: self reference
"""
self.log = gLogger.getSubLogger(self.__class__.__name__, True)
## final states tuple
self.finalStates = ('Canceled', 'Failed', 'Hold', 'Finished',
'FinishedDirty')
## failed states tuple
self.failedStates = ('Canceled', 'Failed', 'Hold', 'FinishedDirty')
## successful states tuple
self.successfulStates = ('Finished', 'Done')
## all file states tuple
self.fileStates = ('Done', 'Active', 'Pending', 'Ready', 'Canceled',
'Failed', 'Finishing', 'Finished', 'Submitted',
'Hold', 'Waiting')
self.newlyCompletedFiles = []
self.newlyFailedFiles = []
self.statusSummary = {}
## request status
self.requestStatus = 'Unknown'
## dict for FTS job files
self.fileDict = {}
## dict for replicas information
self.catalogReplicas = {}
## dict for metadata information
self.catalogMetadata = {}
## dict for files that failed to register
self.failedRegistrations = {}
## placehoder for CatalogInterface reference
self.oCatalog = None
## submit timestamp
self.submitTime = ''
## placeholder FTS job GUID
self.ftsGUID = ''
## placeholder for FTS server URL
self.ftsServer = ''
## not used
self.priority = 3
## flag marking FTS job completness
self.isTerminal = False
## completness percentage
self.percentageComplete = 0.0
## source SE name
self.sourceSE = ''
## flag marking source SE validity
self.sourceValid = False
## source space token
self.sourceToken = ''
## target SE name
self.targetSE = ''
## flag marking target SE validity
self.targetValid = False
## target space token
self.targetToken = ''
## whatever
self.dumpStr = ''
## placeholder for surl file
self.surlFile = None
## placeholder for target StorageElement
self.oTargetSE = None
## placeholder for source StorageElement
self.oSourceSE = None
## checksum type, set it to default
self.__cksmType = self.__defaultCksmType
## disable checksum test by default
self.__cksmTest = False
## replica manager handler
self.replicaManager = ReplicaManager()
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE(self, se):
""" set SE for source
:param self: self reference
:param str se: source SE name
"""
if se == self.targetSE:
return S_ERROR("SourceSE is TargetSE")
self.sourceSE = se
self.oSourceSE = StorageElement(self.sourceSE)
return self.__checkSourceSE()
def getSourceSE(self):
""" source SE getter
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR("Source SE not defined")
return S_OK(self.sourceSE)
def setSourceToken(self, token):
""" set source space token
:param self: self reference
:param str token: source space token
"""
self.sourceToken = token
return S_OK()
def getSourceToken(self):
""" source space token getter
:param self: self reference
"""
if not self.sourceToken:
return S_ERROR("Source token not defined")
return S_OK(self.sourceToken)
def __checkSourceSE(self):
""" check source SE availability
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR("SourceSE not set")
res = self.oSourceSE.isValid('Read')
if not res['OK']:
return S_ERROR("SourceSE not available for reading")
res = self.__getSESpaceToken(self.oSourceSE)
if not res['OK']:
self.log.error(
"FTSRequest failed to get SRM Space Token for SourceSE",
res['Message'])
return S_ERROR("SourceSE does not support FTS transfers")
if self.__cksmTest:
res = self.oSourceSE.getChecksumType()
if not res["OK"]:
self.log.error(
"Unable to get checksum type for SourceSE %s: %s" %
(self.sourceSE, res["Message"]))
cksmType = res["Value"]
if cksmType in ("NONE", "NULL"):
self.log.warn(
"Checksum type set to %s at SourceSE %s, disabling checksum test"
% (cksmType, self.sourceSE))
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn(
"Checksum type mismatch, disabling checksum test")
self.__cksmTest = False
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE(self, se):
""" set target SE
:param self: self reference
:param str se: target SE name
"""
if se == self.sourceSE:
return S_ERROR("TargetSE is SourceSE")
self.targetSE = se
self.oTargetSE = StorageElement(self.targetSE)
return self.__checkTargetSE()
def getTargetSE(self):
""" target SE getter
:param self: self reference
"""
if not self.targetSE:
return S_ERROR("Target SE not defined")
return S_OK(self.targetSE)
def setTargetToken(self, token):
""" target space token setter
:param self: self reference
:param str token: target space token
"""
self.targetToken = token
return S_OK()
def getTargetToken(self):
""" target space token getter
:param self: self reference
"""
if not self.targetToken:
return S_ERROR("Target token not defined")
return S_OK(self.targetToken)
def __checkTargetSE(self):
""" check target SE availability
:param self: self reference
"""
if not self.targetSE:
return S_ERROR("TargetSE not set")
res = self.oTargetSE.isValid('Write')
if not res['OK']:
return S_ERROR("TargetSE not available for writing")
res = self.__getSESpaceToken(self.oTargetSE)
if not res['OK']:
self.log.error(
"FTSRequest failed to get SRM Space Token for TargetSE",
res['Message'])
return S_ERROR("TargetSE does not support FTS transfers")
## check checksum types
if self.__cksmTest:
res = self.oTargetSE.getChecksumType()
if not res["OK"]:
self.log.error(
"Unable to get checksum type for TargetSE %s: %s" %
(self.targetSE, res["Message"]))
cksmType = res["Value"]
if cksmType in ("NONE", "NULL"):
self.log.warn(
"Checksum type set to %s at TargetSE %s, disabling checksum test"
% (cksmType, self.targetSE))
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn(
"Checksum type mismatch, disabling checksum test")
self.__cksmTest = False
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
@staticmethod
def __getSESpaceToken(oSE):
""" get space token from StorageElement instance
:param self: self reference
:param StorageElement oSE: StorageElement instance
"""
res = oSE.getStorageParameters("SRM2")
if not res['OK']:
return res
return S_OK(res['Value'].get('SpaceToken'))
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID(self, guid):
""" FTS job GUID setter
:param self: self reference
:param str guid: string containg GUID
"""
if not checkGuid(guid):
return S_ERROR("Incorrect GUID format")
self.ftsGUID = guid
return S_OK()
def getFTSGUID(self):
""" FTS job GUID getter
:param self: self refenece
"""
if not self.ftsGUID:
return S_ERROR("FTSGUID not set")
return S_OK(self.ftsGUID)
def setFTSServer(self, server):
""" FTS server setter
:param self: self reference
:param str server: FTS server URL
"""
self.ftsServer = server
return S_OK()
def getFTSServer(self):
""" FTS server getter
:param self: self reference
"""
if not self.ftsServer:
return S_ERROR("FTSServer not set")
return S_OK(self.ftsServer)
def setPriority(self, priority):
""" set priority for FTS job
:param self: self reference
:param int priority: a new priority
"""
if not type(priority) in (IntType, LongType):
return S_ERROR("Priority must be integer")
if priority < 0:
priority = 0
elif priority > 5:
priority = 5
self.priority = priority
return S_OK(self.priority)
def getPriority(self):
""" FTS job priority getter
:param self: self reference
"""
return S_OK(self.priority)
def getPercentageComplete(self):
""" get completness percentage
:param self: self reference
"""
completedFiles = 0
totalFiles = 0
for state in (self.statusSummary.keys()):
if state in self.successfulStates:
completedFiles += self.statusSummary[state]
totalFiles += self.statusSummary[state]
self.percentageComplete = (float(completedFiles) *
100.0) / float(totalFiles)
return S_OK(self.percentageComplete)
def isRequestTerminal(self):
""" check if FTS job has terminated
:param self: self reference
"""
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK(self.isTerminal)
def getStatus(self):
""" get FTS job status
:param self: self reference
"""
return S_OK(self.requestStatus)
def setCksmType(self, cksm=None):
""" set checksum type to use
:param self: self reference
:param mixed cksm: checksum type, should be one of 'Adler32', 'md5', 'sha1', None
"""
if str(cksm).upper() not in ("ADLER32", "MD5", "SHA1", "NONE"):
return S_ERROR("Not supported checksum type: %s" % str(cksm))
if not cksm:
self.__cksmType = None
return S_OK(False)
self.__cksmType = str(cksm).upper()
return S_OK(True)
def getCksmType(self):
""" get checksum type
:param self: self reference
"""
return S_OK(self.__cksmType)
def setCksmTest(self, cksmTest=False):
""" set cksm test
:param self: self reference
:param bool cksmTest: flag to enable/disable checksum test
"""
self.__cksmTest = bool(cksmTest)
return S_OK(self.__cksmTest)
def getCksmTest(self):
""" get cksm test flag
:param self: self reference
"""
return S_OK(self.__cksmTest)
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN(self, lfn):
""" add LFN :lfn: to :fileDict:
:param self: self reference
:param str lfn: LFN to add to
"""
if lfn not in self.fileDict:
self.fileDict[lfn] = {}
return S_OK()
def setSourceSURL(self, lfn, surl):
""" source SURL setter
:param self: self reference
:param str lfn: LFN
:param str surl: source SURL
"""
target = self.fileDict[lfn].get('Target')
if target == surl:
return S_ERROR("Source and target the same")
self.__setFileParameter(lfn, 'Source', surl)
return S_OK()
def getSourceSURL(self, lfn):
""" get source SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter(lfn, 'Source')
def setTargetSURL(self, lfn, surl):
""" set target SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
:param str surl: target SURL
"""
source = self.fileDict[lfn].get('Source')
if source == surl:
return S_ERROR("Source and target the same")
self.__setFileParameter(lfn, 'Target', surl)
return S_OK()
def getTargetSURL(self, lfn):
""" target SURL getter
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter(lfn, 'Target')
def getFailReason(self, lfn):
""" get fail reason for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter(lfn, 'Reason')
def getRetries(self, lfn):
""" get number of attepmts made to transfer file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter(lfn, 'Retries')
def getTransferTime(self, lfn):
""" get duration of transfer for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter(lfn, 'Duration')
def getFailed(self):
""" get list of wrongly transferred LFNs
:param self: self reference
"""
return S_OK([
lfn for lfn in self.fileDict
if self.fileDict[lfn].get('Status', '') in self.failedStates
])
def getDone(self):
""" get list of succesfully transferred LFNs
:param self: self reference
"""
return S_OK([
lfn for lfn in self.fileDict
if self.fileDict[lfn].get('Status', '') in self.successfulStates
])
def __setFileParameter(self, lfn, paramName, paramValue):
""" set :paramName: to :paramValue: for :lfn: file
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
:param mixed paramValue: a new parameter value
"""
self.setLFN(lfn)
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter(self, lfn, paramName):
""" get value of :paramName: for file :lfn:
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
"""
if lfn not in self.fileDict:
return S_ERROR("Supplied file not set")
if paramName not in self.fileDict[lfn]:
return S_ERROR("%s not set for file" % paramName)
return S_OK(self.fileDict[lfn][paramName])
####################################################################
#
# Methods for submission
#
def submit(self, monitor=False, printOutput=True):
""" submit FTS job
:param self: self reference
:param bool monitor: flag to monitor progress of FTS job
:param bool printOutput: flag to print output of execution to stdout
"""
res = self.__isSubmissionValid()
if not res['OK']:
return res
res = self.__createSURLPairFile()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = {'ftsGUID': self.ftsGUID, 'ftsServer': self.ftsServer}
print "Submitted %s @ %s" % (self.ftsGUID, self.ftsServer)
if monitor:
self.monitor(untilTerminal=True, printOutput=printOutput)
return S_OK(resDict)
def __isSubmissionValid(self):
""" check validity of job before submission
:param self: self reference
"""
if not self.fileDict:
return S_ERROR("No files set")
if not self.sourceValid:
return S_ERROR("SourceSE not valid")
if not self.targetValid:
return S_ERROR("TargetSE not valid")
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR("FTSServer not valid")
self.resolveSource()
self.resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR("No files to submit")
return S_OK()
def __getCatalogObject(self):
""" CatalogInterface instance facade
:param self: self reference
"""
try:
if not self.oCatalog:
self.oCatalog = CatalogInterface()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache(self, lfns=None, overwrite=False):
""" update replica cache for list of :lfns:
:param self: self reference
:param mixed lfns: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [
lfn for lfn in lfns
if (lfn not in self.catalogReplicas) or overwrite
]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogReplicas(toUpdate)
if not res['OK']:
return S_ERROR("Failed to update replica cache: %s" %
res['Message'])
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache(self, lfns=None, overwrite=False):
""" update metadata cache for list of LFNs
:param self: self reference
:param list lnfs: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [
lfn for lfn in lfns
if (lfn not in self.catalogMetadata) or overwrite
]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogFileMetadata(toUpdate)
if not res['OK']:
return S_ERROR("Failed to get source catalog metadata: %s" %
res['Message'])
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def resolveSource(self):
""" resolve source SE eligible for submission
:param self: self reference
"""
toResolve = [lfn for lfn in self.fileDict]
if not toResolve:
return S_OK()
res = self.__updateMetadataCache(toResolve)
if not res['OK']:
return res
res = self.__updateReplicaCache(toResolve)
if not res['OK']:
return res
for lfn in toResolve:
if self.fileDict[lfn].get("Status", "") == "Failed":
continue
replicas = self.catalogReplicas.get(lfn, {})
if self.sourceSE not in replicas:
gLogger.warn(
"resolveSource: skipping %s - not replicas at SourceSE %s"
% (lfn, self.sourceSE))
self.__setFileParameter(lfn, 'Reason',
"No replica at SourceSE")
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.oSourceSE.getPfnForProtocol(replicas[self.sourceSE],
'SRM2',
withPort=True)
if not res['OK']:
gLogger.warn("resolveSource: skipping %s - %s" %
(lfn, res["Message"]))
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.setSourceSURL(lfn, res['Value'])
if not res['OK']:
gLogger.warn("resolveSource: skipping %s - %s" %
(lfn, res["Message"]))
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
toResolve = {}
for lfn in self.fileDict:
if "Source" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Source']] = lfn
if not toResolve:
return S_ERROR("No eligible Source files")
res = self.oSourceSE.getFileMetadata(toResolve.keys())
if not res['OK']:
return S_ERROR("Failed to check source file metadata")
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
if re.search('File does not exist', error):
gLogger.warn(
"resolveSource: skipping %s - source file does not exists"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Source file does not exist")
self.__setFileParameter(lfn, 'Status', 'Failed')
else:
gLogger.warn(
"resolveSource: skipping %s - failed to get source metadata"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Failed to get Source metadata")
self.__setFileParameter(lfn, 'Status', 'Failed')
for pfn, metadata in res['Value']['Successful'].items():
lfn = toResolve[pfn]
if metadata['Unavailable']:
gLogger.warn(
"resolveSource: skipping %s - source file unavailable" %
lfn)
self.__setFileParameter(lfn, 'Reason',
"Source file Unavailable")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif metadata['Lost']:
gLogger.warn("resolveSource: skipping %s - source file lost" %
lfn)
self.__setFileParameter(lfn, 'Reason', "Source file Lost")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif not metadata['Cached']:
gLogger.warn(
"resolveSource: source file %s not cached, prestaging..." %
lfn)
stage = self.replicaManager.prestageStorageFile(
pfn, self.sourceSE, singleFile=True)
if not stage["OK"]:
gLogger.warn("resolveSource: skipping %s - %s" %
(lfn, stage["Message"]))
self.__setFileParameter(lfn, 'Reason', stage["Message"])
self.__setFileParameter(lfn, 'Status', 'Failed')
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
gLogger.warn(
"resolveSource: skipping %s - source file size mismatch" %
lfn)
self.__setFileParameter(lfn, 'Reason', "Source size mismatch")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \
not ( compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ) ):
gLogger.warn(
"resolveSource: skipping %s - source file checksum mismatch"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Source checksum mismatch")
self.__setFileParameter(lfn, 'Status', 'Failed')
return S_OK()
def resolveTarget(self):
""" find target SE eligible for submission
:param self: self reference
"""
toResolve = [lfn for lfn in self.fileDict]
if not toResolve:
return S_OK()
res = self.__updateReplicaCache(toResolve)
if not res['OK']:
return res
for lfn in toResolve:
if (self.fileDict[lfn].get('Status') == 'Failed'):
continue
res = self.oTargetSE.getPfnForLfn(lfn)
if not res['OK']:
gLogger.warn(
"resolveTarget: skipping %s - failed to create target pfn"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Failed to create Target")
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.oTargetSE.getPfnForProtocol(res['Value'],
'SRM2',
withPort=True)
if not res['OK']:
gLogger.warn("resolveTarget: skipping %s - %s" %
(lfn, res["Message"]))
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.setTargetSURL(lfn, res['Value'])
if not res['OK']:
gLogger.warn("resolveTarget: skipping %s - %s" %
(lfn, res["Message"]))
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
toResolve = {}
for lfn in self.fileDict:
if "Target" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Target']] = lfn
if not toResolve:
return S_ERROR("No eligible Target files")
res = self.oTargetSE.exists(toResolve.keys())
if not res['OK']:
return S_ERROR("Failed to check target existence")
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
toRemove = []
for pfn, exists in res['Value']['Successful'].items():
if exists:
lfn = toResolve[pfn]
res = self.getSourceSURL(lfn)
if not res['OK']:
gLogger.warn("resolveTarget: skipping %s - target exists" %
lfn)
self.__setFileParameter(lfn, 'Reason', "Target exists")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif res['Value'] == pfn:
gLogger.warn(
"resolveTarget: skipping %s - source and target pfns are teh same"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Source and Target the same")
self.__setFileParameter(lfn, 'Status', 'Failed')
else:
toRemove.append(pfn)
if toRemove:
self.oTargetSE.removeFile(toRemove)
return S_OK()
def __filesToSubmit(self):
"""
check if there is at least one file to submit
:return: S_OK if at least one file is present, S_ERROR otherwise
"""
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get('Status')
source = self.fileDict[lfn].get('Source')
target = self.fileDict[lfn].get('Target')
if (lfnStatus != 'Failed') and (lfnStatus !=
'Done') and source and target:
return S_OK()
return S_ERROR()
def __createSURLPairFile(self):
""" create LFNs file for glite-transfer-submit command
This file consists one line for each fiel to be transferred:
sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM]
:param self: self reference
"""
fd, fileName = tempfile.mkstemp()
surlFile = os.fdopen(fd, 'w')
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get('Status')
source = self.fileDict[lfn].get('Source')
target = self.fileDict[lfn].get('Target')
if (lfnStatus not in ('Failed', 'Done')) and source and target:
cksmStr = ""
## add chsmType:cksm only if cksmType is specified, else let FTS decide by itself
if self.__cksmTest and self.__cksmType:
if lfn in self.catalogMetadata and "Checksum" in self.catalogMetadata[
lfn]:
cksmStr = " %s:%s" % (self.__cksmType, self.
catalogMetadata[lfn]["Checksum"])
surlFile.write("%s %s%s\n" % (source, target, cksmStr))
surlFile.close()
self.surlFile = fileName
return S_OK()
def __submitFTSTransfer(self):
""" create and execute glite-transfer-submit CLI command
:param self: self reference
"""
comm = [
'glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile,
'-o'
]
if self.targetToken:
comm.append('-t')
comm.append(self.targetToken)
if self.sourceToken:
comm.append('-S')
comm.append(self.sourceToken)
if self.__cksmTest:
comm.append("--compare-checksums")
res = executeGridCommand('', comm)
os.remove(self.surlFile)
if not res['OK']:
return res
returnCode, output, errStr = res['Value']
if not returnCode == 0:
return S_ERROR(errStr)
guid = output.replace('\n', '')
if not checkGuid(guid):
return S_ERROR('Wrong GUID format returned')
self.ftsGUID = guid
#if self.priority != 3:
# comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)]
# executeGridCommand('',comm)
return res
def __resolveFTSServer(self):
"""
resolve FTS server to use, it should be the closest one from target SE
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR("Source SE not set")
if not self.targetSE:
return S_ERROR("Target SE not set")
res = getSitesForSE(self.sourceSE, 'LCG')
if not res['OK'] or not res['Value']:
return S_ERROR("Could not determine source site")
sourceSite = res['Value'][0]
res = getSitesForSE(self.targetSE, 'LCG')
if not res['OK'] or not res['Value']:
return S_ERROR("Could not determine target site")
targetSite = res['Value'][0]
if (sourceSite == 'LCG.CERN.ch') or (targetSite == 'LCG.CERN.ch'):
ep = 'LCG.CERN.ch'
else:
# Target site FTS server should be used
ep = targetSite
try:
configPath = '/Resources/FTSEndpoints/%s' % ep
endpointURL = gConfig.getValue(configPath)
if not endpointURL:
errStr = "FTSRequest.__resolveFTSServer: Failed to find FTS endpoint, check CS entry for '%s'." % ep
return S_ERROR(errStr)
self.ftsServer = endpointURL
return S_OK(endpointURL)
except Exception, x:
return S_ERROR(
'FTSRequest.__resolveFTSServer: Failed to obtain endpoint details from CS'
)
class FTSRequest(object):
"""
.. class:: FTSRequest
Helper class for FTS job submission and monitoring.
"""
# # default checksum type
__defaultCksmType = "ADLER32"
# # flag to disablr/enable checksum test, default: disabled
__cksmTest = False
def __init__(self):
"""c'tor
:param self: self reference
"""
self.log = gLogger.getSubLogger(self.__class__.__name__, True)
# # final states tuple
self.finalStates = ('Canceled', 'Failed', 'Hold', 'Finished',
'FinishedDirty')
# # failed states tuple
self.failedStates = ('Canceled', 'Failed', 'Hold', 'FinishedDirty')
# # successful states tuple
self.successfulStates = ('Finished', 'Done')
# # all file states tuple
self.fileStates = ('Done', 'Active', 'Pending', 'Ready', 'Canceled',
'Failed', 'Finishing', 'Finished', 'Submitted',
'Hold', 'Waiting')
self.statusSummary = {}
# # request status
self.requestStatus = 'Unknown'
# # dict for FTS job files
self.fileDict = {}
# # dict for replicas information
self.catalogReplicas = {}
# # dict for metadata information
self.catalogMetadata = {}
# # dict for files that failed to register
self.failedRegistrations = {}
# # placehoder for FileCatalog reference
self.oCatalog = None
# # submit timestamp
self.submitTime = ''
# # placeholder FTS job GUID
self.ftsGUID = ''
# # placeholder for FTS server URL
self.ftsServer = ''
# # flag marking FTS job completness
self.isTerminal = False
# # completness percentage
self.percentageComplete = 0.0
# # source SE name
self.sourceSE = ''
# # flag marking source SE validity
self.sourceValid = False
# # source space token
self.sourceToken = ''
# # target SE name
self.targetSE = ''
# # flag marking target SE validity
self.targetValid = False
# # target space token
self.targetToken = ''
# # placeholder for target StorageElement
self.oTargetSE = None
# # placeholder for source StorageElement
self.oSourceSE = None
# # checksum type, set it to default
self.__cksmType = self.__defaultCksmType
# # disable checksum test by default
self.__cksmTest = False
# # statuses that prevent submitting to FTS
self.noSubmitStatus = ('Failed', 'Done', 'Staging')
# # were sources resolved?
self.sourceResolved = False
# # Number of file transfers actually submitted
self.submittedFiles = 0
self.transferTime = 0
self.submitCommand = Operations().getValue(
'DataManagement/FTSPlacement/FTS2/SubmitCommand',
'glite-transfer-submit')
self.monitorCommand = Operations().getValue(
'DataManagement/FTSPlacement/FTS2/MonitorCommand',
'glite-transfer-status')
self.ftsJob = None
self.ftsFiles = []
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE(self, se):
""" set SE for source
:param self: self reference
:param str se: source SE name
"""
if se == self.targetSE:
return S_ERROR("SourceSE is TargetSE")
self.sourceSE = se
self.oSourceSE = StorageElement(self.sourceSE)
return self.__checkSourceSE()
def __checkSourceSE(self):
""" check source SE availability
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR("SourceSE not set")
res = self.oSourceSE.isValid('Read')
if not res['OK']:
return S_ERROR("SourceSE not available for reading")
res = self.__getSESpaceToken(self.oSourceSE)
if not res['OK']:
self.log.error(
"FTSRequest failed to get SRM Space Token for SourceSE",
res['Message'])
return S_ERROR("SourceSE does not support FTS transfers")
if self.__cksmTest:
res = self.oSourceSE.getChecksumType()
if not res["OK"]:
self.log.error(
"Unable to get checksum type for SourceSE %s: %s" %
(self.sourceSE, res["Message"]))
cksmType = res["Value"]
if cksmType in ("NONE", "NULL"):
self.log.warn(
"Checksum type set to %s at SourceSE %s, disabling checksum test"
% (cksmType, self.sourceSE))
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn(
"Checksum type mismatch, disabling checksum test")
self.__cksmTest = False
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE(self, se):
""" set target SE
:param self: self reference
:param str se: target SE name
"""
if se == self.sourceSE:
return S_ERROR("TargetSE is SourceSE")
self.targetSE = se
self.oTargetSE = StorageElement(self.targetSE)
return self.__checkTargetSE()
def setTargetToken(self, token):
""" target space token setter
:param self: self reference
:param str token: target space token
"""
self.targetToken = token
return S_OK()
def __checkTargetSE(self):
""" check target SE availability
:param self: self reference
"""
if not self.targetSE:
return S_ERROR("TargetSE not set")
res = self.oTargetSE.isValid('Write')
if not res['OK']:
return S_ERROR("TargetSE not available for writing")
res = self.__getSESpaceToken(self.oTargetSE)
if not res['OK']:
self.log.error(
"FTSRequest failed to get SRM Space Token for TargetSE",
res['Message'])
return S_ERROR("TargetSE does not support FTS transfers")
# # check checksum types
if self.__cksmTest:
res = self.oTargetSE.getChecksumType()
if not res["OK"]:
self.log.error(
"Unable to get checksum type for TargetSE %s: %s" %
(self.targetSE, res["Message"]))
cksmType = res["Value"]
if cksmType in ("NONE", "NULL"):
self.log.warn(
"Checksum type set to %s at TargetSE %s, disabling checksum test"
% (cksmType, self.targetSE))
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn(
"Checksum type mismatch, disabling checksum test")
self.__cksmTest = False
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
@staticmethod
def __getSESpaceToken(oSE):
""" get space token from StorageElement instance
:param self: self reference
:param StorageElement oSE: StorageElement instance
"""
res = oSE.getStorageParameters("SRM2")
if not res['OK']:
return res
return S_OK(res['Value'].get('SpaceToken'))
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID(self, guid):
""" FTS job GUID setter
:param self: self reference
:param str guid: string containg GUID
"""
if not checkGuid(guid):
return S_ERROR("Incorrect GUID format")
self.ftsGUID = guid
return S_OK()
def setFTSServer(self, server):
""" FTS server setter
:param self: self reference
:param str server: FTS server URL
"""
self.ftsServer = server
return S_OK()
def isRequestTerminal(self):
""" check if FTS job has terminated
:param self: self reference
"""
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK(self.isTerminal)
def setCksmTest(self, cksmTest=False):
""" set cksm test
:param self: self reference
:param bool cksmTest: flag to enable/disable checksum test
"""
self.__cksmTest = bool(cksmTest)
return S_OK(self.__cksmTest)
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN(self, lfn):
""" add LFN :lfn: to :fileDict:
:param self: self reference
:param str lfn: LFN to add to
"""
self.fileDict.setdefault(lfn, {'Status': 'Waiting'})
return S_OK()
def setSourceSURL(self, lfn, surl):
""" source SURL setter
:param self: self reference
:param str lfn: LFN
:param str surl: source SURL
"""
target = self.fileDict[lfn].get('Target')
if target == surl:
return S_ERROR("Source and target the same")
return self.__setFileParameter(lfn, 'Source', surl)
def getSourceSURL(self, lfn):
""" get source SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter(lfn, 'Source')
def setTargetSURL(self, lfn, surl):
""" set target SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
:param str surl: target SURL
"""
source = self.fileDict[lfn].get('Source')
if source == surl:
return S_ERROR("Source and target the same")
return self.__setFileParameter(lfn, 'Target', surl)
def getFailReason(self, lfn):
""" get fail reason for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter(lfn, 'Reason')
def getRetries(self, lfn):
""" get number of attepmts made to transfer file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter(lfn, 'Retries')
def getTransferTime(self, lfn):
""" get duration of transfer for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter(lfn, 'Duration')
def getFailed(self):
""" get list of wrongly transferred LFNs
:param self: self reference
"""
return S_OK([
lfn for lfn in self.fileDict
if self.fileDict[lfn].get('Status', '') in self.failedStates
])
def getStaging(self):
""" get files set for prestaging """
return S_OK([
lfn for lfn in self.fileDict
if self.fileDict[lfn].get('Status', '') == 'Staging'
])
def getDone(self):
""" get list of succesfully transferred LFNs
:param self: self reference
"""
return S_OK([
lfn for lfn in self.fileDict
if self.fileDict[lfn].get('Status', '') in self.successfulStates
])
def __setFileParameter(self, lfn, paramName, paramValue):
""" set :paramName: to :paramValue: for :lfn: file
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
:param mixed paramValue: a new parameter value
"""
self.setLFN(lfn)
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter(self, lfn, paramName):
""" get value of :paramName: for file :lfn:
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
"""
if lfn not in self.fileDict:
return S_ERROR("Supplied file not set")
if paramName not in self.fileDict[lfn]:
return S_ERROR("%s not set for file" % paramName)
return S_OK(self.fileDict[lfn][paramName])
####################################################################
#
# Methods for submission
#
def submit(self, monitor=False, printOutput=True):
""" submit FTS job
:param self: self reference
:param bool monitor: flag to monitor progress of FTS job
:param bool printOutput: flag to print output of execution to stdout
"""
res = self.__prepareForSubmission()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = {
'ftsGUID': self.ftsGUID,
'ftsServer': self.ftsServer,
'submittedFiles': self.submittedFiles
}
if monitor or printOutput:
gLogger.always("Submitted %s@%s" % (self.ftsGUID, self.ftsServer))
if monitor:
self.monitor(untilTerminal=True,
printOutput=printOutput,
full=False)
return S_OK(resDict)
def __prepareForSubmission(self):
""" check validity of job before submission
:param self: self reference
"""
if not self.fileDict:
return S_ERROR("No files set")
if not self.sourceValid:
return S_ERROR("SourceSE not valid")
if not self.targetValid:
return S_ERROR("TargetSE not valid")
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR("FTSServer not valid")
self.resolveSource()
self.resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR("No files to submit")
return S_OK()
def __getCatalogObject(self):
""" CatalogInterface instance facade
:param self: self reference
"""
try:
if not self.oCatalog:
self.oCatalog = FileCatalog()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache(self, lfns=None, overwrite=False):
""" update replica cache for list of :lfns:
:param self: self reference
:param mixed lfns: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [
lfn for lfn in lfns
if (lfn not in self.catalogReplicas) or overwrite
]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getReplicas(toUpdate)
if not res['OK']:
return S_ERROR("Failed to update replica cache: %s" %
res['Message'])
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache(self, lfns=None):
""" update metadata cache for list of LFNs
:param self: self reference
:param list lnfs: list of LFNs
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [lfn for lfn in lfns if lfn not in self.catalogMetadata]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getFileMetadata(toUpdate)
if not res['OK']:
return S_ERROR("Failed to get source catalog metadata: %s" %
res['Message'])
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def resolveSource(self):
""" resolve source SE eligible for submission
:param self: self reference
"""
# Avoid resolving sources twice
if self.sourceResolved:
return S_OK()
# Only resolve files that need a transfer
toResolve = [
lfn for lfn in self.fileDict
if self.fileDict[lfn].get("Status", "") != "Failed"
]
if not toResolve:
return S_OK()
res = self.__updateMetadataCache(toResolve)
if not res['OK']:
return res
res = self.__updateReplicaCache(toResolve)
if not res['OK']:
return res
# Define the source URLs
for lfn in toResolve:
replicas = self.catalogReplicas.get(lfn, {})
if self.sourceSE not in replicas:
gLogger.warn(
"resolveSource: skipping %s - not replicas at SourceSE %s"
% (lfn, self.sourceSE))
self.__setFileParameter(lfn, 'Reason',
"No replica at SourceSE")
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
# Fix first the PFN
pfn = self.oSourceSE.getPfnForLfn(lfn).get('Value', {}).get(
'Successful', {}).get(lfn, replicas[self.sourceSE])
res = returnSingleResult(
self.oSourceSE.getPfnForProtocol(pfn,
protocol='SRM2',
withPort=True))
if not res['OK']:
gLogger.warn("resolveSource: skipping %s - %s" %
(lfn, res["Message"]))
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.setSourceSURL(lfn, res['Value'])
if not res['OK']:
gLogger.warn("resolveSource: skipping %s - %s" %
(lfn, res["Message"]))
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
toResolve = {}
for lfn in self.fileDict:
if "Source" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Source']] = lfn
if not toResolve:
return S_ERROR("No eligible Source files")
# Get metadata of the sources, to check for existance, availability and caching
res = self.oSourceSE.getFileMetadata(toResolve.keys())
if not res['OK']:
return S_ERROR("Failed to check source file metadata")
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
if re.search('File does not exist', error):
gLogger.warn(
"resolveSource: skipping %s - source file does not exists"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Source file does not exist")
self.__setFileParameter(lfn, 'Status', 'Failed')
else:
gLogger.warn(
"resolveSource: skipping %s - failed to get source metadata"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Failed to get Source metadata")
self.__setFileParameter(lfn, 'Status', 'Failed')
toStage = []
nbStagedFiles = 0
for pfn, metadata in res['Value']['Successful'].items():
lfn = toResolve[pfn]
lfnStatus = self.fileDict.get(lfn, {}).get('Status')
if metadata['Unavailable']:
gLogger.warn(
"resolveSource: skipping %s - source file unavailable" %
lfn)
self.__setFileParameter(lfn, 'Reason',
"Source file Unavailable")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif metadata['Lost']:
gLogger.warn("resolveSource: skipping %s - source file lost" %
lfn)
self.__setFileParameter(lfn, 'Reason', "Source file Lost")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif not metadata['Cached']:
if lfnStatus != 'Staging':
toStage.append(pfn)
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
gLogger.warn(
"resolveSource: skipping %s - source file size mismatch" %
lfn)
self.__setFileParameter(lfn, 'Reason', "Source size mismatch")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \
not compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ):
gLogger.warn(
"resolveSource: skipping %s - source file checksum mismatch"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Source checksum mismatch")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif lfnStatus == 'Staging':
# file that was staging is now cached
self.__setFileParameter(lfn, 'Status', 'Waiting')
nbStagedFiles += 1
# Some files were being staged
if nbStagedFiles:
self.log.info('resolveSource: %d files have been staged' %
nbStagedFiles)
# Launching staging of files not in cache
if toStage:
gLogger.warn(
"resolveSource: %s source files not cached, prestaging..." %
len(toStage))
stage = self.oSourceSE.prestageFile(toStage)
if not stage["OK"]:
gLogger.error("resolveSource: error is prestaging - %s" %
stage["Message"])
for pfn in toStage:
lfn = toResolve[pfn]
self.__setFileParameter(lfn, 'Reason', stage["Message"])
self.__setFileParameter(lfn, 'Status', 'Failed')
else:
for pfn in toStage:
lfn = toResolve[pfn]
if pfn in stage['Value']['Successful']:
self.__setFileParameter(lfn, 'Status', 'Staging')
elif pfn in stage['Value']['Failed']:
self.__setFileParameter(lfn, 'Reason',
stage['Value']['Failed'][pfn])
self.__setFileParameter(lfn, 'Status', 'Failed')
self.sourceResolved = True
return S_OK()
def resolveTarget(self):
""" find target SE eligible for submission
:param self: self reference
"""
toResolve = [
lfn for lfn in self.fileDict
if self.fileDict[lfn].get('Status') not in self.noSubmitStatus
]
if not toResolve:
return S_OK()
res = self.__updateReplicaCache(toResolve)
if not res['OK']:
return res
for lfn in toResolve:
res = self.oTargetSE.getPfnForLfn(lfn)
if not res['OK'] or lfn not in res['Value']['Successful']:
gLogger.warn(
"resolveTarget: skipping %s - failed to create target pfn"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Failed to create Target")
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
pfn = res['Value']['Successful'][lfn]
res = self.oTargetSE.getPfnForProtocol(pfn,
protocol='SRM2',
withPort=True)
if not res['OK'] or pfn not in res['Value']['Successful']:
reason = res.get(
'Message',
res.get('Value', {}).get('Failed', {}).get(pfn))
gLogger.warn("resolveTarget: skipping %s - %s" % (lfn, reason))
self.__setFileParameter(lfn, 'Reason', reason)
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
pfn = res['Value']['Successful'][pfn]
res = self.setTargetSURL(lfn, pfn)
if not res['OK']:
gLogger.warn("resolveTarget: skipping %s - %s" %
(lfn, res["Message"]))
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
toResolve = {}
for lfn in self.fileDict:
if "Target" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Target']] = lfn
if not toResolve:
return S_ERROR("No eligible Target files")
res = self.oTargetSE.exists(toResolve.keys())
if not res['OK']:
return S_ERROR("Failed to check target existence")
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
toRemove = []
for pfn, exists in res['Value']['Successful'].items():
if exists:
lfn = toResolve[pfn]
res = self.getSourceSURL(lfn)
if not res['OK']:
gLogger.warn("resolveTarget: skipping %s - target exists" %
lfn)
self.__setFileParameter(lfn, 'Reason', "Target exists")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif res['Value'] == pfn:
gLogger.warn(
"resolveTarget: skipping %s - source and target pfns are the same"
% lfn)
self.__setFileParameter(lfn, 'Reason',
"Source and Target the same")
self.__setFileParameter(lfn, 'Status', 'Failed')
else:
toRemove.append(pfn)
if toRemove:
self.oTargetSE.removeFile(toRemove)
return S_OK()
def __filesToSubmit(self):
"""
check if there is at least one file to submit
:return: S_OK if at least one file is present, S_ERROR otherwise
"""
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get('Status')
source = self.fileDict[lfn].get('Source')
target = self.fileDict[lfn].get('Target')
if lfnStatus not in self.noSubmitStatus and source and target:
return S_OK()
return S_ERROR()
def __createFTSFiles(self):
""" create LFNs file for glite-transfer-submit command
This file consists one line for each fiel to be transferred:
sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM]
:param self: self reference
"""
self.__updateMetadataCache()
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get('Status')
if lfnStatus not in self.noSubmitStatus:
cksmStr = ""
# # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself
if self.__cksmTest and self.__cksmType:
checkSum = self.catalogMetadata.get(lfn,
{}).get('Checksum')
if checkSum:
cksmStr = " %s:%s" % (self.__cksmType,
intAdlerToHex(
hexAdlerToInt(checkSum)))
ftsFile = FTSFile()
ftsFile.LFN = lfn
ftsFile.SourceSURL = self.fileDict[lfn].get('Source')
ftsFile.TargetSURL = self.fileDict[lfn].get('Target')
ftsFile.SourceSE = self.sourceSE
ftsFile.TargetSE = self.targetSE
ftsFile.Status = self.fileDict[lfn].get('Status')
ftsFile.Checksum = cksmStr
ftsFile.Size = self.catalogMetadata.get(lfn, {}).get('Size')
self.ftsFiles.append(ftsFile)
self.submittedFiles += 1
return S_OK()
def __createFTSJob(self, guid=None):
self.__createFTSFiles()
ftsJob = FTSJob()
ftsJob.RequestID = 0
ftsJob.OperationID = 0
ftsJob.SourceSE = self.sourceSE
ftsJob.TargetSE = self.targetSE
ftsJob.SourceToken = self.sourceToken
ftsJob.TargetToken = self.targetToken
ftsJob.FTSServer = self.ftsServer
if guid:
ftsJob.FTSGUID = guid
for ftsFile in self.ftsFiles:
ftsFile.Attempt += 1
ftsFile.Error = ""
ftsJob.addFile(ftsFile)
self.ftsJob = ftsJob
def __submitFTSTransfer(self):
""" create and execute glite-transfer-submit CLI command
:param self: self reference
"""
log = gLogger.getSubLogger('Submit')
self.__createFTSJob()
submit = self.ftsJob.submitFTS2(command=self.submitCommand)
if not submit["OK"]:
log.error("unable to submit FTSJob: %s" % submit["Message"])
return submit
log.info("FTSJob '%s'@'%s' has been submitted" %
(self.ftsJob.FTSGUID, self.ftsJob.FTSServer))
# # update statuses for job files
for ftsFile in self.ftsJob:
ftsFile.FTSGUID = self.ftsJob.FTSGUID
ftsFile.Status = "Submitted"
ftsFile.Attempt += 1
log.info("FTSJob '%s'@'%s' has been submitted" %
(self.ftsJob.FTSGUID, self.ftsJob.FTSServer))
self.ftsGUID = self.ftsJob.FTSGUID
return S_OK()
def __resolveFTSServer(self):
"""
resolve FTS server to use, it should be the closest one from target SE
:param self: self reference
"""
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTSServersForSites
if not self.targetSE:
return S_ERROR("Target SE not set")
res = getSitesForSE(self.targetSE)
if not res['OK'] or not res['Value']:
return S_ERROR("Could not determine target site")
targetSites = res['Value']
targetSite = ''
for targetSite in targetSites:
targetFTS = getFTSServersForSites([targetSite])
if targetFTS['OK']:
ftsTarget = targetFTS['Value'][targetSite]
if ftsTarget:
self.ftsServer = ftsTarget
return S_OK(self.ftsServer)
else:
return targetFTS
return S_ERROR('No FTS server found for %s' % targetSite)
####################################################################
#
# Methods for monitoring
#
def summary(self, untilTerminal=False, printOutput=False):
""" summary of FTS job
:param self: self reference
:param bool untilTerminal: flag to monitor FTS job to its final state
:param bool printOutput: flag to print out monitoring information to the stdout
"""
res = self.__isSummaryValid()
if not res['OK']:
return res
while not self.isTerminal:
res = self.__parseOutput(full=True)
if not res['OK']:
return res
if untilTerminal:
self.__print()
self.isRequestTerminal()
if res['Value'] or (not untilTerminal):
break
time.sleep(1)
if untilTerminal:
print ""
if printOutput and (not untilTerminal):
return self.dumpSummary(printOutput=printOutput)
return S_OK()
def monitor(self, untilTerminal=False, printOutput=False, full=True):
""" monitor FTS job
:param self: self reference
:param bool untilTerminal: flag to monitor FTS job to its final state
:param bool printOutput: flag to print out monitoring information to the stdout
"""
if not self.ftsJob:
self.resolveSource()
self.__createFTSJob(self.ftsGUID)
res = self.__isSummaryValid()
if not res['OK']:
return res
if untilTerminal:
res = self.summary(untilTerminal=untilTerminal,
printOutput=printOutput)
if not res['OK']:
return res
res = self.__parseOutput(full=full)
if not res['OK']:
return res
if untilTerminal:
self.finalize()
if printOutput:
self.dump()
return res
def dumpSummary(self, printOutput=False):
""" get FTS job summary as str
:param self: self reference
:param bool printOutput: print summary to stdout
"""
outStr = ''
for status in sorted(self.statusSummary):
if self.statusSummary[status]:
outStr = '%s\t%-10s : %-10s\n' % (
outStr, status, str(self.statusSummary[status]))
outStr = outStr.rstrip('\n')
if printOutput:
print outStr
return S_OK(outStr)
def __print(self):
""" print progress bar of FTS job completeness to stdout
:param self: self reference
"""
width = 100
bits = int((width * self.percentageComplete) / 100)
outStr = "|%s>%s| %.1f%s %s %s" % ("=" * bits, " " * (width - bits),
self.percentageComplete, "%",
self.requestStatus, " " * 10)
sys.stdout.write("%s\r" % (outStr))
sys.stdout.flush()
def dump(self):
""" print FTS job parameters and files to stdout
:param self: self reference
"""
print "%-10s : %-10s" % ("Status", self.requestStatus)
print "%-10s : %-10s" % ("Source", self.sourceSE)
print "%-10s : %-10s" % ("Target", self.targetSE)
print "%-10s : %-128s" % ("Server", self.ftsServer)
print "%-10s : %-128s" % ("GUID", self.ftsGUID)
for lfn in sorted(self.fileDict):
print "\n %-15s : %-128s" % ('LFN', lfn)
for key in ['Source', 'Target', 'Status', 'Reason', 'Duration']:
print " %-15s : %-128s" % (key,
str(self.fileDict[lfn].get(key)))
return S_OK()
def __isSummaryValid(self):
""" check validity of FTS job summary report
:param self: self reference
"""
if not self.ftsServer:
return S_ERROR("FTSServer not set")
if not self.ftsGUID:
return S_ERROR("FTSGUID not set")
return S_OK()
def __parseOutput(self, full=False):
""" execute glite-transfer-status command and parse its output
:param self: self reference
:param bool full: glite-transfer-status verbosity level, when set, collect information of files as well
"""
monitor = self.ftsJob.monitorFTS2(command=self.monitorCommand,
full=full)
if not monitor['OK']:
return monitor
self.percentageComplete = self.ftsJob.Completeness
self.requestStatus = self.ftsJob.Status
self.submitTime = self.ftsJob.SubmitTime
statusSummary = monitor['Value']
if statusSummary:
for state in statusSummary:
self.statusSummary[state] = statusSummary[state]
self.transferTime = 0
for ftsFile in self.ftsJob:
lfn = ftsFile.LFN
self.__setFileParameter(lfn, 'Status', ftsFile.Status)
self.__setFileParameter(lfn, 'Reason', ftsFile.Error)
self.__setFileParameter(lfn, 'Duration', ftsFile._duration)
targetURL = self.__getFileParameter(lfn, 'Target')
if not targetURL['OK']:
self.__setFileParameter(lfn, 'Target', ftsFile.TargetSURL)
self.transferTime += int(ftsFile._duration)
return S_OK()
####################################################################
#
# Methods for finalization
#
def finalize(self):
""" finalize FTS job
:param self: self reference
"""
self.__updateMetadataCache()
transEndTime = dateTime()
regStartTime = time.time()
res = self.getTransferStatistics()
transDict = res['Value']
res = self.__registerSuccessful(transDict['transLFNs'])
regSuc, regTotal = res['Value']
regTime = time.time() - regStartTime
if self.sourceSE and self.targetSE:
self.__sendAccounting(regSuc, regTotal, regTime, transEndTime,
transDict)
return S_OK()
def getTransferStatistics(self):
""" collect information of Transfers that can be used by Accounting
:param self: self reference
"""
transDict = {
'transTotal': len(self.fileDict),
'transLFNs': [],
'transOK': 0,
'transSize': 0
}
for lfn in self.fileDict:
if self.fileDict[lfn].get('Status') in self.successfulStates:
if self.fileDict[lfn].get('Duration', 0):
transDict['transLFNs'].append(lfn)
transDict['transOK'] += 1
if lfn in self.catalogMetadata:
transDict['transSize'] += self.catalogMetadata[
lfn].get('Size', 0)
return S_OK(transDict)
def getFailedRegistrations(self):
""" get failed registrations dict
:param self: self reference
"""
return S_OK(self.failedRegistrations)
def __registerSuccessful(self, transLFNs):
""" register successfully transferred files to the catalogs,
fill failedRegistrations dict for files that failed to register
:param self: self reference
:param list transLFNs: LFNs in FTS job
"""
self.failedRegistrations = {}
toRegister = {}
for lfn in transLFNs:
res = returnSingleResult(
self.oTargetSE.getPfnForProtocol(
self.fileDict[lfn].get('Target'),
protocol='SRM2',
withPort=False))
if not res['OK']:
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
else:
toRegister[lfn] = {'PFN': res['Value'], 'SE': self.targetSE}
if not toRegister:
return S_OK((0, 0))
res = self.__getCatalogObject()
if not res['OK']:
for lfn in toRegister:
self.failedRegistrations = toRegister
self.log.error('Failed to get Catalog Object', res['Message'])
return S_OK((0, len(toRegister)))
res = self.oCatalog.addReplica(toRegister)
if not res['OK']:
self.failedRegistrations = toRegister
self.log.error('Failed to get Catalog Object', res['Message'])
return S_OK((0, len(toRegister)))
for lfn, error in res['Value']['Failed'].items():
self.failedRegistrations[lfn] = toRegister[lfn]
self.log.error('Registration of Replica failed',
'%s : %s' % (lfn, str(error)))
return S_OK((len(res['Value']['Successful']), len(toRegister)))
def __sendAccounting(self, regSuc, regTotal, regTime, transEndTime,
transDict):
""" send accounting record
:param self: self reference
:param regSuc: number of files successfully registered
:param regTotal: number of files attepted to register
:param regTime: time stamp at the end of registration
:param transEndTime: time stamp at the end of FTS job
:param dict transDict: dict holding couters for files being transerred, their sizes and successfull transfers
"""
oAccounting = DataOperation()
oAccounting.setEndTime(transEndTime)
oAccounting.setStartTime(self.submitTime)
accountingDict = {}
accountingDict['OperationType'] = 'replicateAndRegister'
result = getProxyInfo()
if not result['OK']:
userName = '******'
else:
userName = result['Value'].get('username', 'unknown')
accountingDict['User'] = userName
accountingDict[
'Protocol'] = 'FTS' if 'fts3' not in self.ftsServer else 'FTS3'
accountingDict['RegistrationTime'] = regTime
accountingDict['RegistrationOK'] = regSuc
accountingDict['RegistrationTotal'] = regTotal
accountingDict['TransferOK'] = transDict['transOK']
accountingDict['TransferTotal'] = transDict['transTotal']
accountingDict['TransferSize'] = transDict['transSize']
accountingDict['FinalStatus'] = self.requestStatus
accountingDict['Source'] = self.sourceSE
accountingDict['Destination'] = self.targetSE
accountingDict['TransferTime'] = self.transferTime
oAccounting.setValuesFromDict(accountingDict)
self.log.verbose("Attempting to commit accounting message...")
oAccounting.commit()
self.log.verbose("...committed.")
return S_OK()
def execute(self):
IntegrityDB = RPCClient('DataManagement/DataIntegrity')
res = self.RequestDBClient.getRequest('integrity')
if not res['OK']:
gLogger.info(
"SEvsLFCAgent.execute: Failed to get request from database.")
return S_OK()
elif not res['Value']:
gLogger.info(
"SEvsLFCAgent.execute: No requests to be executed found.")
return S_OK()
requestString = res['Value']['requestString']
requestName = res['Value']['requestName']
sourceServer = res['Value']['Server']
gLogger.info("SEvsLFCAgent.execute: Obtained request %s" % requestName)
oRequest = RequestContainer(request=requestString)
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests('integrity')
if not res['OK']:
errStr = "SEvsLFCAgent.execute: Failed to obtain number of integrity subrequests."
gLogger.error(errStr, res['Message'])
return S_OK()
gLogger.info("SEvsLFCAgent.execute: Found %s sub requests." %
res['Value'])
################################################
# For all the sub-requests in the request
for ind in range(res['Value']):
gLogger.info("SEvsLFCAgent.execute: Processing sub-request %s." %
ind)
subRequestAttributes = oRequest.getSubRequestAttributes(
ind, 'integrity')['Value']
if subRequestAttributes['Status'] == 'Waiting':
subRequestFiles = oRequest.getSubRequestFiles(
ind, 'integrity')['Value']
operation = subRequestAttributes['Operation']
################################################
# If the sub-request is a lfcvsse operation
if operation == 'SEvsLFC':
gLogger.info(
"SEvsLFCAgent.execute: Attempting to execute %s sub-request."
% operation)
storageElementName = subRequestAttributes['StorageElement']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = subRequestFile['LFN']
storageElement = StorageElement(storageElementName)
res = storageElement.isValid()
if not res['OK']:
errStr = "SEvsLFCAgent.execute: Failed to instantiate destination StorageElement."
gLogger.error(errStr, storageElement)
else:
res = storageElement.getPfnForLfn(lfn)
if not res['OK']:
gLogger.info('shit bugger do something.')
else:
oNamespaceBrowser = NamespaceBrowser(
res['Value'])
# Loop over all the directories and sub-directories
while (oNamespaceBrowser.isActive()):
currentDir = oNamespaceBrowser.getActiveDir(
)
gLogger.info(
"SEvsLFCAgent.execute: Attempting to list the contents of %s."
% currentDir)
res = storageElement.listDirectory(
currentDir)
if not res['Value'][
'Successful'].has_key(
currentDir):
gLogger.error(
"SEvsLFCAgent.execute: Failed to list the directory contents.",
"%s %s" %
(currentDir,
res['Value']['Successful']
['Failed'][currentDir]))
subDirs = [currentDir]
else:
subDirs = []
files = {}
for surl, surlDict in res['Value'][
'Successful'][currentDir][
'Files'].items():
pfnRes = storageElement.getPfnForProtocol(
surl,
'SRM2',
withPort=False)
surl = pfnRes['Value']
files[surl] = surlDict
for surl, surlDict in res['Value'][
'Successful'][currentDir][
'SubDirs'].items():
pfnRes = storageElement.getPfnForProtocol(
surl,
'SRM2',
withPort=False)
surl = pfnRes['Value']
subDirs.append(surl)
#subDirs = res['Value']['Successful'][currentDir]['SubDirs']
gLogger.info(
"SEvsLFCAgent.execute: Successfully obtained %s sub-directories."
% len(subDirs))
#files = res['Value']['Successful'][currentDir]['Files']
gLogger.info(
"SEvsLFCAgent.execute: Successfully obtained %s files."
% len(files))
selectedLfns = []
lfnPfnDict = {}
pfnSize = {}
for pfn, pfnDict in files.items():
res = storageElement.getPfnPath(
pfn)
if not res['OK']:
gLogger.error(
"SEvsLFCAgent.execute: Failed to get determine LFN from pfn.",
"%s %s" %
(pfn, res['Message']))
fileMetadata = {
'Prognosis':
'NonConventionPfn',
'LFN': '',
'PFN': pfn,
'StorageElement':
storageElementName,
'Size': pfnDict['Size']
}
res = IntegrityDB.insertProblematic(
AGENT_NAME,
fileMetadata)
if res['OK']:
gLogger.info(
"SEvsLFCAgent.execute: Successfully added to IntegrityDB."
)
gLogger.error(
"Change the status in the LFC,ProcDB...."
)
else:
gLogger.error(
"Shit, f**k, bugger. Add the failover."
)
else:
lfn = res['Value']
selectedLfns.append(lfn)
lfnPfnDict[lfn] = pfn
pfnSize[pfn] = pfnDict[
'Size']
res = self.ReplicaManager.getCatalogFileMetadata(
selectedLfns)
if not res['OK']:
subDirs = [currentDir]
else:
for lfn in res['Value'][
'Failed'].keys():
gLogger.error(
"SEvsLFCAgent.execute: Failed to get metadata.",
"%s %s" %
(lfn, res['Value']
['Failed'][lfn]))
pfn = lfnPfnDict[lfn]
fileMetadata = {
'Prognosis':
'SEPfnNoLfn',
'LFN': lfn,
'PFN': pfn,
'StorageElement':
storageElementName,
'Size': pfnSize[pfn]
}
res = IntegrityDB.insertProblematic(
AGENT_NAME,
fileMetadata)
if res['OK']:
gLogger.info(
"SEvsLFCAgent.execute: Successfully added to IntegrityDB."
)
gLogger.error(
"Change the status in the LFC,ProcDB...."
)
else:
gLogger.error(
"Shit, f**k, bugger. Add the failover."
)
for lfn, lfnDict in res[
'Value'][
'Successful'].items(
):
pfn = lfnPfnDict[lfn]
storageSize = pfnSize[pfn]
catalogSize = lfnDict[
'Size']
if int(catalogSize) == int(
storageSize):
gLogger.info(
"SEvsLFCAgent.execute: Catalog and storage sizes match.",
"%s %s" %
(pfn,
storageElementName
))
gLogger.info(
"Change the status in the LFC"
)
elif int(storageSize) == 0:
gLogger.error(
"SEvsLFCAgent.execute: Physical file size is 0.",
"%s %s" %
(pfn,
storageElementName
))
fileMetadata = {
'Prognosis':
'ZeroSizePfn',
'LFN':
lfn,
'PFN':
pfn,
'StorageElement':
storageElementName
}
res = IntegrityDB.insertProblematic(
AGENT_NAME,
fileMetadata)
if res['OK']:
gLogger.info(
"SEvsLFCAgent.execute: Successfully added to IntegrityDB."
)
gLogger.error(
"Change the status in the LFC,ProcDB...."
)
else:
gLogger.error(
"Shit, f**k, bugger. Add the failover."
)
else:
gLogger.error(
"SEvsLFCAgent.execute: Catalog and storage size mis-match.",
"%s %s" %
(pfn,
storageElementName
))
fileMetadata = {
'Prognosis':
'PfnSizeMismatch',
'LFN':
lfn,
'PFN':
pfn,
'StorageElement':
storageElementName
}
res = IntegrityDB.insertProblematic(
AGENT_NAME,
fileMetadata)
if res['OK']:
gLogger.info(
"SEvsLFCAgent.execute: Successfully added to IntegrityDB."
)
gLogger.error(
"Change the status in the LFC,ProcDB...."
)
else:
gLogger.error(
"Shit, f**k, bugger. Add the failover."
)
res = self.ReplicaManager.getCatalogReplicas(
selectedLfns)
if not res['OK']:
subDirs = [currentDir]
else:
for lfn in res['Value'][
'Failed'].keys():
gLogger.error(
"SEvsLFCAgent.execute: Failed to get replica information.",
"%s %s" %
(lfn, res['Value']
['Failed'][lfn]))
pfn = lfnPfnDict[lfn]
fileMetadata = {
'Prognosis':
'PfnNoReplica',
'LFN': lfn,
'PFN': pfn,
'StorageElement':
storageElementName,
'Size':
pfnSize[pfn]
}
res = IntegrityDB.insertProblematic(
AGENT_NAME,
fileMetadata)
if res['OK']:
gLogger.info(
"SEvsLFCAgent.execute: Successfully added to IntegrityDB."
)
gLogger.error(
"Change the status in the LFC,ProcDB...."
)
else:
gLogger.error(
"Shit, f**k, bugger. Add the failover."
)
for lfn, repDict in res[
'Value'][
'Successful'].items(
):
pfn = lfnPfnDict[lfn]
registeredPfns = repDict.values(
)
if not pfn in registeredPfns:
gLogger.error(
"SEvsLFCAgent.execute: SE PFN not registered.",
"%s %s" %
(lfn, pfn))
fileMetadata = {
'Prognosis':
'PfnNoReplica',
'LFN':
lfn,
'PFN':
pfn,
'StorageElement':
storageElementName
}
res = IntegrityDB.insertProblematic(
AGENT_NAME,
fileMetadata)
if res['OK']:
gLogger.info(
"SEvsLFCAgent.execute: Successfully added to IntegrityDB."
)
gLogger.error(
"Change the status in the LFC,ProcDB...."
)
else:
gLogger.error(
"Shit, f**k, bugger. Add the failover."
)
else:
gLogger.info(
"SEvsLFCAgent.execute: SE Pfn verified.",
pfn)
oNamespaceBrowser.updateDirs(subDirs)
oRequest.setSubRequestFileAttributeValue(
ind, 'integrity', lfn, 'Status',
'Done')
################################################
# If the sub-request is none of the above types
else:
gLogger.info(
"SEvsLFCAgent.execute: Operation not supported.",
operation)
################################################
# Determine whether there are any active files
if oRequest.isSubRequestEmpty(ind, 'integrity')['Value']:
oRequest.setSubRequestStatus(ind, 'integrity', 'Done')
################################################
# If the sub-request is already in terminal state
else:
gLogger.info(
"SEvsLFCAgent.execute: Sub-request %s is status '%s' and not to be executed."
% (ind, subRequestAttributes['Status']))
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
res = self.RequestDBClient.updateRequest(requestName, requestString,
sourceServer)
return S_OK()
class FTSRequest:
def __init__(self):
self.gridEnv = '/afs/cern.ch/project/gd/LCG-share/3.2.8-0/etc/profile.d/grid-env'
self.finalStates = [
'Canceled', 'Failed', 'Hold', 'Finished', 'FinishedDirty'
]
self.failedStates = ['Canceled', 'Failed', 'Hold', 'FinishedDirty']
self.successfulStates = ['Finished', 'Done']
self.fileStates = [
'Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed',
'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting'
]
self.newlyCompletedFiles = []
self.newlyFailedFiles = []
self.statusSummary = {}
self.requestStatus = 'Unknown'
self.fileDict = {}
self.catalogReplicas = {}
self.catalogMetadata = {}
self.oCatalog = None
self.submitTime = ''
self.ftsGUID = ''
self.ftsServer = ''
self.priority = 3
self.isTerminal = False
self.percentageComplete = 0.0
self.sourceSE = ''
self.sourceValid = False
self.sourceToken = ''
self.targetSE = ''
self.targetValid = False
self.targetToken = ''
self.dumpStr = ''
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE(self, se):
if se == self.targetSE:
return S_ERROR("SourceSE is TargetSE")
self.sourceSE = se
self.oSourceSE = StorageElement(self.sourceSE)
return self.__checkSourceSE()
def getSourceSE(self):
if not self.sourceSE:
return S_ERROR("Source SE not defined")
return S_OK(self.sourceSE)
def setSourceToken(self, token):
self.sourceToken = token
return S_OK()
def getSourceToken(self):
if not self.sourceToken:
return S_ERROR("Source token not defined")
return S_OK(self.sourceToken)
def __checkSourceSE(self):
if not self.sourceSE:
return S_ERROR("SourceSE not set")
res = self.oSourceSE.isValid('Read')
if not res['OK']:
return S_ERROR("SourceSE not available for reading")
res = self.__getSESpaceToken(self.oSourceSE)
if not res['OK']:
gLogger.error(
"FTSRequest failed to get SRM Space Token for SourceSE",
res['Message'])
return S_ERROR("SourceSE does not support FTS transfers")
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE(self, se):
if se == self.sourceSE:
return S_ERROR("TargetSE is SourceSE")
self.targetSE = se
self.oTargetSE = StorageElement(self.targetSE)
return self.__checkTargetSE()
def getTargetSE(self):
if not self.targetSE:
return S_ERROR("Target SE not defined")
return S_OK(self.targetSE)
def setTargetToken(self, token):
self.targetToken = token
return S_OK()
def getTargetToken(self):
if not self.targetToken:
return S_ERROR("Target token not defined")
return S_OK(self.targetToken)
def __checkTargetSE(self):
if not self.targetSE:
return S_ERROR("TargetSE not set")
res = self.oTargetSE.isValid('Write')
if not res['OK']:
return S_ERROR("TargetSE not available for writing")
res = self.__getSESpaceToken(self.oTargetSE)
if not res['OK']:
gLogger.error(
"FTSRequest failed to get SRM Space Token for TargetSE",
res['Message'])
return S_ERROR("SourceSE does not support FTS transfers")
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
def __getSESpaceToken(self, oSE):
res = oSE.getStorageParameters("SRM2")
if not res['OK']:
return res
return S_OK(res['Value'].get('SpaceToken'))
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID(self, guid):
if not checkGuid(guid):
return S_ERROR("Incorrect GUID format")
self.ftsGUID = guid
return S_OK()
def getFTSGUID(self):
if not self.ftsGUID:
return S_ERROR("FTSGUID not set")
return S_OK(self.ftsGUID)
def setFTSServer(self, server):
self.ftsServer = server
return S_OK()
def getFTSServer(self):
if not self.ftsServer:
return S_ERROR("FTSServer not set")
return S_OK(self.ftsServer)
def setPriority(self, priority):
if not type(priority) in [types.IntType, types.LongType]:
return S_ERROR("Priority must be integer")
if priority < 0:
priority = 0
elif priority > 5:
priority = 5
self.priority = priority
return S_OK(self.priority)
def getPriority(self):
return S_OK(self.priority)
def getPercentageComplete(self):
completedFiles = 0
totalFiles = 0
for state in (self.statusSummary.keys()):
if state in self.successfulStates:
completedFiles += self.statusSummary[state]
totalFiles += self.statusSummary[state]
self.percentageComplete = (float(completedFiles) *
100.0) / float(totalFiles)
return S_OK(self.percentageComplete)
def isRequestTerminal(self):
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK(self.isTerminal)
def getStatus(self):
return S_OK(self.requestStatus)
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN(self, lfn):
if not self.fileDict.has_key(lfn):
self.fileDict[lfn] = {}
return S_OK()
def setSourceSURL(self, lfn, surl):
target = self.fileDict[lfn].get('Target')
if target == surl:
return S_ERROR("Source and target the same")
self.__setFileParameter(lfn, 'Source', surl)
return S_OK()
def getSourceSURL(self, lfn):
return self.__getFileParameter(lfn, 'Source')
def setTargetSURL(self, lfn, surl):
source = self.fileDict[lfn].get('Source')
if source == surl:
return S_ERROR("Source and target the same")
self.__setFileParameter(lfn, 'Target', surl)
return S_OK()
def getTargetSURL(self, lfn):
return self.__getFileParameter(lfn, 'Target')
def getFailReason(self, lfn):
return self.__getFileParameter(lfn, 'Reason')
def getRetries(self, lfn):
return self.__getFileParameter(lfn, 'Retries')
def getTransferTime(self, lfn):
return self.__getFileParameter(lfn, 'Duration')
def getFailed(self):
failed = []
for lfn in self.fileDict.keys():
status = self.fileDict[lfn].get('Status', '')
if status in self.failedStates:
failed.append(lfn)
return S_OK(failed)
def getDone(self):
done = []
for lfn in self.fileDict.keys():
status = self.fileDict[lfn].get('Status', '')
if status in self.successfulStates:
done.append(lfn)
return S_OK(done)
def __setFileParameter(self, lfn, paramName, paramValue):
self.setLFN(lfn)
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter(self, lfn, paramName):
if not self.fileDict.has_key(lfn):
return S_ERROR("Supplied file not set")
if not self.fileDict[lfn].has_key(paramName):
return S_ERROR("%s not set for file" % paramName)
return S_OK(self.fileDict[lfn][paramName])
####################################################################
#
# Methods for submission
#
def submit(self, monitor=False, printOutput=True):
res = self.__isSubmissionValid()
if not res['OK']:
return res
res = self.__createSURLPairFile()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = {'ftsGUID': self.ftsGUID, 'ftsServer': self.ftsServer}
print "Submitted %s @ %s" % (self.ftsGUID, self.ftsServer)
if monitor:
self.monitor(untilTerminal=True, printOutput=printOutput)
return S_OK(resDict)
def __isSubmissionValid(self):
if not self.fileDict:
return S_ERROR("No files set")
if not self.sourceValid:
return S_ERROR("SourceSE not valid")
if not self.targetValid:
return S_ERROR("TargetSE not valid")
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR("FTSServer not valid")
self.__resolveSource()
self.__resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR("No files to submit")
return S_OK()
def __getCatalogObject(self):
try:
if not self.oCatalog:
self.oCatalog = CatalogInterface()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache(self, lfns=[], overwrite=False):
if not lfns:
lfns = self.fileDict.keys()
toUpdate = []
for lfn in lfns:
if (not lfn in self.catalogReplicas.keys()) or overwrite:
toUpdate.append(lfn)
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogReplicas(toUpdate)
if not res['OK']:
return S_ERROR("Failed to update replica cache", res['Message'])
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache(self, lfns=[], overwrite=False):
if not lfns:
lfns = self.fileDict.keys()
toUpdate = []
for lfn in lfns:
if (not lfn in self.catalogMetadata.keys()) or overwrite:
toUpdate.append(lfn)
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getCatalogFileMetadata(toUpdate)
if not res['OK']:
return S_ERROR("Failed to get source catalog metadata",
res['Message'])
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def __resolveSource(self):
toResolve = []
for lfn in self.fileDict.keys():
if (not self.fileDict[lfn].has_key('Source')) and (
self.fileDict[lfn].get('Status') != 'Failed'):
toResolve.append(lfn)
if not toResolve:
return S_OK()
res = self.__updateMetadataCache(toResolve)
if not res['OK']:
return res
res = self.__updateReplicaCache(toResolve)
if not res['OK']:
return res
for lfn in toResolve:
if self.fileDict[lfn].get('Status') == 'Failed':
continue
replicas = self.catalogReplicas.get(lfn, {})
if not replicas.has_key(self.sourceSE):
self.__setFileParameter(lfn, 'Reason',
"No replica at SourceSE")
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.oSourceSE.getPfnForProtocol(replicas[self.sourceSE],
'SRM2',
withPort=True)
if not res['OK']:
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.setSourceSURL(lfn, res['Value'])
if not res['OK']:
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
toResolve = {}
for lfn in self.fileDict.keys():
if self.fileDict[lfn].has_key('Source'):
toResolve[self.fileDict[lfn]['Source']] = lfn
if not toResolve:
return S_ERROR("No eligible Source files")
res = self.oSourceSE.getFileMetadata(toResolve.keys())
if not res['OK']:
return S_ERROR("Failed to check source file metadata")
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
if re.search('File does not exist', error):
self.__setFileParameter(lfn, 'Reason',
"Source file does not exist")
self.__setFileParameter(lfn, 'Status', 'Failed')
else:
self.__setFileParameter(lfn, 'Reason',
"Failed to get Source metadata")
self.__setFileParameter(lfn, 'Status', 'Failed')
for pfn, metadata in res['Value']['Successful'].items():
lfn = toResolve[pfn]
if metadata['Unavailable']:
self.__setFileParameter(lfn, 'Reason',
"Source file Unavailable")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif metadata['Lost']:
self.__setFileParameter(lfn, 'Reason', "Source file Lost")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif not metadata['Cached']:
self.__setFileParameter(lfn, 'Reason',
"Source file not Cached")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
self.__setFileParameter(lfn, 'Reason', "Source size mismatch")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif self.catalogMetadata[lfn]['Checksum'] and metadata[
'Checksum'] and not (compareAdler(
metadata['Checksum'],
self.catalogMetadata[lfn]['Checksum'])):
self.__setFileParameter(lfn, 'Reason',
"Source checksum mismatch")
self.__setFileParameter(lfn, 'Status', 'Failed')
return S_OK()
def __resolveTarget(self):
toResolve = []
for lfn in self.fileDict.keys():
if not self.fileDict[lfn].has_key('Target') and (
self.fileDict[lfn].get('Status') != 'Failed'):
toResolve.append(lfn)
if not toResolve:
return S_OK()
res = self.__updateReplicaCache(toResolve)
if not res['OK']:
return res
atTarget = []
for lfn in sortList(toResolve):
if self.fileDict[lfn].get('Status') == 'Failed':
continue
replicas = self.catalogReplicas.get(lfn, {})
if replicas.has_key(self.targetSE):
self.__setFileParameter(lfn, 'Reason',
"File already at Target")
self.__setFileParameter(lfn, 'Status', 'Done')
atTarget.append(lfn)
for lfn in toResolve:
if (self.fileDict[lfn].get('Status') == 'Failed') or (lfn
in atTarget):
continue
res = self.oTargetSE.getPfnForLfn(lfn)
if not res['OK']:
self.__setFileParameter(lfn, 'Reason',
"Failed to create Target")
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.oTargetSE.getPfnForProtocol(res['Value'],
'SRM2',
withPort=True)
if not res['OK']:
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
res = self.setTargetSURL(lfn, res['Value'])
if not res['OK']:
self.__setFileParameter(lfn, 'Reason', res['Message'])
self.__setFileParameter(lfn, 'Status', 'Failed')
continue
toResolve = {}
for lfn in self.fileDict.keys():
if self.fileDict[lfn].has_key('Target'):
toResolve[self.fileDict[lfn]['Target']] = lfn
if not toResolve:
return S_ERROR("No eligible Target files")
res = self.oTargetSE.exists(toResolve.keys())
if not res['OK']:
return S_ERROR("Failed to check target existence")
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
self.__setFileParameter(lfn, 'Reason', error)
self.__setFileParameter(lfn, 'Status', 'Failed')
toRemove = []
for pfn, exists in res['Value']['Successful'].items():
if exists:
lfn = toResolve[pfn]
res = self.getSourceSURL(lfn)
if not res['OK']:
self.__setFileParameter(lfn, 'Reason', "Target exists")
self.__setFileParameter(lfn, 'Status', 'Failed')
elif res['Value'] == pfn:
self.__setFileParameter(lfn, 'Reason',
"Source and Target the same")
self.__setFileParameter(lfn, 'Status', 'Failed')
else:
toRemove.append(pfn)
if toRemove:
self.oTargetSE.removeFile(toRemove)
return S_OK()
def __filesToSubmit(self):
for lfn in self.fileDict.keys():
lfnStatus = self.fileDict[lfn].get('Status')
source = self.fileDict[lfn].get('Source')
target = self.fileDict[lfn].get('Target')
if (lfnStatus != 'Failed') and (lfnStatus !=
'Done') and source and target:
return S_OK()
return S_ERROR()
def __createSURLPairFile(self):
fd, fileName = tempfile.mkstemp()
surlFile = os.fdopen(fd, 'w')
for lfn in self.fileDict.keys():
lfnStatus = self.fileDict[lfn].get('Status')
source = self.fileDict[lfn].get('Source')
target = self.fileDict[lfn].get('Target')
if (lfnStatus != 'Failed') and (lfnStatus !=
'Done') and source and target:
surlString = '%s %s\n' % (source, target)
surlFile.write(surlString)
surlFile.close()
self.surlFile = fileName
return S_OK()
def __submitFTSTransfer(self):
comm = [
'glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile,
'-o'
]
if self.targetToken:
comm.append('-t')
comm.append(self.targetToken)
if self.sourceToken:
comm.append('-S')
comm.append(self.sourceToken)
res = executeGridCommand('', comm, self.gridEnv)
os.remove(self.surlFile)
if not res['OK']:
return res
returnCode, output, errStr = res['Value']
if not returnCode == 0:
return S_ERROR(errStr)
guid = output.replace('\n', '')
if not checkGuid(guid):
return S_ERROR('Wrong GUID format returned')
self.ftsGUID = guid
#if self.priority != 3:
# comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)]
# executeGridCommand('',comm,self.gridEnv)
return res
def __resolveFTSServer(self):
if not self.sourceSE:
return S_ERROR("Source SE not set")
if not self.targetSE:
return S_ERROR("Target SE not set")
res = getSitesForSE(self.sourceSE, 'LCG')
if not res['OK'] or not res['Value']:
return S_ERROR("Could not determine source site")
sourceSite = res['Value'][0]
res = getSitesForSE(self.targetSE, 'LCG')
if not res['OK'] or not res['Value']:
return S_ERROR("Could not determine target site")
targetSite = res['Value'][0]
if (sourceSite == 'LCG.CERN.ch') or (targetSite == 'LCG.CERN.ch'):
ep = 'LCG.CERN.ch'
else:
# Target site FTS server should be used
ep = targetSite
try:
configPath = '/Resources/FTSEndpoints/%s' % ep
endpointURL = gConfig.getValue(configPath)
if not endpointURL:
errStr = "FTSRequest.__resolveFTSEndpoint: Failed to find FTS endpoint, check CS entry for '%s'." % ep
return S_ERROR(errStr)
self.ftsServer = endpointURL
return S_OK(endpointURL)
except Exception, x:
return S_ERROR(
'FTSRequest.__resolveFTSEndpoint: Failed to obtain endpoint details from CS'
)
def execute( self ):
IntegrityDB = RPCClient( 'DataManagement/DataIntegrity' )
res = self.RequestDBClient.getRequest( 'integrity' )
if not res['OK']:
gLogger.info( "SEvsLFCAgent.execute: Failed to get request from database." )
return S_OK()
elif not res['Value']:
gLogger.info( "SEvsLFCAgent.execute: No requests to be executed found." )
return S_OK()
requestString = res['Value']['requestString']
requestName = res['Value']['requestName']
sourceServer = res['Value']['Server']
gLogger.info( "SEvsLFCAgent.execute: Obtained request %s" % requestName )
oRequest = RequestContainer( request = requestString )
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests( 'integrity' )
if not res['OK']:
errStr = "SEvsLFCAgent.execute: Failed to obtain number of integrity subrequests."
gLogger.error( errStr, res['Message'] )
return S_OK()
gLogger.info( "SEvsLFCAgent.execute: Found %s sub requests." % res['Value'] )
################################################
# For all the sub-requests in the request
for ind in range( res['Value'] ):
gLogger.info( "SEvsLFCAgent.execute: Processing sub-request %s." % ind )
subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'integrity' )['Value']
if subRequestAttributes['Status'] == 'Waiting':
subRequestFiles = oRequest.getSubRequestFiles( ind, 'integrity' )['Value']
operation = subRequestAttributes['Operation']
################################################
# If the sub-request is a lfcvsse operation
if operation == 'SEvsLFC':
gLogger.info( "SEvsLFCAgent.execute: Attempting to execute %s sub-request." % operation )
storageElementName = subRequestAttributes['StorageElement']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = subRequestFile['LFN']
storageElement = StorageElement( storageElementName )
res = storageElement.isValid()
if not res['OK']:
errStr = "SEvsLFCAgent.execute: Failed to instantiate destination StorageElement."
gLogger.error( errStr, storageElement )
else:
res = storageElement.getPfnForLfn( lfn )
if not res['OK']:
gLogger.info( 'shit bugger do something.' )
else:
oNamespaceBrowser = NamespaceBrowser( res['Value'] )
# Loop over all the directories and sub-directories
while ( oNamespaceBrowser.isActive() ):
currentDir = oNamespaceBrowser.getActiveDir()
gLogger.info( "SEvsLFCAgent.execute: Attempting to list the contents of %s." % currentDir )
res = storageElement.listDirectory( currentDir )
if not res['Value']['Successful'].has_key( currentDir ):
gLogger.error( "SEvsLFCAgent.execute: Failed to list the directory contents.", "%s %s" % ( currentDir, res['Value']['Successful']['Failed'][currentDir] ) )
subDirs = [currentDir]
else:
subDirs = []
files = {}
for surl, surlDict in res['Value']['Successful'][currentDir]['Files'].items():
pfnRes = storageElement.getPfnForProtocol( surl, 'SRM2', withPort = False )
surl = pfnRes['Value']
files[surl] = surlDict
for surl, surlDict in res['Value']['Successful'][currentDir]['SubDirs'].items():
pfnRes = storageElement.getPfnForProtocol( surl, 'SRM2', withPort = False )
surl = pfnRes['Value']
subDirs.append( surl )
#subDirs = res['Value']['Successful'][currentDir]['SubDirs']
gLogger.info( "SEvsLFCAgent.execute: Successfully obtained %s sub-directories." % len( subDirs ) )
#files = res['Value']['Successful'][currentDir]['Files']
gLogger.info( "SEvsLFCAgent.execute: Successfully obtained %s files." % len( files ) )
selectedLfns = []
lfnPfnDict = {}
pfnSize = {}
for pfn, pfnDict in files.items():
res = storageElement.getPfnPath( pfn )
if not res['OK']:
gLogger.error( "SEvsLFCAgent.execute: Failed to get determine LFN from pfn.", "%s %s" % ( pfn, res['Message'] ) )
fileMetadata = {'Prognosis':'NonConventionPfn', 'LFN':'', 'PFN':pfn, 'StorageElement':storageElementName, 'Size':pfnDict['Size']}
res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
if res['OK']:
gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
gLogger.error( "Change the status in the LFC,ProcDB...." )
else:
gLogger.error( "Shit, f**k, bugger. Add the failover." )
else:
lfn = res['Value']
selectedLfns.append( lfn )
lfnPfnDict[lfn] = pfn
pfnSize[pfn] = pfnDict['Size']
res = self.ReplicaManager.getCatalogFileMetadata( selectedLfns )
if not res['OK']:
subDirs = [currentDir]
else:
for lfn in res['Value']['Failed'].keys():
gLogger.error( "SEvsLFCAgent.execute: Failed to get metadata.", "%s %s" % ( lfn, res['Value']['Failed'][lfn] ) )
pfn = lfnPfnDict[lfn]
fileMetadata = {'Prognosis':'SEPfnNoLfn', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName, 'Size':pfnSize[pfn]}
res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
if res['OK']:
gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
gLogger.error( "Change the status in the LFC,ProcDB...." )
else:
gLogger.error( "Shit, f**k, bugger. Add the failover." )
for lfn, lfnDict in res['Value']['Successful'].items():
pfn = lfnPfnDict[lfn]
storageSize = pfnSize[pfn]
catalogSize = lfnDict['Size']
if int( catalogSize ) == int( storageSize ):
gLogger.info( "SEvsLFCAgent.execute: Catalog and storage sizes match.", "%s %s" % ( pfn, storageElementName ) )
gLogger.info( "Change the status in the LFC" )
elif int( storageSize ) == 0:
gLogger.error( "SEvsLFCAgent.execute: Physical file size is 0.", "%s %s" % ( pfn, storageElementName ) )
fileMetadata = {'Prognosis':'ZeroSizePfn', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName}
res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
if res['OK']:
gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
gLogger.error( "Change the status in the LFC,ProcDB...." )
else:
gLogger.error( "Shit, f**k, bugger. Add the failover." )
else:
gLogger.error( "SEvsLFCAgent.execute: Catalog and storage size mis-match.", "%s %s" % ( pfn, storageElementName ) )
fileMetadata = {'Prognosis':'PfnSizeMismatch', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName}
res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
if res['OK']:
gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
gLogger.error( "Change the status in the LFC,ProcDB...." )
else:
gLogger.error( "Shit, f**k, bugger. Add the failover." )
res = self.ReplicaManager.getCatalogReplicas( selectedLfns )
if not res['OK']:
subDirs = [currentDir]
else:
for lfn in res['Value']['Failed'].keys():
gLogger.error( "SEvsLFCAgent.execute: Failed to get replica information.", "%s %s" % ( lfn, res['Value']['Failed'][lfn] ) )
pfn = lfnPfnDict[lfn]
fileMetadata = {'Prognosis':'PfnNoReplica', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName, 'Size':pfnSize[pfn]}
res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
if res['OK']:
gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
gLogger.error( "Change the status in the LFC,ProcDB...." )
else:
gLogger.error( "Shit, f**k, bugger. Add the failover." )
for lfn, repDict in res['Value']['Successful'].items():
pfn = lfnPfnDict[lfn]
registeredPfns = repDict.values()
if not pfn in registeredPfns:
gLogger.error( "SEvsLFCAgent.execute: SE PFN not registered.", "%s %s" % ( lfn, pfn ) )
fileMetadata = {'Prognosis':'PfnNoReplica', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName}
res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
if res['OK']:
gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
gLogger.error( "Change the status in the LFC,ProcDB...." )
else:
gLogger.error( "Shit, f**k, bugger. Add the failover." )
else:
gLogger.info( "SEvsLFCAgent.execute: SE Pfn verified.", pfn )
oNamespaceBrowser.updateDirs( subDirs )
oRequest.setSubRequestFileAttributeValue( ind, 'integrity', lfn, 'Status', 'Done' )
################################################
# If the sub-request is none of the above types
else:
gLogger.info( "SEvsLFCAgent.execute: Operation not supported.", operation )
################################################
# Determine whether there are any active files
if oRequest.isSubRequestEmpty( ind, 'integrity' )['Value']:
oRequest.setSubRequestStatus( ind, 'integrity', 'Done' )
################################################
# If the sub-request is already in terminal state
else:
gLogger.info( "SEvsLFCAgent.execute: Sub-request %s is status '%s' and not to be executed." % ( ind, subRequestAttributes['Status'] ) )
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
res = self.RequestDBClient.updateRequest( requestName, requestString, sourceServer )
return S_OK()
def __getStorageDirectoryContents( self, lfnDir, storageElement ):
""" Obtians the contents of the supplied directory on the storage
"""
gLogger.info( 'Obtaining the contents for %s directories at %s' % ( len( lfnDir ), storageElement ) )
se = StorageElement( storageElement )
res = se.getPfnForLfn( lfnDir )
if not res['OK']:
gLogger.error( "Failed to get PFNs for directories", res['Message'] )
return res
for directory, error in res['Value']['Failed'].items():
gLogger.error( 'Failed to obtain directory PFN from LFNs', '%s %s' % ( directory, error ) )
if res['Value']['Failed']:
return S_ERROR( 'Failed to obtain directory PFN from LFNs' )
storageDirectories = res['Value']['Successful'].values()
res = se.exists( storageDirectories )
if not res['OK']:
gLogger.error( "Failed to obtain existance of directories", res['Message'] )
return res
for directory, error in res['Value']['Failed'].items():
gLogger.error( 'Failed to determine existance of directory', '%s %s' % ( directory, error ) )
if res['Value']['Failed']:
return S_ERROR( 'Failed to determine existance of directory' )
directoryExists = res['Value']['Successful']
activeDirs = []
for directory in sortList( directoryExists.keys() ):
exists = directoryExists[directory]
if exists:
activeDirs.append( directory )
allFiles = {}
while len( activeDirs ) > 0:
currentDir = activeDirs[0]
res = se.listDirectory( currentDir )
activeDirs.remove( currentDir )
if not res['OK']:
gLogger.error( 'Failed to get directory contents', res['Message'] )
return res
elif res['Value']['Failed'].has_key( currentDir ):
gLogger.error( 'Failed to get directory contents', '%s %s' % ( currentDir, res['Value']['Failed'][currentDir] ) )
return S_ERROR( res['Value']['Failed'][currentDir] )
else:
dirContents = res['Value']['Successful'][currentDir]
activeDirs.extend( dirContents['SubDirs'] )
fileMetadata = dirContents['Files']
# RF_NOTE This ugly trick is needed because se.getPfnPath does not follow the Successful/Failed convention
# res = { "Successful" : {}, "Failed" : {} }
# for pfn in fileMetadata:
# inRes = se.getPfnPath( pfn )
# if inRes["OK"]:
# res["Successful"][pfn] = inRes["Value"]
# else:
# res["Failed"][pfn] = inRes["Message"]
res = se.getLfnForPfn( fileMetadata.keys() )
if not res['OK']:
gLogger.error( 'Failed to get directory content LFNs', res['Message'] )
return res
for pfn, error in res['Value']['Failed'].items():
gLogger.error( "Failed to get LFN for PFN", "%s %s" % ( pfn, error ) )
if res['Value']['Failed']:
return S_ERROR( "Failed to get LFNs for PFNs" )
pfnLfns = res['Value']['Successful']
for pfn, lfn in pfnLfns.items():
fileMetadata[pfn]['LFN'] = lfn
allFiles.update( fileMetadata )
zeroSizeFiles = []
lostFiles = []
unavailableFiles = []
for pfn in sortList( allFiles.keys() ):
if os.path.basename( pfn ) == 'dirac_directory':
allFiles.pop( pfn )
else:
metadata = allFiles[pfn]
if metadata['Size'] == 0:
zeroSizeFiles.append( ( metadata['LFN'], pfn, storageElement, 'PFNZeroSize' ) )
# if metadata['Lost']:
# lostFiles.append((metadata['LFN'],pfn,storageElement,'PFNLost'))
# if metadata['Unavailable']:
# unavailableFiles.append((metadata['LFN'],pfn,storageElement,'PFNUnavailable'))
if zeroSizeFiles:
self.__reportProblematicReplicas( zeroSizeFiles, storageElement, 'PFNZeroSize' )
if lostFiles:
self.__reportProblematicReplicas( lostFiles, storageElement, 'PFNLost' )
if unavailableFiles:
self.__reportProblematicReplicas( unavailableFiles, storageElement, 'PFNUnavailable' )
gLogger.info( 'Obtained at total of %s files for directories at %s' % ( len( allFiles ), storageElement ) )
return S_OK( allFiles )
