def ftsSchedule( self, requestID, operationID, opFileList ):
""" schedule lfn for FTS job
:param int requestID: RequestDB.Request.RequestID
:param int operationID: RequestDB.Operation.OperationID
:param list opFileList: list of tuples ( File.toJSON()['Value'], sourcesList, targetList )
"""
# Check whether there are duplicates
fList = []
for fileJSON, sourceSEs, targetSEs in opFileList:
fTuple = ( json.loads( fileJSON ), sourceSEs, targetSEs )
if fTuple not in fList:
fList.append( fTuple )
else:
self.log.warn( 'File list for FTS scheduling has duplicates, fix it:\n', fTuple )
fileIDs = [int( fileJSON.get( 'FileID', 0 ) ) for fileJSON, _sourceSEs, _targetSEs in fList ]
res = self.ftsManager.cleanUpFTSFiles( requestID, fileIDs )
if not res['OK']:
self.log.error( "Failed ftsSchedule", "%s" % res['Message'] )
return S_ERROR( "ftsSchedule: %s" % res['Message'] )
ftsFiles = []
# # this will be returned on success
result = { "Successful": [], "Failed": {} }
for fileJSON, sourceSEs, targetSEs in fList:
lfn = fileJSON.get( "LFN", "" )
size = int( fileJSON.get( "Size", 0 ) )
fileID = int( fileJSON.get( "FileID", 0 ) )
opID = int( fileJSON.get( "OperationID", 0 ) )
self.log.verbose( "ftsSchedule: LFN=%s FileID=%s OperationID=%s sources=%s targets=%s" % ( lfn, fileID, opID,
sourceSEs,
targetSEs ) )
res = self.dataManager.getActiveReplicas( lfn )
if not res['OK']:
self.log.error( "Failed ftsSchedule", "%s" % res['Message'] )
result["Failed"][fileID] = res['Message']
continue
replicaDict = res['Value']
if lfn in replicaDict["Failed"] and lfn not in replicaDict["Successful"]:
result["Failed"][fileID] = "no active replicas found"
continue
replicaDict = replicaDict["Successful"].get( lfn, {} )
# # use valid replicas only
validReplicasDict = dict( [ ( se, pfn ) for se, pfn in replicaDict.items() if se in sourceSEs ] )
if not validReplicasDict:
self.log.warn( "No active replicas found in sources" )
result["Failed"][fileID] = "no active replicas found in sources"
continue
tree = self.ftsManager.getReplicationTree( sourceSEs, targetSEs, size )
if not tree['OK']:
self.log.error( "Failed ftsSchedule", "%s cannot be scheduled: %s" % ( lfn, tree['Message'] ) )
result["Failed"][fileID] = tree['Message']
continue
tree = tree['Value']
self.log.verbose( "LFN=%s tree=%s" % ( lfn, tree ) )
treeBranches = []
printed = False
for repDict in tree.values():
if repDict in treeBranches:
if not printed:
self.log.warn( 'Duplicate tree branch', str( tree ) )
printed = True
else:
treeBranches.append( repDict )
for repDict in treeBranches:
self.log.verbose( "Strategy=%s Ancestor=%s SourceSE=%s TargetSE=%s" % ( repDict["Strategy"],
repDict["Ancestor"],
repDict["SourceSE"],
repDict["TargetSE"] ) )
transferSURLs = self._getTransferURLs( lfn, repDict, sourceSEs, validReplicasDict )
if not transferSURLs['OK']:
result["Failed"][fileID] = transferSURLs['Message']
continue
sourceSURL, targetSURL, fileStatus = transferSURLs['Value']
if sourceSURL == targetSURL:
result["Failed"][fileID] = "sourceSURL equals to targetSURL for %s" % lfn
continue
self.log.verbose( "sourceURL=%s targetURL=%s FTSFile.Status=%s" % ( sourceSURL, targetSURL, fileStatus ) )
ftsFile = FTSFile()
for key in ( "LFN", "FileID", "OperationID", "Checksum", "ChecksumType", "Size" ):
if fileJSON.get( key ):
setattr( ftsFile, key, fileJSON.get( key ) )
ftsFile.RequestID = requestID
ftsFile.OperationID = operationID
ftsFile.SourceSURL = sourceSURL
ftsFile.TargetSURL = targetSURL
ftsFile.SourceSE = repDict["SourceSE"]
ftsFile.TargetSE = repDict["TargetSE"]
ftsFile.Status = fileStatus
ftsFiles.append( ftsFile )
if not ftsFiles:
self.log.info( "ftsSchedule: no FTSFiles to put for request %d" % requestID )
return S_OK( result )
ftsFilesJSONList = [ftsFile.toJSON()['Value'] for ftsFile in ftsFiles]
res = self.ftsManager.putFTSFileList( ftsFilesJSONList )
if not res['OK']:
self.log.error( "Failed ftsSchedule", "%s" % res['Message'] )
return S_ERROR( "ftsSchedule: %s" % res['Message'] )
result['Successful'] += [ fileID for fileID in fileIDs if fileID not in result['Failed']]
# # if we land here some files have been properly scheduled
return S_OK( result )
def ftsSchedule( self, requestID, operationID, opFileList ):
""" schedule lfn for FTS job
:param int requestID: RequestDB.Request.RequestID
:param int operationID: RequestDB.Operation.OperationID
:param list opFileList: list of tuples ( File.toJSON()['Value'], sourcesList, targetList )
"""
fileIDs = [int( fileJSON.get( 'FileID', 0 ) ) for fileJSON, _sourceSEs, _targetSEs in opFileList ]
res = self.ftsManager.cleanUpFTSFiles( requestID, fileIDs )
if not res['OK']:
self.log.error( "ftsSchedule: %s" % res['Message'] )
return S_ERROR( "ftsSchedule: %s" % res['Message'] )
ftsFiles = []
# # this will be returned on success
result = { "Successful": [], "Failed": {} }
for fileJSON, sourceSEs, targetSEs in opFileList:
lfn = fileJSON.get( "LFN", "" )
size = int( fileJSON.get( "Size", 0 ) )
fileID = int( fileJSON.get( "FileID", 0 ) )
opID = int( fileJSON.get( "OperationID", 0 ) )
self.log.verbose( "ftsSchedule: LFN=%s FileID=%s OperationID=%s sources=%s targets=%s" % ( lfn, fileID, opID,
sourceSEs,
targetSEs ) )
res = self.dataManager.getActiveReplicas( lfn )
if not res['OK']:
self.log.error( "ftsSchedule: %s" % res['Message'] )
result["Failed"][fileID] = res['Message']
continue
replicaDict = res['Value']
if lfn in replicaDict["Failed"] and lfn not in replicaDict["Successful"]:
result["Failed"][fileID] = "no active replicas found"
continue
replicaDict = replicaDict["Successful"].get( lfn, {} )
# # use valid replicas only
validReplicasDict = dict( [ ( se, pfn ) for se, pfn in replicaDict.items() if se in sourceSEs ] )
if not validReplicasDict:
self.log.warn( "No active replicas found in sources" )
result["Failed"][fileID] = "no active replicas found in sources"
continue
tree = self.ftsManager.getReplicationTree( sourceSEs, targetSEs, size )
if not tree['OK']:
self.log.error( "ftsSchedule: %s cannot be scheduled: %s" % ( lfn, tree['Message'] ) )
result["Failed"][fileID] = tree['Message']
continue
tree = tree['Value']
self.log.verbose( "LFN=%s tree=%s" % ( lfn, tree ) )
for repDict in tree.values():
self.log.verbose( "Strategy=%s Ancestor=%s SourceSE=%s TargetSE=%s" % ( repDict["Strategy"],
repDict["Ancestor"],
repDict["SourceSE"],
repDict["TargetSE"] ) )
transferSURLs = self._getTransferURLs( lfn, repDict, sourceSEs, validReplicasDict )
if not transferSURLs['OK']:
result["Failed"][fileID] = transferSURLs['Message']
continue
sourceSURL, targetSURL, fileStatus = transferSURLs['Value']
if sourceSURL == targetSURL:
result["Failed"][fileID] = "sourceSURL equals to targetSURL for %s" % lfn
continue
self.log.verbose( "sourceURL=%s targetURL=%s FTSFile.Status=%s" % ( sourceSURL, targetSURL, fileStatus ) )
ftsFile = FTSFile()
for key in ( "LFN", "FileID", "OperationID", "Checksum", "ChecksumType", "Size" ):
if fileJSON.get( key ):
setattr( ftsFile, key, fileJSON.get( key ) )
ftsFile.RequestID = requestID
ftsFile.OperationID = operationID
ftsFile.SourceSURL = sourceSURL
ftsFile.TargetSURL = targetSURL
ftsFile.SourceSE = repDict["SourceSE"]
ftsFile.TargetSE = repDict["TargetSE"]
ftsFile.Status = fileStatus
ftsFiles.append( ftsFile )
if not ftsFiles:
self.log.info( "ftsSchedule: no FTSFiles to put for request %d" % requestID )
return S_OK( result )
ftsFilesJSONList = [ftsFile.toJSON()['Value'] for ftsFile in ftsFiles]
res = self.ftsManager.putFTSFileList( ftsFilesJSONList )
if not res['OK']:
self.log.error( "ftsSchedule: %s" % res['Message'] )
return S_ERROR( "ftsSchedule: %s" % res['Message'] )
result['Successful'] += [ fileID for fileID in fileIDs if fileID not in result['Failed']]
# # if we land here some files have been properly scheduled
return S_OK( result )
def export_ftsSchedule(self, requestID, operationID, fileJSONList):
""" call FTS scheduler
:param int requestID: ReqDB.Request.RequestID
:param int operationID: ReqDB.Operation.OperationID
:param list fileJSONList: [ (fileJSON, [sourceSE, ...], [ targetSE, ...] ) ]
"""
# # this will be returned on success
ret = {"Successful": [], "Failed": {}}
requestID = int(requestID)
operationID = int(operationID)
fileIDs = []
for fileJSON, sourceSEs, targetSEs in fileJSONList:
fileID = int(fileJSON.get("FileID"))
fileIDs.append(fileID)
cleanUpFTSFiles = self.__ftsDB.cleanUpFTSFiles(requestID, fileIDs)
if not cleanUpFTSFiles["OK"]:
self.log.error("ftsSchedule: %s" % cleanUpFTSFiles["Message"])
return S_ERROR(cleanUpFTSFiles["Message"])
ftsFiles = []
for fileJSON, sourceSEs, targetSEs in fileJSONList:
lfn = fileJSON.get("LFN", "")
size = int(fileJSON.get("Size", 0))
fileID = int(fileJSON.get("FileID", 0))
opID = int(fileJSON.get("OperationID", 0))
gLogger.info(
"ftsSchedule: LFN=%s FileID=%s OperationID=%s sources=%s targets=%s"
% (lfn, fileID, opID, sourceSEs, targetSEs))
replicaDict = self.replicaManager().getActiveReplicas(lfn)
if not replicaDict["OK"]:
gLogger.error("ftsSchedule: %s" % replicaDict["Message"])
ret["Failed"][fileID] = replicaDict["Message"]
continue
replicaDict = replicaDict["Value"]
if lfn in replicaDict["Failed"] and lfn not in replicaDict[
"Successful"]:
ret["Failed"][fileID] = "no active replicas found"
continue
replicaDict = replicaDict["Successful"][lfn] if lfn in replicaDict[
"Successful"] else {}
# # use valid replicas only
replicaDict = dict([(se, pfn) for se, pfn in replicaDict.items()
if se in sourceSEs])
if not replicaDict:
ret["Failed"][fileID] = "no active replicas found in sources"
continue
tree = self.ftsStrategy().replicationTree(sourceSEs, targetSEs,
size)
if not tree["OK"]:
gLogger.error("ftsSchedule: %s cannot be scheduled: %s" %
(lfn, tree["Message"]))
ret["Failed"][fileID] = tree["Message"]
continue
tree = tree["Value"]
gLogger.info("LFN=%s tree=%s" % (lfn, tree))
for repDict in tree.values():
gLogger.info(
"Strategy=%s Ancestor=%s SourceSE=%s TargetSE=%s" %
(repDict["Strategy"], repDict["Ancestor"],
repDict["SourceSE"], repDict["TargetSE"]))
transferSURLs = self._getTransferURLs(lfn, repDict, sourceSEs,
replicaDict)
if not transferSURLs["OK"]:
ret["Failed"][fileID] = transferSURLs["Message"]
continue
sourceSURL, targetSURL, fileStatus = transferSURLs["Value"]
if sourceSURL == targetSURL:
ret["Failed"][
fileID] = "sourceSURL equals to targetSURL for %s" % lfn
continue
gLogger.info("sourceURL=%s targetURL=%s FTSFile.Status=%s" %
(sourceSURL, targetSURL, fileStatus))
ftsFile = FTSFile()
for key in ("LFN", "FileID", "OperationID", "Checksum",
"ChecksumType", "Size"):
setattr(ftsFile, key, fileJSON.get(key))
ftsFile.RequestID = requestID
ftsFile.OperationID = operationID
ftsFile.SourceSURL = sourceSURL
ftsFile.TargetSURL = targetSURL
ftsFile.SourceSE = repDict["SourceSE"]
ftsFile.TargetSE = repDict["TargetSE"]
ftsFile.Status = fileStatus
ftsFiles.append(ftsFile)
if not ftsFiles:
return S_ERROR("ftsSchedule: no FTSFiles to put")
put = self.__ftsDB.putFTSFileList(ftsFiles)
if not put["OK"]:
gLogger.error("ftsSchedule: %s" % put["Message"])
return put
for fileJSON, sources, targets in fileJSONList:
lfn = fileJSON.get("LFN", "")
fileID = fileJSON.get("FileID", 0)
if fileID not in ret["Failed"]:
ret["Successful"].append(int(fileID))
# # if we land here some files have been properly scheduled
return S_OK(ret)
def setUp( self ):
""" test case set up """
# ## set some defaults
gConfig.setOptionValue( 'DIRAC/Setup', 'Test' )
gConfig.setOptionValue( '/DIRAC/Setups/Test/DataManagement', 'Test' )
gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/Host', 'localhost' )
gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/DBName', 'FTSDB' )
gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/User', 'Dirac' )
self.ftsSites = [ FTSSite( { "FTSServer": "https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "CERN.ch" } ),
FTSSite( { "FTSServer": "https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "PIC.es" } ),
FTSSite( { "FTSServer": "https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "RAL.uk" } ),
FTSSite( { "FTSServer": "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "SARA.nl" } ),
FTSSite( { "FTSServer": "https://fts.cr.cnaf.infn.it:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "CNAF.it" } ),
FTSSite( { "FTSServer": "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "NIKHEF.nl" } ),
FTSSite( { "FTSServer": "https://fts-fzk.gridka.de:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "GRIDKA.de" } ),
FTSSite( { "FTSServer": "https://cclcgftsprod.in2p3.fr:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "IN2P3.fr" } ) ]
self.ftsFiles = []
for i in range ( 100 ):
ftsFile = FTSFile()
ftsFile.FileID = i + 1
ftsFile.OperationID = 9999
ftsFile.LFN = "/a/b/c/%d" % i
ftsFile.Size = 10
ftsFile.SourceSE = "CERN-USER"
ftsFile.TargetSE = "PIC-USER"
ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN
ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN
ftsFile.Status = "Waiting"
self.ftsFiles.append( ftsFile )
ses = [ "CERN-USER", "RAL-USER" ]
statuses = [ "Submitted", "Finished", "FinishedDirty", "Active", "Ready" ]
self.submitted = 0
self.ftsJobs = []
for i in range( 7200 ):
ftsJob = FTSJob()
ftsJob.FTSGUID = str( uuid.uuid4() )
ftsJob.FTSServer = self.ftsSites[0].FTSServer
ftsJob.Status = statuses[ i % len( statuses ) ]
if ftsJob.Status in FTSJob.FINALSTATES:
ftsJob.Completeness = 100
if ftsJob.Status == "Active":
ftsJob.Completeness = 90
ftsJob.SourceSE = ses[ i % len( ses ) ]
ftsJob.TargetSE = "PIC-USER"
ftsFile = FTSFile()
ftsFile.FileID = i + 1
ftsFile.OperationID = i + 1
ftsFile.LFN = "/a/b/c/%d" % i
ftsFile.Size = 1000000
ftsFile.SourceSE = ftsJob.SourceSE
ftsFile.TargetSE = ftsJob.TargetSE
ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN
ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN
ftsFile.Status = "Waiting" if ftsJob.Status != "FinishedDirty" else "Failed"
ftsFile.FTSGUID = ftsJob.FTSGUID
if ftsJob.Status == "FinishedDirty":
ftsJob.FailedFiles = 1
ftsJob.FailedSize = ftsFile.Size
ftsJob.addFile( ftsFile )
self.ftsJobs.append( ftsJob )
self.submitted = len( [ i for i in self.ftsJobs if i.Status == "Submitted" ] )
def setUp( self ):
""" test case set up """
gLogger.setLevel( 'NOTICE' )
self.ftsSites = [ FTSSite( ftsServer = 'https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer', name = 'CERN.ch' ),
FTSSite( ftsServer = 'https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer', name = 'PIC.es' ),
FTSSite( ftsServer = 'https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer', name = 'RAL.uk' ),
]
self.ses = [ 'CERN-USER', 'RAL-USER' ]
self.statuses = [ 'Submitted', 'Finished', 'FinishedDirty', 'Active', 'Ready' ]
self.submitted = 0
self.numberOfJobs = 10
self.opIDs = []
self.ftsJobs = []
for i in xrange( self.numberOfJobs ):
opID = i % 3
if opID not in self.opIDs:
self.opIDs.append( opID )
ftsJob = FTSJob()
ftsJob.FTSGUID = str( uuid.uuid4() )
ftsJob.FTSServer = self.ftsSites[0].FTSServer
ftsJob.Status = self.statuses[ i % len( self.statuses ) ]
ftsJob.OperationID = opID
if ftsJob.Status in FTSJob.FINALSTATES:
ftsJob.Completeness = 100
if ftsJob.Status == 'Active':
ftsJob.Completeness = 90
ftsJob.SourceSE = self.ses[ i % len( self.ses ) ]
ftsJob.TargetSE = 'PIC-USER'
ftsJob.RequestID = 12345
ftsFile = FTSFile()
ftsFile.FileID = i + 1
ftsFile.OperationID = i + 1
ftsFile.LFN = '/a/b/c/%d' % i
ftsFile.Size = 1000000
ftsFile.OperationID = opID
ftsFile.SourceSE = ftsJob.SourceSE
ftsFile.TargetSE = ftsJob.TargetSE
ftsFile.SourceSURL = 'foo://source.bar.baz/%s' % ftsFile.LFN
ftsFile.TargetSURL = 'foo://target.bar.baz/%s' % ftsFile.LFN
ftsFile.Status = 'Waiting' if ftsJob.Status != 'FinishedDirty' else 'Failed'
ftsFile.RequestID = 12345
ftsFile.Checksum = 'addler'
ftsFile.ChecksumType = 'adler32'
ftsFile.FTSGUID = ftsJob.FTSGUID
if ftsJob.Status == 'FinishedDirty':
ftsJob.FailedFiles = 1
ftsJob.FailedSize = ftsFile.Size
ftsJob.addFile( ftsFile )
self.ftsJobs.append( ftsJob )
self.submitted = len( [ i for i in self.ftsJobs if i.Status == 'Submitted' ] )
self.ftsClient = FTSClient()
self.ftsClient.replicaManager = mock.Mock()
self.ftsClient.replicaManager.getActiveReplicas.return_value = {'OK': True,
'Value': {'Successful': {'/a/b/c/1':{'CERN-USER':'******',
'RAL-USER':'******'},
'/a/b/c/2':{'CERN-USER':'******',
'RAL-USER':'******'},
'/a/b/c/3':{'CERN-USER':'******',
'RAL-USER':'******'}
},
'Failed': {'/a/b/c/4':'/aa/a/b/c/4d',
'/a/b/c/5':'/aa/a/b/c/5d'}
}
}
def setUp(self):
""" test case set up """
# ## set some defaults
gConfig.setOptionValue('DIRAC/Setup', 'Test')
gConfig.setOptionValue('/DIRAC/Setups/Test/DataManagement', 'Test')
gConfig.setOptionValue(
'/Systems/DataManagement/Test/Databases/FTSDB/Host', 'localhost')
gConfig.setOptionValue(
'/Systems/DataManagement/Test/Databases/FTSDB/DBName', 'FTSDB')
gConfig.setOptionValue(
'/Systems/DataManagement/Test/Databases/FTSDB/User', 'Dirac')
self.ftsSites = [
FTSSite({
"FTSServer":
"https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer",
"Name": "CERN.ch"
}),
FTSSite({
"FTSServer":
"https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer",
"Name": "PIC.es"
}),
FTSSite({
"FTSServer":
"https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer",
"Name": "RAL.uk"
}),
FTSSite({
"FTSServer":
"https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer",
"Name": "SARA.nl"
}),
FTSSite({
"FTSServer":
"https://fts.cr.cnaf.infn.it:8443/glite-data-transfer-fts/services/FileTransfer",
"Name": "CNAF.it"
}),
FTSSite({
"FTSServer":
"https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer",
"Name": "NIKHEF.nl"
}),
FTSSite({
"FTSServer":
"https://fts-fzk.gridka.de:8443/glite-data-transfer-fts/services/FileTransfer",
"Name": "GRIDKA.de"
}),
FTSSite({
"FTSServer":
"https://cclcgftsprod.in2p3.fr:8443/glite-data-transfer-fts/services/FileTransfer",
"Name": "IN2P3.fr"
})
]
self.ftsFiles = []
for i in range(100):
ftsFile = FTSFile()
ftsFile.FileID = i + 1
ftsFile.OperationID = 9999
ftsFile.LFN = "/a/b/c/%d" % i
ftsFile.Size = 10
ftsFile.SourceSE = "CERN-USER"
ftsFile.TargetSE = "PIC-USER"
ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN
ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN
ftsFile.Status = "Waiting"
self.ftsFiles.append(ftsFile)
ses = ["CERN-USER", "RAL-USER"]
statuses = [
"Submitted", "Finished", "FinishedDirty", "Active", "Ready"
]
self.submitted = 0
self.ftsJobs = []
for i in range(7200):
ftsJob = FTSJob()
ftsJob.FTSGUID = str(uuid.uuid4())
ftsJob.FTSServer = self.ftsSites[0].FTSServer
ftsJob.Status = statuses[i % len(statuses)]
if ftsJob.Status in FTSJob.FINALSTATES:
ftsJob.Completeness = 100
if ftsJob.Status == "Active":
ftsJob.Completeness = 90
ftsJob.SourceSE = ses[i % len(ses)]
ftsJob.TargetSE = "PIC-USER"
ftsFile = FTSFile()
ftsFile.FileID = i + 1
ftsFile.OperationID = i + 1
ftsFile.LFN = "/a/b/c/%d" % i
ftsFile.Size = 1000000
ftsFile.SourceSE = ftsJob.SourceSE
ftsFile.TargetSE = ftsJob.TargetSE
ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN
ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN
ftsFile.Status = "Waiting" if ftsJob.Status != "FinishedDirty" else "Failed"
ftsFile.FTSGUID = ftsJob.FTSGUID
if ftsJob.Status == "FinishedDirty":
ftsJob.FailedFiles = 1
ftsJob.FailedSize = ftsFile.Size
ftsJob.addFile(ftsFile)
self.ftsJobs.append(ftsJob)
self.submitted = len(
[i for i in self.ftsJobs if i.Status == "Submitted"])
def setUp(self):
""" test case set up """
gLogger.setLevel('NOTICE')
self.ftsSites = [
FTSSite(
ftsServer=
'https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer',
name='CERN.ch'),
FTSSite(
ftsServer=
'https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer',
name='PIC.es'),
FTSSite(
ftsServer=
'https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer',
name='RAL.uk'),
]
self.ses = ['CERN-USER', 'RAL-USER']
self.statuses = [
'Submitted', 'Finished', 'FinishedDirty', 'Active', 'Ready'
]
self.submitted = 0
self.numberOfJobs = 10
self.opIDs = []
self.ftsJobs = []
for i in xrange(self.numberOfJobs):
opID = i % 3
if opID not in self.opIDs:
self.opIDs.append(opID)
ftsJob = FTSJob()
ftsJob.FTSGUID = str(uuid.uuid4())
ftsJob.FTSServer = self.ftsSites[0].FTSServer
ftsJob.Status = self.statuses[i % len(self.statuses)]
ftsJob.OperationID = opID
if ftsJob.Status in FTSJob.FINALSTATES:
ftsJob.Completeness = 100
if ftsJob.Status == 'Active':
ftsJob.Completeness = 90
ftsJob.SourceSE = self.ses[i % len(self.ses)]
ftsJob.TargetSE = 'PIC-USER'
ftsJob.RequestID = 12345
ftsFile = FTSFile()
ftsFile.FileID = i + 1
ftsFile.OperationID = i + 1
ftsFile.LFN = '/a/b/c/%d' % i
ftsFile.Size = 1000000
ftsFile.OperationID = opID
ftsFile.SourceSE = ftsJob.SourceSE
ftsFile.TargetSE = ftsJob.TargetSE
ftsFile.SourceSURL = 'foo://source.bar.baz/%s' % ftsFile.LFN
ftsFile.TargetSURL = 'foo://target.bar.baz/%s' % ftsFile.LFN
ftsFile.Status = 'Waiting' if ftsJob.Status != 'FinishedDirty' else 'Failed'
ftsFile.RequestID = 12345
ftsFile.Checksum = 'addler'
ftsFile.ChecksumType = 'adler32'
ftsFile.FTSGUID = ftsJob.FTSGUID
if ftsJob.Status == 'FinishedDirty':
ftsJob.FailedFiles = 1
ftsJob.FailedSize = ftsFile.Size
ftsJob.addFile(ftsFile)
self.ftsJobs.append(ftsJob)
self.submitted = len(
[i for i in self.ftsJobs if i.Status == 'Submitted'])
self.ftsClient = FTSClient()
def export_ftsSchedule( self, requestID, operationID, fileJSONList ):
""" call FTS scheduler
:param int requestID: ReqDB.Request.RequestID
:param int operationID: ReqDB.Operation.OperationID
:param list fileJSONList: [ (fileJSON, [sourceSE, ...], [ targetSE, ...] ) ]
"""
# # this will be returned on success
ret = { "Successful": [], "Failed": {} }
requestID = int( requestID )
operationID = int( operationID )
fileIDs = []
for fileJSON, sourceSEs, targetSEs in fileJSONList:
fileID = int( fileJSON.get( "FileID" ) )
fileIDs.append( fileID )
cleanUpFTSFiles = self.__ftsDB.cleanUpFTSFiles( requestID, fileIDs )
if not cleanUpFTSFiles["OK"]:
self.log.error( "ftsSchedule: %s" % cleanUpFTSFiles["Message"] )
return S_ERROR( cleanUpFTSFiles["Message"] )
ftsFiles = []
for fileJSON, sourceSEs, targetSEs in fileJSONList:
lfn = fileJSON.get( "LFN", "" )
size = int( fileJSON.get( "Size", 0 ) )
fileID = int( fileJSON.get( "FileID", 0 ) )
opID = int( fileJSON.get( "OperationID", 0 ) )
gLogger.info( "ftsSchedule: LFN=%s FileID=%s OperationID=%s sources=%s targets=%s" % ( lfn, fileID, opID,
sourceSEs, targetSEs ) )
replicaDict = self.replicaManager().getActiveReplicas( lfn )
if not replicaDict["OK"]:
gLogger.error( "ftsSchedule: %s" % replicaDict["Message"] )
ret["Failed"][fileID] = replicaDict["Message"]
continue
replicaDict = replicaDict["Value"]
if lfn in replicaDict["Failed"] and lfn not in replicaDict["Successful"]:
ret["Failed"][fileID] = "no active replicas found"
continue
replicaDict = replicaDict["Successful"][lfn] if lfn in replicaDict["Successful"] else {}
# # use valid replicas only
replicaDict = dict( [ ( se, pfn ) for se, pfn in replicaDict.items() if se in sourceSEs ] )
if not replicaDict:
ret["Failed"][fileID] = "no active replicas found in sources"
continue
tree = self.ftsStrategy().replicationTree( sourceSEs, targetSEs, size )
if not tree["OK"]:
gLogger.error( "ftsSchedule: %s cannot be scheduled: %s" % ( lfn, tree["Message"] ) )
ret["Failed"][fileID] = tree["Message"]
continue
tree = tree["Value"]
gLogger.info( "LFN=%s tree=%s" % ( lfn, tree ) )
for repDict in tree.values():
gLogger.info( "Strategy=%s Ancestor=%s SourceSE=%s TargetSE=%s" % ( repDict["Strategy"], repDict["Ancestor"],
repDict["SourceSE"], repDict["TargetSE"] ) )
transferSURLs = self._getTransferURLs( lfn, repDict, sourceSEs, replicaDict )
if not transferSURLs["OK"]:
ret["Failed"][fileID] = transferSURLs["Message"]
continue
sourceSURL, targetSURL, fileStatus = transferSURLs["Value"]
if sourceSURL == targetSURL:
ret["Failed"][fileID] = "sourceSURL equals to targetSURL for %s" % lfn
continue
gLogger.info( "sourceURL=%s targetURL=%s FTSFile.Status=%s" % ( sourceSURL, targetSURL, fileStatus ) )
ftsFile = FTSFile()
for key in ( "LFN", "FileID", "OperationID", "Checksum", "ChecksumType", "Size" ):
setattr( ftsFile, key, fileJSON.get( key ) )
ftsFile.RequestID = requestID
ftsFile.OperationID = operationID
ftsFile.SourceSURL = sourceSURL
ftsFile.TargetSURL = targetSURL
ftsFile.SourceSE = repDict["SourceSE"]
ftsFile.TargetSE = repDict["TargetSE"]
ftsFile.Status = fileStatus
ftsFiles.append( ftsFile )
if not ftsFiles:
return S_ERROR( "ftsSchedule: no FTSFiles to put" )
put = self.__ftsDB.putFTSFileList( ftsFiles )
if not put["OK"]:
gLogger.error( "ftsSchedule: %s" % put["Message"] )
return put
for fileJSON, sources, targets in fileJSONList:
lfn = fileJSON.get( "LFN", "" )
fileID = fileJSON.get( "FileID", 0 )
if fileID not in ret["Failed"]:
ret["Successful"].append( int( fileID ) )
# # if we land here some files have been properly scheduled
return S_OK( ret )
def ftsSchedule(self, requestID, operationID, opFileList):
""" schedule lfn for FTS job
:param int requestID: RequestDB.Request.RequestID
:param int operationID: RequestDB.Operation.OperationID
:param list opFileList: list of tuples ( File.toJSON()['Value'], sourcesList, targetList )
"""
fileIDs = []
for fileJSON, _sourceSEs, _targetSEs in opFileList:
fileID = int(fileJSON.get('FileID'))
fileIDs.append(fileID)
res = self.ftsManager.cleanUpFTSFiles(requestID, fileIDs)
if not res['OK']:
self.log.error("ftsSchedule: %s" % res['Message'])
return S_ERROR("ftsSchedule: %s" % res['Message'])
ftsFiles = []
# # this will be returned on success
ret = {"Successful": [], "Failed": {}}
for fileJSON, sourceSEs, targetSEs in opFileList:
lfn = fileJSON.get("LFN", "")
size = int(fileJSON.get("Size", 0))
fileID = int(fileJSON.get("FileID", 0))
opID = int(fileJSON.get("OperationID", 0))
gLogger.info(
"ftsSchedule: LFN=%s FileID=%s OperationID=%s sources=%s targets=%s"
% (lfn, fileID, opID, sourceSEs, targetSEs))
res = self.replicaManager.getActiveReplicas(lfn)
if not res['OK']:
gLogger.error("ftsSchedule: %s" % res['Message'])
ret["Failed"][fileID] = res['Message']
continue
replicaDict = res['Value']
if lfn in replicaDict["Failed"] and lfn not in replicaDict[
"Successful"]:
ret["Failed"][fileID] = "no active replicas found"
continue
replicaDict = replicaDict["Successful"][lfn] if lfn in replicaDict[
"Successful"] else {}
# # use valid replicas only
validReplicasDict = dict([(se, pfn)
for se, pfn in replicaDict.items()
if se in sourceSEs])
if not validReplicasDict:
ret["Failed"][fileID] = "no active replicas found in sources"
continue
tree = self.ftsManager.getReplicationTree(sourceSEs, targetSEs,
size)
if not tree['OK']:
gLogger.error("ftsSchedule: %s cannot be scheduled: %s" %
(lfn, tree['Message']))
ret["Failed"][fileID] = tree['Message']
continue
tree = tree['Value']
gLogger.info("LFN=%s tree=%s" % (lfn, tree))
for repDict in tree.values():
gLogger.info(
"Strategy=%s Ancestor=%s SourceSE=%s TargetSE=%s" %
(repDict["Strategy"], repDict["Ancestor"],
repDict["SourceSE"], repDict["TargetSE"]))
transferSURLs = self._getTransferURLs(lfn, repDict, sourceSEs,
validReplicasDict)
if not transferSURLs['OK']:
ret["Failed"][fileID] = transferSURLs['Message']
continue
sourceSURL, targetSURL, fileStatus = transferSURLs['Value']
if sourceSURL == targetSURL:
ret["Failed"][
fileID] = "sourceSURL equals to targetSURL for %s" % lfn
continue
gLogger.info("sourceURL=%s targetURL=%s FTSFile.Status=%s" %
(sourceSURL, targetSURL, fileStatus))
ftsFile = FTSFile()
for key in ("LFN", "FileID", "OperationID", "Checksum",
"ChecksumType", "Size"):
setattr(ftsFile, key, fileJSON.get(key))
ftsFile.RequestID = requestID
ftsFile.OperationID = operationID
ftsFile.SourceSURL = sourceSURL
ftsFile.TargetSURL = targetSURL
ftsFile.SourceSE = repDict["SourceSE"]
ftsFile.TargetSE = repDict["TargetSE"]
ftsFile.Status = fileStatus
ftsFiles.append(ftsFile)
if not ftsFiles:
self.log.error("ftsSchedule: no FTSFiles to put")
return S_ERROR("ftsSchedule: no FTSFiles to put")
res = self.ftsManager.putFTSFileList(ftsFiles)
if not res['OK']:
self.log.error("ftsSchedule: %s" % res['Message'])
return S_ERROR("ftsSchedule: %s" % res['Message'])
for fileJSON, _sources, _targets in opFileList:
lfn = fileJSON.get("LFN", "")
fileID = fileJSON.get("FileID", 0)
if fileID not in ret["Failed"]:
ret["Successful"].append(int(fileID))
# # if we land here some files have been properly scheduled
return S_OK(ret)
