def __createFTSJob( self, guid = None ):
self.__createFTSFiles()
ftsJob = FTSJob()
ftsJob.RequestID = 0
ftsJob.OperationID = 0
ftsJob.SourceSE = self.sourceSE
ftsJob.TargetSE = self.targetSE
ftsJob.SourceToken = self.sourceToken
ftsJob.TargetToken = self.targetToken
ftsJob.FTSServer = self.ftsServer
if guid:
ftsJob.FTSGUID = guid
for ftsFile in self.ftsFiles:
ftsFile.Attempt += 1
ftsFile.Error = ""
ftsJob.addFile( ftsFile )
self.ftsJob = ftsJob
def __createFTSJob( self, guid = None ):
self.__createFTSFiles()
ftsJob = FTSJob()
ftsJob.RequestID = 0
ftsJob.OperationID = 0
ftsJob.SourceSE = self.sourceSE
ftsJob.TargetSE = self.targetSE
ftsJob.SourceToken = self.sourceToken
ftsJob.TargetToken = self.targetToken
ftsJob.FTSServer = self.ftsServer
if guid:
ftsJob.FTSGUID = guid
for ftsFile in self.ftsFiles:
ftsFile.Attempt += 1
ftsFile.Error = ""
ftsJob.addFile( ftsFile )
self.ftsJob = ftsJob
def __submit(self, request, operation, toSubmit):
""" create and submit new FTSJobs using list of FTSFiles
:param Request request: ReqDB.Request instance
:param list ftsFiles: list of FTSFile instances
:return: [ FTSJob, FTSJob, ...]
"""
log = self.log.getSubLogger("%s/submit" % request.RequestName)
bySourceAndTarget = {}
for ftsFile in toSubmit:
if ftsFile.SourceSE not in bySourceAndTarget:
bySourceAndTarget.setdefault(ftsFile.SourceSE, {})
if ftsFile.TargetSE not in bySourceAndTarget[ftsFile.SourceSE]:
bySourceAndTarget[ftsFile.SourceSE].setdefault(
ftsFile.TargetSE, [])
bySourceAndTarget[ftsFile.SourceSE][ftsFile.TargetSE].append(
ftsFile)
ftsJobs = []
for source, targetDict in bySourceAndTarget.items():
for target, ftsFileList in targetDict.items():
log.info("found %s files to submit from %s to %s" %
(len(ftsFileList), source, target))
route = self.__ftsGraph.findRoute(source, target)
if not route["OK"]:
log.error(route["Message"])
continue
route = route["Value"]
sourceRead = route.fromNode.SEs[source]["read"]
if not sourceRead:
log.error("SourceSE %s is banned for reading right now" %
source)
continue
targetWrite = route.toNode.SEs[target]["write"]
if not targetWrite:
log.error("TargetSE %s is banned for writing right now" %
target)
continue
if route.ActiveJobs > route.toNode.MaxActiveJobs:
log.warn(
"unable to submit new FTS job, max active jobs reached"
)
continue
# # create FTSJob
ftsJob = FTSJob()
ftsJob.RequestID = request.RequestID
ftsJob.OperationID = operation.OperationID
ftsJob.SourceSE = source
ftsJob.TargetSE = target
sourceSE = self.getSE(source)
sourceToken = sourceSE.getStorageParameters("SRM2")
if not sourceToken["OK"]:
log.error("unable to get sourceSE '%s' parameters: %s" %
(source, sourceToken["Message"]))
continue
ftsJob.SourceToken = sourceToken["Value"].get("SpaceToken", "")
targetSE = self.getSE(target)
targetToken = targetSE.getStorageParameters("SRM2")
if not targetToken["OK"]:
log.error("unable to get targetSE '%s' parameters: %s" %
(target, targetToken["Message"]))
continue
ftsJob.TargetToken = targetToken["Value"].get("SpaceToken", "")
ftsJob.FTSServer = route.toNode.FTSServer
for ftsFile in ftsFileList:
ftsFile.Attempt += 1
ftsFile.Error = ""
ftsJob.addFile(ftsFile)
submit = ftsJob.submitFTS2(self.STAGE_FILES)
if not submit["OK"]:
log.error("unable to submit FTSJob: %s" %
submit["Message"])
continue
log.info("FTSJob '%s'@'%s' has been submitted" %
(ftsJob.FTSGUID, ftsJob.FTSServer))
# # update statuses for job files
for ftsFile in ftsJob:
ftsFile.FTSGUID = ftsJob.FTSGUID
ftsFile.Status = "Submitted"
ftsFile.Attempt += 1
# # update graph route
try:
self.updateLock().acquire()
route.ActiveJobs += 1
finally:
self.updateLock().release()
ftsJobs.append(ftsJob)
log.info("%s new FTSJobs have been submitted" % len(ftsJobs))
return S_OK(ftsJobs)
def __submit( self, request, operation, toSubmit ):
""" create and submit new FTSJobs using list of FTSFiles
:param Request request: ReqDB.Request instance
:param list ftsFiles: list of FTSFile instances
:return: [ FTSJob, FTSJob, ...]
"""
log = self.log.getSubLogger( "req_%s/%s/submit" % ( request.RequestID, request.RequestName ) )
bySourceAndTarget = {}
for ftsFile in toSubmit:
if ftsFile.SourceSE not in bySourceAndTarget:
bySourceAndTarget.setdefault( ftsFile.SourceSE, {} )
if ftsFile.TargetSE not in bySourceAndTarget[ftsFile.SourceSE]:
bySourceAndTarget[ftsFile.SourceSE].setdefault( ftsFile.TargetSE, [] )
bySourceAndTarget[ftsFile.SourceSE][ftsFile.TargetSE].append( ftsFile )
ftsJobs = []
for source, targetDict in bySourceAndTarget.iteritems():
for target, ftsFileList in targetDict.iteritems():
log.info( "found %s files to submit from %s to %s" % ( len( ftsFileList ), source, target ) )
route = self.__ftsPlacement.findRoute( source, target )
if not route["OK"]:
log.error( route["Message"] )
continue
route = route["Value"]
routeValid = self.__ftsPlacement.isRouteValid( route )
if not routeValid['OK']:
log.error( "Route invalid : %s" % routeValid['Message'] )
continue
sourceSE = StorageElement( source )
sourceToken = sourceSE.getStorageParameters( "SRM2" )
if not sourceToken["OK"]:
log.error( "unable to get sourceSE parameters:", "(%s) %s" % ( source, sourceToken["Message"] ) )
continue
seStatus = sourceSE.getStatus()['Value']
targetSE = StorageElement( target )
targetToken = targetSE.getStorageParameters( "SRM2" )
if not targetToken["OK"]:
log.error( "unable to get targetSE parameters:", "(%s) %s" % ( target, targetToken["Message"] ) )
continue
# # create FTSJob
for fileList in breakListIntoChunks( ftsFileList, self.MAX_FILES_PER_JOB ):
ftsJob = FTSJob()
ftsJob.RequestID = request.RequestID
ftsJob.OperationID = operation.OperationID
ftsJob.SourceSE = source
ftsJob.TargetSE = target
ftsJob.SourceToken = sourceToken["Value"].get( "SpaceToken", "" )
ftsJob.TargetToken = targetToken["Value"].get( "SpaceToken", "" )
ftsJob.FTSServer = route.ftsServer
for ftsFile in fileList:
ftsFile.Attempt += 1
ftsFile.Error = ""
ftsJob.addFile( ftsFile )
submit = ftsJob.submitFTS( self.__ftsVersion, command = self.SUBMIT_COMMAND, pinTime = self.PIN_TIME if seStatus['TapeSE'] else 0 )
if not submit["OK"]:
log.error( "unable to submit FTSJob:", submit["Message"] )
continue
log.info( "FTSJob '%s'@'%s' has been submitted" % ( ftsJob.FTSGUID, ftsJob.FTSServer ) )
# # update statuses for job files
for ftsFile in ftsJob:
ftsFile.FTSGUID = ftsJob.FTSGUID
ftsFile.Status = "Submitted"
ftsFile.Attempt += 1
# # update placement route
try:
self.updateLock().acquire()
self.__ftsPlacement.startTransferOnRoute( route )
finally:
self.updateLock().release()
ftsJobs.append( ftsJob )
log.info( "%s new FTSJobs have been submitted" % len( ftsJobs ) )
return S_OK( ftsJobs )
def __submit( self, request, operation, toSubmit ):
""" create and submit new FTSJobs using list of FTSFiles
:param Request request: ReqDB.Request instance
:param list ftsFiles: list of FTSFile instances
:return: [ FTSJob, FTSJob, ...]
"""
log = self.log.getSubLogger( "%s/submit" % request.RequestName )
bySourceAndTarget = {}
for ftsFile in toSubmit:
if ftsFile.SourceSE not in bySourceAndTarget:
bySourceAndTarget.setdefault( ftsFile.SourceSE, {} )
if ftsFile.TargetSE not in bySourceAndTarget[ftsFile.SourceSE]:
bySourceAndTarget[ftsFile.SourceSE].setdefault( ftsFile.TargetSE, [] )
bySourceAndTarget[ftsFile.SourceSE][ftsFile.TargetSE].append( ftsFile )
ftsJobs = []
for source, targetDict in bySourceAndTarget.items():
for target, ftsFileList in targetDict.items():
log.info( "found %s files to submit from %s to %s" % ( len( ftsFileList ), source, target ) )
route = self.__ftsGraph.findRoute( source, target )
if not route["OK"]:
log.error( route["Message"] )
continue
route = route["Value"]
sourceRead = route.fromNode.SEs[source]["read"]
if not sourceRead:
log.error( "SourceSE %s is banned for reading right now" % source )
continue
targetWrite = route.toNode.SEs[target]["write"]
if not targetWrite:
log.error( "TargetSE %s is banned for writing right now" % target )
continue
if route.ActiveJobs > route.toNode.MaxActiveJobs:
log.warn( "unable to submit new FTS job, max active jobs reached" )
continue
sourceSE = self.getSE( source )
sourceToken = sourceSE.getStorageParameters( "SRM2" )
if not sourceToken["OK"]:
log.error( "unable to get sourceSE '%s' parameters: %s" % ( source, sourceToken["Message"] ) )
continue
seStatus = sourceSE.getStatus()['Value']
targetSE = self.getSE( target )
targetToken = targetSE.getStorageParameters( "SRM2" )
if not targetToken["OK"]:
log.error( "unable to get targetSE '%s' parameters: %s" % ( target, targetToken["Message"] ) )
continue
# # create FTSJob
for fileList in breakListIntoChunks( ftsFileList, self.MAX_FILES_PER_JOB ):
ftsJob = FTSJob()
ftsJob.RequestID = request.RequestID
ftsJob.OperationID = operation.OperationID
ftsJob.SourceSE = source
ftsJob.TargetSE = target
ftsJob.SourceToken = sourceToken["Value"].get( "SpaceToken", "" )
ftsJob.TargetToken = targetToken["Value"].get( "SpaceToken", "" )
ftsJob.FTSServer = route.toNode.FTSServer
for ftsFile in fileList:
ftsFile.Attempt += 1
ftsFile.Error = ""
ftsJob.addFile( ftsFile )
submit = ftsJob.submitFTS2( command = self.SUBMIT_COMMAND, pinTime = self.PIN_TIME if seStatus['TapeSE'] else 0 )
if not submit["OK"]:
log.error( "unable to submit FTSJob: %s" % submit["Message"] )
continue
log.info( "FTSJob '%s'@'%s' has been submitted" % ( ftsJob.FTSGUID, ftsJob.FTSServer ) )
# # update statuses for job files
for ftsFile in ftsJob:
ftsFile.FTSGUID = ftsJob.FTSGUID
ftsFile.Status = "Submitted"
ftsFile.Attempt += 1
# # update graph route
try:
self.updateLock().acquire()
route.ActiveJobs += 1
finally:
self.updateLock().release()
ftsJobs.append( ftsJob )
log.info( "%s new FTSJobs have been submitted" % len( ftsJobs ) )
return S_OK( ftsJobs )
def setUp( self ):
""" test case set up """
gLogger.setLevel( 'NOTICE' )
self.ftsSites = [ FTSSite( ftsServer = 'https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer', name = 'CERN.ch' ),
FTSSite( ftsServer = 'https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer', name = 'PIC.es' ),
FTSSite( ftsServer = 'https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer', name = 'RAL.uk' ),
]
self.ses = [ 'CERN-USER', 'RAL-USER' ]
self.statuses = [ 'Submitted', 'Finished', 'FinishedDirty', 'Active', 'Ready' ]
self.submitted = 0
self.numberOfJobs = 10
self.opIDs = []
self.ftsJobs = []
for i in xrange( self.numberOfJobs ):
opID = i % 3
if opID not in self.opIDs:
self.opIDs.append( opID )
ftsJob = FTSJob()
ftsJob.FTSGUID = str( uuid.uuid4() )
ftsJob.FTSServer = self.ftsSites[0].FTSServer
ftsJob.Status = self.statuses[ i % len( self.statuses ) ]
ftsJob.OperationID = opID
if ftsJob.Status in FTSJob.FINALSTATES:
ftsJob.Completeness = 100
if ftsJob.Status == 'Active':
ftsJob.Completeness = 90
ftsJob.SourceSE = self.ses[ i % len( self.ses ) ]
ftsJob.TargetSE = 'PIC-USER'
ftsJob.RequestID = 12345
ftsFile = FTSFile()
ftsFile.FileID = i + 1
ftsFile.OperationID = i + 1
ftsFile.LFN = '/a/b/c/%d' % i
ftsFile.Size = 1000000
ftsFile.OperationID = opID
ftsFile.SourceSE = ftsJob.SourceSE
ftsFile.TargetSE = ftsJob.TargetSE
ftsFile.SourceSURL = 'foo://source.bar.baz/%s' % ftsFile.LFN
ftsFile.TargetSURL = 'foo://target.bar.baz/%s' % ftsFile.LFN
ftsFile.Status = 'Waiting' if ftsJob.Status != 'FinishedDirty' else 'Failed'
ftsFile.RequestID = 12345
ftsFile.Checksum = 'addler'
ftsFile.ChecksumType = 'adler32'
ftsFile.FTSGUID = ftsJob.FTSGUID
if ftsJob.Status == 'FinishedDirty':
ftsJob.FailedFiles = 1
ftsJob.FailedSize = ftsFile.Size
ftsJob.addFile( ftsFile )
self.ftsJobs.append( ftsJob )
self.submitted = len( [ i for i in self.ftsJobs if i.Status == 'Submitted' ] )
self.ftsClient = FTSClient()
self.ftsClient.replicaManager = mock.Mock()
self.ftsClient.replicaManager.getActiveReplicas.return_value = {'OK': True,
'Value': {'Successful': {'/a/b/c/1':{'CERN-USER':'******',
'RAL-USER':'******'},
'/a/b/c/2':{'CERN-USER':'******',
'RAL-USER':'******'},
'/a/b/c/3':{'CERN-USER':'******',
'RAL-USER':'******'}
},
'Failed': {'/a/b/c/4':'/aa/a/b/c/4d',
'/a/b/c/5':'/aa/a/b/c/5d'}
}
}
def __submit( self, request, operation, toSubmit ):
""" create and submit new FTSJobs using list of FTSFiles
:param Request request: ReqDB.Request instance
:param list ftsFiles: list of FTSFile instances
:return: [ FTSJob, FTSJob, ...]
"""
log = self.log.getSubLogger( "req_%s/%s/submit" % ( request.RequestID, request.RequestName ) )
bySourceAndTarget = {}
for ftsFile in toSubmit:
if ftsFile.SourceSE not in bySourceAndTarget:
bySourceAndTarget.setdefault( ftsFile.SourceSE, {} )
if ftsFile.TargetSE not in bySourceAndTarget[ftsFile.SourceSE]:
bySourceAndTarget[ftsFile.SourceSE].setdefault( ftsFile.TargetSE, [] )
bySourceAndTarget[ftsFile.SourceSE][ftsFile.TargetSE].append( ftsFile )
ftsJobs = []
for source, targetDict in bySourceAndTarget.iteritems():
for target, ftsFileList in targetDict.iteritems():
log.info( "found %s files to submit from %s to %s" % ( len( ftsFileList ), source, target ) )
route = self.__ftsPlacement.findRoute( source, target )
if not route["OK"]:
log.error( route["Message"] )
continue
route = route["Value"]
routeValid = self.__ftsPlacement.isRouteValid( route )
if not routeValid['OK']:
log.error( "Route invalid : %s" % routeValid['Message'] )
continue
sourceSE = StorageElement( source )
sourceToken = sourceSE.getStorageParameters( protocol = 'srm' )
if not sourceToken["OK"]:
log.error( "unable to get sourceSE parameters:", "(%s) %s" % ( source, sourceToken["Message"] ) )
continue
seStatus = sourceSE.getStatus()['Value']
targetSE = StorageElement( target )
targetToken = targetSE.getStorageParameters( protocol = 'srm' )
if not targetToken["OK"]:
log.error( "unable to get targetSE parameters:", "(%s) %s" % ( target, targetToken["Message"] ) )
continue
# # create FTSJob
for fileList in breakListIntoChunks( ftsFileList, self.MAX_FILES_PER_JOB ):
ftsJob = FTSJob()
ftsJob.RequestID = request.RequestID
ftsJob.OperationID = operation.OperationID
ftsJob.SourceSE = source
ftsJob.TargetSE = target
ftsJob.SourceToken = sourceToken["Value"].get( "SpaceToken", "" )
ftsJob.TargetToken = targetToken["Value"].get( "SpaceToken", "" )
ftsJob.FTSServer = route.ftsServer
for ftsFile in fileList:
ftsFile.Attempt += 1
ftsFile.Error = ""
ftsJob.addFile( ftsFile )
submit = ftsJob.submitFTS( self.__ftsVersion, command = self.SUBMIT_COMMAND, pinTime = self.PIN_TIME if seStatus['TapeSE'] else 0 )
if not submit["OK"]:
log.error( "unable to submit FTSJob:", submit["Message"] )
continue
log.info( "FTSJob '%s'@'%s' has been submitted" % ( ftsJob.FTSGUID, ftsJob.FTSServer ) )
# # update statuses for job files
for ftsFile in ftsJob:
ftsFile.FTSGUID = ftsJob.FTSGUID
ftsFile.Status = "Submitted"
ftsFile.Attempt += 1
# # update placement route
try:
self.updateLock().acquire()
self.__ftsPlacement.startTransferOnRoute( route )
finally:
self.updateLock().release()
ftsJobs.append( ftsJob )
log.info( "%s new FTSJobs have been submitted" % len( ftsJobs ) )
return S_OK( ftsJobs )
def setUp(self):
""" test case set up """
gLogger.setLevel('NOTICE')
self.ftsSites = [
FTSSite(
ftsServer=
'https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer',
name='CERN.ch'),
FTSSite(
ftsServer=
'https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer',
name='PIC.es'),
FTSSite(
ftsServer=
'https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer',
name='RAL.uk'),
]
self.ses = ['CERN-USER', 'RAL-USER']
self.statuses = [
'Submitted', 'Finished', 'FinishedDirty', 'Active', 'Ready'
]
self.submitted = 0
self.numberOfJobs = 10
self.opIDs = []
self.ftsJobs = []
for i in xrange(self.numberOfJobs):
opID = i % 3
if opID not in self.opIDs:
self.opIDs.append(opID)
ftsJob = FTSJob()
ftsJob.FTSGUID = str(uuid.uuid4())
ftsJob.FTSServer = self.ftsSites[0].FTSServer
ftsJob.Status = self.statuses[i % len(self.statuses)]
ftsJob.OperationID = opID
if ftsJob.Status in FTSJob.FINALSTATES:
ftsJob.Completeness = 100
if ftsJob.Status == 'Active':
ftsJob.Completeness = 90
ftsJob.SourceSE = self.ses[i % len(self.ses)]
ftsJob.TargetSE = 'PIC-USER'
ftsJob.RequestID = 12345
ftsFile = FTSFile()
ftsFile.FileID = i + 1
ftsFile.OperationID = i + 1
ftsFile.LFN = '/a/b/c/%d' % i
ftsFile.Size = 1000000
ftsFile.OperationID = opID
ftsFile.SourceSE = ftsJob.SourceSE
ftsFile.TargetSE = ftsJob.TargetSE
ftsFile.SourceSURL = 'foo://source.bar.baz/%s' % ftsFile.LFN
ftsFile.TargetSURL = 'foo://target.bar.baz/%s' % ftsFile.LFN
ftsFile.Status = 'Waiting' if ftsJob.Status != 'FinishedDirty' else 'Failed'
ftsFile.RequestID = 12345
ftsFile.Checksum = 'addler'
ftsFile.ChecksumType = 'adler32'
ftsFile.FTSGUID = ftsJob.FTSGUID
if ftsJob.Status == 'FinishedDirty':
ftsJob.FailedFiles = 1
ftsJob.FailedSize = ftsFile.Size
ftsJob.addFile(ftsFile)
self.ftsJobs.append(ftsJob)
self.submitted = len(
[i for i in self.ftsJobs if i.Status == 'Submitted'])
self.ftsClient = FTSClient()
