def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10):
log = self._log.getSubLogger("_prepareNewJobs", child=True)
filesToSubmit = self._getFilesToSubmit(
maxAttemptsPerFile=maxAttemptsPerFile)
log.debug("%s ftsFiles to submit" % len(filesToSubmit))
newJobs = []
# {targetSE : [FTS3Files] }
res = FTS3Utilities.groupFilesByTarget(filesToSubmit)
if not res['OK']:
return res
filesGroupedByTarget = res['Value']
for targetSE, ftsFiles in filesGroupedByTarget.iteritems():
res = self._checkSEAccess(targetSE, 'WriteAccess', vo=self.vo)
if not res['OK']:
# If the SE is currently banned, we just skip it
if cmpError(res, errno.EACCES):
log.info(
"Write access currently not permitted to %s, skipping."
% targetSE)
else:
log.error(res)
for ftsFile in ftsFiles:
ftsFile.attempt += 1
continue
sourceSEs = self.sourceSEs.split(
',') if self.sourceSEs is not None else []
# { sourceSE : [FTSFiles] }
res = FTS3Utilities.selectUniqueRandomSource(
ftsFiles, allowedSources=sourceSEs)
if not res['OK']:
return res
uniqueTransfersBySource = res['Value']
# We don't need to check the source, since it is already filtered by the DataManager
for sourceSE, ftsFiles in uniqueTransfersBySource.iteritems():
for ftsFilesChunk in breakListIntoChunks(
ftsFiles, maxFilesPerJob):
newJob = self._createNewJob('Transfer',
ftsFilesChunk,
targetSE,
sourceSE=sourceSE)
newJobs.append(newJob)
return S_OK(newJobs)
def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10):
log = self._log.getSubLogger("_prepareNewJobs", child=True)
filesToSubmit = self._getFilesToSubmit(maxAttemptsPerFile=maxAttemptsPerFile)
log.debug("%s ftsFiles to submit" % len(filesToSubmit))
newJobs = []
# {targetSE : [FTS3Files] }
res = FTS3Utilities.groupFilesByTarget(filesToSubmit)
if not res['OK']:
return res
filesGroupedByTarget = res['Value']
for targetSE, ftsFiles in filesGroupedByTarget.iteritems():
res = self._checkSEAccess(targetSE, 'WriteAccess', vo=self.vo)
if not res['OK']:
# If the SE is currently banned, we just skip it
if cmpError(res, errno.EACCES):
log.info("Write access currently not permitted to %s, skipping." % targetSE)
else:
log.error(res)
for ftsFile in ftsFiles:
ftsFile.attempt += 1
continue
sourceSEs = self.sourceSEs.split(',') if self.sourceSEs is not None else []
# { sourceSE : [FTSFiles] }
res = FTS3Utilities.selectUniqueRandomSource(ftsFiles, allowedSources=sourceSEs)
if not res['OK']:
return res
uniqueTransfersBySource = res['Value']
# We don't need to check the source, since it is already filtered by the DataManager
for sourceSE, ftsFiles in uniqueTransfersBySource.iteritems():
for ftsFilesChunk in breakListIntoChunks(ftsFiles, maxFilesPerJob):
newJob = self._createNewJob('Transfer', ftsFilesChunk, targetSE, sourceSE=sourceSE)
newJobs.append(newJob)
return S_OK(newJobs)
def __readConf(self):
""" read configurations """
# Getting all the possible servers
res = getFTS3ServerDict()
if not res['OK']:
gLogger.error(res['Message'])
return res
srvDict = res['Value']
serverPolicyType = opHelper().getValue('DataManagement/FTSPlacement/FTS3/ServerPolicy', 'Random')
self._serverPolicy = FTS3Utilities.FTS3ServerPolicy(srvDict, serverPolicy=serverPolicyType)
self.maxNumberOfThreads = self.am_getOption("MaxThreads", 10)
# Number of Operation we treat in one loop
self.operationBulkSize = self.am_getOption("OperationBulkSize", 20)
# Number of Jobs we treat in one loop
self.jobBulkSize = self.am_getOption("JobBulkSize", 20)
self.maxFilesPerJob = self.am_getOption("MaxFilesPerJob", 100)
self.maxAttemptsPerFile = self.am_getOption("MaxAttemptsPerFile", 256)
self.kickDelay = self.am_getOption("KickAssignedHours", 1)
self.maxKick = self.am_getOption("KickLimitPerCycle", 100)
self.deleteDelay = self.am_getOption("DeleteGraceDays", 180)
self.maxDelete = self.am_getOption("DeleteLimitPerCycle", 100)
return S_OK()
def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10):
log = gLogger.getSubLogger("_prepareNewJobs", child=True)
filesToSubmit = self._getFilesToSubmit(maxAttemptsPerFile=maxAttemptsPerFile)
log.debug("%s ftsFiles to submit" % len(filesToSubmit))
newJobs = []
# {targetSE : [FTS3Files] }
filesGroupedByTarget = FTS3Utilities.groupFilesByTarget(filesToSubmit)
for targetSE, ftsFiles in filesGroupedByTarget.iteritems():
res = self._checkSEAccess(targetSE, 'ReadAccess', vo=self.vo)
if not res['OK']:
log.error(res)
continue
for ftsFilesChunk in breakListIntoChunks(ftsFiles, maxFilesPerJob):
newJob = self._createNewJob('Staging', ftsFilesChunk, targetSE, sourceSE=targetSE)
newJobs.append(newJob)
return S_OK(newJobs)
def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10):
log = gLogger.getSubLogger("_prepareNewJobs", child=True)
filesToSubmit = self._getFilesToSubmit(
maxAttemptsPerFile=maxAttemptsPerFile)
log.debug("%s ftsFiles to submit" % len(filesToSubmit))
newJobs = []
# {targetSE : [FTS3Files] }
filesGroupedByTarget = FTS3Utilities.groupFilesByTarget(filesToSubmit)
for targetSE, ftsFiles in filesGroupedByTarget.iteritems():
res = self._checkSEAccess(targetSE, 'ReadAccess', vo=self.vo)
if not res['OK']:
log.error(res)
continue
for ftsFilesChunk in breakListIntoChunks(ftsFiles, maxFilesPerJob):
newJob = self._createNewJob('Staging',
ftsFilesChunk,
targetSE,
sourceSE=targetSE)
newJobs.append(newJob)
return S_OK(newJobs)
def __readConf(self):
"""Read configurations
:return: S_OK()/S_ERROR()
"""
# Getting all the possible servers
res = getFTS3ServerDict()
if not res["OK"]:
gLogger.error(res["Message"])
return res
srvDict = res["Value"]
serverPolicyType = opHelper().getValue("DataManagement/FTSPlacement/FTS3/ServerPolicy", "Random")
self._serverPolicy = FTS3Utilities.FTS3ServerPolicy(srvDict, serverPolicy=serverPolicyType)
self.maxNumberOfThreads = self.am_getOption("MaxThreads", 10)
# Number of Operation we treat in one loop
self.operationBulkSize = self.am_getOption("OperationBulkSize", 20)
# Number of Jobs we treat in one loop
self.jobBulkSize = self.am_getOption("JobBulkSize", 20)
self.maxFilesPerJob = self.am_getOption("MaxFilesPerJob", 100)
self.maxAttemptsPerFile = self.am_getOption("MaxAttemptsPerFile", 256)
self.kickDelay = self.am_getOption("KickAssignedHours", 1)
self.maxKick = self.am_getOption("KickLimitPerCycle", 100)
self.deleteDelay = self.am_getOption("DeleteGraceDays", 180)
self.maxDelete = self.am_getOption("DeleteLimitPerCycle", 100)
# lifetime of the proxy we download to delegate to FTS
self.proxyLifetime = self.am_getOption("ProxyLifetime", PROXY_LIFETIME)
return S_OK()
def init_on_load(self):
"""This method initializes some attributes.
It is called by sqlalchemy (which does not call __init__)
"""
self._vo = None
# Note that in the case of an FTS3Operation created from an RMS
# object, the members here will probably be "wrong" in the sense
# that the VO will not be known by then.
# It does not really matter however, since we do not perform anything
# on an operation created this way, it's just to be then serialized
# in the DB.
self.dManager = DataManager()
self.rssClient = ResourceStatus()
self.fts3Plugin = FTS3Utilities.getFTS3Plugin(vo=self.vo)
opID = getattr(self, "operationID", None)
loggerName = "%s/" % opID if opID else ""
loggerName += "req_%s/op_%s" % (self.rmsReqID, self.rmsOpID)
self._log = gLogger.getSubLogger(loggerName)
def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10):
log = self._log.getSubLogger("_prepareNewJobs", child=True)
filesToSubmit = self._getFilesToSubmit(
maxAttemptsPerFile=maxAttemptsPerFile)
log.debug("%s ftsFiles to submit" % len(filesToSubmit))
newJobs = []
# {targetSE : [FTS3Files] }
res = FTS3Utilities.groupFilesByTarget(filesToSubmit)
if not res['OK']:
return res
filesGroupedByTarget = res['Value']
for targetSE, ftsFiles in filesGroupedByTarget.items():
res = self._checkSEAccess(targetSE, 'WriteAccess', vo=self.vo)
if not res['OK']:
# If the SE is currently banned, we just skip it
if cmpError(res, errno.EACCES):
log.info(
"Write access currently not permitted to %s, skipping."
% targetSE)
else:
log.error(res)
for ftsFile in ftsFiles:
ftsFile.attempt += 1
continue
sourceSEs = self.sourceSEs.split(
',') if self.sourceSEs is not None else []
# { sourceSE : [FTSFiles] }
res = FTS3Utilities.selectUniqueRandomSource(
ftsFiles, allowedSources=sourceSEs)
if not res['OK']:
return res
uniqueTransfersBySource, failedFiles = res['Value']
# Treat the errors of the failed files
for ftsFile, errMsg in failedFiles.items():
log.error("Error when selecting random sources",
"%s, %s" % (ftsFile.lfn, errMsg))
# If the error is that the file does not exist in the catalog
# fail it !
if cmpError(errMsg, errno.ENOENT):
log.error("The file does not exist, setting it Defunct",
"%s" % ftsFile.lfn)
ftsFile.status = 'Defunct'
# We don't need to check the source, since it is already filtered by the DataManager
for sourceSE, ftsFiles in uniqueTransfersBySource.items():
if self.__needsMultiHopStaging(sourceSE, targetSE):
log.verbose(
"Needs multihop staging, max files per job is 1")
maxFilesPerJob = 1
for ftsFilesChunk in breakListIntoChunks(
ftsFiles, maxFilesPerJob):
newJob = self._createNewJob('Transfer',
ftsFilesChunk,
targetSE,
sourceSE=sourceSE)
newJobs.append(newJob)
return S_OK(newJobs)
def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10):
log = self._log.getSubLogger("_prepareNewJobs")
filesToSubmit = self._getFilesToSubmit(
maxAttemptsPerFile=maxAttemptsPerFile)
log.debug("%s ftsFiles to submit" % len(filesToSubmit))
newJobs = []
# {targetSE : [FTS3Files] }
res = FTS3Utilities.groupFilesByTarget(filesToSubmit)
if not res["OK"]:
return res
filesGroupedByTarget = res["Value"]
for targetSE, ftsFiles in filesGroupedByTarget.items():
res = self._checkSEAccess(targetSE, "WriteAccess", vo=self.vo)
if not res["OK"]:
# If the SE is currently banned, we just skip it
if cmpError(res, errno.EACCES):
log.info(
"Write access currently not permitted to %s, skipping."
% targetSE)
else:
log.error(res)
for ftsFile in ftsFiles:
ftsFile.attempt += 1
continue
sourceSEs = self.sourceSEs.split(
",") if self.sourceSEs is not None else []
# { sourceSE : [FTSFiles] }
res = FTS3Utilities.selectUniqueSource(ftsFiles,
self.fts3Plugin,
allowedSources=sourceSEs)
if not res["OK"]:
return res
uniqueTransfersBySource, failedFiles = res["Value"]
# Treat the errors of the failed files
for ftsFile, errMsg in failedFiles.items():
log.error("Error when selecting random sources",
"%s, %s" % (ftsFile.lfn, errMsg))
# If the error is that the file does not exist in the catalog
# fail it !
if cmpError(errMsg, errno.ENOENT):
log.error("The file does not exist, setting it Defunct",
"%s" % ftsFile.lfn)
ftsFile.status = "Defunct"
# We don't need to check the source, since it is already filtered by the DataManager
for sourceSE, ftsFiles in uniqueTransfersBySource.items():
# Checking whether we will need multiHop transfer
multiHopSE = self.fts3Plugin.findMultiHopSEToCoverUpForWLCGFailure(
sourceSE, targetSE)
if multiHopSE:
log.verbose(
"WLCG failure manifestation, use %s for multihop, max files per job is 1"
% multiHopSE)
# Check that we can write and read from it
try:
for accessType in ("Read", "Write"):
res = self._checkSEAccess(multiHopSE,
"%sAccess" % accessType,
vo=self.vo)
if not res["OK"]:
# If the SE is currently banned, we just skip it
if cmpError(res, errno.EACCES):
log.info(
"Access currently not permitted",
"%s to %s" % (accessType, multiHopSE))
else:
log.error("CheckSEAccess error", res)
for ftsFile in ftsFiles:
ftsFile.attempt += 1
# If we have a problem with the multiHop SE,
# we skip the whole loop for the pair
# (targetSE, sourceSE)
raise RuntimeError("MultiHopSE unavailable")
except RuntimeError as e:
log.info(
"Problem with multiHop SE, skipping transfers from %s to %s."
% (sourceSE, targetSE))
continue
maxFilesPerJob = 1
# Check if we need a multihop staging
elif self.__needsMultiHopStaging(sourceSE, targetSE):
log.verbose(
"Needs multihop staging, max files per job is 1")
maxFilesPerJob = 1
for ftsFilesChunk in breakListIntoChunks(
ftsFiles, maxFilesPerJob):
newJob = self._createNewJob("Transfer",
ftsFilesChunk,
targetSE,
sourceSE=sourceSE,
multiHopSE=multiHopSE)
newJobs.append(newJob)
return S_OK(newJobs)