def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10):
log = gLogger.getSubLogger("_prepareNewJobs", child=True)
filesToSubmit = self._getFilesToSubmit(maxAttemptsPerFile=maxAttemptsPerFile)
log.debug("%s ftsFiles to submit" % len(filesToSubmit))
newJobs = []
# {targetSE : [FTS3Files] }
filesGroupedByTarget = FTS3Utilities.groupFilesByTarget(filesToSubmit)
for targetSE, ftsFiles in filesGroupedByTarget.iteritems():
res = self._checkSEAccess(targetSE, 'ReadAccess', vo=self.vo)
if not res['OK']:
log.error(res)
continue
for ftsFilesChunk in breakListIntoChunks(ftsFiles, maxFilesPerJob):
newJob = self._createNewJob('Staging', ftsFilesChunk, targetSE, sourceSE=targetSE)
newJobs.append(newJob)
return S_OK(newJobs)
def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10):
log = gLogger.getSubLogger("_prepareNewJobs", child=True)
filesToSubmit = self._getFilesToSubmit(
maxAttemptsPerFile=maxAttemptsPerFile)
log.debug("%s ftsFiles to submit" % len(filesToSubmit))
newJobs = []
# {targetSE : [FTS3Files] }
filesGroupedByTarget = FTS3Utilities.groupFilesByTarget(filesToSubmit)
for targetSE, ftsFiles in filesGroupedByTarget.iteritems():
res = self._checkSEAccess(targetSE, 'ReadAccess', vo=self.vo)
if not res['OK']:
log.error(res)
continue
for ftsFilesChunk in breakListIntoChunks(ftsFiles, maxFilesPerJob):
newJob = self._createNewJob('Staging',
ftsFilesChunk,
targetSE,
sourceSE=targetSE)
newJobs.append(newJob)
return S_OK(newJobs)
def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10):
log = self._log.getSubLogger("_prepareNewJobs", child=True)
filesToSubmit = self._getFilesToSubmit(
maxAttemptsPerFile=maxAttemptsPerFile)
log.debug("%s ftsFiles to submit" % len(filesToSubmit))
newJobs = []
# {targetSE : [FTS3Files] }
res = FTS3Utilities.groupFilesByTarget(filesToSubmit)
if not res['OK']:
return res
filesGroupedByTarget = res['Value']
for targetSE, ftsFiles in filesGroupedByTarget.iteritems():
res = self._checkSEAccess(targetSE, 'WriteAccess', vo=self.vo)
if not res['OK']:
# If the SE is currently banned, we just skip it
if cmpError(res, errno.EACCES):
log.info(
"Write access currently not permitted to %s, skipping."
% targetSE)
else:
log.error(res)
for ftsFile in ftsFiles:
ftsFile.attempt += 1
continue
sourceSEs = self.sourceSEs.split(
',') if self.sourceSEs is not None else []
# { sourceSE : [FTSFiles] }
res = FTS3Utilities.selectUniqueRandomSource(
ftsFiles, allowedSources=sourceSEs)
if not res['OK']:
return res
uniqueTransfersBySource = res['Value']
# We don't need to check the source, since it is already filtered by the DataManager
for sourceSE, ftsFiles in uniqueTransfersBySource.iteritems():
for ftsFilesChunk in breakListIntoChunks(
ftsFiles, maxFilesPerJob):
newJob = self._createNewJob('Transfer',
ftsFilesChunk,
targetSE,
sourceSE=sourceSE)
newJobs.append(newJob)
return S_OK(newJobs)
def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10):
log = self._log.getSubLogger("_prepareNewJobs", child=True)
filesToSubmit = self._getFilesToSubmit(maxAttemptsPerFile=maxAttemptsPerFile)
log.debug("%s ftsFiles to submit" % len(filesToSubmit))
newJobs = []
# {targetSE : [FTS3Files] }
res = FTS3Utilities.groupFilesByTarget(filesToSubmit)
if not res['OK']:
return res
filesGroupedByTarget = res['Value']
for targetSE, ftsFiles in filesGroupedByTarget.iteritems():
res = self._checkSEAccess(targetSE, 'WriteAccess', vo=self.vo)
if not res['OK']:
# If the SE is currently banned, we just skip it
if cmpError(res, errno.EACCES):
log.info("Write access currently not permitted to %s, skipping." % targetSE)
else:
log.error(res)
for ftsFile in ftsFiles:
ftsFile.attempt += 1
continue
sourceSEs = self.sourceSEs.split(',') if self.sourceSEs is not None else []
# { sourceSE : [FTSFiles] }
res = FTS3Utilities.selectUniqueRandomSource(ftsFiles, allowedSources=sourceSEs)
if not res['OK']:
return res
uniqueTransfersBySource = res['Value']
# We don't need to check the source, since it is already filtered by the DataManager
for sourceSE, ftsFiles in uniqueTransfersBySource.iteritems():
for ftsFilesChunk in breakListIntoChunks(ftsFiles, maxFilesPerJob):
newJob = self._createNewJob('Transfer', ftsFilesChunk, targetSE, sourceSE=sourceSE)
newJobs.append(newJob)
return S_OK(newJobs)
def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10):
log = self._log.getSubLogger("_prepareNewJobs", child=True)
filesToSubmit = self._getFilesToSubmit(
maxAttemptsPerFile=maxAttemptsPerFile)
log.debug("%s ftsFiles to submit" % len(filesToSubmit))
newJobs = []
# {targetSE : [FTS3Files] }
res = FTS3Utilities.groupFilesByTarget(filesToSubmit)
if not res['OK']:
return res
filesGroupedByTarget = res['Value']
for targetSE, ftsFiles in filesGroupedByTarget.items():
res = self._checkSEAccess(targetSE, 'WriteAccess', vo=self.vo)
if not res['OK']:
# If the SE is currently banned, we just skip it
if cmpError(res, errno.EACCES):
log.info(
"Write access currently not permitted to %s, skipping."
% targetSE)
else:
log.error(res)
for ftsFile in ftsFiles:
ftsFile.attempt += 1
continue
sourceSEs = self.sourceSEs.split(
',') if self.sourceSEs is not None else []
# { sourceSE : [FTSFiles] }
res = FTS3Utilities.selectUniqueRandomSource(
ftsFiles, allowedSources=sourceSEs)
if not res['OK']:
return res
uniqueTransfersBySource, failedFiles = res['Value']
# Treat the errors of the failed files
for ftsFile, errMsg in failedFiles.items():
log.error("Error when selecting random sources",
"%s, %s" % (ftsFile.lfn, errMsg))
# If the error is that the file does not exist in the catalog
# fail it !
if cmpError(errMsg, errno.ENOENT):
log.error("The file does not exist, setting it Defunct",
"%s" % ftsFile.lfn)
ftsFile.status = 'Defunct'
# We don't need to check the source, since it is already filtered by the DataManager
for sourceSE, ftsFiles in uniqueTransfersBySource.items():
if self.__needsMultiHopStaging(sourceSE, targetSE):
log.verbose(
"Needs multihop staging, max files per job is 1")
maxFilesPerJob = 1
for ftsFilesChunk in breakListIntoChunks(
ftsFiles, maxFilesPerJob):
newJob = self._createNewJob('Transfer',
ftsFilesChunk,
targetSE,
sourceSE=sourceSE)
newJobs.append(newJob)
return S_OK(newJobs)
def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10):
log = self._log.getSubLogger("_prepareNewJobs")
filesToSubmit = self._getFilesToSubmit(
maxAttemptsPerFile=maxAttemptsPerFile)
log.debug("%s ftsFiles to submit" % len(filesToSubmit))
newJobs = []
# {targetSE : [FTS3Files] }
res = FTS3Utilities.groupFilesByTarget(filesToSubmit)
if not res["OK"]:
return res
filesGroupedByTarget = res["Value"]
for targetSE, ftsFiles in filesGroupedByTarget.items():
res = self._checkSEAccess(targetSE, "WriteAccess", vo=self.vo)
if not res["OK"]:
# If the SE is currently banned, we just skip it
if cmpError(res, errno.EACCES):
log.info(
"Write access currently not permitted to %s, skipping."
% targetSE)
else:
log.error(res)
for ftsFile in ftsFiles:
ftsFile.attempt += 1
continue
sourceSEs = self.sourceSEs.split(
",") if self.sourceSEs is not None else []
# { sourceSE : [FTSFiles] }
res = FTS3Utilities.selectUniqueSource(ftsFiles,
self.fts3Plugin,
allowedSources=sourceSEs)
if not res["OK"]:
return res
uniqueTransfersBySource, failedFiles = res["Value"]
# Treat the errors of the failed files
for ftsFile, errMsg in failedFiles.items():
log.error("Error when selecting random sources",
"%s, %s" % (ftsFile.lfn, errMsg))
# If the error is that the file does not exist in the catalog
# fail it !
if cmpError(errMsg, errno.ENOENT):
log.error("The file does not exist, setting it Defunct",
"%s" % ftsFile.lfn)
ftsFile.status = "Defunct"
# We don't need to check the source, since it is already filtered by the DataManager
for sourceSE, ftsFiles in uniqueTransfersBySource.items():
# Checking whether we will need multiHop transfer
multiHopSE = self.fts3Plugin.findMultiHopSEToCoverUpForWLCGFailure(
sourceSE, targetSE)
if multiHopSE:
log.verbose(
"WLCG failure manifestation, use %s for multihop, max files per job is 1"
% multiHopSE)
# Check that we can write and read from it
try:
for accessType in ("Read", "Write"):
res = self._checkSEAccess(multiHopSE,
"%sAccess" % accessType,
vo=self.vo)
if not res["OK"]:
# If the SE is currently banned, we just skip it
if cmpError(res, errno.EACCES):
log.info(
"Access currently not permitted",
"%s to %s" % (accessType, multiHopSE))
else:
log.error("CheckSEAccess error", res)
for ftsFile in ftsFiles:
ftsFile.attempt += 1
# If we have a problem with the multiHop SE,
# we skip the whole loop for the pair
# (targetSE, sourceSE)
raise RuntimeError("MultiHopSE unavailable")
except RuntimeError as e:
log.info(
"Problem with multiHop SE, skipping transfers from %s to %s."
% (sourceSE, targetSE))
continue
maxFilesPerJob = 1
# Check if we need a multihop staging
elif self.__needsMultiHopStaging(sourceSE, targetSE):
log.verbose(
"Needs multihop staging, max files per job is 1")
maxFilesPerJob = 1
for ftsFilesChunk in breakListIntoChunks(
ftsFiles, maxFilesPerJob):
newJob = self._createNewJob("Transfer",
ftsFilesChunk,
targetSE,
sourceSE=sourceSE,
multiHopSE=multiHopSE)
newJobs.append(newJob)
return S_OK(newJobs)