def _checkExistingFTS3Operations(self):
"""
Check if there are ongoing FTS3Operation for the current RMS Operation
Under some conditions, we can be trying to schedule files while
there is still an FTS transfer going on. This typically happens
when the REA hangs. To prevent further race condition, we check
if there are FTS3Operations in a non Final state matching the
current operation ID. If so, we put the corresponding files in
scheduled mode. We will then wait till the FTS3 Operation performs
the callback
:returns: S_OK with True if we can go on, False if we should stop the processing
"""
res = FTS3Client().getOperationsFromRMSOpID(self.operation.OperationID)
if not res["OK"]:
self.log.debug("Could not get FTS3Operations matching OperationID",
self.operation.OperationID)
return res
existingFTSOperations = res["Value"]
# It is ok to have FTS Operations in a final state, so we
# care only about the others
unfinishedFTSOperations = [
ops for ops in existingFTSOperations
if ops.status not in FTS3TransferOperation.FINAL_STATES
]
if not unfinishedFTSOperations:
self.log.debug("No ongoing FTS3Operations, all good")
return S_OK(True)
self.log.warn(
"Some FTS3Operations already exist for the RMS Operation:",
[op.operationID for op in unfinishedFTSOperations],
)
# This would really be a screwed up situation !
if len(unfinishedFTSOperations) > 1:
self.log.warn("That's a serious problem !!")
# We take the rmsFileID of the files in the Operations,
# find the corresponding File object, and set them scheduled
rmsFileIDsToSetScheduled = set([
ftsFile.rmsFileID for ftsOp in unfinishedFTSOperations
for ftsFile in ftsOp.ftsFiles
])
for opFile in self.operation:
# If it is in the DB, it has a FileID
opFileID = opFile.FileID
if opFileID in rmsFileIDsToSetScheduled:
self.log.warn("Setting RMSFile as already scheduled", opFileID)
opFile.Status = "Scheduled"
# We return here such that the Request is set back to Scheduled in the DB
# With no further modification
return S_OK(False)
def printFTSJobs(request):
""" Prints the FTSJobs associated to a request
:param request: Request object
"""
try:
if request.RequestID:
# We try first the new FTS3 system
from DIRAC.DataManagementSystem.Client.FTS3Client import FTS3Client
fts3Client = FTS3Client()
res = fts3Client.ping()
if res['OK']:
associatedFTS3Jobs = []
for op in request:
res = fts3Client.getOperationsFromRMSOpID(op.OperationID)
if res['OK']:
for fts3Op in res['Value']:
associatedFTS3Jobs.extend(fts3Op.ftsJobs)
if associatedFTS3Jobs:
gLogger.always(
'\n\nFTS3 jobs associated: \n%s' %
'\n'.join('%s@%s (%s)' %
(job.ftsGUID, job.ftsServer, job.status)
for job in associatedFTS3Jobs))
return
# If we are here, the attempt with the new FTS3 system did not work, let's try the old FTS system
gLogger.debug("Could not instantiate FTS3Client", res)
from DIRAC.DataManagementSystem.Client.FTSClient import FTSClient
ftsClient = FTSClient()
res = ftsClient.ping()
if not res['OK']:
gLogger.debug("Could not instantiate FtsClient", res)
return
res = ftsClient.getFTSJobsForRequest(request.RequestID)
if res['OK']:
ftsJobs = res['Value']
if ftsJobs:
gLogger.always(' FTS jobs associated: %s' %
','.join('%s (%s)' %
(job.FTSGUID, job.Status)
for job in ftsJobs))
# ImportError can be thrown for the old client
# AttributeError can be thrown because the deserialization will not have
# happened correctly on the new fts3 (CC7 typically), and the error is not
# properly propagated
except (ImportError, AttributeError) as err:
gLogger.debug("Could not instantiate FtsClient because of Exception",
repr(err))
def printFTSJobs(request):
""" Prints the FTSJobs associated to a request
:param request: Request object
"""
try:
if request.RequestID:
# We try first the new FTS3 system
from DIRAC.DataManagementSystem.Client.FTS3Client import FTS3Client
fts3Client = FTS3Client()
res = fts3Client.ping()
if res['OK']:
associatedFTS3Jobs = []
for op in request:
res = fts3Client.getOperationsFromRMSOpID(op.OperationID)
if res['OK']:
for fts3Op in res['Value']:
associatedFTS3Jobs.extend(fts3Op.ftsJobs)
if associatedFTS3Jobs:
gLogger.always(
'\n\nFTS3 jobs associated: \n%s' %
'\n'.join(
'%s@%s (%s)' %
(job.ftsGUID,
job.ftsServer,
job.status) for job in associatedFTS3Jobs))
return
# If we are here, the attempt with the new FTS3 system did not work, let's try the old FTS system
gLogger.debug("Could not instantiate FTS3Client", res)
from DIRAC.DataManagementSystem.Client.FTSClient import FTSClient
ftsClient = FTSClient()
res = ftsClient.ping()
if not res['OK']:
gLogger.debug("Could not instantiate FtsClient", res)
return
res = ftsClient.getFTSJobsForRequest(request.RequestID)
if res['OK']:
ftsJobs = res['Value']
if ftsJobs:
gLogger.always(' FTS jobs associated: %s' % ','.join('%s (%s)' % (job.FTSGUID, job.Status)
for job in ftsJobs))
except ImportError as err:
gLogger.debug("Could not instantiate FtsClient because of Exception", repr(err))
def printFTSJobs(request):
"""Prints the FTSJobs associated to a request
:param request: Request object
"""
try:
if request.RequestID:
# We try first the new FTS3 system
from DIRAC.DataManagementSystem.Client.FTS3Client import FTS3Client
fts3Client = FTS3Client()
res = fts3Client.ping()
if res["OK"]:
associatedFTS3Jobs = []
for op in request:
res = fts3Client.getOperationsFromRMSOpID(op.OperationID)
if res["OK"]:
for fts3Op in res["Value"]:
associatedFTS3Jobs.extend(fts3Op.ftsJobs)
if associatedFTS3Jobs:
# Display the direct url and the status
gLogger.always(
"\n\nFTS3 jobs associated: \n%s"
% "\n".join(
"%s/fts3/ftsmon/#/job/%s (%s)"
% (
job.ftsServer.replace(":8446", ":8449"), # Submission port is 8446, web port is 8449
job.ftsGUID,
job.status,
)
for job in associatedFTS3Jobs
)
)
return
# AttributeError can be thrown because the deserialization will not have
# happened correctly on the new fts3 (CC7 typically), and the error is not
# properly propagated
except AttributeError as err:
gLogger.debug("Could not instantiate FtsClient because of Exception", repr(err))
def fts3Transfer(self):
""" replicate and register using FTS3 """
self.log.info("scheduling files in FTS3...")
# Check first if we do not have ongoing transfers
res = self._checkExistingFTS3Operations()
if not res['OK']:
return res
# if res['Value'] is False
# it means that there are ongoing transfers
# and we should stop here
if res['Value'] is False:
# return S_OK such that the request is put back
return S_OK()
fts3Files = []
toSchedule = {}
# Dict which maps the FileID to the object
rmsFilesIds = {}
for opFile in self.getWaitingFilesList():
rmsFilesIds[opFile.FileID] = opFile
opFile.Error = ''
gMonitor.addMark("FTSScheduleAtt")
# # check replicas
replicas = self._filterReplicas(opFile)
if not replicas["OK"]:
continue
replicas = replicas["Value"]
validReplicas = replicas["Valid"]
noMetaReplicas = replicas["NoMetadata"]
noReplicas = replicas['NoReplicas']
badReplicas = replicas['Bad']
noPFN = replicas['NoPFN']
if validReplicas:
validTargets = list(
set(self.operation.targetSEList) - set(validReplicas))
if not validTargets:
self.log.info("file %s is already present at all targets" %
opFile.LFN)
opFile.Status = "Done"
else:
toSchedule[opFile.LFN] = [opFile, validTargets]
else:
gMonitor.addMark("FTSScheduleFail")
if noMetaReplicas:
self.log.warn(
"unable to schedule '%s', couldn't get metadata at %s"
% (opFile.LFN, ','.join(noMetaReplicas)))
opFile.Error = "Couldn't get metadata"
elif noReplicas:
self.log.error(
"Unable to schedule transfer",
"File %s doesn't exist at %s" %
(opFile.LFN, ','.join(noReplicas)))
opFile.Error = 'No replicas found'
opFile.Status = 'Failed'
elif badReplicas:
self.log.error(
"Unable to schedule transfer",
"File %s, all replicas have a bad checksum at %s" %
(opFile.LFN, ','.join(badReplicas)))
opFile.Error = 'All replicas have a bad checksum'
opFile.Status = 'Failed'
elif noPFN:
self.log.warn(
"unable to schedule %s, could not get a PFN at %s" %
(opFile.LFN, ','.join(noPFN)))
res = self._addMetadataToFiles(toSchedule)
if not res['OK']:
return res
else:
filesToSchedule = res['Value']
for lfn in filesToSchedule:
opFile = filesToSchedule[lfn]
validTargets = toSchedule[lfn][1]
for targetSE in validTargets:
ftsFile = FTS3File.fromRMSFile(opFile, targetSE)
fts3Files.append(ftsFile)
if fts3Files:
res = Registry.getUsernameForDN(self.request.OwnerDN)
if not res['OK']:
self.log.error(
"Cannot get username for DN",
"%s %s" % (self.request.OwnerDN, res['Message']))
return res
username = res['Value']
fts3Operation = FTS3TransferOperation.fromRMSObjects(
self.request, self.operation, username)
fts3Operation.ftsFiles = fts3Files
ftsSchedule = FTS3Client().persistOperation(fts3Operation)
if not ftsSchedule["OK"]:
self.log.error("Completely failed to schedule to FTS3:",
ftsSchedule["Message"])
return ftsSchedule
# might have nothing to schedule
ftsSchedule = ftsSchedule["Value"]
self.log.info("Scheduled with FTS3Operation id %s" % ftsSchedule)
self.log.info("%d files have been scheduled to FTS3" %
len(fts3Files))
for ftsFile in fts3Files:
opFile = rmsFilesIds[ftsFile.rmsFileID]
gMonitor.addMark("FTSScheduleOK", 1)
opFile.Status = "Scheduled"
self.log.debug("%s has been scheduled for FTS" % opFile.LFN)
else:
self.log.info("No files to schedule after metadata checks")
# Just in case some transfers could not be scheduled, try them with RM
return self.dmTransfer(fromFTS=True)
def setUp(self):
self.client = FTS3Client()
self.db = FTS3DB()
self.fileCounter = 0
def fts3Transfer(self):
"""replicate and register using FTS3"""
self.log.info("scheduling files in FTS3...")
# Check first if we do not have ongoing transfers
res = self._checkExistingFTS3Operations()
if not res["OK"]:
return res
# if res['Value'] is False
# it means that there are ongoing transfers
# and we should stop here
if res["Value"] is False:
# return S_OK such that the request is put back
return S_OK()
fts3Files = []
toSchedule = {}
# Dict which maps the FileID to the object
rmsFilesIds = {}
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord(
self.createRMSRecord("Attempted",
len(self.getWaitingFilesList())))
for opFile in self.getWaitingFilesList():
rmsFilesIds[opFile.FileID] = opFile
opFile.Error = ""
# # check replicas
replicas = self._filterReplicas(opFile)
if not replicas["OK"]:
continue
replicas = replicas["Value"]
validReplicas = replicas["Valid"]
noMetaReplicas = replicas["NoMetadata"]
noReplicas = replicas["NoReplicas"]
badReplicas = replicas["Bad"]
noPFN = replicas["NoPFN"]
if validReplicas:
validTargets = list(
set(self.operation.targetSEList) - set(validReplicas))
if not validTargets:
self.log.info("file %s is already present at all targets" %
opFile.LFN)
opFile.Status = "Done"
else:
toSchedule[opFile.LFN] = [opFile, validTargets]
else:
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord(
self.createRMSRecord("Failed", 1))
if noMetaReplicas:
self.log.warn(
"unable to schedule file",
"'%s': couldn't get metadata at %s" %
(opFile.LFN, ",".join(noMetaReplicas)),
)
opFile.Error = "Couldn't get metadata"
elif noReplicas:
self.log.error(
"Unable to schedule transfer",
"File %s doesn't exist at %s" %
(opFile.LFN, ",".join(noReplicas)),
)
opFile.Error = "No replicas found"
opFile.Status = "Failed"
elif badReplicas:
self.log.error(
"Unable to schedule transfer",
"File %s, all replicas have a bad checksum at %s" %
(opFile.LFN, ",".join(badReplicas)),
)
opFile.Error = "All replicas have a bad checksum"
opFile.Status = "Failed"
elif noPFN:
self.log.warn(
"unable to schedule %s, could not get a PFN at %s" %
(opFile.LFN, ",".join(noPFN)))
if self.rmsMonitoring:
self.rmsMonitoringReporter.commit()
res = self._addMetadataToFiles(toSchedule)
if not res["OK"]:
return res
else:
filesToSchedule = res["Value"]
for lfn in filesToSchedule:
opFile = filesToSchedule[lfn]
validTargets = toSchedule[lfn][1]
for targetSE in validTargets:
ftsFile = FTS3File.fromRMSFile(opFile, targetSE)
fts3Files.append(ftsFile)
if fts3Files:
res = Registry.getUsernameForDN(self.request.OwnerDN)
if not res["OK"]:
self.log.error(
"Cannot get username for DN",
"%s %s" % (self.request.OwnerDN, res["Message"]))
return res
username = res["Value"]
fts3Operation = FTS3TransferOperation.fromRMSObjects(
self.request, self.operation, username)
fts3Operation.ftsFiles = fts3Files
try:
if not fts3Operation.activity:
vo = getVOfromProxyGroup().get("Value")
fts3Plugin = getFTS3Plugin(vo=vo)
fts3Operation.activity = fts3Plugin.inferFTSActivity(
fts3Operation, self.request, self.operation)
except Exception:
pass
ftsSchedule = FTS3Client().persistOperation(fts3Operation)
if not ftsSchedule["OK"]:
self.log.error("Completely failed to schedule to FTS3:",
ftsSchedule["Message"])
return ftsSchedule
# might have nothing to schedule
ftsSchedule = ftsSchedule["Value"]
self.log.info("Scheduled with FTS3Operation id %s" % ftsSchedule)
self.log.info("%d files have been scheduled to FTS3" %
len(fts3Files))
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord(
self.createRMSRecord("Successful", len(fts3Files)))
for ftsFile in fts3Files:
opFile = rmsFilesIds[ftsFile.rmsFileID]
opFile.Status = "Scheduled"
self.log.debug("%s has been scheduled for FTS" % opFile.LFN)
else:
self.log.info("No files to schedule after metadata checks")
if self.rmsMonitoring:
self.rmsMonitoringReporter.commit()
# Just in case some transfers could not be scheduled, try them with RM
return self.dmTransfer(fromFTS=True)