def __init__( self, operation = None, csPath = None ):
"""c'tor
:param self: self reference
:param Operation operation: Operation instance
:param str csPath: CS path for this handler
"""
super( ReplicateAndRegister, self ).__init__( operation, csPath )
# # own gMonitor stuff for files
gMonitor.registerActivity( "ReplicateAndRegisterAtt", "Replicate and register attempted",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "ReplicateOK", "Replications successful",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "ReplicateFail", "Replications failed",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RegisterOK", "Registrations successful",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RegisterFail", "Registrations failed",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
# # for FTS
gMonitor.registerActivity( "FTSScheduleAtt", "Files schedule attempted",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "FTSScheduleOK", "File schedule successful",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "FTSScheduleFail", "File schedule failed",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
# # SE cache
# Clients
self.fc = FileCatalog()
if hasattr( self, "FTSMode" ) and getattr( self, "FTSMode" ):
from DIRAC.DataManagementSystem.Client.FTSClient import FTSClient
self.ftsClient = FTSClient()
def printRequest(request, status=None, full=False, verbose=True, terse=False):
global output
ftsClient = None
try:
from DIRAC.DataManagementSystem.Client.FTSClient import FTSClient
ftsClient = FTSClient()
except Exception, e:
gLogger.debug("Could not instantiate FtsClient", e)
def printFTSJobs(request):
""" Prints the FTSJobs associated to a request
:param request: Request object
"""
try:
if request.RequestID:
# We try first the new FTS3 system
from DIRAC.DataManagementSystem.Client.FTS3Client import FTS3Client
fts3Client = FTS3Client()
res = fts3Client.ping()
if res['OK']:
associatedFTS3Jobs = []
for op in request:
res = fts3Client.getOperationsFromRMSOpID(op.OperationID)
if res['OK']:
for fts3Op in res['Value']:
associatedFTS3Jobs.extend(fts3Op.ftsJobs)
if associatedFTS3Jobs:
gLogger.always(
'\n\nFTS3 jobs associated: \n%s' %
'\n'.join('%s@%s (%s)' %
(job.ftsGUID, job.ftsServer, job.status)
for job in associatedFTS3Jobs))
return
# If we are here, the attempt with the new FTS3 system did not work, let's try the old FTS system
gLogger.debug("Could not instantiate FTS3Client", res)
from DIRAC.DataManagementSystem.Client.FTSClient import FTSClient
ftsClient = FTSClient()
res = ftsClient.ping()
if not res['OK']:
gLogger.debug("Could not instantiate FtsClient", res)
return
res = ftsClient.getFTSJobsForRequest(request.RequestID)
if res['OK']:
ftsJobs = res['Value']
if ftsJobs:
gLogger.always(' FTS jobs associated: %s' %
','.join('%s (%s)' %
(job.FTSGUID, job.Status)
for job in ftsJobs))
# ImportError can be thrown for the old client
# AttributeError can be thrown because the deserialization will not have
# happened correctly on the new fts3 (CC7 typically), and the error is not
# properly propagated
except (ImportError, AttributeError) as err:
gLogger.debug("Could not instantiate FtsClient because of Exception",
repr(err))
def printRequest(request, status=None, full=False, verbose=True, terse=False):
global output
ftsClient = None
try:
if request.RequestID:
from DIRAC.DataManagementSystem.Client.FTSClient import FTSClient
ftsClient = FTSClient()
res = ftsClient.ping()
if not res['OK']:
gLogger.debug("Could not instantiate FtsClient", res)
ftsClient = None
except ImportError as err:
gLogger.debug("Could not instantiate FtsClient because of Exception",
repr(err))
if full:
output = ''
prettyPrint(json.loads(request.toJSON()['Value']))
gLogger.always(output)
else:
if not status:
status = request.Status
gLogger.always(
"Request name='%s' ID=%s Status='%s'%s%s%s" %
(request.RequestName, request.RequestID if hasattr(
request, 'RequestID') else '(not set yet)', request.Status,
" ('%s' in DB)" % status if status != request.Status else '',
(" Error='%s'" % request.Error)
if request.Error and request.Error.strip() else "",
(" Job=%s" % request.JobID) if request.JobID else ""))
gLogger.always("Created %s, Updated %s%s" %
(request.CreationTime, request.LastUpdate,
(", NotBefore %s" %
request.NotBefore) if request.NotBefore else ""))
if request.OwnerDN:
gLogger.always("Owner: '%s', Group: %s" %
(request.OwnerDN, request.OwnerGroup))
for indexOperation in enumerate(request):
op = indexOperation[1]
if not terse or op.Status == 'Failed':
printOperation(indexOperation, verbose, onlyFailed=terse)
if ftsClient:
# Check if FTS job exists
res = ftsClient.getFTSJobsForRequest(request.RequestID)
if res['OK']:
ftsJobs = res['Value']
if ftsJobs:
gLogger.always(' FTS jobs associated: %s' %
','.join('%s (%s)' % (job.FTSGUID, job.Status)
for job in ftsJobs))
def printFTSJobs(request):
""" Prints the FTSJobs associated to a request
:param request: Request object
"""
try:
if request.RequestID:
# We try first the new FTS3 system
from DIRAC.DataManagementSystem.Client.FTS3Client import FTS3Client
fts3Client = FTS3Client()
res = fts3Client.ping()
if res['OK']:
associatedFTS3Jobs = []
for op in request:
res = fts3Client.getOperationsFromRMSOpID(op.OperationID)
if res['OK']:
for fts3Op in res['Value']:
associatedFTS3Jobs.extend(fts3Op.ftsJobs)
if associatedFTS3Jobs:
gLogger.always(
'\n\nFTS3 jobs associated: \n%s' %
'\n'.join(
'%s@%s (%s)' %
(job.ftsGUID,
job.ftsServer,
job.status) for job in associatedFTS3Jobs))
return
# If we are here, the attempt with the new FTS3 system did not work, let's try the old FTS system
gLogger.debug("Could not instantiate FTS3Client", res)
from DIRAC.DataManagementSystem.Client.FTSClient import FTSClient
ftsClient = FTSClient()
res = ftsClient.ping()
if not res['OK']:
gLogger.debug("Could not instantiate FtsClient", res)
return
res = ftsClient.getFTSJobsForRequest(request.RequestID)
if res['OK']:
ftsJobs = res['Value']
if ftsJobs:
gLogger.always(' FTS jobs associated: %s' % ','.join('%s (%s)' % (job.FTSGUID, job.Status)
for job in ftsJobs))
except ImportError as err:
gLogger.debug("Could not instantiate FtsClient because of Exception", repr(err))
def __init__(self, operation=None, csPath=None):
"""c'tor
:param self: self reference
:param Operation operation: Operation instance
:param str csPath: CS path for this handler
"""
super(ReplicateAndRegister, self).__init__(operation, csPath)
# # own gMonitor stuff for files
gMonitor.registerActivity("ReplicateAndRegisterAtt",
"Replicate and register attempted",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("ReplicateOK", "Replications successful",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("ReplicateFail", "Replications failed",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RegisterOK", "Registrations successful",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RegisterFail", "Registrations failed",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
# # for FTS
gMonitor.registerActivity("FTSScheduleAtt", "Files schedule attempted",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("FTSScheduleOK", "File schedule successful",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("FTSScheduleFail", "File schedule failed",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
# # SE cache
self.seCache = {}
# Clients
self.rm = ReplicaManager()
self.ftsClient = FTSClient()
def ftsClient(cls):
""" FTS client """
if not cls.__ftsClient:
cls.__ftsClient = FTSClient()
return cls.__ftsClient
def ftsClient(self):
""" FTSClient getter """
if not self.__ftsClient:
self.__ftsClient = FTSClient()
return self.__ftsClient
def ftsTransfer(self):
""" replicate and register using FTS """
self.log.info("scheduling files in FTS...")
bannedTargets = self.checkSEsRSS()
if not bannedTargets['OK']:
gMonitor.addMark("FTSScheduleAtt")
gMonitor.addMark("FTSScheduleFail")
return bannedTargets
if bannedTargets['Value']:
return S_OK("%s targets are banned for writing" %
",".join(bannedTargets['Value']))
# Can continue now
self.log.verbose("No targets banned for writing")
toSchedule = {}
delayExecution = 0
errors = defaultdict(int)
for opFile in self.getWaitingFilesList():
opFile.Error = ''
gMonitor.addMark("FTSScheduleAtt")
# # check replicas
replicas = self._filterReplicas(opFile)
if not replicas["OK"]:
continue
replicas = replicas["Value"]
validReplicas = replicas.get("Valid")
noMetaReplicas = replicas.get("NoMetadata")
noReplicas = replicas.get('NoReplicas')
badReplicas = replicas.get('Bad')
noActiveReplicas = replicas.get('NoActiveReplicas')
if validReplicas:
validTargets = list(
set(self.operation.targetSEList) - set(validReplicas))
if not validTargets:
self.log.info("file %s is already present at all targets" %
opFile.LFN)
opFile.Status = "Done"
else:
toSchedule[opFile.LFN] = [
opFile, validReplicas, validTargets
]
else:
gMonitor.addMark("FTSScheduleFail")
if noMetaReplicas:
err = "Couldn't get metadata"
errors[err] += 1
self.log.verbose(
"unable to schedule '%s', %s at %s" %
(opFile.LFN, err, ','.join(noMetaReplicas)))
opFile.Error = err
elif noReplicas:
err = "File doesn't exist"
errors[err] += 1
self.log.error(
"Unable to schedule transfer", "%s %s at %s" %
(opFile.LFN, err, ','.join(noReplicas)))
opFile.Error = err
opFile.Status = 'Failed'
elif badReplicas:
err = "All replicas have a bad checksum"
errors[err] += 1
self.log.error(
"Unable to schedule transfer", "%s, %s at %s" %
(opFile.LFN, err, ','.join(badReplicas)))
opFile.Error = err
opFile.Status = 'Failed'
elif noActiveReplicas:
err = "No active replica found"
errors[err] += 1
self.log.verbose(
"Unable to schedule transfer", "%s, %s at %s" %
(opFile.LFN, err, ','.join(noActiveReplicas)))
opFile.Error = err
# All source SEs are banned, delay execution by 1 hour
delayExecution = 60
if delayExecution:
self.log.info("Delay execution of the request by %d minutes" %
delayExecution)
self.request.delayNextExecution(delayExecution)
# Log error counts
for error, count in errors.iteritems():
self.log.error(error, 'for %d files' % count)
filesToScheduleList = []
res = self._addMetadataToFiles(toSchedule)
if not res['OK']:
return res
else:
filesToSchedule = res['Value']
for lfn in filesToSchedule:
filesToScheduleList.append(
(filesToSchedule[lfn][0].toJSON()['Value'],
toSchedule[lfn][1], toSchedule[lfn][2]))
if filesToScheduleList:
ftsSchedule = FTSClient().ftsSchedule(self.request.RequestID,
self.operation.OperationID,
filesToScheduleList)
if not ftsSchedule["OK"]:
self.log.error("Completely failed to schedule to FTS:",
ftsSchedule["Message"])
return ftsSchedule
# might have nothing to schedule
ftsSchedule = ftsSchedule["Value"]
if not ftsSchedule:
return S_OK()
self.log.info("%d files have been scheduled to FTS" %
len(ftsSchedule['Successful']))
for opFile in self.operation:
fileID = opFile.FileID
if fileID in ftsSchedule["Successful"]:
gMonitor.addMark("FTSScheduleOK", 1)
opFile.Status = "Scheduled"
self.log.debug("%s has been scheduled for FTS" %
opFile.LFN)
elif fileID in ftsSchedule["Failed"]:
gMonitor.addMark("FTSScheduleFail", 1)
opFile.Error = ftsSchedule["Failed"][fileID]
if 'sourceSURL equals to targetSURL' in opFile.Error:
# In this case there is no need to continue
opFile.Status = 'Failed'
self.log.warn("unable to schedule %s for FTS: %s" %
(opFile.LFN, opFile.Error))
else:
self.log.info("No files to schedule after metadata checks")
# Just in case some transfers could not be scheduled, try them with RM
return self.dmTransfer(fromFTS=True)
Script.setUsageMessage('\n'.join(
[__doc__, 'Usage:',
' %s [option|cfgfile]' % Script.scriptName]))
from operator import itemgetter
if __name__ == "__main__":
from DIRAC.Core.Base.Script import parseCommandLine
parseCommandLine()
import DIRAC
from DIRAC import gLogger, gConfig
from DIRAC.DataManagementSystem.Client.FTSClient import FTSClient
ftsClient = FTSClient()
ret = ftsClient.getDBSummary()
if not ret["OK"]:
gLogger.error(ret["Message"])
DIRAC.exit(-1)
ret = ret["Value"]
ic = 1
ftsSites = ret.get("FTSSite", None)
if ftsSites:
gLogger.always("[%d] FTSSites:" % ic)
ic += 1
for ftsSite in ftsSites:
gLogger.always("- %-20s (%s)" %
def setUp(self):
""" test case set up """
gLogger.setLevel('NOTICE')
self.ftsSites = [
FTSSite(
ftsServer=
'https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer',
name='CERN.ch'),
FTSSite(
ftsServer=
'https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer',
name='PIC.es'),
FTSSite(
ftsServer=
'https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer',
name='RAL.uk'),
]
self.ses = ['CERN-USER', 'RAL-USER']
self.statuses = [
'Submitted', 'Finished', 'FinishedDirty', 'Active', 'Ready'
]
self.submitted = 0
self.numberOfJobs = 10
self.opIDs = []
self.ftsJobs = []
for i in xrange(self.numberOfJobs):
opID = i % 3
if opID not in self.opIDs:
self.opIDs.append(opID)
ftsJob = FTSJob()
ftsJob.FTSGUID = str(uuid.uuid4())
ftsJob.FTSServer = self.ftsSites[0].FTSServer
ftsJob.Status = self.statuses[i % len(self.statuses)]
ftsJob.OperationID = opID
if ftsJob.Status in FTSJob.FINALSTATES:
ftsJob.Completeness = 100
if ftsJob.Status == 'Active':
ftsJob.Completeness = 90
ftsJob.SourceSE = self.ses[i % len(self.ses)]
ftsJob.TargetSE = 'PIC-USER'
ftsJob.RequestID = 12345
ftsFile = FTSFile()
ftsFile.FileID = i + 1
ftsFile.OperationID = i + 1
ftsFile.LFN = '/a/b/c/%d' % i
ftsFile.Size = 1000000
ftsFile.OperationID = opID
ftsFile.SourceSE = ftsJob.SourceSE
ftsFile.TargetSE = ftsJob.TargetSE
ftsFile.SourceSURL = 'foo://source.bar.baz/%s' % ftsFile.LFN
ftsFile.TargetSURL = 'foo://target.bar.baz/%s' % ftsFile.LFN
ftsFile.Status = 'Waiting' if ftsJob.Status != 'FinishedDirty' else 'Failed'
ftsFile.RequestID = 12345
ftsFile.Checksum = 'addler'
ftsFile.ChecksumType = 'adler32'
ftsFile.FTSGUID = ftsJob.FTSGUID
if ftsJob.Status == 'FinishedDirty':
ftsJob.FailedFiles = 1
ftsJob.FailedSize = ftsFile.Size
ftsJob.addFile(ftsFile)
self.ftsJobs.append(ftsJob)
self.submitted = len(
[i for i in self.ftsJobs if i.Status == 'Submitted'])
self.ftsClient = FTSClient()