def __createFTSFiles( self ):
""" create LFNs file for glite-transfer-submit command
This file consists one line for each fiel to be transferred:
sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM]
:param self: self reference
"""
self.__updateMetadataCache()
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
if lfnStatus not in self.noSubmitStatus:
cksmStr = ""
# # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself
if self.__cksmTest and self.__cksmType:
checkSum = self.catalogMetadata.get( lfn, {} ).get( 'Checksum' )
if checkSum:
cksmStr = " %s:%s" % ( self.__cksmType, intAdlerToHex( hexAdlerToInt( checkSum ) ) )
ftsFile = FTSFile()
ftsFile.LFN = lfn
ftsFile.SourceSURL = self.fileDict[lfn].get( 'Source' )
ftsFile.TargetSURL = self.fileDict[lfn].get( 'Target' )
ftsFile.SourceSE = self.sourceSE
ftsFile.TargetSE = self.targetSE
ftsFile.Status = self.fileDict[lfn].get( 'Status' )
ftsFile.Checksum = cksmStr
ftsFile.Size = self.catalogMetadata.get( lfn, {} ).get( 'Size' )
self.ftsFiles.append( ftsFile )
self.submittedFiles += 1
return S_OK()
def __createFTSFiles( self ):
""" create LFNs file for glite-transfer-submit command
This file consists one line for each fiel to be transferred:
sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM]
:param self: self reference
"""
self.__updateMetadataCache()
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
if lfnStatus not in self.noSubmitStatus:
cksmStr = ""
# # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself
if self.__cksmTest and self.__cksmType:
checkSum = self.catalogMetadata.get( lfn, {} ).get( 'Checksum' )
if checkSum:
cksmStr = " %s:%s" % ( self.__cksmType, intAdlerToHex( hexAdlerToInt( checkSum ) ) )
ftsFile = FTSFile()
ftsFile.LFN = lfn
ftsFile.SourceSURL = self.fileDict[lfn].get( 'Source' )
ftsFile.TargetSURL = self.fileDict[lfn].get( 'Target' )
ftsFile.SourceSE = self.sourceSE
ftsFile.TargetSE = self.targetSE
ftsFile.Status = self.fileDict[lfn].get( 'Status' )
ftsFile.Checksum = cksmStr
ftsFile.Size = self.catalogMetadata.get( lfn, {} ).get( 'Size' )
self.ftsFiles.append( ftsFile )
self.submittedFiles += 1
return S_OK()
def test01Ctor( self ):
""" test ctor and (de-)serialization """
ftsFile = FTSFile( self.fromDict )
self.assertEqual( isinstance( ftsFile, FTSFile ), True )
for k, v in self.fromDict.items():
self.assertEqual( getattr( ftsFile, k ), v )
json = ftsFile.toJSON()
ftsFileJSON = FTSFile( json["Value"] )
self.assertEqual( isinstance( ftsFileJSON, FTSFile ), True )
for k, v in self.fromDict.items():
self.assertEqual( getattr( ftsFileJSON, k ), v )
def test01Ctor(self):
""" test ctor and (de-)serialization """
ftsFile = FTSFile(self.fromDict)
self.assertEqual(isinstance(ftsFile, FTSFile), True)
for k, v in self.fromDict.items():
self.assertEqual(getattr(ftsFile, k), v)
json = ftsFile.toJSON()
ftsFileJSON = FTSFile(json["Value"])
self.assertEqual(isinstance(ftsFileJSON, FTSFile), True)
for k, v in self.fromDict.items():
self.assertEqual(getattr(ftsFileJSON, k), v)
def getFTSJobList(self, statusList=None, limit=500):
""" select FTS jobs with statuses in :statusList: """
statusList = statusList if statusList else list(FTSJob.INITSTATES +
FTSJob.TRANSSTATES)
query = "SELECT * FROM `FTSJob` WHERE `Status` IN (%s) ORDER BY `LastUpdate` DESC LIMIT %s;" % (
stringListToString(statusList), limit)
trn = self._transaction([query])
if not trn['OK']:
self.log.error('Failed ftsJobSQL',
"getFTSJobList: %s" % trn['Message'])
return trn
ftsJobs = [FTSJob(ftsJobDict) for ftsJobDict in trn['Value'][query]]
for ftsJob in ftsJobs:
query = "SELECT * FROM `FTSFile` WHERE `FTSGUID` = '%s';" % ftsJob.FTSGUID
trn = self._transaction(query)
if not trn['OK']:
self.log.error('Failed ftsFileSQL',
"getFTSJobList: %s" % trn['Message'])
return trn
ftsFiles = [
FTSFile(ftsFileDict) for ftsFileDict in trn['Value'][query]
]
for ftsFile in ftsFiles:
ftsJob.addFile(ftsFile)
return S_OK(ftsJobs)
def getFTSJob(self, ftsJobID=None, readOnly=False):
""" get FTSJob given FTSJobID """
getJob = ["SELECT * FROM `FTSJob` WHERE `FTSJobID` = %s;" % ftsJobID]
getJob = self._transaction(getJob)
if not getJob["OK"]:
self.log.error(getJob["Message"])
return getJob
getJob = getJob["Value"]
if not getJob:
return S_OK()
ftsJob = FTSJob(getJob.values()[0][0])
selectFiles = self._transaction([
"SELECT * FROM `FTSFile` WHERE `FTSGUID` = '%s';" % ftsJob.FTSGUID
])
if not selectFiles["OK"]:
self.log.error(selectFiles["Message"])
return selectFiles
selectFiles = selectFiles["Value"]
ftsFiles = [FTSFile(item) for item in selectFiles.values()[0]]
for ftsFile in ftsFiles:
ftsJob.addFile(ftsFile)
if not readOnly:
setAssigned = "UPDATE `FTSJob` SET `Status`='Assigned' WHERE `FTSJobID` = %s;" % ftsJobID
setAssigned = self._query(setAssigned)
if not setAssigned["OK"]:
self.log.error(setAssigned["Message"])
return setAssigned
return S_OK(ftsJob)
def export_putFTSFileList(self, ftsFilesJSONList):
""" put FTS files list
"""
ftsFiles = []
for ftsFileJSON in ftsFilesJSONList:
ftsFiles.append(FTSFile(ftsFileJSON))
return self.ftsDB.putFTSFileList(ftsFiles)
def getFTSJobsForRequest(self, requestID, statusList=None):
""" get list of FTSJobs with status in :statusList: for request given its requestID
TODO: should be more smart, i.e. one query to select all ftsfiles
"""
statusList = statusList if statusList else list(FTSJob.INITSTATES +
FTSJob.TRANSSTATES)
query = "SELECT * FROM `FTSJob` WHERE `RequestID` = %s AND `Status` in (%s)" % (
requestID, stringListToString(statusList))
ftsJobs = self._transaction([query])
if not ftsJobs['OK']:
self.log.error("Failed getFTSJobsForRequest",
"%s" % ftsJobs['Message'])
return ftsJobs
ftsJobs = ftsJobs['Value'][query] if query in ftsJobs['Value'] else []
ftsJobs = [FTSJob(ftsJobDict) for ftsJobDict in ftsJobs]
for ftsJob in ftsJobs:
query = "SELECT * FROM `FTSFile` WHERE `FTSGUID` = '%s' AND `RequestID`=%s" % (
ftsJob.FTSGUID, requestID)
ftsFiles = self._transaction([query])
if not ftsFiles['OK']:
self.log.error("Failed getFTSJobsForRequest",
"%s" % ftsFiles['Message'])
return ftsFiles
ftsFiles = ftsFiles['Value'][query] if query in ftsFiles[
'Value'] else []
for ftsFileDict in ftsFiles:
ftsJob.addFile(FTSFile(ftsFileDict))
return S_OK(ftsJobs)
def __init__(self, fromDict=None):
"""c'tor
:param self: self reference
:param dict fromDict: data dict
"""
Record.__init__(self)
now = datetime.datetime.utcnow().replace(microsecond=0)
self.__data__["CreationTime"] = now
self.__data__["SubmitTime"] = now
self.__data__["LastUpdate"] = now
self.__data__["Status"] = "Submitted"
self.__data__["Completeness"] = 0
self.__data__["FTSJobID"] = 0
self.__files__ = TypedList(allowedTypes=FTSFile)
self._log = gLogger.getSubLogger("FTSJob-%s" % self.FTSJobID, True)
fromDict = fromDict if fromDict else {}
for ftsFileDict in fromDict.get("FTSFiles", []):
self += FTSFile(ftsFileDict)
if "FTSFiles" in fromDict: del fromDict["FTSFiles"]
for key, value in fromDict.items():
if key not in self.__data__:
raise AttributeError("Unknown FTSJob attribute '%s'" % key)
if value:
setattr(self, key, value)
def getFTSFileList(self, statusList=None, limit=1000):
""" get at most :limit: FTSFiles with status in :statusList:
:param statusList: list with FTSFiles statuses
:type statusList: python:list
:param int limit: select query limit
"""
statusList = statusList if statusList else ["Waiting"]
reStatus = []
inStatus = []
for status in statusList:
if "%" in status or ".*" in status or ".+" in status:
reStatus.append(status)
else:
inStatus.append(status)
reQuery = "`Status` REGEXP '%s'" % "|".join(
reStatus) if reStatus else ""
inQuery = "`Status` IN (%s)" % stringListToString(
inStatus) if inStatus else ""
whereClause = " OR ".join([q for q in (reQuery, inQuery) if q])
if whereClause:
whereClause = "WHERE %s" % whereClause
query = "SELECT * FROM `FTSFile` %s ORDER BY `LastUpdate` DESC LIMIT %s;" % (
whereClause, limit)
trn = self._transaction([query])
if not trn['OK']:
self.log.error("Failed getFTSFileList", "%s" % trn['Message'])
return trn
return S_OK([FTSFile(fileDict) for fileDict in trn['Value'][query]])
def _getFTSFileProperties(self, ftsFileID, columnNames=None):
""" select :columnNames: from FTSJobFile table """
columnNames = columnNames if columnNames else FTSFile.tableDesc(
)["Fields"].keys()
columnNames = ",".join(
['`%s`' % str(columnName) for columnName in columnNames])
return "SELECT %s FROM `FTSFile` WHERE `FTSFileID` = %s;" % (
columnNames, int(ftsFileID))
def getFTSFileList( self, statusList = None, limit = None ):
""" get list of FTSFiles with status in statusList """
statusList = statusList if statusList else [ "Waiting" ]
limit = limit if limit else 1000
getFTSFileList = self.ftsManager.getFTSFileList( statusList, limit )
if not getFTSFileList['OK']:
self.log.error( "Failed getFTSFileList", "%s" % getFTSFileList['Message'] )
return getFTSFileList
getFTSFileList = getFTSFileList['Value']
return S_OK( [ FTSFile( ftsFile ) for ftsFile in getFTSFileList ] )
def getAllFTSFilesForRequest( self, requestID ):
""" read FTSFiles for a given :requestID:
:param int requestID: ReqDB.Request.RequestID
"""
ftsFiles = self.ftsManager.getAllFTSFilesForRequest( requestID )
if not ftsFiles['OK']:
self.log.error( "Failed getFTSFilesForRequest", "%s" % ftsFiles['Message'] )
return ftsFiles
return S_OK( [ FTSFile( ftsFileDict ) for ftsFileDict in ftsFiles['Value'] ] )
def getAllFTSFilesForRequest(self, requestID):
""" get FTSFiles with status in :statusList: for request given its :requestID: """
requestID = int(requestID)
query = "SELECT * FROM `FTSFile` WHERE `RequestID` = %s;" % (requestID)
ftsFiles = self._transaction([query])
if not ftsFiles['OK']:
self.log.error("getFTSFilesForRequest: %s" % ftsFiles['Message'])
return ftsFiles
ftsFiles = ftsFiles['Value'].get(query, [])
return S_OK([FTSFile(ftsFileDict) for ftsFileDict in ftsFiles])
def getFTSFileList( self, statusList = None, limit = None ):
""" get list of FTSFiles with status in statusList """
statusList = statusList if statusList else [ "Waiting" ]
limit = limit if limit else 1000
getFTSFileList = self.ftsManager().getFTSFileList( statusList, limit )
if not getFTSFileList["OK"]:
self.log.error( "getFTSFileList: %s" % getFTSFileList["Message"] )
return getFTSFileList
getFTSFileList = getFTSFileList["Value"]
return S_OK( [ FTSFile( ftsFile ) for ftsFile in getFTSFileList ] )
def getFTSFilesForRequest( self, requestID, statusList = None ):
""" read FTSFiles for a given :requestID:
:param int requestID: ReqDB.Request.RequestID
:param list statusList: List of statuses (default: Waiting)
"""
ftsFiles = self.ftsManager.getFTSFilesForRequest( requestID, statusList )
if not ftsFiles['OK']:
self.log.error( "Failed getFTSFilesForRequest", "%s" % ftsFiles['Message'] )
return ftsFiles
return S_OK( [ FTSFile( ftsFileDict ) for ftsFileDict in ftsFiles['Value'] ] )
def getFTSFilesForRequest( self, requestID, operationID = None ):
""" read FTSFiles for a given :requestID:
:param int requestID: ReqDB.Request.RequestID
:param int operationID: ReqDB.Operation.OperationID
"""
ftsFiles = self.ftsManager().getFTSFilesForRequest( requestID, operationID )
if not ftsFiles["OK"]:
self.log.error( "getFTSFilesForRequest: %s" % ftsFiles["Message"] )
return ftsFiles
return S_OK( [ FTSFile( ftsFileDict ) for ftsFileDict in ftsFiles["Value"] ] )
def getFTSFile( self, ftsFileID = None ):
""" get FTSFile
:param int ftsFileID: FTSFileID
"""
getFile = self.ftsManager.getFTSFile( ftsFileID )
if not getFile['OK']:
self.log.error( 'Failed to get FTS file', getFile['Message'] )
# # de-serialize
if getFile['Value']:
ftsFile = FTSFile( getFile['Value'] )
return S_OK( ftsFile )
def getFTSFilesForRequest(self, requestID, statusList=None):
""" get FTSFiles with status in :statusList: for request given its :requestID: """
requestID = int(requestID)
statusList = statusList if statusList else ["Waiting"]
query = "SELECT * FROM `FTSFile` WHERE `RequestID` = %s AND `Status` IN (%s);" % (
requestID, stringListToString(statusList))
ftsFiles = self._transaction([query])
if not ftsFiles['OK']:
self.log.error("getFTSFilesForRequest: %s" % ftsFiles['Message'])
return ftsFiles
ftsFiles = ftsFiles['Value'].get(query, [])
return S_OK([FTSFile(ftsFileDict) for ftsFileDict in ftsFiles])
def export_putFTSFile(cls, ftsFileJSON):
""" put FTSFile into FTSDB """
try:
ftsFile = FTSFile(ftsFileJSON)
isValid = cls.ftsValidator().validate(ftsFile)
if not isValid["OK"]:
gLogger.error(isValid["Message"])
return isValid
return cls.__ftsDB.putFTSFile(ftsFile)
except Exception, error:
gLogger.exception(error)
return S_ERROR(error)
def getFTSFile(self, ftsFileID):
""" read FTSFile from db given FTSFileID """
select = "SELECT * FROM `FTSFile` WHERE `FTSFileID` = %s;" % ftsFileID
select = self._transaction([select])
if not select["OK"]:
self.log.error(select["Message"])
return select
select = select["Value"]
if not select.values()[0]:
return S_OK()
ftsFile = FTSFile(select.values()[0][0])
return S_OK(ftsFile)
def getFTSFile(self, ftsFileID):
""" read FTSFile from db given FTSFileID """
select = "SELECT * FROM `FTSFile` WHERE `FTSFileID` = %s;" % ftsFileID
select = self._transaction([select])
if not select['OK']:
self.log.error('Failed FTS file selection', select['Message'])
return select
select = select['Value']
if not select.values()[0]:
return S_OK()
ftsFile = FTSFile(select.values()[0][0])
return S_OK(ftsFile)
def setUp(self):
""" test set up """
self.fileA = FTSFile({
"LFN": "/a",
"ChecksumType": "ADLER32",
"Checksum": "123456",
"Size": 10,
"SourceSE": "CERN-USER",
"TargetSE": "PIC-USER",
"SourceSURL": "sourceSURL",
"TargetSURL": "targetSURL"
})
self.fileB = FTSFile({
"LFN": "/b",
"ChecksumType": "ADLER32",
"Checksum": "654321",
"Size": 10,
"SourceSE": "CERN-USER",
"TargetSE": "PIC-USER",
"SourceSURL": "sourceSURL",
"TargetSURL": "targetSURL"
})
def getFTSFile( self, ftsFileID = None ):
""" get FTSFile
:param int fileID: FileID
:param int ftsFileID: FTSFileID
"""
getFile = self.ftsManager().getFTSFile( ftsFileID )
if not getFile["OK"]:
self.log.error( getFile["Message"] )
# # de-serialize
if getFile["Value"]:
getFile = FTSFile( getFile["Value"] )
if not getFile["OK"]:
self.log.error( getFile["Message"] )
return getFile
def export_putFTSJob(self, ftsJobJSON):
""" put FTSJob (serialized in JSON into FTSDB """
ftsFiles = []
if "FTSFiles" in ftsJobJSON:
ftsFiles = ftsJobJSON.get("FTSFiles", [])
del ftsJobJSON["FTSFiles"]
try:
ftsJob = FTSJob(ftsJobJSON)
for ftsFile in ftsFiles:
ftsJob.addFile(FTSFile(ftsFile))
except Exception, error:
gLogger.exception(error)
return S_ERROR(error)
def __init__(self, fromDict=None):
"""c'tor
:param self: self reference
:param dict fromDict: data dict
"""
self.__data__ = dict.fromkeys(self.tableDesc()["Fields"].keys(), None)
now = datetime.datetime.utcnow().replace(microsecond=0)
self.__data__["CreationTime"] = now
self.__data__["SubmitTime"] = now
self.__data__["LastUpdate"] = now
self.__data__["Status"] = "Submitted"
self.__data__["Completeness"] = 0
self.__data__["FTSJobID"] = 0
self._regTime = 0.
self._regSuccess = 0
self._regTotal = 0
self.__files__ = TypedList(allowedTypes=FTSFile)
self._fc = FileCatalog()
self._fts3context = None
self._states = tuple(
set(self.INITSTATES + self.TRANSSTATES + self.FAILEDSTATES +
self.FINALSTATES))
fromDict = fromDict if fromDict else {}
for ftsFileDict in fromDict.get("FTSFiles", []):
self += FTSFile(ftsFileDict)
if "FTSFiles" in fromDict:
del fromDict["FTSFiles"]
for key, value in fromDict.items():
if key not in self.__data__:
raise AttributeError("Unknown FTSJob attribute '%s'" % key)
if value:
setattr(self, key, value)
self._log = gLogger.getSubLogger(
"req_%s/FTSJob-%s" % (self.RequestID, self.FTSGUID), True)
def getFTSJob(self, ftsJobID=None):
""" get FTSJob given FTSJobID """
getJob = ["SELECT * FROM `FTSJob` WHERE `FTSJobID` = %s;" % ftsJobID]
getJob = self._transaction(getJob)
if not getJob['OK']:
self.log.error('Failed ftsFileSQL', getJob['Message'])
return getJob
getJob = getJob['Value']
if not getJob:
return S_OK()
ftsJob = FTSJob(getJob.values()[0][0])
selectFiles = self._transaction([
"SELECT * FROM `FTSFile` WHERE `FTSGUID` = '%s';" % ftsJob.FTSGUID
])
if not selectFiles['OK']:
self.log.error('Failed ftsFileSQL', selectFiles['Message'])
return selectFiles
selectFiles = selectFiles['Value']
ftsFiles = [FTSFile(item) for item in selectFiles.values()[0]]
for ftsFile in ftsFiles:
ftsJob.addFile(ftsFile)
return S_OK(ftsJob)
def monitorFTS2(self, command="glite-transfer-status", full=False):
""" monitor fts job """
if not self.FTSGUID:
return S_ERROR("FTSGUID not set, FTS job not submitted?")
monitorCommand = command.split() + \
["--verbose",
"-s",
self.FTSServer,
self.FTSGUID ]
if full:
monitorCommand.append("-l")
monitor = executeGridCommand("", monitorCommand)
if not monitor["OK"]:
return monitor
returnCode, outputStr, errStr = monitor["Value"]
# Returns a non zero status if error
if returnCode != 0:
if 'was not found' in outputStr and not errStr:
errStr = 'Job was not found'
return S_ERROR(errStr)
outputStr = outputStr.replace("'", "").replace("<",
"").replace(">", "")
# # set FTS job status
regExp = re.compile("Status:\\s+(\\S+)")
# with FTS3 this can be uppercase
self.Status = re.search(regExp, outputStr).group(1)
statusSummary = {}
# This is capitalized, even in FTS3!
for state in FTSFile.ALL_STATES:
regExp = re.compile("\\s+%s:\\s+(\\d+)" % state)
if regExp.search(outputStr):
statusSummary[state] = int(
re.search(regExp, outputStr).group(1))
total = sum(statusSummary.values())
completed = sum(
statusSummary.get(state, 0) for state in FTSFile.FINAL_STATES)
self.Completeness = 100 * completed / total if total else 0
if not full:
return S_OK(statusSummary)
# The order of informations is not the same for glite- and fts- !!!
# In order: new fts-, old fts-, glite-
realJob = len(self) != 0
iExptr = None
for iExptr, exptr in enumerate(
('[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Staging:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)',
'[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)',
'[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Retries:[ ]+(\\d+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)'
)):
regExp = re.compile(exptr, re.S)
fileInfo = re.findall(regExp, outputStr)
if fileInfo:
break
if not fileInfo:
return S_ERROR("Error monitoring job (no regexp match)")
for info in fileInfo:
if iExptr == 0:
# version >= 3.2.30
sourceURL, targetURL, fileStatus, reason, duration, _retries, _staging = info
elif iExptr == 1:
# version FTS3 < 3.2.30
sourceURL, targetURL, fileStatus, reason, duration, _retries = info
elif iExptr == 2:
# version FTS2
sourceURL, targetURL, fileStatus, _retries, reason, duration = info
else:
return S_ERROR('Error monitoring job (implement match %d)' %
iExptr)
candidateFile = None
if not realJob:
# This is used by the CLI monitoring of jobs in case no file was specified
candidateFile = FTSFile()
candidateFile.LFN = overlap(sourceURL, targetURL)
candidateFile.SourceSURL = sourceURL
candidateFile.Size = 0
self += candidateFile
else:
for ftsFile in self:
if ftsFile.SourceSURL == sourceURL:
candidateFile = ftsFile
break
if not candidateFile:
continue
# Can be uppercase for FTS3
if not candidateFile.TargetSURL:
candidateFile.TargetSURL = targetURL
candidateFile.Status = fileStatus
candidateFile.Error = reason
candidateFile._duration = duration
if candidateFile.Status == "Failed":
for missingSource in self.missingSourceErrors:
if missingSource.match(reason):
candidateFile.Error = "MissingSource"
# If the staging info was present, record it
if len(info) > 6:
candidateFile._staging = info[6]
# # register successful files
if self.Status in FTSJob.FINALSTATES:
return self.finalize()
return S_OK()
def setUp(self):
""" test case set up """
gLogger.setLevel('NOTICE')
self.ftsSites = [
FTSSite(
ftsServer=
'https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer',
name='CERN.ch'),
FTSSite(
ftsServer=
'https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer',
name='PIC.es'),
FTSSite(
ftsServer=
'https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer',
name='RAL.uk'),
]
self.ses = ['CERN-USER', 'RAL-USER']
self.statuses = [
'Submitted', 'Finished', 'FinishedDirty', 'Active', 'Ready'
]
self.submitted = 0
self.numberOfJobs = 10
self.opIDs = []
self.ftsJobs = []
for i in xrange(self.numberOfJobs):
opID = i % 3
if opID not in self.opIDs:
self.opIDs.append(opID)
ftsJob = FTSJob()
ftsJob.FTSGUID = str(uuid.uuid4())
ftsJob.FTSServer = self.ftsSites[0].FTSServer
ftsJob.Status = self.statuses[i % len(self.statuses)]
ftsJob.OperationID = opID
if ftsJob.Status in FTSJob.FINALSTATES:
ftsJob.Completeness = 100
if ftsJob.Status == 'Active':
ftsJob.Completeness = 90
ftsJob.SourceSE = self.ses[i % len(self.ses)]
ftsJob.TargetSE = 'PIC-USER'
ftsJob.RequestID = 12345
ftsFile = FTSFile()
ftsFile.FileID = i + 1
ftsFile.OperationID = i + 1
ftsFile.LFN = '/a/b/c/%d' % i
ftsFile.Size = 1000000
ftsFile.OperationID = opID
ftsFile.SourceSE = ftsJob.SourceSE
ftsFile.TargetSE = ftsJob.TargetSE
ftsFile.SourceSURL = 'foo://source.bar.baz/%s' % ftsFile.LFN
ftsFile.TargetSURL = 'foo://target.bar.baz/%s' % ftsFile.LFN
ftsFile.Status = 'Waiting' if ftsJob.Status != 'FinishedDirty' else 'Failed'
ftsFile.RequestID = 12345
ftsFile.Checksum = 'addler'
ftsFile.ChecksumType = 'adler32'
ftsFile.FTSGUID = ftsJob.FTSGUID
if ftsJob.Status == 'FinishedDirty':
ftsJob.FailedFiles = 1
ftsJob.FailedSize = ftsFile.Size
ftsJob.addFile(ftsFile)
self.ftsJobs.append(ftsJob)
self.submitted = len(
[i for i in self.ftsJobs if i.Status == 'Submitted'])
self.ftsClient = FTSClient()
def setUp(self):
""" test case set up """
# ## set some defaults
gConfig.setOptionValue('DIRAC/Setup', 'Test')
gConfig.setOptionValue('/DIRAC/Setups/Test/DataManagement', 'Test')
gConfig.setOptionValue(
'/Systems/DataManagement/Test/Databases/FTSDB/Host', 'localhost')
gConfig.setOptionValue(
'/Systems/DataManagement/Test/Databases/FTSDB/DBName', 'FTSDB')
gConfig.setOptionValue(
'/Systems/DataManagement/Test/Databases/FTSDB/User', 'Dirac')
self.ftsSites = [
FTSSite({
"FTSServer":
"https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer",
"Name": "CERN.ch"
}),
FTSSite({
"FTSServer":
"https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer",
"Name": "PIC.es"
}),
FTSSite({
"FTSServer":
"https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer",
"Name": "RAL.uk"
}),
FTSSite({
"FTSServer":
"https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer",
"Name": "SARA.nl"
}),
FTSSite({
"FTSServer":
"https://fts.cr.cnaf.infn.it:8443/glite-data-transfer-fts/services/FileTransfer",
"Name": "CNAF.it"
}),
FTSSite({
"FTSServer":
"https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer",
"Name": "NIKHEF.nl"
}),
FTSSite({
"FTSServer":
"https://fts-fzk.gridka.de:8443/glite-data-transfer-fts/services/FileTransfer",
"Name": "GRIDKA.de"
}),
FTSSite({
"FTSServer":
"https://cclcgftsprod.in2p3.fr:8443/glite-data-transfer-fts/services/FileTransfer",
"Name": "IN2P3.fr"
})
]
self.ftsFiles = []
for i in range(100):
ftsFile = FTSFile()
ftsFile.FileID = i + 1
ftsFile.OperationID = 9999
ftsFile.LFN = "/a/b/c/%d" % i
ftsFile.Size = 10
ftsFile.SourceSE = "CERN-USER"
ftsFile.TargetSE = "PIC-USER"
ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN
ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN
ftsFile.Status = "Waiting"
self.ftsFiles.append(ftsFile)
ses = ["CERN-USER", "RAL-USER"]
statuses = [
"Submitted", "Finished", "FinishedDirty", "Active", "Ready"
]
self.submitted = 0
self.ftsJobs = []
for i in range(7200):
ftsJob = FTSJob()
ftsJob.FTSGUID = str(uuid.uuid4())
ftsJob.FTSServer = self.ftsSites[0].FTSServer
ftsJob.Status = statuses[i % len(statuses)]
if ftsJob.Status in FTSJob.FINALSTATES:
ftsJob.Completeness = 100
if ftsJob.Status == "Active":
ftsJob.Completeness = 90
ftsJob.SourceSE = ses[i % len(ses)]
ftsJob.TargetSE = "PIC-USER"
ftsFile = FTSFile()
ftsFile.FileID = i + 1
ftsFile.OperationID = i + 1
ftsFile.LFN = "/a/b/c/%d" % i
ftsFile.Size = 1000000
ftsFile.SourceSE = ftsJob.SourceSE
ftsFile.TargetSE = ftsJob.TargetSE
ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN
ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN
ftsFile.Status = "Waiting" if ftsJob.Status != "FinishedDirty" else "Failed"
ftsFile.FTSGUID = ftsJob.FTSGUID
if ftsJob.Status == "FinishedDirty":
ftsJob.FailedFiles = 1
ftsJob.FailedSize = ftsFile.Size
ftsJob.addFile(ftsFile)
self.ftsJobs.append(ftsJob)
self.submitted = len(
[i for i in self.ftsJobs if i.Status == "Submitted"])
def _getFTSFileProperties( self, ftsFileID, columnNames = None ): """ select :columnNames: from FTSJobFile table """ columnNames = columnNames if columnNames else FTSFile.tableDesc()["Fields"].keys() columnNames = ",".join( [ '`%s`' % str( columnName ) for columnName in columnNames ] ) return "SELECT %s FROM `FTSFile` WHERE `FTSFileID` = %s;" % ( columnNames, int( ftsFileID ) )
def export_ftsSchedule(self, requestID, operationID, fileJSONList):
""" call FTS scheduler
:param int requestID: ReqDB.Request.RequestID
:param int operationID: ReqDB.Operation.OperationID
:param list fileJSONList: [ (fileJSON, [sourceSE, ...], [ targetSE, ...] ) ]
"""
# # this will be returned on success
ret = {"Successful": [], "Failed": {}}
requestID = int(requestID)
operationID = int(operationID)
fileIDs = []
for fileJSON, sourceSEs, targetSEs in fileJSONList:
fileID = int(fileJSON.get("FileID"))
fileIDs.append(fileID)
cleanUpFTSFiles = self.__ftsDB.cleanUpFTSFiles(requestID, fileIDs)
if not cleanUpFTSFiles["OK"]:
self.log.error("ftsSchedule: %s" % cleanUpFTSFiles["Message"])
return S_ERROR(cleanUpFTSFiles["Message"])
ftsFiles = []
for fileJSON, sourceSEs, targetSEs in fileJSONList:
lfn = fileJSON.get("LFN", "")
size = int(fileJSON.get("Size", 0))
fileID = int(fileJSON.get("FileID", 0))
opID = int(fileJSON.get("OperationID", 0))
gLogger.info(
"ftsSchedule: LFN=%s FileID=%s OperationID=%s sources=%s targets=%s"
% (lfn, fileID, opID, sourceSEs, targetSEs))
replicaDict = self.replicaManager().getActiveReplicas(lfn)
if not replicaDict["OK"]:
gLogger.error("ftsSchedule: %s" % replicaDict["Message"])
ret["Failed"][fileID] = replicaDict["Message"]
continue
replicaDict = replicaDict["Value"]
if lfn in replicaDict["Failed"] and lfn not in replicaDict[
"Successful"]:
ret["Failed"][fileID] = "no active replicas found"
continue
replicaDict = replicaDict["Successful"][lfn] if lfn in replicaDict[
"Successful"] else {}
# # use valid replicas only
replicaDict = dict([(se, pfn) for se, pfn in replicaDict.items()
if se in sourceSEs])
if not replicaDict:
ret["Failed"][fileID] = "no active replicas found in sources"
continue
tree = self.ftsStrategy().replicationTree(sourceSEs, targetSEs,
size)
if not tree["OK"]:
gLogger.error("ftsSchedule: %s cannot be scheduled: %s" %
(lfn, tree["Message"]))
ret["Failed"][fileID] = tree["Message"]
continue
tree = tree["Value"]
gLogger.info("LFN=%s tree=%s" % (lfn, tree))
for repDict in tree.values():
gLogger.info(
"Strategy=%s Ancestor=%s SourceSE=%s TargetSE=%s" %
(repDict["Strategy"], repDict["Ancestor"],
repDict["SourceSE"], repDict["TargetSE"]))
transferSURLs = self._getTransferURLs(lfn, repDict, sourceSEs,
replicaDict)
if not transferSURLs["OK"]:
ret["Failed"][fileID] = transferSURLs["Message"]
continue
sourceSURL, targetSURL, fileStatus = transferSURLs["Value"]
if sourceSURL == targetSURL:
ret["Failed"][
fileID] = "sourceSURL equals to targetSURL for %s" % lfn
continue
gLogger.info("sourceURL=%s targetURL=%s FTSFile.Status=%s" %
(sourceSURL, targetSURL, fileStatus))
ftsFile = FTSFile()
for key in ("LFN", "FileID", "OperationID", "Checksum",
"ChecksumType", "Size"):
setattr(ftsFile, key, fileJSON.get(key))
ftsFile.RequestID = requestID
ftsFile.OperationID = operationID
ftsFile.SourceSURL = sourceSURL
ftsFile.TargetSURL = targetSURL
ftsFile.SourceSE = repDict["SourceSE"]
ftsFile.TargetSE = repDict["TargetSE"]
ftsFile.Status = fileStatus
ftsFiles.append(ftsFile)
if not ftsFiles:
return S_ERROR("ftsSchedule: no FTSFiles to put")
put = self.__ftsDB.putFTSFileList(ftsFiles)
if not put["OK"]:
gLogger.error("ftsSchedule: %s" % put["Message"])
return put
for fileJSON, sources, targets in fileJSONList:
lfn = fileJSON.get("LFN", "")
fileID = fileJSON.get("FileID", 0)
if fileID not in ret["Failed"]:
ret["Successful"].append(int(fileID))
# # if we land here some files have been properly scheduled
return S_OK(ret)
def ftsSchedule( self, requestID, operationID, opFileList ):
""" schedule lfn for FTS job
:param int requestID: RequestDB.Request.RequestID
:param int operationID: RequestDB.Operation.OperationID
:param list opFileList: list of tuples ( File.toJSON()['Value'], sourcesList, targetList )
"""
fileIDs = [int( fileJSON.get( 'FileID', 0 ) ) for fileJSON, _sourceSEs, _targetSEs in opFileList ]
res = self.ftsManager.cleanUpFTSFiles( requestID, fileIDs )
if not res['OK']:
self.log.error( "ftsSchedule: %s" % res['Message'] )
return S_ERROR( "ftsSchedule: %s" % res['Message'] )
ftsFiles = []
# # this will be returned on success
result = { "Successful": [], "Failed": {} }
for fileJSON, sourceSEs, targetSEs in opFileList:
lfn = fileJSON.get( "LFN", "" )
size = int( fileJSON.get( "Size", 0 ) )
fileID = int( fileJSON.get( "FileID", 0 ) )
opID = int( fileJSON.get( "OperationID", 0 ) )
self.log.verbose( "ftsSchedule: LFN=%s FileID=%s OperationID=%s sources=%s targets=%s" % ( lfn, fileID, opID,
sourceSEs,
targetSEs ) )
res = self.dataManager.getActiveReplicas( lfn )
if not res['OK']:
self.log.error( "ftsSchedule: %s" % res['Message'] )
result["Failed"][fileID] = res['Message']
continue
replicaDict = res['Value']
if lfn in replicaDict["Failed"] and lfn not in replicaDict["Successful"]:
result["Failed"][fileID] = "no active replicas found"
continue
replicaDict = replicaDict["Successful"].get( lfn, {} )
# # use valid replicas only
validReplicasDict = dict( [ ( se, pfn ) for se, pfn in replicaDict.items() if se in sourceSEs ] )
if not validReplicasDict:
self.log.warn( "No active replicas found in sources" )
result["Failed"][fileID] = "no active replicas found in sources"
continue
tree = self.ftsManager.getReplicationTree( sourceSEs, targetSEs, size )
if not tree['OK']:
self.log.error( "ftsSchedule: %s cannot be scheduled: %s" % ( lfn, tree['Message'] ) )
result["Failed"][fileID] = tree['Message']
continue
tree = tree['Value']
self.log.verbose( "LFN=%s tree=%s" % ( lfn, tree ) )
for repDict in tree.values():
self.log.verbose( "Strategy=%s Ancestor=%s SourceSE=%s TargetSE=%s" % ( repDict["Strategy"],
repDict["Ancestor"],
repDict["SourceSE"],
repDict["TargetSE"] ) )
transferSURLs = self._getTransferURLs( lfn, repDict, sourceSEs, validReplicasDict )
if not transferSURLs['OK']:
result["Failed"][fileID] = transferSURLs['Message']
continue
sourceSURL, targetSURL, fileStatus = transferSURLs['Value']
if sourceSURL == targetSURL:
result["Failed"][fileID] = "sourceSURL equals to targetSURL for %s" % lfn
continue
self.log.verbose( "sourceURL=%s targetURL=%s FTSFile.Status=%s" % ( sourceSURL, targetSURL, fileStatus ) )
ftsFile = FTSFile()
for key in ( "LFN", "FileID", "OperationID", "Checksum", "ChecksumType", "Size" ):
if fileJSON.get( key ):
setattr( ftsFile, key, fileJSON.get( key ) )
ftsFile.RequestID = requestID
ftsFile.OperationID = operationID
ftsFile.SourceSURL = sourceSURL
ftsFile.TargetSURL = targetSURL
ftsFile.SourceSE = repDict["SourceSE"]
ftsFile.TargetSE = repDict["TargetSE"]
ftsFile.Status = fileStatus
ftsFiles.append( ftsFile )
if not ftsFiles:
self.log.info( "ftsSchedule: no FTSFiles to put for request %d" % requestID )
return S_OK( result )
ftsFilesJSONList = [ftsFile.toJSON()['Value'] for ftsFile in ftsFiles]
res = self.ftsManager.putFTSFileList( ftsFilesJSONList )
if not res['OK']:
self.log.error( "ftsSchedule: %s" % res['Message'] )
return S_ERROR( "ftsSchedule: %s" % res['Message'] )
result['Successful'] += [ fileID for fileID in fileIDs if fileID not in result['Failed']]
# # if we land here some files have been properly scheduled
return S_OK( result )
def setUp( self ):
""" test case set up """
# ## set some defaults
gConfig.setOptionValue( 'DIRAC/Setup', 'Test' )
gConfig.setOptionValue( '/DIRAC/Setups/Test/DataManagement', 'Test' )
gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/Host', 'localhost' )
gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/DBName', 'FTSDB' )
gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/User', 'Dirac' )
self.ftsSites = [ FTSSite( { "FTSServer": "https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "CERN.ch" } ),
FTSSite( { "FTSServer": "https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "PIC.es" } ),
FTSSite( { "FTSServer": "https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "RAL.uk" } ),
FTSSite( { "FTSServer": "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "SARA.nl" } ),
FTSSite( { "FTSServer": "https://fts.cr.cnaf.infn.it:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "CNAF.it" } ),
FTSSite( { "FTSServer": "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "NIKHEF.nl" } ),
FTSSite( { "FTSServer": "https://fts-fzk.gridka.de:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "GRIDKA.de" } ),
FTSSite( { "FTSServer": "https://cclcgftsprod.in2p3.fr:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "IN2P3.fr" } ) ]
self.ftsFiles = []
for i in range ( 100 ):
ftsFile = FTSFile()
ftsFile.FileID = i + 1
ftsFile.OperationID = 9999
ftsFile.LFN = "/a/b/c/%d" % i
ftsFile.Size = 10
ftsFile.SourceSE = "CERN-USER"
ftsFile.TargetSE = "PIC-USER"
ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN
ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN
ftsFile.Status = "Waiting"
self.ftsFiles.append( ftsFile )
ses = [ "CERN-USER", "RAL-USER" ]
statuses = [ "Submitted", "Finished", "FinishedDirty", "Active", "Ready" ]
self.submitted = 0
self.ftsJobs = []
for i in range( 7200 ):
ftsJob = FTSJob()
ftsJob.FTSGUID = str( uuid.uuid4() )
ftsJob.FTSServer = self.ftsSites[0].FTSServer
ftsJob.Status = statuses[ i % len( statuses ) ]
if ftsJob.Status in FTSJob.FINALSTATES:
ftsJob.Completeness = 100
if ftsJob.Status == "Active":
ftsJob.Completeness = 90
ftsJob.SourceSE = ses[ i % len( ses ) ]
ftsJob.TargetSE = "PIC-USER"
ftsFile = FTSFile()
ftsFile.FileID = i + 1
ftsFile.OperationID = i + 1
ftsFile.LFN = "/a/b/c/%d" % i
ftsFile.Size = 1000000
ftsFile.SourceSE = ftsJob.SourceSE
ftsFile.TargetSE = ftsJob.TargetSE
ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN
ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN
ftsFile.Status = "Waiting" if ftsJob.Status != "FinishedDirty" else "Failed"
ftsFile.FTSGUID = ftsJob.FTSGUID
if ftsJob.Status == "FinishedDirty":
ftsJob.FailedFiles = 1
ftsJob.FailedSize = ftsFile.Size
ftsJob.addFile( ftsFile )
self.ftsJobs.append( ftsJob )
self.submitted = len( [ i for i in self.ftsJobs if i.Status == "Submitted" ] )
def export_ftsSchedule( self, requestID, operationID, fileJSONList ):
""" call FTS scheduler
:param int requestID: ReqDB.Request.RequestID
:param int operationID: ReqDB.Operation.OperationID
:param list fileJSONList: [ (fileJSON, [sourceSE, ...], [ targetSE, ...] ) ]
"""
# # this will be returned on success
ret = { "Successful": [], "Failed": {} }
requestID = int( requestID )
operationID = int( operationID )
fileIDs = []
for fileJSON, sourceSEs, targetSEs in fileJSONList:
fileID = int( fileJSON.get( "FileID" ) )
fileIDs.append( fileID )
cleanUpFTSFiles = self.__ftsDB.cleanUpFTSFiles( requestID, fileIDs )
if not cleanUpFTSFiles["OK"]:
self.log.error( "ftsSchedule: %s" % cleanUpFTSFiles["Message"] )
return S_ERROR( cleanUpFTSFiles["Message"] )
ftsFiles = []
for fileJSON, sourceSEs, targetSEs in fileJSONList:
lfn = fileJSON.get( "LFN", "" )
size = int( fileJSON.get( "Size", 0 ) )
fileID = int( fileJSON.get( "FileID", 0 ) )
opID = int( fileJSON.get( "OperationID", 0 ) )
gLogger.info( "ftsSchedule: LFN=%s FileID=%s OperationID=%s sources=%s targets=%s" % ( lfn, fileID, opID,
sourceSEs, targetSEs ) )
replicaDict = self.replicaManager().getActiveReplicas( lfn )
if not replicaDict["OK"]:
gLogger.error( "ftsSchedule: %s" % replicaDict["Message"] )
ret["Failed"][fileID] = replicaDict["Message"]
continue
replicaDict = replicaDict["Value"]
if lfn in replicaDict["Failed"] and lfn not in replicaDict["Successful"]:
ret["Failed"][fileID] = "no active replicas found"
continue
replicaDict = replicaDict["Successful"][lfn] if lfn in replicaDict["Successful"] else {}
# # use valid replicas only
replicaDict = dict( [ ( se, pfn ) for se, pfn in replicaDict.items() if se in sourceSEs ] )
if not replicaDict:
ret["Failed"][fileID] = "no active replicas found in sources"
continue
tree = self.ftsStrategy().replicationTree( sourceSEs, targetSEs, size )
if not tree["OK"]:
gLogger.error( "ftsSchedule: %s cannot be scheduled: %s" % ( lfn, tree["Message"] ) )
ret["Failed"][fileID] = tree["Message"]
continue
tree = tree["Value"]
gLogger.info( "LFN=%s tree=%s" % ( lfn, tree ) )
for repDict in tree.values():
gLogger.info( "Strategy=%s Ancestor=%s SourceSE=%s TargetSE=%s" % ( repDict["Strategy"], repDict["Ancestor"],
repDict["SourceSE"], repDict["TargetSE"] ) )
transferSURLs = self._getTransferURLs( lfn, repDict, sourceSEs, replicaDict )
if not transferSURLs["OK"]:
ret["Failed"][fileID] = transferSURLs["Message"]
continue
sourceSURL, targetSURL, fileStatus = transferSURLs["Value"]
if sourceSURL == targetSURL:
ret["Failed"][fileID] = "sourceSURL equals to targetSURL for %s" % lfn
continue
gLogger.info( "sourceURL=%s targetURL=%s FTSFile.Status=%s" % ( sourceSURL, targetSURL, fileStatus ) )
ftsFile = FTSFile()
for key in ( "LFN", "FileID", "OperationID", "Checksum", "ChecksumType", "Size" ):
setattr( ftsFile, key, fileJSON.get( key ) )
ftsFile.RequestID = requestID
ftsFile.OperationID = operationID
ftsFile.SourceSURL = sourceSURL
ftsFile.TargetSURL = targetSURL
ftsFile.SourceSE = repDict["SourceSE"]
ftsFile.TargetSE = repDict["TargetSE"]
ftsFile.Status = fileStatus
ftsFiles.append( ftsFile )
if not ftsFiles:
return S_ERROR( "ftsSchedule: no FTSFiles to put" )
put = self.__ftsDB.putFTSFileList( ftsFiles )
if not put["OK"]:
gLogger.error( "ftsSchedule: %s" % put["Message"] )
return put
for fileJSON, sources, targets in fileJSONList:
lfn = fileJSON.get( "LFN", "" )
fileID = fileJSON.get( "FileID", 0 )
if fileID not in ret["Failed"]:
ret["Successful"].append( int( fileID ) )
# # if we land here some files have been properly scheduled
return S_OK( ret )
def setUp( self ):
""" test case set up """
gLogger.setLevel( 'NOTICE' )
self.ftsSites = [ FTSSite( ftsServer = 'https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer', name = 'CERN.ch' ),
FTSSite( ftsServer = 'https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer', name = 'PIC.es' ),
FTSSite( ftsServer = 'https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer', name = 'RAL.uk' ),
]
self.ses = [ 'CERN-USER', 'RAL-USER' ]
self.statuses = [ 'Submitted', 'Finished', 'FinishedDirty', 'Active', 'Ready' ]
self.submitted = 0
self.numberOfJobs = 10
self.opIDs = []
self.ftsJobs = []
for i in xrange( self.numberOfJobs ):
opID = i % 3
if opID not in self.opIDs:
self.opIDs.append( opID )
ftsJob = FTSJob()
ftsJob.FTSGUID = str( uuid.uuid4() )
ftsJob.FTSServer = self.ftsSites[0].FTSServer
ftsJob.Status = self.statuses[ i % len( self.statuses ) ]
ftsJob.OperationID = opID
if ftsJob.Status in FTSJob.FINALSTATES:
ftsJob.Completeness = 100
if ftsJob.Status == 'Active':
ftsJob.Completeness = 90
ftsJob.SourceSE = self.ses[ i % len( self.ses ) ]
ftsJob.TargetSE = 'PIC-USER'
ftsJob.RequestID = 12345
ftsFile = FTSFile()
ftsFile.FileID = i + 1
ftsFile.OperationID = i + 1
ftsFile.LFN = '/a/b/c/%d' % i
ftsFile.Size = 1000000
ftsFile.OperationID = opID
ftsFile.SourceSE = ftsJob.SourceSE
ftsFile.TargetSE = ftsJob.TargetSE
ftsFile.SourceSURL = 'foo://source.bar.baz/%s' % ftsFile.LFN
ftsFile.TargetSURL = 'foo://target.bar.baz/%s' % ftsFile.LFN
ftsFile.Status = 'Waiting' if ftsJob.Status != 'FinishedDirty' else 'Failed'
ftsFile.RequestID = 12345
ftsFile.Checksum = 'addler'
ftsFile.ChecksumType = 'adler32'
ftsFile.FTSGUID = ftsJob.FTSGUID
if ftsJob.Status == 'FinishedDirty':
ftsJob.FailedFiles = 1
ftsJob.FailedSize = ftsFile.Size
ftsJob.addFile( ftsFile )
self.ftsJobs.append( ftsJob )
self.submitted = len( [ i for i in self.ftsJobs if i.Status == 'Submitted' ] )
self.ftsClient = FTSClient()
self.ftsClient.replicaManager = mock.Mock()
self.ftsClient.replicaManager.getActiveReplicas.return_value = {'OK': True,
'Value': {'Successful': {'/a/b/c/1':{'CERN-USER':'******',
'RAL-USER':'******'},
'/a/b/c/2':{'CERN-USER':'******',
'RAL-USER':'******'},
'/a/b/c/3':{'CERN-USER':'******',
'RAL-USER':'******'}
},
'Failed': {'/a/b/c/4':'/aa/a/b/c/4d',
'/a/b/c/5':'/aa/a/b/c/5d'}
}
}
def monitorFTS2( self, command = "glite-transfer-status", full = False ):
""" monitor fts job """
if not self.FTSGUID:
return S_ERROR( "FTSGUID not set, FTS job not submitted?" )
monitorCommand = command.split() + \
["--verbose",
"-s",
self.FTSServer,
self.FTSGUID ]
if full:
monitorCommand.append( "-l" )
monitor = executeGridCommand( "", monitorCommand )
if not monitor["OK"]:
return monitor
returnCode, outputStr, errStr = monitor["Value"]
# Returns a non zero status if error
if returnCode != 0:
if 'was not found' in outputStr and not errStr:
errStr = 'Job was not found'
return S_ERROR( errStr )
outputStr = outputStr.replace( "'" , "" ).replace( "<", "" ).replace( ">", "" )
# # set FTS job status
regExp = re.compile( "Status:\\s+(\\S+)" )
# with FTS3 this can be uppercase
self.Status = re.search( regExp, outputStr ).group( 1 )
statusSummary = {}
# This is capitalized, even in FTS3!
for state in FTSFile.ALL_STATES:
regExp = re.compile( "\\s+%s:\\s+(\\d+)" % state )
if regExp.search( outputStr ):
statusSummary[state] = int( re.search( regExp, outputStr ).group( 1 ) )
total = sum( statusSummary.values() )
completed = sum( statusSummary.get( state, 0 ) for state in FTSFile.FINAL_STATES )
self.Completeness = 100 * completed / total if total else 0
if not full:
return S_OK( statusSummary )
# The order of informations is not the same for glite- and fts- !!!
# In order: new fts-, old fts-, glite-
realJob = len( self ) != 0
iExptr = None
for iExptr, exptr in enumerate( (
'[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Staging:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)',
'[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)',
'[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Retries:[ ]+(\\d+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)'
) ):
regExp = re.compile( exptr, re.S )
fileInfo = re.findall( regExp, outputStr )
if fileInfo:
break
if not fileInfo:
return S_ERROR( "Error monitoring job (no regexp match)" )
for info in fileInfo:
if iExptr == 0:
# version >= 3.2.30
sourceURL, targetURL, fileStatus, reason, duration, _retries, _staging = info
elif iExptr == 1:
# version FTS3 < 3.2.30
sourceURL, targetURL, fileStatus, reason, duration, _retries = info
elif iExptr == 2:
# version FTS2
sourceURL, targetURL, fileStatus, _retries, reason, duration = info
else:
return S_ERROR( 'Error monitoring job (implement match %d)' % iExptr )
candidateFile = None
if not realJob:
# This is used by the CLI monitoring of jobs in case no file was specified
candidateFile = FTSFile()
candidateFile.LFN = overlap( sourceURL, targetURL )
candidateFile.SourceSURL = sourceURL
candidateFile.Size = 0
self +=candidateFile
else:
for ftsFile in self:
if ftsFile.SourceSURL == sourceURL:
candidateFile = ftsFile
break
if not candidateFile:
continue
# Can be uppercase for FTS3
if not candidateFile.TargetSURL:
candidateFile.TargetSURL = targetURL
candidateFile.Status = fileStatus
candidateFile.Error = reason
candidateFile._duration = duration
if candidateFile.Status == "Failed":
for missingSource in self.missingSourceErrors:
if missingSource.match( reason ):
candidateFile.Error = "MissingSource"
# If the staging info was present, record it
if len( info ) > 6:
candidateFile._staging = info[6]
# # register successful files
if self.Status in FTSJob.FINALSTATES:
return self.finalize()
return S_OK()
def ftsSchedule( self, requestID, operationID, opFileList ):
""" schedule lfn for FTS job
:param int requestID: RequestDB.Request.RequestID
:param int operationID: RequestDB.Operation.OperationID
:param list opFileList: list of tuples ( File.toJSON()['Value'], sourcesList, targetList )
"""
# Check whether there are duplicates
fList = []
for fileJSON, sourceSEs, targetSEs in opFileList:
fTuple = ( json.loads( fileJSON ), sourceSEs, targetSEs )
if fTuple not in fList:
fList.append( fTuple )
else:
self.log.warn( 'File list for FTS scheduling has duplicates, fix it:\n', fTuple )
fileIDs = [int( fileJSON.get( 'FileID', 0 ) ) for fileJSON, _sourceSEs, _targetSEs in fList ]
res = self.ftsManager.cleanUpFTSFiles( requestID, fileIDs )
if not res['OK']:
self.log.error( "Failed ftsSchedule", "%s" % res['Message'] )
return S_ERROR( "ftsSchedule: %s" % res['Message'] )
ftsFiles = []
# # this will be returned on success
result = { "Successful": [], "Failed": {} }
for fileJSON, sourceSEs, targetSEs in fList:
lfn = fileJSON.get( "LFN", "" )
size = int( fileJSON.get( "Size", 0 ) )
fileID = int( fileJSON.get( "FileID", 0 ) )
opID = int( fileJSON.get( "OperationID", 0 ) )
self.log.verbose( "ftsSchedule: LFN=%s FileID=%s OperationID=%s sources=%s targets=%s" % ( lfn, fileID, opID,
sourceSEs,
targetSEs ) )
res = self.dataManager.getActiveReplicas( lfn )
if not res['OK']:
self.log.error( "Failed ftsSchedule", "%s" % res['Message'] )
result["Failed"][fileID] = res['Message']
continue
replicaDict = res['Value']
if lfn in replicaDict["Failed"] and lfn not in replicaDict["Successful"]:
result["Failed"][fileID] = "no active replicas found"
continue
replicaDict = replicaDict["Successful"].get( lfn, {} )
# # use valid replicas only
validReplicasDict = dict( [ ( se, pfn ) for se, pfn in replicaDict.items() if se in sourceSEs ] )
if not validReplicasDict:
self.log.warn( "No active replicas found in sources" )
result["Failed"][fileID] = "no active replicas found in sources"
continue
tree = self.ftsManager.getReplicationTree( sourceSEs, targetSEs, size )
if not tree['OK']:
self.log.error( "Failed ftsSchedule", "%s cannot be scheduled: %s" % ( lfn, tree['Message'] ) )
result["Failed"][fileID] = tree['Message']
continue
tree = tree['Value']
self.log.verbose( "LFN=%s tree=%s" % ( lfn, tree ) )
treeBranches = []
printed = False
for repDict in tree.values():
if repDict in treeBranches:
if not printed:
self.log.warn( 'Duplicate tree branch', str( tree ) )
printed = True
else:
treeBranches.append( repDict )
for repDict in treeBranches:
self.log.verbose( "Strategy=%s Ancestor=%s SourceSE=%s TargetSE=%s" % ( repDict["Strategy"],
repDict["Ancestor"],
repDict["SourceSE"],
repDict["TargetSE"] ) )
transferSURLs = self._getTransferURLs( lfn, repDict, sourceSEs, validReplicasDict )
if not transferSURLs['OK']:
result["Failed"][fileID] = transferSURLs['Message']
continue
sourceSURL, targetSURL, fileStatus = transferSURLs['Value']
if sourceSURL == targetSURL:
result["Failed"][fileID] = "sourceSURL equals to targetSURL for %s" % lfn
continue
self.log.verbose( "sourceURL=%s targetURL=%s FTSFile.Status=%s" % ( sourceSURL, targetSURL, fileStatus ) )
ftsFile = FTSFile()
for key in ( "LFN", "FileID", "OperationID", "Checksum", "ChecksumType", "Size" ):
if fileJSON.get( key ):
setattr( ftsFile, key, fileJSON.get( key ) )
ftsFile.RequestID = requestID
ftsFile.OperationID = operationID
ftsFile.SourceSURL = sourceSURL
ftsFile.TargetSURL = targetSURL
ftsFile.SourceSE = repDict["SourceSE"]
ftsFile.TargetSE = repDict["TargetSE"]
ftsFile.Status = fileStatus
ftsFiles.append( ftsFile )
if not ftsFiles:
self.log.info( "ftsSchedule: no FTSFiles to put for request %d" % requestID )
return S_OK( result )
ftsFilesJSONList = [ftsFile.toJSON()['Value'] for ftsFile in ftsFiles]
res = self.ftsManager.putFTSFileList( ftsFilesJSONList )
if not res['OK']:
self.log.error( "Failed ftsSchedule", "%s" % res['Message'] )
return S_ERROR( "ftsSchedule: %s" % res['Message'] )
result['Successful'] += [ fileID for fileID in fileIDs if fileID not in result['Failed']]
# # if we land here some files have been properly scheduled
return S_OK( result )