def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(" sourceSE: source SE")
Script.registerArgument(
" LFN: LFN or file containing a List of LFNs")
Script.registerArgument(["targetSE: target SEs"])
Script.parseCommandLine()
import DIRAC
from DIRAC import gLogger
# parseCommandLine show help when mandatory arguments are not specified or incorrect argument
args = Script.getPositionalArgs()
sourceSE = args[0]
lfnList = getLFNList(args[1])
targetSEs = list(
set([se for targetSE in args[2:] for se in targetSE.split(",")]))
gLogger.info("Will create request with 'MoveReplica' "
"operation using %s lfns and %s target SEs" %
(len(lfnList), len(targetSEs)))
from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
from DIRAC.RequestManagementSystem.Client.Request import Request
from DIRAC.RequestManagementSystem.Client.Operation import Operation
from DIRAC.RequestManagementSystem.Client.File import File
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.Core.Utilities.List import breakListIntoChunks
lfnChunks = breakListIntoChunks(lfnList, 100)
multiRequests = len(lfnChunks) > 1
error = 0
count = 0
reqClient = ReqClient()
fc = FileCatalog()
for lfnChunk in lfnChunks:
metaDatas = fc.getFileMetadata(lfnChunk)
if not metaDatas["OK"]:
gLogger.error("unable to read metadata for lfns: %s" %
metaDatas["Message"])
error = -1
continue
metaDatas = metaDatas["Value"]
for failedLFN, reason in metaDatas["Failed"].items():
gLogger.error("skipping %s: %s" % (failedLFN, reason))
lfnChunk = set(metaDatas["Successful"])
if not lfnChunk:
gLogger.error("LFN list is empty!!!")
error = -1
continue
if len(lfnChunk) > Operation.MAX_FILES:
gLogger.error(
"too many LFNs, max number of files per operation is %s" %
Operation.MAX_FILES)
error = -1
continue
count += 1
request = Request()
request.RequestName = "%s_%s" % (
md5(repr(time.time()).encode()).hexdigest()[:16],
md5(repr(time.time()).encode()).hexdigest()[:16],
)
moveReplica = Operation()
moveReplica.Type = "MoveReplica"
moveReplica.SourceSE = sourceSE
moveReplica.TargetSE = ",".join(targetSEs)
for lfn in lfnChunk:
metaDict = metaDatas["Successful"][lfn]
opFile = File()
opFile.LFN = lfn
opFile.Size = metaDict["Size"]
if "Checksum" in metaDict:
# # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD'
opFile.Checksum = metaDict["Checksum"]
opFile.ChecksumType = "ADLER32"
moveReplica.addFile(opFile)
request.addOperation(moveReplica)
result = reqClient.putRequest(request)
if not result["OK"]:
gLogger.error("Failed to submit Request: %s" % (result["Message"]))
error = -1
continue
if not multiRequests:
gLogger.always("Request %d submitted successfully" %
result["Value"])
if multiRequests:
gLogger.always("%d requests have been submitted" % (count))
DIRAC.exit(error)
def test__cleanUp(mocker):
mocker.patch("LHCbDIRAC.Workflow.Modules.ModuleBase.RequestValidator",
side_effect=MagicMock())
f1 = File()
f1.LFN = '/a/1.txt'
f2 = File()
f2.LFN = '/a/2.txt'
f3 = File()
f3.LFN = '/a/3.txt'
o1 = Operation()
o1.Type = 'RegisterFile'
o1.addFile(f1)
o2 = Operation()
o2.Type = 'RegisterFile'
o2.addFile(f2)
o3 = Operation()
o3.Type = 'ForwardDISET'
o4 = Operation()
o4.Type = 'RegisterFile'
o4.addFile(f1)
o4.addFile(f3)
r = Request()
r.addOperation(o4)
r.addOperation(o1)
r.addOperation(o2)
r.addOperation(o3)
uod = UploadOutputData(bkClient=bkc_mock, dm=dm_mock)
uod.failoverTransfer = ft_mock
uod.request = r
expected = Request()
expected.addOperation(o3)
removeOp = Operation()
removeOp.Type = 'RemoveFile'
fileRemove1 = File()
fileRemove1.LFN = '/a/1.txt'
fileRemove2 = File()
fileRemove2.LFN = '/a/2.txt'
fileRemove3 = File()
fileRemove3.LFN = '/a/notPresent.txt'
removeOp.addFile(fileRemove1)
removeOp.addFile(fileRemove3)
removeOp.addFile(fileRemove2)
expected.addOperation(removeOp)
uod._cleanUp({
'1.txt': {
'lfn': '/a/1.txt'
},
'2.txt': {
'lfn': '/a/2.txt'
},
'notPresent.txt': {
'lfn': '/a/notPresent.txt'
}
})
for opsR, opsE in itertools.izip(uod.request, expected):
assert str(opsR) == str(opsE)
def createRequest(self, requestName, archiveLFN, lfnChunk):
"""Create the Request."""
request = Request()
request.RequestName = requestName
self._checkReplicaSites(request, lfnChunk)
archiveFiles = Operation()
archiveFiles.Type = "ArchiveFiles"
archiveFiles.Arguments = DEncode.encode({
"SourceSE":
self.sourceSEs[0],
"TarballSE":
self.switches["TarballSE"],
"RegisterDescendent":
self.switches["RegisterDescendent"],
"ArchiveLFN":
archiveLFN,
})
self.addLFNs(archiveFiles, lfnChunk)
request.addOperation(archiveFiles)
# Replicate the Tarball, ArchiveFiles will upload it
if self.switches.get("ReplicateTarball"):
replicateAndRegisterTarBall = Operation()
replicateAndRegisterTarBall.Type = "ReplicateAndRegister"
replicateAndRegisterTarBall.TargetSE = self.targetSE
opFile = File()
opFile.LFN = archiveLFN
replicateAndRegisterTarBall.addFile(opFile)
request.addOperation(replicateAndRegisterTarBall)
if self.switches.get("CheckMigration"):
checkMigrationTarBall = Operation()
checkMigrationTarBall.Type = "CheckMigration"
migrationTarget = self.targetSE if self.switches.get(
"ReplicateTarball") else self.switches["TarballSE"]
checkMigrationTarBall.TargetSE = migrationTarget
opFile = File()
opFile.LFN = archiveLFN
checkMigrationTarBall.addFile(opFile)
request.addOperation(checkMigrationTarBall)
# Register Archive Replica for LFNs
if self.switches.get("ArchiveSE"):
registerArchived = Operation()
registerArchived.Type = "RegisterReplica"
registerArchived.TargetSE = self.switches.get("ArchiveSE")
self.addLFNs(registerArchived, lfnChunk, addPFN=True)
request.addOperation(registerArchived)
# Remove all Other Replicas for LFNs
if self.switches.get("RemoveReplicas"):
removeArchiveReplicas = Operation()
removeArchiveReplicas.Type = "RemoveReplica"
removeArchiveReplicas.TargetSE = ",".join(self.replicaSEs)
self.addLFNs(removeArchiveReplicas, lfnChunk)
request.addOperation(removeArchiveReplicas)
# Remove all Replicas for LFNs
if self.switches.get("RemoveFiles"):
removeArchiveFiles = Operation()
removeArchiveFiles.Type = "RemoveFile"
self.addLFNs(removeArchiveFiles, lfnChunk)
request.addOperation(removeArchiveFiles)
# Remove Original tarball replica
if self.switches.get("ReplicateTarball"):
removeTarballOrg = Operation()
removeTarballOrg.Type = "RemoveReplica"
removeTarballOrg.TargetSE = self.sourceSEs[0]
opFile = File()
opFile.LFN = archiveLFN
removeTarballOrg.addFile(opFile)
request.addOperation(removeTarballOrg)
return request
def getRequest(self, requestName='', assigned=True):
""" read request for execution
:param str requestName: request's name (default None)
"""
requestID = None
if requestName:
self.log.info("getRequest: selecting request '%s'" % requestName)
reqIDQuery = "SELECT `RequestID`, `Status` FROM `Request` WHERE `RequestName` = '%s';" % str(
requestName)
reqID = self._transaction(reqIDQuery)
if not reqID["OK"]:
self.log.error("getRequest: %s" % reqID["Message"])
return reqID
requestID = reqID["Value"][reqIDQuery][0][
"RequestID"] if "RequestID" in reqID["Value"][reqIDQuery][
0] else None
status = reqID["Value"][reqIDQuery][0][
"Status"] if "Status" in reqID["Value"][reqIDQuery][0] else None
if not all((requestID, status)):
return S_ERROR("getRequest: request '%s' not exists" %
requestName)
if requestID and status and status == "Assigned" and assigned:
return S_ERROR(
"getRequest: status of request '%s' is 'Assigned', request cannot be selected"
% requestName)
else:
reqIDsQuery = "SELECT `RequestID` FROM `Request` WHERE `Status` = 'Waiting' ORDER BY `LastUpdate` ASC LIMIT 100;"
reqIDs = self._transaction(reqIDsQuery)
if not reqIDs["OK"]:
self.log.error("getRequest: %s" % reqIDs["Message"])
return reqIDs
reqIDs = reqIDs["Value"][reqIDsQuery]
reqIDs = [reqID["RequestID"] for reqID in reqIDs]
if not reqIDs:
return S_OK()
random.shuffle(reqIDs)
requestID = reqIDs[0]
selectQuery = [
"SELECT * FROM `Request` WHERE `RequestID` = %s;" % requestID,
"SELECT * FROM `Operation` WHERE `RequestID` = %s;" % requestID
]
selectReq = self._transaction(selectQuery)
if not selectReq["OK"]:
self.log.error("getRequest: %s" % selectReq["Message"])
return S_ERROR(selectReq["Message"])
selectReq = selectReq["Value"]
request = Request(selectReq[selectQuery[0]][0])
for records in sorted(selectReq[selectQuery[1]],
key=lambda k: k["Order"]):
# # order is ro, remove
del records["Order"]
operation = Operation(records)
getFilesQuery = "SELECT * FROM `File` WHERE `OperationID` = %s;" % operation.OperationID
getFiles = self._transaction(getFilesQuery)
if not getFiles["OK"]:
self.log.error("getRequest: %s" % getFiles["Message"])
return getFiles
getFiles = getFiles["Value"][getFilesQuery]
for getFile in getFiles:
getFileDict = dict([(key, value)
for key, value in getFile.items()
if value != None])
operation.addFile(File(getFileDict))
request.addOperation(operation)
if assigned:
setAssigned = self._transaction(
"UPDATE `Request` SET `Status` = 'Assigned' WHERE RequestID = %s;"
% requestID)
if not setAssigned["OK"]:
self.log.error("getRequest: %s" % setAssigned["Message"])
return setAssigned
return S_OK(request)
def setUp(self):
""" test setup """
self.request = Request()
self.operation = Operation()
self.file = File()
def test05FTS(self):
""" FTS state machine """
req = Request()
req.RequestName = "FTSTest"
ftsTransfer = Operation()
ftsTransfer.Type = "ReplicateAndRegister"
ftsTransfer.TargetSE = "CERN-USER"
ftsFile = File()
ftsFile.LFN = "/a/b/c"
ftsFile.Checksum = "123456"
ftsFile.ChecksumType = "Adler32"
ftsTransfer.addFile(ftsFile)
req.addOperation(ftsTransfer)
self.assertEqual(req.Status, "Waiting",
"1. wrong request status: %s" % req.Status)
self.assertEqual(ftsTransfer.Status, "Waiting",
"1. wrong ftsStatus status: %s" % ftsTransfer.Status)
# # scheduled
ftsFile.Status = "Scheduled"
self.assertEqual(
ftsTransfer.Status, "Scheduled",
"2. wrong status for ftsTransfer: %s" % ftsTransfer.Status)
self.assertEqual(req.Status, "Scheduled",
"2. wrong status for request: %s" % req.Status)
# # add new operation before FTS
insertBefore = Operation()
insertBefore.Type = "RegisterReplica"
insertBefore.TargetSE = "CERN-USER"
insertFile = File()
insertFile.LFN = "/a/b/c"
insertFile.PFN = "http://foo/bar"
insertBefore.addFile(insertFile)
req.insertBefore(insertBefore, ftsTransfer)
self.assertEqual(
insertBefore.Status, "Waiting",
"3. wrong status for insertBefore: %s" % insertBefore.Status)
self.assertEqual(
ftsTransfer.Status, "Scheduled",
"3. wrong status for ftsStatus: %s" % ftsTransfer.Status)
self.assertEqual(req.Status, "Waiting",
"3. wrong status for request: %s" % req.Status)
# # prev done
insertFile.Status = "Done"
self.assertEqual(
insertBefore.Status, "Done",
"4. wrong status for insertBefore: %s" % insertBefore.Status)
self.assertEqual(
ftsTransfer.Status, "Scheduled",
"4. wrong status for ftsStatus: %s" % ftsTransfer.Status)
self.assertEqual(req.Status, "Scheduled",
"4. wrong status for request: %s" % req.Status)
# # reschedule
ftsFile.Status = "Waiting"
self.assertEqual(
insertBefore.Status, "Done",
"5. wrong status for insertBefore: %s" % insertBefore.Status)
self.assertEqual(
ftsTransfer.Status, "Waiting",
"5. wrong status for ftsStatus: %s" % ftsTransfer.Status)
self.assertEqual(req.Status, "Waiting",
"5. wrong status for request: %s" % req.Status)
# # fts done
ftsFile.Status = "Done"
self.assertEqual(
insertBefore.Status, "Done",
"5. wrong status for insertBefore: %s" % insertBefore.Status)
self.assertEqual(
ftsTransfer.Status, "Done",
"5. wrong status for ftsStatus: %s" % ftsTransfer.Status)
self.assertEqual(req.Status, "Done",
"5. wrong status for request: %s" % req.Status)
def main():
from DIRAC.Core.Base.Script import parseCommandLine
parseCommandLine()
import DIRAC
from DIRAC import gLogger
args = Script.getPositionalArgs()
requestName = None
LFN = None
PFN = None
targetSE = None
if len(args) != 4:
Script.showHelp()
else:
requestName = args[0]
LFN = args[1]
PFN = args[2]
targetSE = args[3]
if not os.path.isabs(LFN):
gLogger.error("LFN should be absolute path!!!")
DIRAC.exit(-1)
gLogger.info("will create request '%s' with 'PutAndRegister' "
"operation using %s pfn and %s target SE" %
(requestName, PFN, targetSE))
from DIRAC.RequestManagementSystem.Client.Request import Request
from DIRAC.RequestManagementSystem.Client.Operation import Operation
from DIRAC.RequestManagementSystem.Client.File import File
from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
from DIRAC.Core.Utilities.Adler import fileAdler
if not os.path.exists(PFN):
gLogger.error("%s does not exist" % PFN)
DIRAC.exit(-1)
if not os.path.isfile(PFN):
gLogger.error("%s is not a file" % PFN)
DIRAC.exit(-1)
PFN = os.path.abspath(PFN)
size = os.path.getsize(PFN)
adler32 = fileAdler(PFN)
request = Request()
request.RequestName = requestName
putAndRegister = Operation()
putAndRegister.Type = "PutAndRegister"
putAndRegister.TargetSE = targetSE
opFile = File()
opFile.LFN = LFN
opFile.PFN = PFN
opFile.Size = size
opFile.Checksum = adler32
opFile.ChecksumType = "ADLER32"
putAndRegister.addFile(opFile)
request.addOperation(putAndRegister)
reqClient = ReqClient()
putRequest = reqClient.putRequest(request)
if not putRequest["OK"]:
gLogger.error("unable to put request '%s': %s" %
(requestName, putRequest["Message"]))
DIRAC.exit(-1)
gLogger.always("Request '%s' has been put to ReqDB for execution." %
requestName)
gLogger.always(
"You can monitor its status using command: 'dirac-rms-request %s'" %
requestName)
DIRAC.exit(0)
def getRequest(self, requestName='', assigned=True):
""" read request for execution
:param str requestName: request's name (default None)
"""
requestID = None
log = self.log.getSubLogger(
'getRequest' if assigned else 'peekRequest')
if requestName:
log.verbose("selecting request '%s'%s" %
(requestName, ' (Assigned)' if assigned else ''))
reqIDQuery = "SELECT `RequestID`, `Status` FROM `Request` WHERE `RequestName` = '%s';" % str(
requestName)
reqID = self._transaction(reqIDQuery)
if not reqID["OK"]:
log.error(reqID["Message"])
return reqID
reqID = reqID["Value"].get(reqIDQuery, [])
if reqID:
reqID = reqID[0]
else:
reqID = {}
requestID = reqID.get("RequestID")
status = reqID.get("Status")
if not all((requestID, status)):
return S_ERROR("getRequest: Request '%s' does not exist" %
requestName)
if requestID and status and status == "Assigned" and assigned:
return S_ERROR(
"getRequest: status of request '%s' is 'Assigned', request cannot be selected"
% requestName)
else:
reqIDsQuery = "SELECT `RequestID` FROM `Request` WHERE `Status` = 'Waiting' ORDER BY `LastUpdate` ASC LIMIT 100;"
reqAscIDs = self._transaction(reqIDsQuery)
if not reqAscIDs['OK']:
log.error(reqAscIDs["Message"])
return reqAscIDs
reqIDs = set([
reqID['RequestID'] for reqID in reqAscIDs["Value"][reqIDsQuery]
])
reqIDsQuery = "SELECT `RequestID` FROM `Request` WHERE `Status` = 'Waiting' ORDER BY `LastUpdate` DESC LIMIT 50;"
reqDescIDs = self._transaction(reqIDsQuery)
if not reqDescIDs['OK']:
log.error(reqDescIDs["Message"])
return reqDescIDs
reqIDs |= set([
reqID['RequestID']
for reqID in reqDescIDs["Value"][reqIDsQuery]
])
if not reqIDs:
return S_OK()
reqIDs = list(reqIDs)
random.shuffle(reqIDs)
requestID = reqIDs[0]
selectQuery = [
"SELECT * FROM `Request` WHERE `RequestID` = %s;" % requestID,
"SELECT * FROM `Operation` WHERE `RequestID` = %s;" % requestID
]
selectReq = self._transaction(selectQuery)
if not selectReq["OK"]:
log.error(selectReq["Message"])
return S_ERROR(selectReq["Message"])
selectReq = selectReq["Value"]
request = Request(selectReq[selectQuery[0]][0])
origLastUpdate = request.LastUpdate
if not requestName:
log.verbose(
"selected request '%s'%s" %
(request.RequestName, ' (Assigned)' if assigned else ''))
for records in sorted(selectReq[selectQuery[1]],
key=lambda k: k["Order"]):
# # order is ro, remove
del records["Order"]
operation = Operation(records)
getFilesQuery = "SELECT * FROM `File` WHERE `OperationID` = %s;" % operation.OperationID
getFiles = self._transaction(getFilesQuery)
if not getFiles["OK"]:
log.error(getFiles["Message"])
return getFiles
getFiles = getFiles["Value"][getFilesQuery]
for getFile in getFiles:
getFileDict = dict([(key, value)
for key, value in getFile.items()
if value != None])
operation.addFile(File(getFileDict))
request.addOperation(operation)
request.LastUpdate = origLastUpdate
if assigned:
setAssigned = self._transaction(
"UPDATE `Request` SET `Status` = 'Assigned', `LastUpdate`=UTC_TIMESTAMP() WHERE RequestID = %s;"
% requestID)
if not setAssigned["OK"]:
log.error("%s" % setAssigned["Message"])
return setAssigned
return S_OK(request)
def prepareTransformationTasks(self,
transBody,
taskDict,
owner='',
ownerGroup='',
ownerDN=''):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if (not owner) or (not ownerGroup):
res = getProxyInfo(False, False)
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername(owner)
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split(';')
except AttributeError:
pass
for taskID in sorted(taskDict):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if type(paramDict['InputData']) == type([]):
files = paramDict['InputData']
elif type(paramDict['InputData']) == type(''):
files = paramDict['InputData'].split(';')
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile(trFile)
oRequest.addOperation(transfer)
oRequest.RequestName = _requestName(transID, taskID)
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = RequestValidator().validate(oRequest)
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK(taskDict)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument("requestName: a request name")
Script.registerArgument("LFN: logical file name")
Script.registerArgument("localFile: local file you want to put")
Script.registerArgument("targetSE: target SE")
Script.parseCommandLine()
import DIRAC
from DIRAC import gLogger
# parseCommandLine show help when mandatory arguments are not specified or incorrect argument
requestName, LFN, PFN, targetSE = Script.getPositionalArgs(group=True)
if not os.path.isabs(LFN):
gLogger.error("LFN should be absolute path!!!")
DIRAC.exit(-1)
gLogger.info("will create request '%s' with 'PutAndRegister' "
"operation using %s pfn and %s target SE" %
(requestName, PFN, targetSE))
from DIRAC.RequestManagementSystem.Client.Request import Request
from DIRAC.RequestManagementSystem.Client.Operation import Operation
from DIRAC.RequestManagementSystem.Client.File import File
from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
from DIRAC.Core.Utilities.Adler import fileAdler
if not os.path.exists(PFN):
gLogger.error("%s does not exist" % PFN)
DIRAC.exit(-1)
if not os.path.isfile(PFN):
gLogger.error("%s is not a file" % PFN)
DIRAC.exit(-1)
PFN = os.path.abspath(PFN)
size = os.path.getsize(PFN)
adler32 = fileAdler(PFN)
request = Request()
request.RequestName = requestName
putAndRegister = Operation()
putAndRegister.Type = "PutAndRegister"
putAndRegister.TargetSE = targetSE
opFile = File()
opFile.LFN = LFN
opFile.PFN = PFN
opFile.Size = size
opFile.Checksum = adler32
opFile.ChecksumType = "ADLER32"
putAndRegister.addFile(opFile)
request.addOperation(putAndRegister)
reqClient = ReqClient()
putRequest = reqClient.putRequest(request)
if not putRequest["OK"]:
gLogger.error("unable to put request '%s': %s" %
(requestName, putRequest["Message"]))
DIRAC.exit(-1)
gLogger.always("Request '%s' has been put to ReqDB for execution." %
requestName)
gLogger.always(
"You can monitor its status using command: 'dirac-rms-request %s'" %
requestName)
DIRAC.exit(0)
def main():
catalog = None
Script.registerSwitch("C:", "Catalog=", "Catalog to use")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(" requestName: a request name")
Script.registerArgument(" LFNs: single LFN or file with LFNs")
Script.registerArgument(["targetSE: target SE"])
Script.parseCommandLine()
for switch in Script.getUnprocessedSwitches():
if switch[0] == "C" or switch[0].lower() == "catalog":
catalog = switch[1]
args = Script.getPositionalArgs()
requestName = None
targetSEs = None
if len(args) < 3:
Script.showHelp(exitCode=1)
requestName = args[0]
lfnList = getLFNList(args[1])
targetSEs = list(
set([se for targetSE in args[2:] for se in targetSE.split(",")]))
gLogger.info("Will create request '%s' with 'ReplicateAndRegister' "
"operation using %s lfns and %s target SEs" %
(requestName, len(lfnList), len(targetSEs)))
from DIRAC.RequestManagementSystem.Client.Request import Request
from DIRAC.RequestManagementSystem.Client.Operation import Operation
from DIRAC.RequestManagementSystem.Client.File import File
from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.Core.Utilities.List import breakListIntoChunks
lfnChunks = breakListIntoChunks(lfnList, 100)
multiRequests = len(lfnChunks) > 1
error = 0
count = 0
reqClient = ReqClient()
fc = FileCatalog()
requestIDs = []
for lfnChunk in lfnChunks:
metaDatas = fc.getFileMetadata(lfnChunk)
if not metaDatas["OK"]:
gLogger.error("unable to read metadata for lfns: %s" %
metaDatas["Message"])
error = -1
continue
metaDatas = metaDatas["Value"]
for failedLFN, reason in metaDatas["Failed"].items():
gLogger.error("skipping %s: %s" % (failedLFN, reason))
lfnChunk = set(metaDatas["Successful"])
if not lfnChunk:
gLogger.error("LFN list is empty!!!")
error = -1
continue
if len(lfnChunk) > Operation.MAX_FILES:
gLogger.error(
"too many LFNs, max number of files per operation is %s" %
Operation.MAX_FILES)
error = -1
continue
count += 1
request = Request()
request.RequestName = requestName if not multiRequests else "%s_%d" % (
requestName, count)
replicateAndRegister = Operation()
replicateAndRegister.Type = "ReplicateAndRegister"
replicateAndRegister.TargetSE = ",".join(targetSEs)
if catalog is not None:
replicateAndRegister.Catalog = catalog
for lfn in lfnChunk:
metaDict = metaDatas["Successful"][lfn]
opFile = File()
opFile.LFN = lfn
opFile.Size = metaDict["Size"]
if "Checksum" in metaDict:
# # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD'
opFile.Checksum = metaDict["Checksum"]
opFile.ChecksumType = "ADLER32"
replicateAndRegister.addFile(opFile)
request.addOperation(replicateAndRegister)
putRequest = reqClient.putRequest(request)
if not putRequest["OK"]:
gLogger.error("unable to put request '%s': %s" %
(request.RequestName, putRequest["Message"]))
error = -1
continue
requestIDs.append(str(putRequest["Value"]))
if not multiRequests:
gLogger.always(
"Request '%s' has been put to ReqDB for execution." %
request.RequestName)
if multiRequests:
gLogger.always(
"%d requests have been put to ReqDB for execution, with name %s_<num>"
% (count, requestName))
if requestIDs:
gLogger.always("RequestID(s): %s" % " ".join(requestIDs))
gLogger.always(
"You can monitor requests' status using command: 'dirac-rms-request <requestName/ID>'"
)
DIRAC.exit(error)
def createRequest(self, requestName, archiveLFN, lfnChunk):
"""Create the Request."""
request = Request()
request.RequestName = requestName
self._checkReplicaSites(request, lfnChunk)
archiveFiles = Operation()
archiveFiles.Type = 'ArchiveFiles'
archiveFiles.Arguments = DEncode.encode({
'SourceSE':
self.sourceSEs[0],
'TarballSE':
self.switches['TarballSE'],
'RegisterDescendent':
self.switches['RegisterDescendent'],
'ArchiveLFN':
archiveLFN
})
self.addLFNs(archiveFiles, lfnChunk)
request.addOperation(archiveFiles)
# Replicate the Tarball, ArchiveFiles will upload it
if self.switches.get('ReplicateTarball'):
replicateAndRegisterTarBall = Operation()
replicateAndRegisterTarBall.Type = 'ReplicateAndRegister'
replicateAndRegisterTarBall.TargetSE = self.targetSE
opFile = File()
opFile.LFN = archiveLFN
replicateAndRegisterTarBall.addFile(opFile)
request.addOperation(replicateAndRegisterTarBall)
if self.switches.get('CheckMigration'):
checkMigrationTarBall = Operation()
checkMigrationTarBall.Type = 'CheckMigration'
migrationTarget = self.targetSE if self.switches.get(
'ReplicateTarball') else self.switches['TarballSE']
checkMigrationTarBall.TargetSE = migrationTarget
opFile = File()
opFile.LFN = archiveLFN
checkMigrationTarBall.addFile(opFile)
request.addOperation(checkMigrationTarBall)
# Register Archive Replica for LFNs
if self.switches.get('ArchiveSE'):
registerArchived = Operation()
registerArchived.Type = 'RegisterReplica'
registerArchived.TargetSE = self.switches.get('ArchiveSE')
self.addLFNs(registerArchived, lfnChunk, addPFN=True)
request.addOperation(registerArchived)
# Remove all Other Replicas for LFNs
if self.switches.get('RemoveReplicas'):
removeArchiveReplicas = Operation()
removeArchiveReplicas.Type = 'RemoveReplica'
removeArchiveReplicas.TargetSE = ','.join(self.replicaSEs)
self.addLFNs(removeArchiveReplicas, lfnChunk)
request.addOperation(removeArchiveReplicas)
# Remove all Replicas for LFNs
if self.switches.get('RemoveFiles'):
removeArchiveFiles = Operation()
removeArchiveFiles.Type = 'RemoveFile'
self.addLFNs(removeArchiveFiles, lfnChunk)
request.addOperation(removeArchiveFiles)
# Remove Original tarball replica
if self.switches.get('ReplicateTarball'):
removeTarballOrg = Operation()
removeTarballOrg.Type = 'RemoveReplica'
removeTarballOrg.TargetSE = self.sourceSEs[0]
opFile = File()
opFile.LFN = archiveLFN
removeTarballOrg.addFile(opFile)
request.addOperation(removeTarballOrg)
return request
def test__addMetadataToFiles(self):
resMeta = {
'OK': True,
'Value': {
'Failed': {},
'Successful': {
'/lhcb/1.dst': {
'ChecksumType':
'AD',
'Checksum':
'123456',
'CreationDate':
datetime.datetime(2013, 12, 11, 20, 20, 21),
'GUID':
'92F9CE97-7A62-E311-8401-0025907FD430',
'Mode':
436,
'ModificationDate':
datetime.datetime(2013, 12, 11, 20, 20, 21),
'NumberOfLinks':
1,
'Size':
5846023777,
'Status':
'-'
},
'/lhcb/2.dst': {
'ChecksumType':
'AD',
'Checksum':
'987654',
'CreationDate':
datetime.datetime(2013, 12, 12, 6, 26, 52),
'GUID':
'DAE4933A-C162-E311-8A6B-003048FEAF04',
'Mode':
436,
'ModificationDate':
datetime.datetime(2013, 12, 12, 6, 26, 52),
'NumberOfLinks':
1,
'Size':
5893396937,
'Status':
'-'
}
}
}
}
self.rr.fc.getFileMetadata.return_value = resMeta
file1 = File()
file1.LFN = '/lhcb/1.dst'
file2 = File()
file2.LFN = '/lhcb/2.dst'
toSchedule = {
'/lhcb/1.dst': [file1, ['SE1'], ['SE2', 'SE3']],
'/lhcb/2.dst': [file2, ['SE4'], ['SE5', 'SE6']]
}
res = self.rr._addMetadataToFiles(toSchedule)
self.assert_(res['OK'])
self.assertEqual(
json.loads(res['Value'][0][0])['LFN'],
resMeta['Value']['Successful'].keys()[0])
self.assertEqual(
json.loads(res['Value'][0][0])['Size'],
resMeta['Value']['Successful'].values()[0]['Size'])
self.assertEqual(
json.loads(res['Value'][1][0])['LFN'],
resMeta['Value']['Successful'].keys()[1])
self.assertEqual(
json.loads(res['Value'][1][0])['Size'],
resMeta['Value']['Successful'].values()[1]['Size'])
jr_mock.setJobParameter.return_value = {'OK': True, 'Value': 'pippo'}
fr_mock = MagicMock()
fr_mock.getFiles.return_value = {}
fr_mock.setFileStatus.return_value = {'OK': True, 'Value': ''}
fr_mock.commit.return_value = {'OK': True, 'Value': ''}
fr_mock.generateForwardDISET.return_value = {'OK': True, 'Value': Operation()}
rc_mock = Request()
rc_mock.RequestName = 'aRequestName'
rc_mock.OwnerDN = 'pippo'
rc_mock.OwnerGroup = 'pippoGRP'
rOp = Operation()
rOp.Type = 'PutAndRegister'
rOp.TargetSE = 'anSE'
f = File()
f.LFN = '/foo/bar.py'
f.PFN = '/foo/bar.py'
rOp.addFile(f)
rc_mock.addOperation(rOp)
wf_commons = [{'PRODUCTION_ID': prod_id, 'JOB_ID': prod_job_id, 'eventType': '123456789', 'jobType': 'merge',
'configName': 'aConfigName', 'configVersion': 'aConfigVersion', 'outputDataFileMask': '',
'BookkeepingLFNs': 'aa', 'ProductionOutputData': 'ProductionOutputData', 'numberOfEvents': '100',
'JobReport': jr_mock, 'Request': rc_mock, 'AccountingReport': ar_mock, 'FileReport': fr_mock,
'SystemConfig': 'sys_config', 'runNumber': 'Unknown', 'gaudiSteps': ['someApp_1'],
'outputSEs':{"DAVINCIHIST": "CERN-HIST", "TXT": "SE1"}},
{'PRODUCTION_ID': prod_id, 'JOB_ID': prod_job_id,
'configName': 'aConfigName', 'configVersion': 'aConfigVersion',
'outputDataFileMask': '', 'jobType': 'merge',
def test02props(self):
""" test properties """
# # valid values
operation = Operation()
operation.Arguments = "foobar"
self.assertEqual(operation.Arguments, "foobar", "wrong Arguments")
operation.SourceSE = "CERN-RAW"
self.assertEqual(operation.SourceSE, "CERN-RAW", "wrong SourceSE")
operation.TargetSE = "CERN-RAW"
self.assertEqual(operation.TargetSE, "CERN-RAW", "wrong TargetSE")
operation.Catalog = ""
self.assertEqual(operation.Catalog, "", "wrong Catalog")
operation.Catalog = "BookkeepingDB"
self.assertEqual(operation.Catalog, "BookkeepingDB", "wrong Catalog")
operation.Error = "error"
self.assertEqual(operation.Error, "error", "wrong Error")
# # wrong props
try:
operation.RequestID = "foo"
except Exception as error:
self.assertEqual(type(error), AttributeError, "wrong exc raised")
self.assertEqual(str(error), "can't set attribute",
"wrong exc reason")
try:
operation.OperationID = "foo"
except Exception as error:
self.assertEqual(type(error), ValueError, "wrong exc raised")
# # timestamps
try:
operation.SubmitTime = "foo"
except Exception as error:
self.assertEqual(type(error), ValueError, "wrong exp raised")
self.assertEqual(
str(error),
"time data 'foo' does not match format '%Y-%m-%d %H:%M:%S'",
"wrong exc reason")
try:
operation.LastUpdate = "foo"
except Exception as error:
self.assertEqual(type(error), ValueError, "wrong exc raised")
self.assertEqual(
str(error),
"time data 'foo' does not match format '%Y-%m-%d %H:%M:%S'",
"wrong exc reason")
# # Status
operation = Operation()
try:
operation.Status = "foo"
except Exception as error:
self.assertEqual(type(error), ValueError, "wrong exc raised")
self.assertEqual(str(error), "unknown Status 'foo'",
"wrong exc reason")
operation.addFile(File({"Status": "Waiting", "LFN": "/a"}))
def createRequest(reqType):
r = Request()
# Simple failover
op1 = Operation()
f = File()
f.LFN = '/This/is/an/LFN'
op1.addFile(f)
op1.Type = 'ReplicateAndRegister'
op1.SourceSE = 'CERN-FAILOVER'
op1.TargetSE = 'CERN-BUFFER'
r.addOperation(op1)
# You cannot reuse the same File object,
# since it is a different entry in the DB
fr = File()
fr.LFN = '/This/is/an/LFN'
op2 = Operation()
op2.addFile(fr)
op2.Type = 'RemoveReplica'
op2.TargetSE = 'CERN-FAILOVER'
r.addOperation(op2)
if reqType == 0:
return r
# two files for Failover
f1 = File()
f1.LFN = '/This/is/a/second/LFN'
op3 = Operation()
op3.addFile(f1)
op3.Type = 'ReplicateAndRegister'
op3.SourceSE = 'CERN-FAILOVER'
op3.TargetSE = 'CERN-BUFFER'
r.addOperation(op3)
f1r = File()
f1r.LFN = '/This/is/a/second/LFN'
op3 = Operation()
op3.addFile(f1r)
op3.Type = 'RemoveReplica'
op3.TargetSE = 'CERN-FAILOVER'
r.addOperation(op3)
if reqType == 1:
return r
op = Operation()
op.Type = 'ForwardDiset'
if reqType == 2:
r.addOperation(op)
return r
r.insertBefore(op, r[0])
if reqType == 3:
return r
op4 = Operation()
op4.Type = 'ForwardDiset'
r.addOperation(op4)
if reqType == 4:
return r
# 2 different FAILOVER SEs: removal not optimized
r[1].SourceSE = 'RAL-FAILOVER'
r[2].SourceSE = 'RAL-FAILOVER'
if reqType == 5:
return r
# 2 different destinations, same FAILOVER: replication not optimized
r[3].SourceSE = 'RAL-FAILOVER'
r[4].SourceSE = 'RAL-FAILOVER'
r[3].TargetSE = 'RAL-BUFFER'
if reqType == 6:
return r
print 'This should not happen, reqType =', reqType
replicateAndRegister.TargetSE = targetSE
res = fc.getFileMetadata(lfnList)
if not res['OK']:
print "Can't get file metadata: %s" % res['Message']
DIRAC.exit(1)
if res['Value']['Failed']:
print "Could not get the file metadata of the following, so skipping them:"
for fFile in res['Value']['Failed']:
print fFile
lfnMetadata = res['Value']['Successful']
for lfn in lfnMetadata:
try:
rarFile = File()
rarFile.LFN = lfn
rarFile.Size = lfnMetadata[lfn]['Size']
rarFile.Checksum = lfnMetadata[lfn]['Checksum']
rarFile.GUID = lfnMetadata[lfn]['GUID']
rarFile.ChecksumType = 'ADLER32'
replicateAndRegister.addFile(rarFile)
except ValueError as err:
print "Error", str(err), lfn
continue
oRequest.addOperation(replicateAndRegister)
isValid = RequestValidator().validate(oRequest)
if not isValid['OK']:
print "Request is not valid: ", isValid['Message']
DIRAC.exit(1)
def getBulkRequests(self, numberOfRequest=10, assigned=True):
""" read as many requests as requested for execution
:param int numberOfRequest: Number of Request we want (default 10)
:param bool assigned: if True, the status of the selected requests are set to assign
:returns a dictionary of Request objects indexed on the RequestID
"""
# r_RequestID : RequestID, r_LastUpdate : LastUpdate...
requestAttrDict = dict(
("r_%s" % r, r) for r in Request.tableDesc()["Fields"])
# o_RequestID : RequestID, o_OperationID : OperationID...
operationAttrDict = dict(
("o_%s" % o, o) for o in Operation.tableDesc()["Fields"])
# f_OperationID : OperationID, f_FileID : FileID...
fileAttrDict = dict(
("f_%s" % f, f) for f in File.tableDesc()["Fields"])
# o.OperationID as o_OperationID, ..., r_RequestID, ..., f_FileID, ...
allFieldsStr = ",".join( [ "o.%s as %s" % ( operationAttrDict[o], o ) for o in operationAttrDict]\
+ requestAttrDict.keys() + fileAttrDict.keys() )
# RequestID as r_RequestID, LastUpdate as r_LastUpdate, ...
requestAttrStr = ",".join(
["%s as %s" % (requestAttrDict[r], r) for r in requestAttrDict])
# OperationID as f_OperationID, FileID as f_FileID...
fileAttrStr = ",".join(
["%s as %s" % (fileAttrDict[f], f) for f in fileAttrDict])
# Selects all the Request (limited to numberOfRequest, sorted by LastUpdate) , Operation and File information.
# The entries are sorted by the LastUpdate of the Requests, RequestID if several requests were update the last time
# at the same time, and finally according to the Operation Order
query = "SELECT %s FROM Operation o \
INNER JOIN (SELECT %s FROM Request WHERE Status = 'Waiting' ORDER BY `LastUpdate` ASC limit %s) r\
ON r_RequestID = o.RequestID\
INNER JOIN (SELECT %s from File) f ON f_OperationId = o.OperationId\
ORDER BY r_LastUpdate, r_RequestId, o_Order;" \
% ( allFieldsStr, requestAttrStr, numberOfRequest, fileAttrStr )
queryResult = self._transaction(query)
if not queryResult["OK"]:
self.log.error("RequestDB.getRequests: %s" %
queryResult["Message"])
return queryResult
allResults = queryResult["Value"][query]
# We now construct a dict of Request indexed by their ID, and the same for Operation
requestDict = {}
operationDict = {}
for entry in allResults:
requestID = int(entry["r_RequestID"])
# If the object already exists, we get it, otherwise we create it and assign it
requestObj = requestDict.setdefault(
requestID,
Request(
dict((requestAttrDict[r], entry[r])
for r in requestAttrDict)))
operationID = int(entry["o_OperationID"])
operationObj = operationDict.get(operationID, None)
# If the Operation object does not exist yet, we create it, and add it to the Request
if not operationObj:
operationObj = Operation(
dict((operationAttrDict[o], entry[o])
for o in operationAttrDict))
operationDict[operationID] = operationObj
requestObj.addOperation(operationObj)
fileObj = File(
dict((fileAttrDict[f], entry[f]) for f in fileAttrDict))
operationObj.addFile(fileObj)
if assigned and len(requestDict):
listOfReqId = ",".join(str(rId) for rId in requestDict)
setAssigned = self._transaction(
"UPDATE `Request` SET `Status` = 'Assigned' WHERE RequestID IN (%s);"
% listOfReqId)
if not setAssigned["OK"]:
self.log.error("getRequests: %s" % setAssigned["Message"])
return setAssigned
return S_OK(requestDict)
except Exception, error:
self.assertEqual(type(error), ValueError, "wrong exc raised")
self.assertEqual(
str(error),
"time data 'foo' does not match format '%Y-%m-%d %H:%M:%S'",
"wrong exc reason")
# # Status
operation = Operation()
try:
operation.Status = "foo"
except Exception, error:
self.assertEqual(type(error), ValueError, "wrong exc raised")
self.assertEqual(str(error), "unknown Status 'foo'",
"wrong exc reason")
operation.addFile(File({"Status": "Waiting", "LFN": "/a"}))
def test04StateMachine(self):
""" state machine """
op = Operation()
self.assertEqual(op.Status, "Queued", "1. wrong status %s" % op.Status)
op.addFile(File({"Status": "Waiting"}))
self.assertEqual(op.Status, "Queued", "2. wrong status %s" % op.Status)
op.addFile(File({"Status": "Scheduled"}))
self.assertEqual(op.Status, "Scheduled",
"3. wrong status %s" % op.Status)
op.addFile(File({"Status": "Done"}))
self.assertEqual(op.Status, "Scheduled",
def test04Operations(self):
""" operations arithmetic and state machine """
req = Request()
self.assertEqual(len(req), 0)
transfer = Operation()
transfer.Type = "ReplicateAndRegister"
transfer.addFile(File({"LFN": "/a/b/c", "Status": "Waiting"}))
getWaiting = req.getWaiting()
self.assertEqual(getWaiting["OK"], True)
self.assertEqual(getWaiting["Value"], None)
req.addOperation(transfer)
self.assertEqual(len(req), 1)
self.assertEqual(transfer.Order, req.Order)
self.assertEqual(transfer.Status, "Waiting")
getWaiting = req.getWaiting()
self.assertEqual(getWaiting["OK"], True)
self.assertEqual(getWaiting["Value"], transfer)
removal = Operation({"Type": "RemoveFile"})
removal.addFile(File({"LFN": "/a/b/c", "Status": "Waiting"}))
req.insertBefore(removal, transfer)
getWaiting = req.getWaiting()
self.assertEqual(getWaiting["OK"], True)
self.assertEqual(getWaiting["Value"], removal)
self.assertEqual(len(req), 2)
self.assertEqual([op.Status for op in req], ["Waiting", "Queued"])
self.assertEqual(req.subStatusList(), ["Waiting", "Queued"])
self.assertEqual(removal.Order, 0)
self.assertEqual(removal.Order, req.Order)
self.assertEqual(transfer.Order, 1)
self.assertEqual(removal.Status, "Waiting")
self.assertEqual(transfer.Status, "Queued")
for subFile in removal:
subFile.Status = "Done"
removal.Status = "Done"
self.assertEqual(removal.Status, "Done")
self.assertEqual(transfer.Status, "Waiting")
self.assertEqual(transfer.Order, req.Order)
# # len, looping
self.assertEqual(len(req), 2)
self.assertEqual([op.Status for op in req], ["Done", "Waiting"])
self.assertEqual(req.subStatusList(), ["Done", "Waiting"])
digest = req.toJSON()
self.assertEqual(digest["OK"], True)
getWaiting = req.getWaiting()
self.assertEqual(getWaiting["OK"], True)
self.assertEqual(getWaiting["Value"], transfer)
def test06Dirty(self):
""" dirty records """
db = RequestDB()
r = Request()
r.RequestName = "dirty"
op1 = Operation({
"Type": "ReplicateAndRegister",
"TargetSE": "CERN-USER"
})
op1 += File({
"LFN": "/a/b/c/1",
"Status": "Scheduled",
"Checksum": "123456",
"ChecksumType": "ADLER32"
})
op2 = Operation({
"Type": "ReplicateAndRegister",
"TargetSE": "CERN-USER"
})
op2 += File({
"LFN": "/a/b/c/2",
"Status": "Scheduled",
"Checksum": "123456",
"ChecksumType": "ADLER32"
})
op3 = Operation({
"Type": "ReplicateAndRegister",
"TargetSE": "CERN-USER"
})
op3 += File({
"LFN": "/a/b/c/3",
"Status": "Scheduled",
"Checksum": "123456",
"ChecksumType": "ADLER32"
})
r += op1
r += op2
r += op3
put = db.putRequest(r)
self.assertEqual(put["OK"], True,
"1. putRequest failed: %s" % put.get("Message", ""))
reqID = put['Value']
r = db.getRequest(reqID)
self.assertEqual(r["OK"], True,
"1. getRequest failed: %s" % r.get("Message", ""))
r = r["Value"]
del r[0]
self.assertEqual(len(r), 2, "1. len wrong")
put = db.putRequest(r)
self.assertEqual(put["OK"], True,
"2. putRequest failed: %s" % put.get("Message", ""))
reqID = put['Value']
r = db.getRequest(reqID)
self.assertEqual(r["OK"], True,
"2. getRequest failed: %s" % r.get("Message", ""))
r = r["Value"]
self.assertEqual(len(r), 2, "2. len wrong")
op4 = Operation({
"Type": "ReplicateAndRegister",
"TargetSE": "CERN-USER"
})
op4 += File({
"LFN": "/a/b/c/4",
"Status": "Scheduled",
"Checksum": "123456",
"ChecksumType": "ADLER32"
})
r[0] = op4
put = db.putRequest(r)
self.assertEqual(put["OK"], True,
"3. putRequest failed: %s" % put.get("Message", ""))
reqID = put['Value']
r = db.getRequest(reqID)
self.assertEqual(r["OK"], True,
"3. getRequest failed: %s" % r.get("Message", ""))
r = r["Value"]
self.assertEqual(len(r), 2, "3. len wrong")
delete = db.deleteRequest(reqID)
self.assertEqual(delete["OK"], True,
delete['Message'] if 'Message' in delete else 'OK')
if len( lfnChunk ) > Operation.MAX_FILES:
gLogger.error( "too many LFNs, max number of files per operation is %s" % Operation.MAX_FILES )
error = -1
continue
count += 1
request = Request()
request.RequestName = requestName if not multiRequests else '%s_%d' % ( requestName, count )
replicateAndRegister = Operation()
replicateAndRegister.Type = "ReplicateAndRegister"
replicateAndRegister.TargetSE = ",".join( targetSEs )
for lfn in lfnChunk:
metaDict = metaDatas["Successful"][lfn]
opFile = File()
opFile.LFN = lfn
opFile.Size = metaDict["Size"]
if "Checksum" in metaDict:
# # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD'
opFile.Checksum = metaDict["Checksum"]
opFile.ChecksumType = "ADLER32"
replicateAndRegister.addFile( opFile )
request.addOperation( replicateAndRegister )
putRequest = reqClient.putRequest( request )
if not putRequest["OK"]:
gLogger.error( "unable to put request '%s': %s" % ( request.RequestName, putRequest["Message"] ) )
error = -1
def test__addMetadataToFiles(self):
resMeta = {
"OK": True,
"Value": {
"Failed": {},
"Successful": {
"/lhcb/1.dst": {
"ChecksumType":
"AD",
"Checksum":
"123456",
"CreationDate":
datetime.datetime(2013, 12, 11, 20, 20, 21),
"GUID":
"92F9CE97-7A62-E311-8401-0025907FD430",
"Mode":
436,
"ModificationDate":
datetime.datetime(2013, 12, 11, 20, 20, 21),
"NumberOfLinks":
1,
"Size":
5846023777,
"Status":
"-",
},
"/lhcb/2.dst": {
"ChecksumType":
"AD",
"Checksum":
"987654",
"CreationDate":
datetime.datetime(2013, 12, 12, 6, 26, 52),
"GUID":
"DAE4933A-C162-E311-8A6B-003048FEAF04",
"Mode":
436,
"ModificationDate":
datetime.datetime(2013, 12, 12, 6, 26, 52),
"NumberOfLinks":
1,
"Size":
5893396937,
"Status":
"-",
},
},
},
}
self.rr.fc.getFileMetadata.return_value = resMeta
file1 = File()
file1.LFN = "/lhcb/1.dst"
file2 = File()
file2.LFN = "/lhcb/2.dst"
toSchedule = {
"/lhcb/1.dst": [file1, ["SE1"], ["SE2", "SE3"]],
"/lhcb/2.dst": [file2, ["SE4"], ["SE5", "SE6"]]
}
res = self.rr._addMetadataToFiles(toSchedule)
self.assertTrue(res["OK"])
for lfn in toSchedule:
self.assertEqual(res["Value"][lfn].LFN, lfn)
for attr in ("GUID", "Size", "Checksum"):
self.assertEqual(getattr(res["Value"][lfn], attr),
resMeta["Value"]["Successful"][lfn][attr])
# AD should be transformed into Adler32
self.assertEqual(res["Value"][lfn].ChecksumType, "ADLER32")
