def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument(" sourceSE: source SE") Script.registerArgument( " LFN: LFN or file containing a List of LFNs") Script.registerArgument(["targetSE: target SEs"]) Script.parseCommandLine() import DIRAC from DIRAC import gLogger # parseCommandLine show help when mandatory arguments are not specified or incorrect argument args = Script.getPositionalArgs() sourceSE = args[0] lfnList = getLFNList(args[1]) targetSEs = list( set([se for targetSE in args[2:] for se in targetSE.split(",")])) gLogger.info("Will create request with 'MoveReplica' " "operation using %s lfns and %s target SEs" % (len(lfnList), len(targetSEs))) from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient from DIRAC.RequestManagementSystem.Client.Request import Request from DIRAC.RequestManagementSystem.Client.Operation import Operation from DIRAC.RequestManagementSystem.Client.File import File from DIRAC.Resources.Catalog.FileCatalog import FileCatalog from DIRAC.Core.Utilities.List import breakListIntoChunks lfnChunks = breakListIntoChunks(lfnList, 100) multiRequests = len(lfnChunks) > 1 error = 0 count = 0 reqClient = ReqClient() fc = FileCatalog() for lfnChunk in lfnChunks: metaDatas = fc.getFileMetadata(lfnChunk) if not metaDatas["OK"]: gLogger.error("unable to read metadata for lfns: %s" % metaDatas["Message"]) error = -1 continue metaDatas = metaDatas["Value"] for failedLFN, reason in metaDatas["Failed"].items(): gLogger.error("skipping %s: %s" % (failedLFN, reason)) lfnChunk = set(metaDatas["Successful"]) if not lfnChunk: gLogger.error("LFN list is empty!!!") error = -1 continue if len(lfnChunk) > Operation.MAX_FILES: gLogger.error( "too many LFNs, max number of files per operation is %s" % Operation.MAX_FILES) error = -1 continue count += 1 request = Request() request.RequestName = "%s_%s" % ( md5(repr(time.time()).encode()).hexdigest()[:16], md5(repr(time.time()).encode()).hexdigest()[:16], ) moveReplica = Operation() moveReplica.Type = "MoveReplica" moveReplica.SourceSE = sourceSE moveReplica.TargetSE = ",".join(targetSEs) for lfn in lfnChunk: metaDict = metaDatas["Successful"][lfn] opFile = File() opFile.LFN = lfn opFile.Size = metaDict["Size"] if "Checksum" in metaDict: # # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD' opFile.Checksum = metaDict["Checksum"] opFile.ChecksumType = "ADLER32" moveReplica.addFile(opFile) request.addOperation(moveReplica) result = reqClient.putRequest(request) if not result["OK"]: gLogger.error("Failed to submit Request: %s" % (result["Message"])) error = -1 continue if not multiRequests: gLogger.always("Request %d submitted successfully" % result["Value"]) if multiRequests: gLogger.always("%d requests have been submitted" % (count)) DIRAC.exit(error)
def test__cleanUp(mocker): mocker.patch("LHCbDIRAC.Workflow.Modules.ModuleBase.RequestValidator", side_effect=MagicMock()) f1 = File() f1.LFN = '/a/1.txt' f2 = File() f2.LFN = '/a/2.txt' f3 = File() f3.LFN = '/a/3.txt' o1 = Operation() o1.Type = 'RegisterFile' o1.addFile(f1) o2 = Operation() o2.Type = 'RegisterFile' o2.addFile(f2) o3 = Operation() o3.Type = 'ForwardDISET' o4 = Operation() o4.Type = 'RegisterFile' o4.addFile(f1) o4.addFile(f3) r = Request() r.addOperation(o4) r.addOperation(o1) r.addOperation(o2) r.addOperation(o3) uod = UploadOutputData(bkClient=bkc_mock, dm=dm_mock) uod.failoverTransfer = ft_mock uod.request = r expected = Request() expected.addOperation(o3) removeOp = Operation() removeOp.Type = 'RemoveFile' fileRemove1 = File() fileRemove1.LFN = '/a/1.txt' fileRemove2 = File() fileRemove2.LFN = '/a/2.txt' fileRemove3 = File() fileRemove3.LFN = '/a/notPresent.txt' removeOp.addFile(fileRemove1) removeOp.addFile(fileRemove3) removeOp.addFile(fileRemove2) expected.addOperation(removeOp) uod._cleanUp({ '1.txt': { 'lfn': '/a/1.txt' }, '2.txt': { 'lfn': '/a/2.txt' }, 'notPresent.txt': { 'lfn': '/a/notPresent.txt' } }) for opsR, opsE in itertools.izip(uod.request, expected): assert str(opsR) == str(opsE)
def createRequest(self, requestName, archiveLFN, lfnChunk): """Create the Request.""" request = Request() request.RequestName = requestName self._checkReplicaSites(request, lfnChunk) archiveFiles = Operation() archiveFiles.Type = "ArchiveFiles" archiveFiles.Arguments = DEncode.encode({ "SourceSE": self.sourceSEs[0], "TarballSE": self.switches["TarballSE"], "RegisterDescendent": self.switches["RegisterDescendent"], "ArchiveLFN": archiveLFN, }) self.addLFNs(archiveFiles, lfnChunk) request.addOperation(archiveFiles) # Replicate the Tarball, ArchiveFiles will upload it if self.switches.get("ReplicateTarball"): replicateAndRegisterTarBall = Operation() replicateAndRegisterTarBall.Type = "ReplicateAndRegister" replicateAndRegisterTarBall.TargetSE = self.targetSE opFile = File() opFile.LFN = archiveLFN replicateAndRegisterTarBall.addFile(opFile) request.addOperation(replicateAndRegisterTarBall) if self.switches.get("CheckMigration"): checkMigrationTarBall = Operation() checkMigrationTarBall.Type = "CheckMigration" migrationTarget = self.targetSE if self.switches.get( "ReplicateTarball") else self.switches["TarballSE"] checkMigrationTarBall.TargetSE = migrationTarget opFile = File() opFile.LFN = archiveLFN checkMigrationTarBall.addFile(opFile) request.addOperation(checkMigrationTarBall) # Register Archive Replica for LFNs if self.switches.get("ArchiveSE"): registerArchived = Operation() registerArchived.Type = "RegisterReplica" registerArchived.TargetSE = self.switches.get("ArchiveSE") self.addLFNs(registerArchived, lfnChunk, addPFN=True) request.addOperation(registerArchived) # Remove all Other Replicas for LFNs if self.switches.get("RemoveReplicas"): removeArchiveReplicas = Operation() removeArchiveReplicas.Type = "RemoveReplica" removeArchiveReplicas.TargetSE = ",".join(self.replicaSEs) self.addLFNs(removeArchiveReplicas, lfnChunk) request.addOperation(removeArchiveReplicas) # Remove all Replicas for LFNs if self.switches.get("RemoveFiles"): removeArchiveFiles = Operation() removeArchiveFiles.Type = "RemoveFile" self.addLFNs(removeArchiveFiles, lfnChunk) request.addOperation(removeArchiveFiles) # Remove Original tarball replica if self.switches.get("ReplicateTarball"): removeTarballOrg = Operation() removeTarballOrg.Type = "RemoveReplica" removeTarballOrg.TargetSE = self.sourceSEs[0] opFile = File() opFile.LFN = archiveLFN removeTarballOrg.addFile(opFile) request.addOperation(removeTarballOrg) return request
def getRequest(self, requestName='', assigned=True): """ read request for execution :param str requestName: request's name (default None) """ requestID = None if requestName: self.log.info("getRequest: selecting request '%s'" % requestName) reqIDQuery = "SELECT `RequestID`, `Status` FROM `Request` WHERE `RequestName` = '%s';" % str( requestName) reqID = self._transaction(reqIDQuery) if not reqID["OK"]: self.log.error("getRequest: %s" % reqID["Message"]) return reqID requestID = reqID["Value"][reqIDQuery][0][ "RequestID"] if "RequestID" in reqID["Value"][reqIDQuery][ 0] else None status = reqID["Value"][reqIDQuery][0][ "Status"] if "Status" in reqID["Value"][reqIDQuery][0] else None if not all((requestID, status)): return S_ERROR("getRequest: request '%s' not exists" % requestName) if requestID and status and status == "Assigned" and assigned: return S_ERROR( "getRequest: status of request '%s' is 'Assigned', request cannot be selected" % requestName) else: reqIDsQuery = "SELECT `RequestID` FROM `Request` WHERE `Status` = 'Waiting' ORDER BY `LastUpdate` ASC LIMIT 100;" reqIDs = self._transaction(reqIDsQuery) if not reqIDs["OK"]: self.log.error("getRequest: %s" % reqIDs["Message"]) return reqIDs reqIDs = reqIDs["Value"][reqIDsQuery] reqIDs = [reqID["RequestID"] for reqID in reqIDs] if not reqIDs: return S_OK() random.shuffle(reqIDs) requestID = reqIDs[0] selectQuery = [ "SELECT * FROM `Request` WHERE `RequestID` = %s;" % requestID, "SELECT * FROM `Operation` WHERE `RequestID` = %s;" % requestID ] selectReq = self._transaction(selectQuery) if not selectReq["OK"]: self.log.error("getRequest: %s" % selectReq["Message"]) return S_ERROR(selectReq["Message"]) selectReq = selectReq["Value"] request = Request(selectReq[selectQuery[0]][0]) for records in sorted(selectReq[selectQuery[1]], key=lambda k: k["Order"]): # # order is ro, remove del records["Order"] operation = Operation(records) getFilesQuery = "SELECT * FROM `File` WHERE `OperationID` = %s;" % operation.OperationID getFiles = self._transaction(getFilesQuery) if not getFiles["OK"]: self.log.error("getRequest: %s" % getFiles["Message"]) return getFiles getFiles = getFiles["Value"][getFilesQuery] for getFile in getFiles: getFileDict = dict([(key, value) for key, value in getFile.items() if value != None]) operation.addFile(File(getFileDict)) request.addOperation(operation) if assigned: setAssigned = self._transaction( "UPDATE `Request` SET `Status` = 'Assigned' WHERE RequestID = %s;" % requestID) if not setAssigned["OK"]: self.log.error("getRequest: %s" % setAssigned["Message"]) return setAssigned return S_OK(request)
def setUp(self): """ test setup """ self.request = Request() self.operation = Operation() self.file = File()
def test05FTS(self): """ FTS state machine """ req = Request() req.RequestName = "FTSTest" ftsTransfer = Operation() ftsTransfer.Type = "ReplicateAndRegister" ftsTransfer.TargetSE = "CERN-USER" ftsFile = File() ftsFile.LFN = "/a/b/c" ftsFile.Checksum = "123456" ftsFile.ChecksumType = "Adler32" ftsTransfer.addFile(ftsFile) req.addOperation(ftsTransfer) self.assertEqual(req.Status, "Waiting", "1. wrong request status: %s" % req.Status) self.assertEqual(ftsTransfer.Status, "Waiting", "1. wrong ftsStatus status: %s" % ftsTransfer.Status) # # scheduled ftsFile.Status = "Scheduled" self.assertEqual( ftsTransfer.Status, "Scheduled", "2. wrong status for ftsTransfer: %s" % ftsTransfer.Status) self.assertEqual(req.Status, "Scheduled", "2. wrong status for request: %s" % req.Status) # # add new operation before FTS insertBefore = Operation() insertBefore.Type = "RegisterReplica" insertBefore.TargetSE = "CERN-USER" insertFile = File() insertFile.LFN = "/a/b/c" insertFile.PFN = "http://foo/bar" insertBefore.addFile(insertFile) req.insertBefore(insertBefore, ftsTransfer) self.assertEqual( insertBefore.Status, "Waiting", "3. wrong status for insertBefore: %s" % insertBefore.Status) self.assertEqual( ftsTransfer.Status, "Scheduled", "3. wrong status for ftsStatus: %s" % ftsTransfer.Status) self.assertEqual(req.Status, "Waiting", "3. wrong status for request: %s" % req.Status) # # prev done insertFile.Status = "Done" self.assertEqual( insertBefore.Status, "Done", "4. wrong status for insertBefore: %s" % insertBefore.Status) self.assertEqual( ftsTransfer.Status, "Scheduled", "4. wrong status for ftsStatus: %s" % ftsTransfer.Status) self.assertEqual(req.Status, "Scheduled", "4. wrong status for request: %s" % req.Status) # # reschedule ftsFile.Status = "Waiting" self.assertEqual( insertBefore.Status, "Done", "5. wrong status for insertBefore: %s" % insertBefore.Status) self.assertEqual( ftsTransfer.Status, "Waiting", "5. wrong status for ftsStatus: %s" % ftsTransfer.Status) self.assertEqual(req.Status, "Waiting", "5. wrong status for request: %s" % req.Status) # # fts done ftsFile.Status = "Done" self.assertEqual( insertBefore.Status, "Done", "5. wrong status for insertBefore: %s" % insertBefore.Status) self.assertEqual( ftsTransfer.Status, "Done", "5. wrong status for ftsStatus: %s" % ftsTransfer.Status) self.assertEqual(req.Status, "Done", "5. wrong status for request: %s" % req.Status)
def main(): from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() import DIRAC from DIRAC import gLogger args = Script.getPositionalArgs() requestName = None LFN = None PFN = None targetSE = None if len(args) != 4: Script.showHelp() else: requestName = args[0] LFN = args[1] PFN = args[2] targetSE = args[3] if not os.path.isabs(LFN): gLogger.error("LFN should be absolute path!!!") DIRAC.exit(-1) gLogger.info("will create request '%s' with 'PutAndRegister' " "operation using %s pfn and %s target SE" % (requestName, PFN, targetSE)) from DIRAC.RequestManagementSystem.Client.Request import Request from DIRAC.RequestManagementSystem.Client.Operation import Operation from DIRAC.RequestManagementSystem.Client.File import File from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient from DIRAC.Core.Utilities.Adler import fileAdler if not os.path.exists(PFN): gLogger.error("%s does not exist" % PFN) DIRAC.exit(-1) if not os.path.isfile(PFN): gLogger.error("%s is not a file" % PFN) DIRAC.exit(-1) PFN = os.path.abspath(PFN) size = os.path.getsize(PFN) adler32 = fileAdler(PFN) request = Request() request.RequestName = requestName putAndRegister = Operation() putAndRegister.Type = "PutAndRegister" putAndRegister.TargetSE = targetSE opFile = File() opFile.LFN = LFN opFile.PFN = PFN opFile.Size = size opFile.Checksum = adler32 opFile.ChecksumType = "ADLER32" putAndRegister.addFile(opFile) request.addOperation(putAndRegister) reqClient = ReqClient() putRequest = reqClient.putRequest(request) if not putRequest["OK"]: gLogger.error("unable to put request '%s': %s" % (requestName, putRequest["Message"])) DIRAC.exit(-1) gLogger.always("Request '%s' has been put to ReqDB for execution." % requestName) gLogger.always( "You can monitor its status using command: 'dirac-rms-request %s'" % requestName) DIRAC.exit(0)
def getRequest(self, requestName='', assigned=True): """ read request for execution :param str requestName: request's name (default None) """ requestID = None log = self.log.getSubLogger( 'getRequest' if assigned else 'peekRequest') if requestName: log.verbose("selecting request '%s'%s" % (requestName, ' (Assigned)' if assigned else '')) reqIDQuery = "SELECT `RequestID`, `Status` FROM `Request` WHERE `RequestName` = '%s';" % str( requestName) reqID = self._transaction(reqIDQuery) if not reqID["OK"]: log.error(reqID["Message"]) return reqID reqID = reqID["Value"].get(reqIDQuery, []) if reqID: reqID = reqID[0] else: reqID = {} requestID = reqID.get("RequestID") status = reqID.get("Status") if not all((requestID, status)): return S_ERROR("getRequest: Request '%s' does not exist" % requestName) if requestID and status and status == "Assigned" and assigned: return S_ERROR( "getRequest: status of request '%s' is 'Assigned', request cannot be selected" % requestName) else: reqIDsQuery = "SELECT `RequestID` FROM `Request` WHERE `Status` = 'Waiting' ORDER BY `LastUpdate` ASC LIMIT 100;" reqAscIDs = self._transaction(reqIDsQuery) if not reqAscIDs['OK']: log.error(reqAscIDs["Message"]) return reqAscIDs reqIDs = set([ reqID['RequestID'] for reqID in reqAscIDs["Value"][reqIDsQuery] ]) reqIDsQuery = "SELECT `RequestID` FROM `Request` WHERE `Status` = 'Waiting' ORDER BY `LastUpdate` DESC LIMIT 50;" reqDescIDs = self._transaction(reqIDsQuery) if not reqDescIDs['OK']: log.error(reqDescIDs["Message"]) return reqDescIDs reqIDs |= set([ reqID['RequestID'] for reqID in reqDescIDs["Value"][reqIDsQuery] ]) if not reqIDs: return S_OK() reqIDs = list(reqIDs) random.shuffle(reqIDs) requestID = reqIDs[0] selectQuery = [ "SELECT * FROM `Request` WHERE `RequestID` = %s;" % requestID, "SELECT * FROM `Operation` WHERE `RequestID` = %s;" % requestID ] selectReq = self._transaction(selectQuery) if not selectReq["OK"]: log.error(selectReq["Message"]) return S_ERROR(selectReq["Message"]) selectReq = selectReq["Value"] request = Request(selectReq[selectQuery[0]][0]) origLastUpdate = request.LastUpdate if not requestName: log.verbose( "selected request '%s'%s" % (request.RequestName, ' (Assigned)' if assigned else '')) for records in sorted(selectReq[selectQuery[1]], key=lambda k: k["Order"]): # # order is ro, remove del records["Order"] operation = Operation(records) getFilesQuery = "SELECT * FROM `File` WHERE `OperationID` = %s;" % operation.OperationID getFiles = self._transaction(getFilesQuery) if not getFiles["OK"]: log.error(getFiles["Message"]) return getFiles getFiles = getFiles["Value"][getFilesQuery] for getFile in getFiles: getFileDict = dict([(key, value) for key, value in getFile.items() if value != None]) operation.addFile(File(getFileDict)) request.addOperation(operation) request.LastUpdate = origLastUpdate if assigned: setAssigned = self._transaction( "UPDATE `Request` SET `Status` = 'Assigned', `LastUpdate`=UTC_TIMESTAMP() WHERE RequestID = %s;" % requestID) if not setAssigned["OK"]: log.error("%s" % setAssigned["Message"]) return setAssigned return S_OK(request)
def prepareTransformationTasks(self, transBody, taskDict, owner='', ownerGroup='', ownerDN=''): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if (not owner) or (not ownerGroup): res = getProxyInfo(False, False) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername(owner) if not res['OK']: return res ownerDN = res['Value'][0] requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split(';') except AttributeError: pass for taskID in sorted(taskDict): paramDict = taskDict[taskID] if paramDict['InputData']: transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] if type(paramDict['InputData']) == type([]): files = paramDict['InputData'] elif type(paramDict['InputData']) == type(''): files = paramDict['InputData'].split(';') for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile(trFile) oRequest.addOperation(transfer) oRequest.RequestName = _requestName(transID, taskID) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = RequestValidator().validate(oRequest) if not isValid['OK']: return isValid taskDict[taskID]['TaskObject'] = oRequest return S_OK(taskDict)
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument("requestName: a request name") Script.registerArgument("LFN: logical file name") Script.registerArgument("localFile: local file you want to put") Script.registerArgument("targetSE: target SE") Script.parseCommandLine() import DIRAC from DIRAC import gLogger # parseCommandLine show help when mandatory arguments are not specified or incorrect argument requestName, LFN, PFN, targetSE = Script.getPositionalArgs(group=True) if not os.path.isabs(LFN): gLogger.error("LFN should be absolute path!!!") DIRAC.exit(-1) gLogger.info("will create request '%s' with 'PutAndRegister' " "operation using %s pfn and %s target SE" % (requestName, PFN, targetSE)) from DIRAC.RequestManagementSystem.Client.Request import Request from DIRAC.RequestManagementSystem.Client.Operation import Operation from DIRAC.RequestManagementSystem.Client.File import File from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient from DIRAC.Core.Utilities.Adler import fileAdler if not os.path.exists(PFN): gLogger.error("%s does not exist" % PFN) DIRAC.exit(-1) if not os.path.isfile(PFN): gLogger.error("%s is not a file" % PFN) DIRAC.exit(-1) PFN = os.path.abspath(PFN) size = os.path.getsize(PFN) adler32 = fileAdler(PFN) request = Request() request.RequestName = requestName putAndRegister = Operation() putAndRegister.Type = "PutAndRegister" putAndRegister.TargetSE = targetSE opFile = File() opFile.LFN = LFN opFile.PFN = PFN opFile.Size = size opFile.Checksum = adler32 opFile.ChecksumType = "ADLER32" putAndRegister.addFile(opFile) request.addOperation(putAndRegister) reqClient = ReqClient() putRequest = reqClient.putRequest(request) if not putRequest["OK"]: gLogger.error("unable to put request '%s': %s" % (requestName, putRequest["Message"])) DIRAC.exit(-1) gLogger.always("Request '%s' has been put to ReqDB for execution." % requestName) gLogger.always( "You can monitor its status using command: 'dirac-rms-request %s'" % requestName) DIRAC.exit(0)
def main(): catalog = None Script.registerSwitch("C:", "Catalog=", "Catalog to use") # Registering arguments will automatically add their description to the help menu Script.registerArgument(" requestName: a request name") Script.registerArgument(" LFNs: single LFN or file with LFNs") Script.registerArgument(["targetSE: target SE"]) Script.parseCommandLine() for switch in Script.getUnprocessedSwitches(): if switch[0] == "C" or switch[0].lower() == "catalog": catalog = switch[1] args = Script.getPositionalArgs() requestName = None targetSEs = None if len(args) < 3: Script.showHelp(exitCode=1) requestName = args[0] lfnList = getLFNList(args[1]) targetSEs = list( set([se for targetSE in args[2:] for se in targetSE.split(",")])) gLogger.info("Will create request '%s' with 'ReplicateAndRegister' " "operation using %s lfns and %s target SEs" % (requestName, len(lfnList), len(targetSEs))) from DIRAC.RequestManagementSystem.Client.Request import Request from DIRAC.RequestManagementSystem.Client.Operation import Operation from DIRAC.RequestManagementSystem.Client.File import File from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient from DIRAC.Resources.Catalog.FileCatalog import FileCatalog from DIRAC.Core.Utilities.List import breakListIntoChunks lfnChunks = breakListIntoChunks(lfnList, 100) multiRequests = len(lfnChunks) > 1 error = 0 count = 0 reqClient = ReqClient() fc = FileCatalog() requestIDs = [] for lfnChunk in lfnChunks: metaDatas = fc.getFileMetadata(lfnChunk) if not metaDatas["OK"]: gLogger.error("unable to read metadata for lfns: %s" % metaDatas["Message"]) error = -1 continue metaDatas = metaDatas["Value"] for failedLFN, reason in metaDatas["Failed"].items(): gLogger.error("skipping %s: %s" % (failedLFN, reason)) lfnChunk = set(metaDatas["Successful"]) if not lfnChunk: gLogger.error("LFN list is empty!!!") error = -1 continue if len(lfnChunk) > Operation.MAX_FILES: gLogger.error( "too many LFNs, max number of files per operation is %s" % Operation.MAX_FILES) error = -1 continue count += 1 request = Request() request.RequestName = requestName if not multiRequests else "%s_%d" % ( requestName, count) replicateAndRegister = Operation() replicateAndRegister.Type = "ReplicateAndRegister" replicateAndRegister.TargetSE = ",".join(targetSEs) if catalog is not None: replicateAndRegister.Catalog = catalog for lfn in lfnChunk: metaDict = metaDatas["Successful"][lfn] opFile = File() opFile.LFN = lfn opFile.Size = metaDict["Size"] if "Checksum" in metaDict: # # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD' opFile.Checksum = metaDict["Checksum"] opFile.ChecksumType = "ADLER32" replicateAndRegister.addFile(opFile) request.addOperation(replicateAndRegister) putRequest = reqClient.putRequest(request) if not putRequest["OK"]: gLogger.error("unable to put request '%s': %s" % (request.RequestName, putRequest["Message"])) error = -1 continue requestIDs.append(str(putRequest["Value"])) if not multiRequests: gLogger.always( "Request '%s' has been put to ReqDB for execution." % request.RequestName) if multiRequests: gLogger.always( "%d requests have been put to ReqDB for execution, with name %s_<num>" % (count, requestName)) if requestIDs: gLogger.always("RequestID(s): %s" % " ".join(requestIDs)) gLogger.always( "You can monitor requests' status using command: 'dirac-rms-request <requestName/ID>'" ) DIRAC.exit(error)
def createRequest(self, requestName, archiveLFN, lfnChunk): """Create the Request.""" request = Request() request.RequestName = requestName self._checkReplicaSites(request, lfnChunk) archiveFiles = Operation() archiveFiles.Type = 'ArchiveFiles' archiveFiles.Arguments = DEncode.encode({ 'SourceSE': self.sourceSEs[0], 'TarballSE': self.switches['TarballSE'], 'RegisterDescendent': self.switches['RegisterDescendent'], 'ArchiveLFN': archiveLFN }) self.addLFNs(archiveFiles, lfnChunk) request.addOperation(archiveFiles) # Replicate the Tarball, ArchiveFiles will upload it if self.switches.get('ReplicateTarball'): replicateAndRegisterTarBall = Operation() replicateAndRegisterTarBall.Type = 'ReplicateAndRegister' replicateAndRegisterTarBall.TargetSE = self.targetSE opFile = File() opFile.LFN = archiveLFN replicateAndRegisterTarBall.addFile(opFile) request.addOperation(replicateAndRegisterTarBall) if self.switches.get('CheckMigration'): checkMigrationTarBall = Operation() checkMigrationTarBall.Type = 'CheckMigration' migrationTarget = self.targetSE if self.switches.get( 'ReplicateTarball') else self.switches['TarballSE'] checkMigrationTarBall.TargetSE = migrationTarget opFile = File() opFile.LFN = archiveLFN checkMigrationTarBall.addFile(opFile) request.addOperation(checkMigrationTarBall) # Register Archive Replica for LFNs if self.switches.get('ArchiveSE'): registerArchived = Operation() registerArchived.Type = 'RegisterReplica' registerArchived.TargetSE = self.switches.get('ArchiveSE') self.addLFNs(registerArchived, lfnChunk, addPFN=True) request.addOperation(registerArchived) # Remove all Other Replicas for LFNs if self.switches.get('RemoveReplicas'): removeArchiveReplicas = Operation() removeArchiveReplicas.Type = 'RemoveReplica' removeArchiveReplicas.TargetSE = ','.join(self.replicaSEs) self.addLFNs(removeArchiveReplicas, lfnChunk) request.addOperation(removeArchiveReplicas) # Remove all Replicas for LFNs if self.switches.get('RemoveFiles'): removeArchiveFiles = Operation() removeArchiveFiles.Type = 'RemoveFile' self.addLFNs(removeArchiveFiles, lfnChunk) request.addOperation(removeArchiveFiles) # Remove Original tarball replica if self.switches.get('ReplicateTarball'): removeTarballOrg = Operation() removeTarballOrg.Type = 'RemoveReplica' removeTarballOrg.TargetSE = self.sourceSEs[0] opFile = File() opFile.LFN = archiveLFN removeTarballOrg.addFile(opFile) request.addOperation(removeTarballOrg) return request
def test__addMetadataToFiles(self): resMeta = { 'OK': True, 'Value': { 'Failed': {}, 'Successful': { '/lhcb/1.dst': { 'ChecksumType': 'AD', 'Checksum': '123456', 'CreationDate': datetime.datetime(2013, 12, 11, 20, 20, 21), 'GUID': '92F9CE97-7A62-E311-8401-0025907FD430', 'Mode': 436, 'ModificationDate': datetime.datetime(2013, 12, 11, 20, 20, 21), 'NumberOfLinks': 1, 'Size': 5846023777, 'Status': '-' }, '/lhcb/2.dst': { 'ChecksumType': 'AD', 'Checksum': '987654', 'CreationDate': datetime.datetime(2013, 12, 12, 6, 26, 52), 'GUID': 'DAE4933A-C162-E311-8A6B-003048FEAF04', 'Mode': 436, 'ModificationDate': datetime.datetime(2013, 12, 12, 6, 26, 52), 'NumberOfLinks': 1, 'Size': 5893396937, 'Status': '-' } } } } self.rr.fc.getFileMetadata.return_value = resMeta file1 = File() file1.LFN = '/lhcb/1.dst' file2 = File() file2.LFN = '/lhcb/2.dst' toSchedule = { '/lhcb/1.dst': [file1, ['SE1'], ['SE2', 'SE3']], '/lhcb/2.dst': [file2, ['SE4'], ['SE5', 'SE6']] } res = self.rr._addMetadataToFiles(toSchedule) self.assert_(res['OK']) self.assertEqual( json.loads(res['Value'][0][0])['LFN'], resMeta['Value']['Successful'].keys()[0]) self.assertEqual( json.loads(res['Value'][0][0])['Size'], resMeta['Value']['Successful'].values()[0]['Size']) self.assertEqual( json.loads(res['Value'][1][0])['LFN'], resMeta['Value']['Successful'].keys()[1]) self.assertEqual( json.loads(res['Value'][1][0])['Size'], resMeta['Value']['Successful'].values()[1]['Size'])
jr_mock.setJobParameter.return_value = {'OK': True, 'Value': 'pippo'} fr_mock = MagicMock() fr_mock.getFiles.return_value = {} fr_mock.setFileStatus.return_value = {'OK': True, 'Value': ''} fr_mock.commit.return_value = {'OK': True, 'Value': ''} fr_mock.generateForwardDISET.return_value = {'OK': True, 'Value': Operation()} rc_mock = Request() rc_mock.RequestName = 'aRequestName' rc_mock.OwnerDN = 'pippo' rc_mock.OwnerGroup = 'pippoGRP' rOp = Operation() rOp.Type = 'PutAndRegister' rOp.TargetSE = 'anSE' f = File() f.LFN = '/foo/bar.py' f.PFN = '/foo/bar.py' rOp.addFile(f) rc_mock.addOperation(rOp) wf_commons = [{'PRODUCTION_ID': prod_id, 'JOB_ID': prod_job_id, 'eventType': '123456789', 'jobType': 'merge', 'configName': 'aConfigName', 'configVersion': 'aConfigVersion', 'outputDataFileMask': '', 'BookkeepingLFNs': 'aa', 'ProductionOutputData': 'ProductionOutputData', 'numberOfEvents': '100', 'JobReport': jr_mock, 'Request': rc_mock, 'AccountingReport': ar_mock, 'FileReport': fr_mock, 'SystemConfig': 'sys_config', 'runNumber': 'Unknown', 'gaudiSteps': ['someApp_1'], 'outputSEs':{"DAVINCIHIST": "CERN-HIST", "TXT": "SE1"}}, {'PRODUCTION_ID': prod_id, 'JOB_ID': prod_job_id, 'configName': 'aConfigName', 'configVersion': 'aConfigVersion', 'outputDataFileMask': '', 'jobType': 'merge',
def test02props(self): """ test properties """ # # valid values operation = Operation() operation.Arguments = "foobar" self.assertEqual(operation.Arguments, "foobar", "wrong Arguments") operation.SourceSE = "CERN-RAW" self.assertEqual(operation.SourceSE, "CERN-RAW", "wrong SourceSE") operation.TargetSE = "CERN-RAW" self.assertEqual(operation.TargetSE, "CERN-RAW", "wrong TargetSE") operation.Catalog = "" self.assertEqual(operation.Catalog, "", "wrong Catalog") operation.Catalog = "BookkeepingDB" self.assertEqual(operation.Catalog, "BookkeepingDB", "wrong Catalog") operation.Error = "error" self.assertEqual(operation.Error, "error", "wrong Error") # # wrong props try: operation.RequestID = "foo" except Exception as error: self.assertEqual(type(error), AttributeError, "wrong exc raised") self.assertEqual(str(error), "can't set attribute", "wrong exc reason") try: operation.OperationID = "foo" except Exception as error: self.assertEqual(type(error), ValueError, "wrong exc raised") # # timestamps try: operation.SubmitTime = "foo" except Exception as error: self.assertEqual(type(error), ValueError, "wrong exp raised") self.assertEqual( str(error), "time data 'foo' does not match format '%Y-%m-%d %H:%M:%S'", "wrong exc reason") try: operation.LastUpdate = "foo" except Exception as error: self.assertEqual(type(error), ValueError, "wrong exc raised") self.assertEqual( str(error), "time data 'foo' does not match format '%Y-%m-%d %H:%M:%S'", "wrong exc reason") # # Status operation = Operation() try: operation.Status = "foo" except Exception as error: self.assertEqual(type(error), ValueError, "wrong exc raised") self.assertEqual(str(error), "unknown Status 'foo'", "wrong exc reason") operation.addFile(File({"Status": "Waiting", "LFN": "/a"}))
def createRequest(reqType): r = Request() # Simple failover op1 = Operation() f = File() f.LFN = '/This/is/an/LFN' op1.addFile(f) op1.Type = 'ReplicateAndRegister' op1.SourceSE = 'CERN-FAILOVER' op1.TargetSE = 'CERN-BUFFER' r.addOperation(op1) # You cannot reuse the same File object, # since it is a different entry in the DB fr = File() fr.LFN = '/This/is/an/LFN' op2 = Operation() op2.addFile(fr) op2.Type = 'RemoveReplica' op2.TargetSE = 'CERN-FAILOVER' r.addOperation(op2) if reqType == 0: return r # two files for Failover f1 = File() f1.LFN = '/This/is/a/second/LFN' op3 = Operation() op3.addFile(f1) op3.Type = 'ReplicateAndRegister' op3.SourceSE = 'CERN-FAILOVER' op3.TargetSE = 'CERN-BUFFER' r.addOperation(op3) f1r = File() f1r.LFN = '/This/is/a/second/LFN' op3 = Operation() op3.addFile(f1r) op3.Type = 'RemoveReplica' op3.TargetSE = 'CERN-FAILOVER' r.addOperation(op3) if reqType == 1: return r op = Operation() op.Type = 'ForwardDiset' if reqType == 2: r.addOperation(op) return r r.insertBefore(op, r[0]) if reqType == 3: return r op4 = Operation() op4.Type = 'ForwardDiset' r.addOperation(op4) if reqType == 4: return r # 2 different FAILOVER SEs: removal not optimized r[1].SourceSE = 'RAL-FAILOVER' r[2].SourceSE = 'RAL-FAILOVER' if reqType == 5: return r # 2 different destinations, same FAILOVER: replication not optimized r[3].SourceSE = 'RAL-FAILOVER' r[4].SourceSE = 'RAL-FAILOVER' r[3].TargetSE = 'RAL-BUFFER' if reqType == 6: return r print 'This should not happen, reqType =', reqType
replicateAndRegister.TargetSE = targetSE res = fc.getFileMetadata(lfnList) if not res['OK']: print "Can't get file metadata: %s" % res['Message'] DIRAC.exit(1) if res['Value']['Failed']: print "Could not get the file metadata of the following, so skipping them:" for fFile in res['Value']['Failed']: print fFile lfnMetadata = res['Value']['Successful'] for lfn in lfnMetadata: try: rarFile = File() rarFile.LFN = lfn rarFile.Size = lfnMetadata[lfn]['Size'] rarFile.Checksum = lfnMetadata[lfn]['Checksum'] rarFile.GUID = lfnMetadata[lfn]['GUID'] rarFile.ChecksumType = 'ADLER32' replicateAndRegister.addFile(rarFile) except ValueError as err: print "Error", str(err), lfn continue oRequest.addOperation(replicateAndRegister) isValid = RequestValidator().validate(oRequest) if not isValid['OK']: print "Request is not valid: ", isValid['Message'] DIRAC.exit(1)
def getBulkRequests(self, numberOfRequest=10, assigned=True): """ read as many requests as requested for execution :param int numberOfRequest: Number of Request we want (default 10) :param bool assigned: if True, the status of the selected requests are set to assign :returns a dictionary of Request objects indexed on the RequestID """ # r_RequestID : RequestID, r_LastUpdate : LastUpdate... requestAttrDict = dict( ("r_%s" % r, r) for r in Request.tableDesc()["Fields"]) # o_RequestID : RequestID, o_OperationID : OperationID... operationAttrDict = dict( ("o_%s" % o, o) for o in Operation.tableDesc()["Fields"]) # f_OperationID : OperationID, f_FileID : FileID... fileAttrDict = dict( ("f_%s" % f, f) for f in File.tableDesc()["Fields"]) # o.OperationID as o_OperationID, ..., r_RequestID, ..., f_FileID, ... allFieldsStr = ",".join( [ "o.%s as %s" % ( operationAttrDict[o], o ) for o in operationAttrDict]\ + requestAttrDict.keys() + fileAttrDict.keys() ) # RequestID as r_RequestID, LastUpdate as r_LastUpdate, ... requestAttrStr = ",".join( ["%s as %s" % (requestAttrDict[r], r) for r in requestAttrDict]) # OperationID as f_OperationID, FileID as f_FileID... fileAttrStr = ",".join( ["%s as %s" % (fileAttrDict[f], f) for f in fileAttrDict]) # Selects all the Request (limited to numberOfRequest, sorted by LastUpdate) , Operation and File information. # The entries are sorted by the LastUpdate of the Requests, RequestID if several requests were update the last time # at the same time, and finally according to the Operation Order query = "SELECT %s FROM Operation o \ INNER JOIN (SELECT %s FROM Request WHERE Status = 'Waiting' ORDER BY `LastUpdate` ASC limit %s) r\ ON r_RequestID = o.RequestID\ INNER JOIN (SELECT %s from File) f ON f_OperationId = o.OperationId\ ORDER BY r_LastUpdate, r_RequestId, o_Order;" \ % ( allFieldsStr, requestAttrStr, numberOfRequest, fileAttrStr ) queryResult = self._transaction(query) if not queryResult["OK"]: self.log.error("RequestDB.getRequests: %s" % queryResult["Message"]) return queryResult allResults = queryResult["Value"][query] # We now construct a dict of Request indexed by their ID, and the same for Operation requestDict = {} operationDict = {} for entry in allResults: requestID = int(entry["r_RequestID"]) # If the object already exists, we get it, otherwise we create it and assign it requestObj = requestDict.setdefault( requestID, Request( dict((requestAttrDict[r], entry[r]) for r in requestAttrDict))) operationID = int(entry["o_OperationID"]) operationObj = operationDict.get(operationID, None) # If the Operation object does not exist yet, we create it, and add it to the Request if not operationObj: operationObj = Operation( dict((operationAttrDict[o], entry[o]) for o in operationAttrDict)) operationDict[operationID] = operationObj requestObj.addOperation(operationObj) fileObj = File( dict((fileAttrDict[f], entry[f]) for f in fileAttrDict)) operationObj.addFile(fileObj) if assigned and len(requestDict): listOfReqId = ",".join(str(rId) for rId in requestDict) setAssigned = self._transaction( "UPDATE `Request` SET `Status` = 'Assigned' WHERE RequestID IN (%s);" % listOfReqId) if not setAssigned["OK"]: self.log.error("getRequests: %s" % setAssigned["Message"]) return setAssigned return S_OK(requestDict)
except Exception, error: self.assertEqual(type(error), ValueError, "wrong exc raised") self.assertEqual( str(error), "time data 'foo' does not match format '%Y-%m-%d %H:%M:%S'", "wrong exc reason") # # Status operation = Operation() try: operation.Status = "foo" except Exception, error: self.assertEqual(type(error), ValueError, "wrong exc raised") self.assertEqual(str(error), "unknown Status 'foo'", "wrong exc reason") operation.addFile(File({"Status": "Waiting", "LFN": "/a"})) def test04StateMachine(self): """ state machine """ op = Operation() self.assertEqual(op.Status, "Queued", "1. wrong status %s" % op.Status) op.addFile(File({"Status": "Waiting"})) self.assertEqual(op.Status, "Queued", "2. wrong status %s" % op.Status) op.addFile(File({"Status": "Scheduled"})) self.assertEqual(op.Status, "Scheduled", "3. wrong status %s" % op.Status) op.addFile(File({"Status": "Done"})) self.assertEqual(op.Status, "Scheduled",
def test04Operations(self): """ operations arithmetic and state machine """ req = Request() self.assertEqual(len(req), 0) transfer = Operation() transfer.Type = "ReplicateAndRegister" transfer.addFile(File({"LFN": "/a/b/c", "Status": "Waiting"})) getWaiting = req.getWaiting() self.assertEqual(getWaiting["OK"], True) self.assertEqual(getWaiting["Value"], None) req.addOperation(transfer) self.assertEqual(len(req), 1) self.assertEqual(transfer.Order, req.Order) self.assertEqual(transfer.Status, "Waiting") getWaiting = req.getWaiting() self.assertEqual(getWaiting["OK"], True) self.assertEqual(getWaiting["Value"], transfer) removal = Operation({"Type": "RemoveFile"}) removal.addFile(File({"LFN": "/a/b/c", "Status": "Waiting"})) req.insertBefore(removal, transfer) getWaiting = req.getWaiting() self.assertEqual(getWaiting["OK"], True) self.assertEqual(getWaiting["Value"], removal) self.assertEqual(len(req), 2) self.assertEqual([op.Status for op in req], ["Waiting", "Queued"]) self.assertEqual(req.subStatusList(), ["Waiting", "Queued"]) self.assertEqual(removal.Order, 0) self.assertEqual(removal.Order, req.Order) self.assertEqual(transfer.Order, 1) self.assertEqual(removal.Status, "Waiting") self.assertEqual(transfer.Status, "Queued") for subFile in removal: subFile.Status = "Done" removal.Status = "Done" self.assertEqual(removal.Status, "Done") self.assertEqual(transfer.Status, "Waiting") self.assertEqual(transfer.Order, req.Order) # # len, looping self.assertEqual(len(req), 2) self.assertEqual([op.Status for op in req], ["Done", "Waiting"]) self.assertEqual(req.subStatusList(), ["Done", "Waiting"]) digest = req.toJSON() self.assertEqual(digest["OK"], True) getWaiting = req.getWaiting() self.assertEqual(getWaiting["OK"], True) self.assertEqual(getWaiting["Value"], transfer)
def test06Dirty(self): """ dirty records """ db = RequestDB() r = Request() r.RequestName = "dirty" op1 = Operation({ "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER" }) op1 += File({ "LFN": "/a/b/c/1", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" }) op2 = Operation({ "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER" }) op2 += File({ "LFN": "/a/b/c/2", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" }) op3 = Operation({ "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER" }) op3 += File({ "LFN": "/a/b/c/3", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" }) r += op1 r += op2 r += op3 put = db.putRequest(r) self.assertEqual(put["OK"], True, "1. putRequest failed: %s" % put.get("Message", "")) reqID = put['Value'] r = db.getRequest(reqID) self.assertEqual(r["OK"], True, "1. getRequest failed: %s" % r.get("Message", "")) r = r["Value"] del r[0] self.assertEqual(len(r), 2, "1. len wrong") put = db.putRequest(r) self.assertEqual(put["OK"], True, "2. putRequest failed: %s" % put.get("Message", "")) reqID = put['Value'] r = db.getRequest(reqID) self.assertEqual(r["OK"], True, "2. getRequest failed: %s" % r.get("Message", "")) r = r["Value"] self.assertEqual(len(r), 2, "2. len wrong") op4 = Operation({ "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER" }) op4 += File({ "LFN": "/a/b/c/4", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" }) r[0] = op4 put = db.putRequest(r) self.assertEqual(put["OK"], True, "3. putRequest failed: %s" % put.get("Message", "")) reqID = put['Value'] r = db.getRequest(reqID) self.assertEqual(r["OK"], True, "3. getRequest failed: %s" % r.get("Message", "")) r = r["Value"] self.assertEqual(len(r), 2, "3. len wrong") delete = db.deleteRequest(reqID) self.assertEqual(delete["OK"], True, delete['Message'] if 'Message' in delete else 'OK')
if len( lfnChunk ) > Operation.MAX_FILES: gLogger.error( "too many LFNs, max number of files per operation is %s" % Operation.MAX_FILES ) error = -1 continue count += 1 request = Request() request.RequestName = requestName if not multiRequests else '%s_%d' % ( requestName, count ) replicateAndRegister = Operation() replicateAndRegister.Type = "ReplicateAndRegister" replicateAndRegister.TargetSE = ",".join( targetSEs ) for lfn in lfnChunk: metaDict = metaDatas["Successful"][lfn] opFile = File() opFile.LFN = lfn opFile.Size = metaDict["Size"] if "Checksum" in metaDict: # # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD' opFile.Checksum = metaDict["Checksum"] opFile.ChecksumType = "ADLER32" replicateAndRegister.addFile( opFile ) request.addOperation( replicateAndRegister ) putRequest = reqClient.putRequest( request ) if not putRequest["OK"]: gLogger.error( "unable to put request '%s': %s" % ( request.RequestName, putRequest["Message"] ) ) error = -1
def test__addMetadataToFiles(self): resMeta = { "OK": True, "Value": { "Failed": {}, "Successful": { "/lhcb/1.dst": { "ChecksumType": "AD", "Checksum": "123456", "CreationDate": datetime.datetime(2013, 12, 11, 20, 20, 21), "GUID": "92F9CE97-7A62-E311-8401-0025907FD430", "Mode": 436, "ModificationDate": datetime.datetime(2013, 12, 11, 20, 20, 21), "NumberOfLinks": 1, "Size": 5846023777, "Status": "-", }, "/lhcb/2.dst": { "ChecksumType": "AD", "Checksum": "987654", "CreationDate": datetime.datetime(2013, 12, 12, 6, 26, 52), "GUID": "DAE4933A-C162-E311-8A6B-003048FEAF04", "Mode": 436, "ModificationDate": datetime.datetime(2013, 12, 12, 6, 26, 52), "NumberOfLinks": 1, "Size": 5893396937, "Status": "-", }, }, }, } self.rr.fc.getFileMetadata.return_value = resMeta file1 = File() file1.LFN = "/lhcb/1.dst" file2 = File() file2.LFN = "/lhcb/2.dst" toSchedule = { "/lhcb/1.dst": [file1, ["SE1"], ["SE2", "SE3"]], "/lhcb/2.dst": [file2, ["SE4"], ["SE5", "SE6"]] } res = self.rr._addMetadataToFiles(toSchedule) self.assertTrue(res["OK"]) for lfn in toSchedule: self.assertEqual(res["Value"][lfn].LFN, lfn) for attr in ("GUID", "Size", "Checksum"): self.assertEqual(getattr(res["Value"][lfn], attr), resMeta["Value"]["Successful"][lfn][attr]) # AD should be transformed into Adler32 self.assertEqual(res["Value"][lfn].ChecksumType, "ADLER32")