def test02props(self): """ test properties """ # # valid values operation = Operation() operation.OperationID = 1 self.assertEqual(operation.OperationID, 1, "wrong OperationID") operation.OperationID = "1" self.assertEqual(operation.OperationID, 1, "wrong OperationID") operation.Arguments = "foobar" self.assertEqual(operation.Arguments, "foobar", "wrong Arguments") operation.SourceSE = "CERN-RAW" self.assertEqual(operation.SourceSE, "CERN-RAW", "wrong SourceSE") operation.TargetSE = "CERN-RAW" self.assertEqual(operation.TargetSE, "CERN-RAW", "wrong TargetSE") operation.Catalog = "" self.assertEqual(operation.Catalog, "", "wrong Catalog") operation.Catalog = "BookkeepingDB" self.assertEqual(operation.Catalog, "BookkeepingDB", "wrong Catalog") operation.Error = "error" self.assertEqual(operation.Error, "error", "wrong Error") # # wrong props try: operation.RequestID = "foo" except Exception, error: self.assertEqual(type(error), AttributeError, "wrong exc raised") self.assertEqual(str(error), "can't set attribute", "wrong exc reason")
def test02props( self ): """ test properties """ # # valid values operation = Operation() operation.Arguments = "foobar" self.assertEqual( operation.Arguments, "foobar", "wrong Arguments" ) operation.SourceSE = "CERN-RAW" self.assertEqual( operation.SourceSE, "CERN-RAW", "wrong SourceSE" ) operation.TargetSE = "CERN-RAW" self.assertEqual( operation.TargetSE, "CERN-RAW", "wrong TargetSE" ) operation.Catalog = "" self.assertEqual( operation.Catalog, "", "wrong Catalog" ) operation.Catalog = "BookkeepingDB" self.assertEqual( operation.Catalog, "BookkeepingDB", "wrong Catalog" ) operation.Error = "error" self.assertEqual( operation.Error, "error", "wrong Error" ) # # wrong props try: operation.RequestID = "foo" except Exception, error: self.assertEqual( type( error ), AttributeError, "wrong exc raised" ) self.assertEqual( str( error ), "can't set attribute", "wrong exc reason" )
def getRegisterOperation(self, opFile, targetSE, type='RegisterFile', catalog=None): """ add RegisterReplica operation for file :param File opFile: operation file :param str targetSE: target SE """ # # add RegisterReplica operation registerOperation = Operation() registerOperation.Type = type registerOperation.TargetSE = targetSE if catalog: registerOperation.Catalog = catalog registerFile = File() registerFile.LFN = opFile.LFN registerFile.PFN = StorageElement(targetSE).getPfnForLfn( opFile.LFN).get('Value', {}).get('Successful', {}).get(opFile.LFN) registerFile.GUID = opFile.GUID registerFile.Checksum = opFile.Checksum registerFile.ChecksumType = opFile.ChecksumType registerFile.Size = opFile.Size registerOperation.addFile(registerFile) return registerOperation
def setBKRegistrationRequest(self, lfn, error='', metaData={'Checksum': 'justSomething', 'ChecksumType': 'ADLER32', 'GUID': 'aGUID'}): """ Set a BK registration request for changing the replica flag. Uses the global request object (self.request). """ if error: self.log.info('BK registration for %s failed with message: "%s" setting failover request' % (lfn, error)) else: self.log.info('Setting BK registration request for %s' % (lfn)) regFile = Operation() regFile.Type = 'RegisterFile' regFile.Catalog = 'BookkeepingDB' bkFile = File() bkFile.LFN = lfn # this should NOT be needed... but RMS complains! bkFile.PFN = lfn bkFile.GUID = metaData['GUID'] bkFile.Checksum = metaData['Checksum'] bkFile.ChecksumType = metaData['ChecksumType'] regFile.addFile(bkFile) res = self.request.addOperation(regFile) if not res['OK']: raise RuntimeError(res['Message'])
def getRegisterOperation(self, opFile, targetSE, type="RegisterFile", catalog=None): """add RegisterReplica operation for file :param ~DIRAC.RequestManagementSystem.Client.File.File opFile: operation file :param str targetSE: target SE """ # # add RegisterReplica operation registerOperation = Operation() registerOperation.Type = type registerOperation.TargetSE = targetSE if catalog: registerOperation.Catalog = catalog registerFile = File() registerFile.LFN = opFile.LFN registerFile.PFN = ( StorageElement(targetSE) .getURL(opFile.LFN, protocol=self.registrationProtocols) .get("Value", {}) .get("Successful", {}) .get(opFile.LFN) ) registerFile.GUID = opFile.GUID registerFile.Checksum = opFile.Checksum registerFile.ChecksumType = opFile.ChecksumType registerFile.Size = opFile.Size registerOperation.addFile(registerFile) return registerOperation
def __setRegistrationRequest( self, lfn, targetSE, fileDict, catalog ): """ Sets a registration request :param str lfn: LFN :param list se: list of SE :param list catalog: list of catalogs to use :param dict fileDict: file metadata """ self.log.info( 'Setting registration request for %s at %s.' % ( lfn, targetSE ) ) for cat in catalog: register = Operation() register.Type = "RegisterFile" register.Catalog = cat register.TargetSE = targetSE regFile = File() regFile.LFN = lfn regFile.Checksum = fileDict.get( "Checksum", "" ) regFile.ChecksumType = fileDict.get( "ChecksumType", "" ) regFile.Size = fileDict.get( "Size", 0 ) regFile.GUID = fileDict.get( "GUID", "" ) se = StorageElement( targetSE ) pfn = se.getPfnForLfn( lfn ) if not pfn["OK"]: self.log.error( "unable to get PFN for LFN: %s" % pfn["Message"] ) return pfn regFile.PFN = pfn["Value"] register.addFile( regFile ) self.request.addOperation( register ) return S_OK()
def test_valid_properties(): operation = Operation() operation.Arguments = "foobar" assert operation.Arguments == b"foobar", "wrong Arguments" operation.SourceSE = "CERN-RAW" assert operation.SourceSE == "CERN-RAW", "wrong SourceSE" operation.TargetSE = "CERN-RAW" assert operation.TargetSE == "CERN-RAW", "wrong TargetSE" operation.Catalog = "" assert operation.Catalog == "", "wrong Catalog" operation.Catalog = "BookkeepingDB" assert operation.Catalog == "BookkeepingDB", "wrong Catalog" operation.Error = "error" assert operation.Error == "error", "wrong Error" toJSON = operation.toJSON() assert toJSON["OK"]
def _setRegistrationRequest(self, lfn, targetSE, fileDict, catalog): """ Sets a registration request :param str lfn: LFN :param list se: list of SE (or just string) :param list catalog: list (or string) of catalogs to use :param dict fileDict: file metadata """ self.log.info('Setting registration request for %s at %s.' % (lfn, targetSE)) if not type(catalog) == type([]): catalog = [catalog] for cat in catalog: register = Operation() register.Type = "RegisterFile" register.Catalog = cat register.TargetSE = targetSE regFile = File() regFile.LFN = lfn regFile.Checksum = fileDict.get("Checksum", "") regFile.ChecksumType = fileDict.get("ChecksumType", self.defaultChecksumType) regFile.Size = fileDict.get("Size", 0) regFile.GUID = fileDict.get("GUID", "") se = StorageElement(targetSE) pfn = se.getPfnForLfn(lfn) if not pfn["OK"] or lfn not in pfn["Value"]['Successful']: self.log.error( "unable to get PFN for LFN: %s" % pfn.get('Message', pfn.get('Value', {}).get('Failed', {}).get(lfn))) return pfn regFile.PFN = pfn["Value"]['Successful'][lfn] register.addFile(regFile) self.request.addOperation(register) return S_OK()
def _setRegistrationRequest(self, lfn, targetSE, fileDict, catalog): """Sets a registration request :param str lfn: LFN :param list se: list of SE (or just string) :param list catalog: list (or string) of catalogs to use :param dict fileDict: file metadata """ self.log.info("Setting registration request", "for %s at %s." % (lfn, targetSE)) if not isinstance(catalog, list): catalog = [catalog] for cat in catalog: register = Operation() register.Type = "RegisterFile" register.Catalog = cat register.TargetSE = targetSE regFile = File() regFile.LFN = lfn regFile.Checksum = fileDict.get("Checksum", "") regFile.ChecksumType = fileDict.get("ChecksumType", self.defaultChecksumType) regFile.Size = fileDict.get("Size", 0) regFile.GUID = fileDict.get("GUID", "") se = StorageElement(targetSE) res = returnSingleResult(se.getURL(lfn, self.registrationProtocols)) if not res["OK"]: self.log.error("Unable to get PFN for LFN", res["Message"]) return res regFile.PFN = res["Value"] register.addFile(regFile) self.request.addOperation(register) return S_OK()
def _setRegistrationRequest( self, lfn, targetSE, fileDict, catalog ): """ Sets a registration request :param str lfn: LFN :param list se: list of SE (or just string) :param list catalog: list (or string) of catalogs to use :param dict fileDict: file metadata """ self.log.info( 'Setting registration request for %s at %s.' % ( lfn, targetSE ) ) if not isinstance( catalog, list ): catalog = [catalog] for cat in catalog: register = Operation() register.Type = "RegisterFile" register.Catalog = cat register.TargetSE = targetSE regFile = File() regFile.LFN = lfn regFile.Checksum = fileDict.get( "Checksum", "" ) regFile.ChecksumType = fileDict.get( "ChecksumType", self.defaultChecksumType ) regFile.Size = fileDict.get( "Size", 0 ) regFile.GUID = fileDict.get( "GUID", "" ) se = StorageElement( targetSE ) pfn = se.getURL( lfn, self.registrationProtocols ) if not pfn["OK"] or lfn not in pfn["Value"]['Successful']: self.log.error( "Unable to get PFN for LFN", "%s" % pfn.get( 'Message', pfn.get( 'Value', {} ).get( 'Failed', {} ).get( lfn ) ) ) return pfn regFile.PFN = pfn["Value"]['Successful'][lfn] register.addFile( regFile ) self.request.addOperation( register ) return S_OK()
def getRegisterOperation( self, opFile, targetSE, type = 'RegisterFile', catalog = None ): """ add RegisterReplica operation for file :param File opFile: operation file :param str targetSE: target SE """ # # add RegisterReplica operation registerOperation = Operation() registerOperation.Type = type registerOperation.TargetSE = targetSE if catalog: registerOperation.Catalog = catalog registerFile = File() registerFile.LFN = opFile.LFN registerFile.PFN = StorageElement( targetSE ).getURL( opFile.LFN, protocol = self.registrationProtocols ).get( 'Value', {} ).get( 'Successful', {} ).get( opFile.LFN ) registerFile.GUID = opFile.GUID registerFile.Checksum = opFile.Checksum registerFile.ChecksumType = opFile.ChecksumType registerFile.Size = opFile.Size registerOperation.addFile( registerFile ) return registerOperation
continue if len( lfnChunk ) > Operation.MAX_FILES: gLogger.error( "too many LFNs, max number of files per operation is %s" % Operation.MAX_FILES ) error = -1 continue count += 1 request = Request() request.RequestName = requestName if not multiRequests else '%s_%d' % ( requestName, count ) replicateAndRegister = Operation() replicateAndRegister.Type = "ReplicateAndRegister" replicateAndRegister.TargetSE = ",".join( targetSEs ) if catalog is not None: replicateAndRegister.Catalog = catalog for lfn in lfnChunk: metaDict = metaDatas["Successful"][lfn] opFile = File() opFile.LFN = lfn opFile.Size = metaDict["Size"] if "Checksum" in metaDict: # # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD' opFile.Checksum = metaDict["Checksum"] opFile.ChecksumType = "ADLER32" replicateAndRegister.addFile( opFile ) request.addOperation( replicateAndRegister )
gLogger.error( "too many LFNs, max number of files per operation is %s" % Operation.MAX_FILES) error = -1 continue count += 1 request = Request() request.RequestName = requestName if not multiRequests else '%s_%d' % ( requestName, count) replicateAndRegister = Operation() replicateAndRegister.Type = "ReplicateAndRegister" replicateAndRegister.TargetSE = ",".join(targetSEs) if catalog is not None: replicateAndRegister.Catalog = catalog for lfn in lfnChunk: metaDict = metaDatas["Successful"][lfn] opFile = File() opFile.LFN = lfn opFile.Size = metaDict["Size"] if "Checksum" in metaDict: # # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD' opFile.Checksum = metaDict["Checksum"] opFile.ChecksumType = "ADLER32" replicateAndRegister.addFile(opFile) request.addOperation(replicateAndRegister)
def test02props(self): """ test properties """ # # valid values operation = Operation() operation.Arguments = "foobar" self.assertEqual(operation.Arguments, "foobar", "wrong Arguments") operation.SourceSE = "CERN-RAW" self.assertEqual(operation.SourceSE, "CERN-RAW", "wrong SourceSE") operation.TargetSE = "CERN-RAW" self.assertEqual(operation.TargetSE, "CERN-RAW", "wrong TargetSE") operation.Catalog = "" self.assertEqual(operation.Catalog, "", "wrong Catalog") operation.Catalog = "BookkeepingDB" self.assertEqual(operation.Catalog, "BookkeepingDB", "wrong Catalog") operation.Error = "error" self.assertEqual(operation.Error, "error", "wrong Error") # # wrong props try: operation.RequestID = "foo" except Exception as error: self.assertEqual(type(error), AttributeError, "wrong exc raised") self.assertEqual(str(error), "can't set attribute", "wrong exc reason") try: operation.OperationID = "foo" except Exception as error: self.assertEqual(type(error), ValueError, "wrong exc raised") # # timestamps try: operation.SubmitTime = "foo" except Exception as error: self.assertEqual(type(error), ValueError, "wrong exp raised") self.assertEqual( str(error), "time data 'foo' does not match format '%Y-%m-%d %H:%M:%S'", "wrong exc reason") try: operation.LastUpdate = "foo" except Exception as error: self.assertEqual(type(error), ValueError, "wrong exc raised") self.assertEqual( str(error), "time data 'foo' does not match format '%Y-%m-%d %H:%M:%S'", "wrong exc reason") # # Status operation = Operation() try: operation.Status = "foo" except Exception as error: self.assertEqual(type(error), ValueError, "wrong exc raised") self.assertEqual(str(error), "unknown Status 'foo'", "wrong exc reason") operation.addFile(File({"Status": "Waiting", "LFN": "/a"}))
def main(): catalog = None Script.registerSwitch("C:", "Catalog=", "Catalog to use") # Registering arguments will automatically add their description to the help menu Script.registerArgument(" requestName: a request name") Script.registerArgument(" LFNs: single LFN or file with LFNs") Script.registerArgument(["targetSE: target SE"]) Script.parseCommandLine() for switch in Script.getUnprocessedSwitches(): if switch[0] == "C" or switch[0].lower() == "catalog": catalog = switch[1] args = Script.getPositionalArgs() requestName = None targetSEs = None if len(args) < 3: Script.showHelp(exitCode=1) requestName = args[0] lfnList = getLFNList(args[1]) targetSEs = list( set([se for targetSE in args[2:] for se in targetSE.split(",")])) gLogger.info("Will create request '%s' with 'ReplicateAndRegister' " "operation using %s lfns and %s target SEs" % (requestName, len(lfnList), len(targetSEs))) from DIRAC.RequestManagementSystem.Client.Request import Request from DIRAC.RequestManagementSystem.Client.Operation import Operation from DIRAC.RequestManagementSystem.Client.File import File from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient from DIRAC.Resources.Catalog.FileCatalog import FileCatalog from DIRAC.Core.Utilities.List import breakListIntoChunks lfnChunks = breakListIntoChunks(lfnList, 100) multiRequests = len(lfnChunks) > 1 error = 0 count = 0 reqClient = ReqClient() fc = FileCatalog() requestIDs = [] for lfnChunk in lfnChunks: metaDatas = fc.getFileMetadata(lfnChunk) if not metaDatas["OK"]: gLogger.error("unable to read metadata for lfns: %s" % metaDatas["Message"]) error = -1 continue metaDatas = metaDatas["Value"] for failedLFN, reason in metaDatas["Failed"].items(): gLogger.error("skipping %s: %s" % (failedLFN, reason)) lfnChunk = set(metaDatas["Successful"]) if not lfnChunk: gLogger.error("LFN list is empty!!!") error = -1 continue if len(lfnChunk) > Operation.MAX_FILES: gLogger.error( "too many LFNs, max number of files per operation is %s" % Operation.MAX_FILES) error = -1 continue count += 1 request = Request() request.RequestName = requestName if not multiRequests else "%s_%d" % ( requestName, count) replicateAndRegister = Operation() replicateAndRegister.Type = "ReplicateAndRegister" replicateAndRegister.TargetSE = ",".join(targetSEs) if catalog is not None: replicateAndRegister.Catalog = catalog for lfn in lfnChunk: metaDict = metaDatas["Successful"][lfn] opFile = File() opFile.LFN = lfn opFile.Size = metaDict["Size"] if "Checksum" in metaDict: # # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD' opFile.Checksum = metaDict["Checksum"] opFile.ChecksumType = "ADLER32" replicateAndRegister.addFile(opFile) request.addOperation(replicateAndRegister) putRequest = reqClient.putRequest(request) if not putRequest["OK"]: gLogger.error("unable to put request '%s': %s" % (request.RequestName, putRequest["Message"])) error = -1 continue requestIDs.append(str(putRequest["Value"])) if not multiRequests: gLogger.always( "Request '%s' has been put to ReqDB for execution." % request.RequestName) if multiRequests: gLogger.always( "%d requests have been put to ReqDB for execution, with name %s_<num>" % (count, requestName)) if requestIDs: gLogger.always("RequestID(s): %s" % " ".join(requestIDs)) gLogger.always( "You can monitor requests' status using command: 'dirac-rms-request <requestName/ID>'" ) DIRAC.exit(error)