def myRequest(): """Create a request and put it to the db""" request = Request() request.RequestName = 'myAwesomeRemovalRequest.xml' request.JobID = 0 request.SourceComponent = "myScript" remove = Operation() remove.Type = "RemoveFile" lfn = "/ilc/user/s/sailer/test.txt" rmFile = File() rmFile.LFN = lfn remove.addFile( rmFile ) request.addOperation( remove ) isValid = RequestValidator().validate( request ) if not isValid['OK']: raise RuntimeError( "Failover request is not valid: %s" % isValid['Message'] ) else: print("It is a GOGOGO") requestClient = ReqClient() result = requestClient.putRequest( request ) print(result)
def __setFileReplicationRequest( self, lfn, se, fileMetaDict ): """ Sets a registration request. """ self.log.info( 'Setting replication request for %s to %s' % ( lfn, se ) ) transfer = Operation() transfer.Type = "ReplicateAndRegister" transfer.TargetSE = se trFile = File() trFile.LFN = lfn cksm = fileMetaDict.get( "Checksum", None ) cksmType = fileMetaDict.get( "ChecksumType", None ) if cksm and cksmType: trFile.Checksum = cksm trFile.ChecksumType = cksmType size = fileMetaDict.get( "Size", 0 ) if size: trFile.Size = size guid = fileMetaDict.get( "GUID", "" ) if guid: trFile.GUID = guid transfer.addFile( trFile ) self.request.addOperation( transfer ) return S_OK()
def _createLogUploadRequest(self, targetSE, logFileLFN, uploadedSE): """ Set a request to upload job log files from the output sandbox Changed to be similar to LHCb createLogUploadRequest using LHCb LogUpload Request and Removal Request """ self.log.info('Setting log upload request for %s at %s' %(targetSE, logFileLFN)) request = self._getRequestContainer() logUpload = Operation() logUpload.Type = "LogUpload" logUpload.TargetSE = targetSE upFile = File() upFile.LFN = logFileLFN logUpload.addFile( upFile ) logRemoval = Operation() logRemoval.Type = 'RemoveFile' logRemoval.TargetSE = uploadedSE logRemoval.addFile( upFile ) request.addOperation ( logUpload ) request.addOperation ( logRemoval ) self.workflow_commons['Request'] = request return S_OK()
def test01ctor( self ): """ test constructors and (de)serialisation """ # # empty ctor self.assertEqual( isinstance( Operation(), Operation ), True, "empty ctor failed" ) # # using fromDict operation = Operation( self.fromDict ) self.assertEqual( isinstance( operation, Operation ), True, "fromDict ctor failed" ) for key, value in self.fromDict.items(): self.assertEqual( getattr( operation, key ), value, "wrong attr value %s (%s) %s" % ( key, getattr( operation, key ), value ) ) # # same with file operation = Operation( self.fromDict ) operation.addFile( self.subFile ) for key, value in self.fromDict.items(): self.assertEqual( getattr( operation, key ), value, "wrong attr value %s (%s) %s" % ( key, getattr( operation, key ), value ) ) toJSON = operation.toJSON() self.assertEqual( toJSON["OK"], True, "JSON serialization failed" )
def test02props( self ): """ test properties """ # # valid values operation = Operation() operation.OperationID = 1 self.assertEqual( operation.OperationID, 1, "wrong OperationID" ) operation.OperationID = "1" self.assertEqual( operation.OperationID, 1, "wrong OperationID" ) operation.Arguments = "foobar" self.assertEqual( operation.Arguments, "foobar", "wrong Arguments" ) operation.SourceSE = "CERN-RAW" self.assertEqual( operation.SourceSE, "CERN-RAW", "wrong SourceSE" ) operation.TargetSE = "CERN-RAW" self.assertEqual( operation.TargetSE, "CERN-RAW", "wrong TargetSE" ) operation.Catalog = "" self.assertEqual( operation.Catalog, "", "wrong Catalog" ) operation.Catalog = "BookkeepingDB" self.assertEqual( operation.Catalog, "BookkeepingDB", "wrong Catalog" ) operation.Error = "error" self.assertEqual( operation.Error, "error", "wrong Error" ) # # wrong props try: operation.RequestID = "foo" except Exception, error: self.assertEqual( type( error ), AttributeError, "wrong exc raised" ) self.assertEqual( str( error ), "can't set attribute", "wrong exc reason" )
def __deleteSandboxFromExternalBackend( self, SEName, SEPFN ): if self.getCSOption( "DelayedExternalDeletion", True ): gLogger.info( "Setting deletion request" ) try: request = Request() request.RequestName = "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time() ) physicalRemoval = Operation() physicalRemoval.Type = "PhysicalRemoval" physicalRemoval.TargetSE = SEName fileToRemove = File() fileToRemove.PFN = SEPFN physicalRemoval.addFile( fileToRemove ) request.addOperation( physicalRemoval ) return ReqClient().putRequest( request ) except Exception as e: gLogger.exception( "Exception while setting deletion request" ) return S_ERROR( "Cannot set deletion request: %s" % str( e ) ) else: gLogger.info( "Deleting external Sandbox" ) try: return StorageElement( SEName ).removeFile( SEPFN ) except Exception as e: gLogger.exception( "RM raised an exception while trying to delete a remote sandbox" ) return S_ERROR( "RM raised an exception while trying to delete a remote sandbox" )
class MoveReplicaSuccess( ReqOpsTestCase ): def setUp(self): self.op = Operation() self.op.Type = "MoveFile" self.op.SourceSE = "%s,%s" % ( "sourceSE1", "sourceSE2" ) self.op.TargetSE = "%s,%s" % ( "targetSE1", "targetSE2" ) self.File = File() self.File.LFN = '/cta/file1' self.File.Size = 2L self.File.Checksum = '011300a2' self.File.ChecksumType = "adler32" self.op.addFile( self.File ) self.req = Request() self.req.addOperation( self.op ) self.mr = MoveReplica( self.op ) self.mr.dm = MagicMock() self.mr.fc = MagicMock() # This test needs to be fixed. It currently fails because StorageElement is not mocked '''def test__dmTransfer( self ): successful = {} for sourceSE in self.op.sourceSEList: successful[sourceSE] = 'dips://' + sourceSE.lower() + ':9148/DataManagement/StorageElement' + self.File.LFN res = {'OK': True, 'Value': {'Successful': {self.File.LFN : successful}, 'Failed': {}}} self.mr.dm.getActiveReplicas.return_value = res res = {'OK': True, 'Value': {'Successful': {self.File.LFN : {'register': 0.1228799819946289, 'replicate': 9.872732877731323}}, 'Failed': {}}} self.mr.dm.replicateAndRegister.return_value = res res = self.mr.dmTransfer( self.File ) self.assertTrue( res['OK'] ) self.assertEqual( self.mr.operation.__files__[0].Status, 'Waiting' ) self.assertEqual( self.mr.operation.Status, 'Waiting' ) self.assertEqual( self.mr.request.Status, 'Waiting' )''' def test__dmRemoval( self ): res = {'OK': True, 'Value': {'Successful': { self.File.LFN : {'DIRACFileCatalog': True}}, 'Failed': {}}} self.mr.dm.removeReplica.return_value = res toRemoveDict = {self.File.LFN: self.File} targetSEs = self.op.sourceSEList res = self.mr.dmRemoval( toRemoveDict, targetSEs ) self.assertTrue( res['OK'] ) resvalue = dict( [ ( targetSE, '' ) for targetSE in targetSEs ] ) self.assertEqual( res['Value'], {self.File.LFN: resvalue} ) self.assertEqual( self.mr.operation.__files__[0].Status, 'Done' ) self.assertEqual( self.mr.operation.Status, 'Done' ) self.assertEqual( self.mr.request.Status, 'Done' )
def createRequest(requestID, opType, opStatus, fileStatus, lfnError=" ", lfn="/ilc/fake/lfn"): """Create a request.""" req = Request({"RequestID": requestID}) op = Operation({"Type": opType, "Status": opStatus}) op.addFile(File({"LFN": lfn, "Status": fileStatus, "Error": lfnError})) req.addOperation(op) return req
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if ( not owner ) or ( not ownerGroup ): res = getProxyInfo( False, False ) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername( owner ) if not res['OK']: return res ownerDN = res['Value'][0] requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass for taskID in sorted( taskDict ): paramDict = taskDict[taskID] if paramDict['InputData']: transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] if isinstance( paramDict['InputData'], list ): files = paramDict['InputData'] elif isinstance( paramDict['InputData'], basestring ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = _requestName( transID, taskID ) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate( oRequest ) if not isValid['OK']: return isValid taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict )
def test04StateMachine( self ): """ state machine """ op = Operation() self.assertEqual( op.Status, "Queued", "1. wrong status %s" % op.Status ) op.addFile( File( {"Status": "Waiting"} ) ) self.assertEqual( op.Status, "Queued", "2. wrong status %s" % op.Status ) op.addFile( File( {"Status": "Scheduled" } ) ) self.assertEqual( op.Status, "Scheduled", "3. wrong status %s" % op.Status ) op.addFile( File( {"Status": "Done" } ) ) self.assertEqual( op.Status, "Scheduled", "4. wrong status %s" % op.Status ) op.addFile( File( { "Status": "Failed" } ) ) self.assertEqual( op.Status, "Scheduled", "5. wrong status %s" % op.Status ) op[3].Status = "Scheduled" self.assertEqual( op.Status, "Scheduled", "6. wrong status %s" % op.Status ) op[0].Status = "Scheduled" self.assertEqual( op.Status, "Scheduled", "7. wrong status %s" % op.Status ) op[0].Status = "Waiting" self.assertEqual( op.Status, "Scheduled", "8. wrong status %s" % op.Status ) for f in op: f.Status = "Done" self.assertEqual( op.Status, "Done", "9. wrong status %s" % op.Status ) for f in op: f.Status = "Failed" self.assertEqual( op.Status, "Failed", "9. wrong status %s" % op.Status )
def _sendToFailover( rpcStub ): """ Create a ForwardDISET operation for failover """ request = Request() request.RequestName = "Accounting.DataStore.%s.%s" % ( time.time(), random.random() ) forwardDISETOp = Operation() forwardDISETOp.Type = "ForwardDISET" forwardDISETOp.Arguments = DEncode.encode( rpcStub ) request.addOperation( forwardDISETOp ) return ReqClient().putRequest( request )
def generateForwardDISET( self ): """ Commit the accumulated records and generate request eventually """ result = self.commit() commitOp = None if not result['OK']: # Generate Request commitOp = Operation() commitOp.Type = 'SetFileStatus' commitOp.Arguments = DEncode.encode( {'transformation':self.transformation, 'statusDict':self.statusDict, 'force':self.force} ) return S_OK( commitOp )
def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup): """ deal with a Request that has multiple operations :param transJson: list of lists of string and dictionaries, e.g.: .. code :: python body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }), ( "RemoveReplica", { "TargetSE":"FOO-SRM" } ), ] :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ failedTasks = [] for taskID, task in taskDict.items(): transID = task['TransformationID'] if not task.get('InputData'): self._logError("Error creating request for task", "%s, No input data" % taskID, transID=transID) taskDict.pop(taskID) continue files = [] oRequest = Request() if isinstance(task['InputData'], list): files = task['InputData'] elif isinstance(task['InputData'], basestring): files = task['InputData'].split(';') # create the operations from the json structure for operationTuple in transJson: op = Operation() op.Type = operationTuple[0] for parameter, value in operationTuple[1].iteritems(): setattr(op, parameter, value) for lfn in files: opFile = File() opFile.LFN = lfn op.addFile(opFile) oRequest.addOperation(op) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result['OK']: failedTasks.append(taskID) # Remove failed tasks for taskID in failedTasks: taskDict.pop(taskID)
def addRemovalRequests(self, lfnList): """Create removalRequests for lfns in lfnList and add it to the common request""" request = self._getRequestContainer() remove = Operation() remove.Type = "RemoveFile" for lfn in lfnList: rmFile = File() rmFile.LFN = lfn remove.addFile( rmFile ) request.addOperation( remove ) self.workflow_commons['Request'] = request
class RequestManagerHandlerTests( unittest.TestCase ): """ .. class:: RequestManagerHandlerTests """ def setUp( self ): """ test setup :param self: self reference """ self.request = Request() self.request.RequestName = "RequestManagerHandlerTests" self.request.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" self.request.OwnerGroup = "dirac_user" self.operation = Operation() self.operation.Type = "ReplicateAndRegister" self.operation.TargetSE = "CERN-USER" self.file = File() self.file.LFN = "/lhcb/user/c/cibak/testFile" self.file.Checksum = "123456" self.file.ChecksumType = "ADLER32" self.request.addOperation( self.operation ) self.operation.addFile( self.file ) # # xml representation of a whole request self.xmlStr = self.request.toXML( True )["Value"] # # request client self.requestClient = RequestClient() def tearDown( self ): """ test case tear down """ del self.request del self.operation del self.file del self.xmlStr def test01PutRequest( self ): """ test set request """ put = self.requestClient.putRequest( self.request ) self.assertEqual( put["OK"], True, "put failed" ) def test02GetRequest( self ): """ test get request """ get = self.requestClient.getRequest( self.request.RequestName ) self.assertEqual( get["OK"], True, "get failed" ) def test03DeleteRequest( self ): """ test delete request """ delete = self.requestClient.deleteRequest( "test" ) self.assertEqual( delete["OK"], True, "delete failed" )
def __setFileRemovalRequest( self, lfn, se = '', pfn = '' ): """ Sets a removal request for a file including all replicas. """ remove = Operation() remove.Type = "RemoveFile" if se: remove.TargetSE = se rmFile = File() rmFile.LFN = lfn if pfn: rmFile.PFN = pfn remove.addFile( rmFile ) self.request.addOperation( remove ) return S_OK()
def generateForwardDISET( self ): """ Commit the accumulated records and generate request eventually """ result = self.commit() forwardDISETOp = None if not result['OK']: # Generate Request if "FailedResults" in result: for res in result['FailedResults']: if 'rpcStub' in res: forwardDISETOp = Operation() forwardDISETOp.Type = "ForwardDISET" forwardDISETOp.Arguments = DEncode.encode( res['rpcStub'] ) return S_OK( forwardDISETOp )
class ReqClientTestCase( unittest.TestCase ): """ .. class:: ReqClientTestCase """ def setUp( self ): """ test case set up """ gLogger.setLevel( 'NOTICE' ) self.file = File() self.file.LFN = "/lhcb/user/c/cibak/testFile" self.file.Checksum = "123456" self.file.ChecksumType = "ADLER32" self.file2 = File() self.file2.LFN = "/lhcb/user/f/fstagni/testFile" self.file2.Checksum = "654321" self.file2.ChecksumType = "ADLER32" self.operation = Operation() self.operation.Type = "ReplicateAndRegister" self.operation.TargetSE = "CERN-USER" self.operation.addFile( self.file ) self.operation.addFile( self.file2 ) self.request = Request() self.request.RequestName = "RequestManagerHandlerTests" self.request.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" self.request.OwnerGroup = "dirac_user" self.request.JobID = 123 self.request.addOperation( self.operation ) # # JSON representation of a whole request self.jsonStr = self.request.toJSON()['Value'] # # request client self.requestClient = ReqClient() self.stressRequests = 1000 self.bulkRequest = 1000 def tearDown( self ): """ clean up """ del self.request del self.operation del self.file del self.jsonStr
class ReqClientTestCase(unittest.TestCase): """ .. class:: ReqClientTestCase """ def setUp(self): """ test case set up """ gLogger.setLevel('INFO') self.file = File() self.file.LFN = "/lhcb/user/c/cibak/testFile" self.file.Checksum = "123456" self.file.ChecksumType = "ADLER32" self.file2 = File() self.file2.LFN = "/lhcb/user/f/fstagni/testFile" self.file2.Checksum = "654321" self.file2.ChecksumType = "ADLER32" self.operation = Operation() self.operation.Type = "ReplicateAndRegister" self.operation.TargetSE = "CERN-USER" self.operation.addFile(self.file) self.operation.addFile(self.file2) proxyInfo = getProxyInfo()['Value'] self.request = Request() self.request.RequestName = "RequestManagerHandlerTests" self.request.OwnerDN = proxyInfo['identity'] self.request.OwnerGroup = proxyInfo['group'] self.request.JobID = 123 self.request.addOperation(self.operation) # # JSON representation of a whole request self.jsonStr = self.request.toJSON()['Value'] # # request client self.requestClient = ReqClient() self.stressRequests = 1000 self.bulkRequest = 1000 def tearDown(self): """ clean up """ del self.request del self.operation del self.file del self.jsonStr
def test03sql( self ): """ sql insert or update """ operation = Operation() operation.Type = "ReplicateAndRegister" request = Request() request.RequestName = "testRequest" request.RequestID = 1 # # no parent request set try: operation.toSQL() except Exception, error: self.assertEqual( isinstance( error, AttributeError ), True, "wrong exc raised" ) self.assertEqual( str( error ), "RequestID not set", "wrong exc reason" )
def generateForwardDISET( self ): """ Commit the accumulated records and generate request eventually """ result = self.commit() forwardDISETOp = None if not result['OK']: # Generate Request if result.has_key( 'rpcStub' ): forwardDISETOp = Operation() forwardDISETOp.Type = "ForwardDISET" forwardDISETOp.Arguments = DEncode.encode( result['rpcStub'] ) else: return S_ERROR( 'Could not create ForwardDISET operation' ) return S_OK( forwardDISETOp )
def _sendToFailover( rpcStub ): """ Create a ForwardDISET operation for failover """ try: request = Request() request.RequestName = "Accounting.DataStore.%s.%s" % ( time.time(), random.random() ) forwardDISETOp = Operation() forwardDISETOp.Type = "ForwardDISET" forwardDISETOp.Arguments = DEncode.encode( rpcStub ) request.addOperation( forwardDISETOp ) return ReqClient().putRequest( request ) # We catch all the exceptions, because it should never crash except Exception as e: # pylint: disable=broad-except return S_ERROR( ERMSUKN, "Exception sending accounting failover request: %s" % repr( e ) )
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup): """ deal with a Request that has just one operation, as it was sofar :param transBody: string, can be an empty string :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split(';') except AttributeError: pass failedTasks = [] # Do not remove sorted, we might pop elements in the loop for taskID, task in taskDict.iteritems(): transID = task['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = task['TargetSE'] # If there are input files if task.get('InputData'): if isinstance(task['InputData'], list): files = task['InputData'] elif isinstance(task['InputData'], basestring): files = task['InputData'].split(';') for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile(trFile) oRequest.addOperation(transfer) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result['OK']: failedTasks.append(taskID) # Remove failed tasks for taskID in failedTasks: taskDict.pop(taskID)
def test01TableDesc(self): """ table description """ tableDict = RequestDB.getTableMeta() self.assertEqual("Request" in tableDict, True) self.assertEqual("Operation" in tableDict, True) self.assertEqual("File" in tableDict, True) self.assertEqual(tableDict["Request"], Request.tableDesc()) self.assertEqual(tableDict["Operation"], Operation.tableDesc()) self.assertEqual(tableDict["File"], File.tableDesc())
def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ): """ Set removal request with the given credentials """ oRequest = Request() oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup oRequest.RequestName = os.path.basename( lfn ).strip() + '_removal_request.xml' oRequest.SourceComponent = 'JobCleaningAgent' removeFile = Operation() removeFile.Type = 'RemoveFile' removedFile = File() removedFile.LFN = lfn removeFile.addFile( removedFile ) oRequest.addOperation( removeFile ) return ReqClient().putRequest( oRequest )
def generateForwardDISET( self ): """ Generate and return failover requests for the operations in the internal cache """ fowardDISETOp = None result = self.sendStoredStatusInfo() if not result['OK']: if 'rpcStub' in result: rpcStub = result['rpcStub'] forwardDISETOp = Operation() forwardDISETOp.Type = "ForwardDISET" forwardDISETOp.Arguments = DEncode.encode( rpcStub ) else: return S_ERROR( 'Could not create ForwardDISET operation' ) return S_OK( fowardDISETOp )
def generateForwardDISET(self): """ Generate and return failover requests for the operations in the internal cache """ fowardDISETOp = None result = self.sendStoredStatusInfo() if not result["OK"]: if "rpcStub" in result: rpcStub = result["rpcStub"] forwardDISETOp = Operation() forwardDISETOp.Type = "ForwardDISET" forwardDISETOp.Arguments = DEncode.encode(rpcStub) else: return S_ERROR("Could not create job parameters sub-request") return S_OK(fowardDISETOp)
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass for taskID in sorted( taskDict ): paramDict = taskDict[taskID] if paramDict['InputData']: transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] if type( paramDict['InputData'] ) == type( [] ): files = paramDict['InputData'] elif type( paramDict['InputData'] ) == type( '' ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 ) oRequest.OwnerDN = owner oRequest.OwnerGroup = ownerGroup isValid = gRequestValidator.validate( oRequest ) if not isValid['OK']: return isValid taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict )
def setUp(self): """ test case setup """ self.request = Request({"RequestName": "test1", "JobID": 1}) self.operation1 = Operation({"Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"}) self.file = File({"LFN": "/a/b/c", "ChecksumType": "ADLER32", "Checksum": "123456"}) self.request.addOperation(self.operation1) self.operation1.addFile(self.file) self.operation2 = Operation() self.operation2.Type = "RemoveFile" self.operation2.addFile(File({"LFN": "/c/d/e"})) self.request.addOperation(self.operation2) # ## set some defaults gConfig.setOptionValue("DIRAC/Setup", "Test") gConfig.setOptionValue("/DIRAC/Setups/Test/RequestManagement", "Test") gConfig.setOptionValue("/Systems/RequestManagement/Test/Databases/ReqDB/Host", "localhost") gConfig.setOptionValue("/Systems/RequestManagement/Test/Databases/ReqDB/DBName", "ReqDB") gConfig.setOptionValue("/Systems/RequestManagement/Test/Databases/ReqDB/User", "Dirac") self.i = 1000
def test07Authorization(self): """ Test whether request sets on behalf of others are rejected""" request = Request({"RequestName": "unauthorized"}) request.OwnerDN = 'NotMe' request.OwnerDN = 'AnotherGroup' op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"}) op += File({"LFN": "/lhcb/user/c/cibak/foo"}) request += op res = self.requestClient.putRequest(request) self.assertFalse(res['OK'], res)
def __setReplicaRemovalRequest( self, lfn, se ): """ Sets a removal request for a replica. :param str lfn: LFN :param se: """ if type( se ) == str: se = ",".join( [ se.strip() for se in se.split( "," ) if se.strip() ] ) removeReplica = Operation() removeReplica.Type = "RemoveReplica" removeReplica.TargetSE = se replicaToRemove = File() replicaToRemove.LFN = lfn removeReplica.addFile( replicaToRemove ) self.request.addOperation( removeReplica ) return S_OK()
def generateForwardDISET(self): """ Generate and return failover requests for the operations in the internal cache """ forwardDISETOp = None result = self.sendStoredStatusInfo() if not result['OK']: gLogger.error("Error while sending the job status", result['Message']) if 'rpcStub' in result: rpcStub = result['rpcStub'] forwardDISETOp = Operation() forwardDISETOp.Type = "ForwardDISET" forwardDISETOp.Arguments = DEncode.encode(rpcStub) else: return S_ERROR('Could not create ForwardDISET operation') return S_OK(forwardDISETOp)
def setBKRegistrationRequest(self, lfn, error='', metaData={'Checksum': 'justSomething', 'ChecksumType': 'ADLER32', 'GUID': 'aGUID'}): """ Set a BK registration request for changing the replica flag. Uses the global request object (self.request). """ if error: self.log.info('BK registration for %s failed with message: "%s" setting failover request' % (lfn, error)) else: self.log.info('Setting BK registration request for %s' % (lfn)) regFile = Operation() regFile.Type = 'RegisterFile' regFile.Catalog = 'BookkeepingDB' bkFile = File() bkFile.LFN = lfn # this should NOT be needed... but RMS complains! bkFile.PFN = lfn bkFile.GUID = metaData['GUID'] bkFile.Checksum = metaData['Checksum'] bkFile.ChecksumType = metaData['ChecksumType'] regFile.addFile(bkFile) res = self.request.addOperation(regFile) if not res['OK']: raise RuntimeError(res['Message'])
def __submitRMSOp(self, target_se, lfns_chunk_dict, whichRMSOp='ReplicateAndRegister' ): """ target_se : SE name to which to replicate lfns_chunk_dict : LFNS dict with 100 lfns as key andeEach lfn has 'Size', 'Checksum' whichRMSOp: Choose from RMP operation - ReplicateAndRegister, ReplicateAndRemove, PutAndRegister """ ## Setup request request = Request() request.RequestName = "DDM_"+ str(target_se) + datetime.datetime.now().strftime("_%Y%m%d_%H%M%S") myOp = Operation() myOp.Type = whichRMSOp myOp.TargetSE = target_se ## Add LFNS to operations for lfn in lfns_chunk_dict.keys(): opFile = File() opFile.LFN = lfn opFile.Size = lfns_chunk_dict[lfn]['Size'] if "Checksum" in lfns_chunk_dict[lfn]: opFile.Checksum = lfns_chunk_dict[lfn]['Checksum'] opFile.ChecksumType = 'ADLER32' ## Add file to operation myOp.addFile( opFile ) request.addOperation( myOp ) reqClient = ReqClient() putRequest = reqClient.putRequest( request ) if not putRequest["OK"]: gLogger.error( "Unable to put request '%s': %s" % ( request.RequestName, putRequest["Message"] ) ) return S_ERROR("Problem submitting to RMS.")
def archiveRequestAndOp(listOfLFNs): """Return a tuple of the request and operation.""" req = Request() req.RequestName = "MyRequest" op = Operation() switches = {} archiveLFN = "/vo/tars/myTar.tar" op.Arguments = DEncode.encode({ "SourceSE": switches.get("SourceSE", "SOURCE-SE"), "TarballSE": switches.get("TarballSE", "TARBALL-SE"), "RegisterDescendent": False, "ArchiveLFN": archiveLFN, }) op.Type = "ArchiveFiles" for index, lfn in enumerate(listOfLFNs): oFile = File() oFile.LFN = lfn oFile.Size = index oFile.Checksum = "01130a%0d" % index oFile.ChecksumType = "adler32" op.addFile(oFile) req.addOperation(op) return req, op
def archiveRequestAndOp(listOfLFNs): """Return a tuple of the request and operation.""" req = Request() req.RequestName = 'MyRequest' op = Operation() switches = {} archiveLFN = '/vo/tars/myTar.tar' op.Arguments = DEncode.encode({ 'SourceSE': switches.get('SourceSE', 'SOURCE-SE'), 'TarballSE': switches.get('TarballSE', 'TARBALL-SE'), 'RegisterDescendent': False, 'ArchiveLFN': archiveLFN }) op.Type = 'ArchiveFiles' for index, lfn in enumerate(listOfLFNs): oFile = File() oFile.LFN = lfn oFile.Size = index oFile.Checksum = '01130a%0d' % index oFile.ChecksumType = 'adler32' op.addFile(oFile) req.addOperation(op) return req, op
def __deleteSandboxFromExternalBackend(self, SEName, SEPFN): if self.getCSOption("DelayedExternalDeletion", True): gLogger.info("Setting deletion request") try: request = Request() request.RequestName = "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time()) physicalRemoval = Operation() physicalRemoval.Type = "PhysicalRemoval" physicalRemoval.TargetSE = SEName fileToRemove = File() fileToRemove.PFN = SEPFN physicalRemoval.addFile(fileToRemove) request.addOperation(physicalRemoval) return ReqClient().putRequest(request) except Exception as e: gLogger.exception("Exception while setting deletion request") return S_ERROR("Cannot set deletion request: %s" % str(e)) else: gLogger.info("Deleting external Sandbox") try: return StorageElement(SEName).removeFile(SEPFN) except Exception as e: gLogger.exception( "RM raised an exception while trying to delete a remote sandbox" ) return S_ERROR( "RM raised an exception while trying to delete a remote sandbox" )
def __init__(self, fromDict=None): """c'tor :param self: self reference :param fromDict: if false, new request. Can be json string that represents the object, or the dictionary directly """ self.__waiting = None now = datetime.datetime.utcnow().replace(microsecond=0) self._CreationTime = now self._SubmitTime = now self._LastUpdate = now # the time before which the request should not be executed # If None, no delay self._NotBefore = now self._Status = "Done" self.JobID = 0 self.Error = None self.DIRACSetup = None self.OwnerDN = None self.RequestName = None self.OwnerGroup = None self._SourceComponent = None self.dmsHelper = DMSHelpers() proxyInfo = getProxyInfo() if proxyInfo["OK"]: proxyInfo = proxyInfo["Value"] if proxyInfo["validGroup"] and proxyInfo["validDN"]: self.OwnerDN = proxyInfo["identity"] self.OwnerGroup = proxyInfo["group"] self.__operations__ = [] if isinstance(fromDict, str): fromDict = json.loads(fromDict) elif not isinstance(fromDict, dict): fromDict = {} if "Operations" in fromDict: for opDict in fromDict.get("Operations", []): self += Operation(opDict) del fromDict["Operations"] for key, value in fromDict.items(): if value: setattr(self, key, value) self._notify()
def __setRemovalRequest(self, lfn, ownerDN, ownerGroup): """Set removal request with the given credentials""" oRequest = Request() oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup oRequest.RequestName = os.path.basename( lfn).strip() + "_removal_request.xml" oRequest.SourceComponent = "JobCleaningAgent" removeFile = Operation() removeFile.Type = "RemoveFile" removedFile = File() removedFile.LFN = lfn removeFile.addFile(removedFile) oRequest.addOperation(removeFile) # put the request with the owner certificate to make sure it's still a valid DN return ReqClient(useCertificates=True, delegatedDN=ownerDN, delegatedGroup=ownerGroup).putRequest(oRequest)
def test04StateMachine(self): """ state machine """ op = Operation() self.assertEqual(op.Status, "Queued", "1. wrong status %s" % op.Status) op.addFile(File({"Status": "Waiting"})) self.assertEqual(op.Status, "Queued", "2. wrong status %s" % op.Status) op.addFile(File({"Status": "Scheduled"})) self.assertEqual(op.Status, "Scheduled", "3. wrong status %s" % op.Status) op.addFile(File({"Status": "Done"})) self.assertEqual(op.Status, "Scheduled", "4. wrong status %s" % op.Status) op.addFile(File({"Status": "Failed"})) self.assertEqual(op.Status, "Scheduled", "5. wrong status %s" % op.Status) op[3].Status = "Scheduled" self.assertEqual(op.Status, "Scheduled", "6. wrong status %s" % op.Status) op[0].Status = "Scheduled" self.assertEqual(op.Status, "Scheduled", "7. wrong status %s" % op.Status) op[0].Status = "Waiting" self.assertEqual(op.Status, "Scheduled", "8. wrong status %s" % op.Status) for f in op: f.Status = "Done" self.assertEqual(op.Status, "Done", "9. wrong status %s" % op.Status) for f in op: f.Status = "Failed" self.assertEqual(op.Status, "Failed", "9. wrong status %s" % op.Status)
def setUp( self ): """ test setup :param self: self reference """ self.request = Request() self.request.RequestName = "RequestManagerHandlerTests" self.request.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" self.request.OwnerGroup = "dirac_user" self.operation = Operation() self.operation.Type = "ReplicateAndRegister" self.operation.TargetSE = "CERN-USER" self.file = File() self.file.LFN = "/lhcb/user/c/cibak/testFile" self.file.Checksum = "123456" self.file.ChecksumType = "ADLER32" self.request.addOperation( self.operation ) self.operation.addFile( self.file ) # # xml representation of a whole request self.xmlStr = self.request.toXML( True )["Value"] # # request client self.requestClient = RequestClient()
def test_List(): """getitem, setitem, delitem and dirty""" op = Operation() files = [] for _ in range(5): f = File() files.append(f) op += f for i in range(len(op)): assert op[i] == files[i], "__getitem__ failed" for i in range(len(op)): op[i] = File({"LFN": "/%s" % i}) assert op[i].LFN == "/%s" % i, "__setitem__ failed" del op[0] assert len(op) == 4, "__delitem__ failed" # opID set op.OperationID = 1 del op[0]
def test05List(self): """ getitem, setitem, delitem and dirty """ op = Operation() files = [] for i in range(5): f = File() files.append(f) op += f for i in range(len(op)): self.assertEqual(op[i], files[i], "__getitem__ failed") for i in range(len(op)): op[i] = File({"LFN": "/%s" % i}) self.assertEqual(op[i].LFN, "/%s" % i, "__setitem__ failed") del op[0] self.assertEqual(len(op), 4, "__delitem__ failed") # # opID set op.OperationID = 1 del op[0]
def _cleanUp(self, final): """ Clean up uploaded data for the LFNs in the list """ lfnList = [] for _fileName, metadata in final.items(): lfnList.append(metadata['lfn']) self.log.verbose("Cleaning up the request, for LFNs: %s" % ', '.join(lfnList)) newRequest = Request() for op in self.request: add = True if op.Type in [ 'PutAndRegister', 'ReplicateAndRegister', 'RegisterFile', 'RegisterReplica', 'RemoveReplica' ]: for files in op: if files.LFN in lfnList: add = False if add: newRequest.addOperation(op) self.request = newRequest self.log.verbose( "And adding RemoveFile operation for LFNs: %s, just in case" % ', '.join(lfnList)) removeFiles = Operation() removeFiles.Type = 'RemoveFile' for lfn in lfnList: removedFile = File() removedFile.LFN = lfn removeFiles.addFile(removedFile) self.request.addOperation(removeFiles)
def setUp(self): """ test set up """ self.hiArgs = (("RequestManagement/RequestManager", { "keepAliveLapse": 10, "timeout": 5 }), "foo", (12345, { "Hi": "There!" })) self.req = Request({"RequestName": "testRequest"}) self.op = Operation({ "Type": "ForwardDISET", "Arguments": DEncode.encode(self.hiArgs) }) self.req += self.op
def setUp(self): """ test case set up """ gLogger.setLevel('INFO') self.file = File() self.file.LFN = "/lhcb/user/c/cibak/testFile" self.file.Checksum = "123456" self.file.ChecksumType = "ADLER32" self.file2 = File() self.file2.LFN = "/lhcb/user/f/fstagni/testFile" self.file2.Checksum = "654321" self.file2.ChecksumType = "ADLER32" self.operation = Operation() self.operation.Type = "ReplicateAndRegister" self.operation.TargetSE = "CERN-USER" self.operation.addFile(self.file) self.operation.addFile(self.file2) proxyInfo = getProxyInfo()['Value'] self.request = Request() self.request.RequestName = "RequestManagerHandlerTests" self.request.OwnerDN = proxyInfo['identity'] self.request.OwnerGroup = proxyInfo['group'] self.request.JobID = 123 self.request.addOperation(self.operation) # # JSON representation of a whole request self.jsonStr = self.request.toJSON()['Value'] # # request client self.requestClient = ReqClient() self.stressRequests = 1000 self.bulkRequest = 1000
def test_valid_properties(): operation = Operation() operation.Arguments = "foobar" assert operation.Arguments == b"foobar", "wrong Arguments" operation.SourceSE = "CERN-RAW" assert operation.SourceSE == "CERN-RAW", "wrong SourceSE" operation.TargetSE = "CERN-RAW" assert operation.TargetSE == "CERN-RAW", "wrong TargetSE" operation.Catalog = "" assert operation.Catalog == "", "wrong Catalog" operation.Catalog = "BookkeepingDB" assert operation.Catalog == "BookkeepingDB", "wrong Catalog" operation.Error = "error" assert operation.Error == "error", "wrong Error" toJSON = operation.toJSON() assert toJSON["OK"]
def _setFileRemovalRequest(self, lfn, se="", pfn=""): """Sets a removal request for a file including all replicas.""" remove = Operation() remove.Type = "RemoveFile" if se: remove.TargetSE = se rmFile = File() rmFile.LFN = lfn if pfn: rmFile.PFN = pfn remove.addFile(rmFile) self.request.addOperation(remove) return S_OK()
def test01StressBulk(self): """ stress test bulk """ db = RequestDB() reqIDs = [] for i in xrange(self.stressRequests): request = Request({"RequestName": "test-%d" % i}) op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"}) op += File({"LFN": "/lhcb/user/c/cibak/foo"}) request += op put = db.putRequest(request) self.assertEqual(put["OK"], True) reqIDs.append(put['Value']) loops = self.stressRequests // self.bulkRequest + \ (1 if (self.stressRequests % self.bulkRequest) else 0) totalSuccessful = 0 time.sleep(1) startTime = time.time() for i in xrange(loops): get = db.getBulkRequests(self.bulkRequest, True) if "Message" in get: print(get["Message"]) self.assertEqual(get["OK"], True, "get failed") totalSuccessful += len(get["Value"]) endTime = time.time() print("getRequests duration %s " % (endTime - startTime)) self.assertEqual( totalSuccessful, self.stressRequests, "Did not retrieve all the requests: %s instead of %s" % (totalSuccessful, self.stressRequests)) for reqID in reqIDs: delete = db.deleteRequest(reqID) self.assertEqual( delete["OK"], True, delete['Message'] if 'Message' in delete else 'OK')
def __init__( self, fromDict = None ): """c'tor :param self: self reference """ Record.__init__( self ) self.__waiting = None now = datetime.datetime.utcnow().replace( microsecond = 0 ) self.__data__["CreationTime"] = now self.__data__["SubmitTime"] = now self.__data__["LastUpdate"] = now self.__data__["Status"] = "Done" self.__data__["JobID"] = 0 self.__data__["RequestID"] = 0 proxyInfo = getProxyInfo() if proxyInfo["OK"]: proxyInfo = proxyInfo["Value"] if proxyInfo["validGroup"] and proxyInfo["validDN"]: self.OwnerDN = proxyInfo["identity"] self.OwnerGroup = proxyInfo["group"] self.__dirty = [] self.__operations__ = TypedList( allowedTypes = Operation ) fromDict = fromDict if fromDict else {} self.__dirty = fromDict.get( "__dirty", [] ) if "__dirty" in fromDict: del fromDict["__dirty"] for opDict in fromDict.get( "Operations", [] ): self +=Operation( opDict ) if "Operations" in fromDict: del fromDict["Operations"] for key, value in fromDict.items(): if key not in self.__data__: raise AttributeError( "Unknown Request attribute '%s'" % key ) if value: setattr( self, key, value ) self._notify()
def setUp(self): """test case set up""" self.handlersDict = {"ForwardDISET": "DIRAC/RequestManagementSystem/private/ForwardDISET"} self.req = Request() self.req.RequestName = "foobarbaz" self.req.OwnerGroup = "lhcb_user" self.req.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=chaen/CN=705305/CN=Christophe Haen" self.op = Operation({"Type": "ForwardDISET", "Arguments": "tts10:helloWorldee"}) self.req.addOperation(self.op) self.task = None self.mockRC = MagicMock() self.mockObjectOps = MagicMock() self.mockObjectOps.getSections.return_value = {"OK": True, "Value": ["DataProcessing", "DataManager"]} self.mockObjectOps.getOptionsDict.return_value = { "OK": True, "Value": {"Group": "lhcb_user", "User": "******"}, } self.mockOps = MagicMock() self.mockOps.return_value = self.mockObjectOps
def test_db(self): """ table description """ tableDict = RequestDB.getTableMeta() self.assertEqual("Request" in tableDict, True) self.assertEqual("Operation" in tableDict, True) self.assertEqual("File" in tableDict, True) self.assertEqual(tableDict["Request"], Request.tableDesc()) self.assertEqual(tableDict["Operation"], Operation.tableDesc()) self.assertEqual(tableDict["File"], File.tableDesc()) # # empty DB at that stage ret = RequestDB().getDBSummary() self.assertEqual(ret, { 'OK': True, 'Value': { 'Operation': {}, 'Request': {}, 'File': {} } })
def checkRequestAndOp(listOfLFNs): req = Request() req.RequestName = "MyRequest" op = Operation() op.Type = "CheckMigration" op.TargetSE = "Foo-SE" for index, lfn in enumerate(listOfLFNs): oFile = File() oFile.LFN = lfn oFile.Size = index oFile.Checksum = "01130a%0d" % index oFile.ChecksumType = "adler32" op.addFile(oFile) req.addOperation(op) return req, op
def _setRegistrationRequest(self, lfn, targetSE, fileDict, catalog): """ Sets a registration request :param str lfn: LFN :param list se: list of SE (or just string) :param list catalog: list (or string) of catalogs to use :param dict fileDict: file metadata """ self.log.info('Setting registration request for %s at %s.' % (lfn, targetSE)) if not type(catalog) == type([]): catalog = [catalog] for cat in catalog: register = Operation() register.Type = "RegisterFile" register.Catalog = cat register.TargetSE = targetSE regFile = File() regFile.LFN = lfn regFile.Checksum = fileDict.get("Checksum", "") regFile.ChecksumType = fileDict.get("ChecksumType", self.defaultChecksumType) regFile.Size = fileDict.get("Size", 0) regFile.GUID = fileDict.get("GUID", "") se = StorageElement(targetSE) pfn = se.getPfnForLfn(lfn) if not pfn["OK"] or lfn not in pfn["Value"]['Successful']: self.log.error( "unable to get PFN for LFN: %s" % pfn.get('Message', pfn.get('Value', {}).get('Failed', {}).get(lfn))) return pfn regFile.PFN = pfn["Value"]['Successful'][lfn] register.addFile(regFile) self.request.addOperation(register) return S_OK()
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup): """deal with a Request that has just one operation, as it was sofar :param transBody: string, can be an empty string :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ requestOperation = "ReplicateAndRegister" if transBody: try: _requestType, requestOperation = transBody.split(";") except AttributeError: pass failedTasks = [] # Do not remove sorted, we might pop elements in the loop for taskID, task in taskDict.items(): transID = task["TransformationID"] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = task["TargetSE"] # If there are input files if task.get("InputData"): if isinstance(task["InputData"], list): files = task["InputData"] elif isinstance(task["InputData"], six.string_types): files = task["InputData"].split(";") for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile(trFile) oRequest.addOperation(transfer) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result["OK"]: failedTasks.append(taskID) # Remove failed tasks for taskID in failedTasks: taskDict.pop(taskID)
def test02Authorization(self): """ Test whether request sets on behalf of others are rejected, unless done with Delegation properties This test is kind of stupid though, since we do the same thing than the server... not a real test ! """ request = Request({"RequestName": "unauthorized"}) request.OwnerDN = 'NotMe' request.OwnerDN = 'AnotherGroup' op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"}) op += File({"LFN": "/lhcb/user/c/cibak/foo"}) request += op res = self.requestClient.putRequest(request) credProperties = getProxyInfo()['Value']['groupProperties'] # If the proxy with which we test has delegation, it should work if FULL_DELEGATION in credProperties or LIMITED_DELEGATION in credProperties: self.assertTrue(res['OK'], res) self.requestClient.deleteRequest(res['Value']) # otherwise no else: self.assertFalse(res['OK'], res)
def __deleteSandboxFromExternalBackend(self, SEName, SEPFN): if self.getCSOption("DelayedExternalDeletion", True): gLogger.info("Setting deletion request") try: # We need the hostDN used in order to pass these credentials to the # SandboxStoreDB.. hostCertLocation, _ = Locations.getHostCertificateAndKeyLocation( ) hostCert = X509Certificate.X509Certificate() hostCert.loadFromFile(hostCertLocation) hostDN = hostCert.getSubjectDN().get("Value") # use the host authentication to fetch the data result = self.sandboxDB.getSandboxOwner( SEName, SEPFN, hostDN, "hosts") if not result["OK"]: return result _owner, ownerDN, ownerGroup = result["Value"] request = Request() request.RequestName = "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time()) request.OwnerDN = ownerDN request.OwnerGroup = ownerGroup physicalRemoval = Operation() physicalRemoval.Type = "PhysicalRemoval" physicalRemoval.TargetSE = SEName fileToRemove = File() fileToRemove.PFN = SEPFN physicalRemoval.addFile(fileToRemove) request.addOperation(physicalRemoval) return ReqClient().putRequest(request) except Exception as e: gLogger.exception("Exception while setting deletion request") return S_ERROR(f"Cannot set deletion request: {e}") else: gLogger.info("Deleting external Sandbox") try: return StorageElement(SEName).removeFile(SEPFN) except Exception: gLogger.exception( "RM raised an exception while trying to delete a remote sandbox" ) return S_ERROR( "RM raised an exception while trying to delete a remote sandbox" )
def test02Scheduled(self): """ scheduled request r/w """ db = RequestDB() req = Request({"RequestName": "FTSTest"}) op = Operation({ "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER" }) op += File({ "LFN": "/a/b/c", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" }) req += op put = db.putRequest(req) self.assertEqual(put["OK"], True, put['Message'] if 'Message' in put else 'OK') reqID = put['Value'] peek = db.peekRequest(reqID) self.assertEqual(peek["OK"], True, peek['Message'] if 'Message' in peek else 'OK') peek = peek["Value"] for op in peek: opId = op.OperationID getFTS = db.getScheduledRequest(opId) self.assertEqual(getFTS["OK"], True, "getScheduled failed") self.assertEqual(getFTS["Value"].RequestName, "FTSTest", "wrong request selected") delete = db.deleteRequest(reqID) self.assertEqual(delete["OK"], True, delete['Message'] if 'Message' in delete else 'OK')
def test04Stress(self): """ stress test """ db = RequestDB() for i in range(self.i): request = Request({"RequestName": "test-%d" % i}) op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"}) op += File({"LFN": "/lhcb/user/c/cibak/foo"}) request += op put = db.putRequest(request) self.assertEqual(put["OK"], True, "put failed") for i in range(self.i): get = db.getRequest("test-%s" % i, False) if "Message" in get: print get["Message"] self.assertEqual(get["OK"], True, "get failed") for i in range(self.i): delete = db.deleteRequest("test-%s" % i) self.assertEqual(delete["OK"], True, "delete failed")