def __submitRMSOp(self, target_se, lfns_chunk_dict, whichRMSOp='ReplicateAndRegister' ): """ target_se : SE name to which to replicate lfns_chunk_dict : LFNS dict with 100 lfns as key andeEach lfn has 'Size', 'Checksum' whichRMSOp: Choose from RMP operation - ReplicateAndRegister, ReplicateAndRemove, PutAndRegister """ ## Setup request request = Request() request.RequestName = "DDM_"+ str(target_se) + datetime.datetime.now().strftime("_%Y%m%d_%H%M%S") myOp = Operation() myOp.Type = whichRMSOp myOp.TargetSE = target_se ## Add LFNS to operations for lfn in lfns_chunk_dict.keys(): opFile = File() opFile.LFN = lfn opFile.Size = lfns_chunk_dict[lfn]['Size'] if "Checksum" in lfns_chunk_dict[lfn]: opFile.Checksum = lfns_chunk_dict[lfn]['Checksum'] opFile.ChecksumType = 'ADLER32' ## Add file to operation myOp.addFile( opFile ) request.addOperation( myOp ) reqClient = ReqClient() putRequest = reqClient.putRequest( request ) if not putRequest["OK"]: gLogger.error( "Unable to put request '%s': %s" % ( request.RequestName, putRequest["Message"] ) ) return S_ERROR("Problem submitting to RMS.")
def archiveRequestAndOp(listOfLFNs): """Return a tuple of the request and operation.""" req = Request() req.RequestName = "MyRequest" op = Operation() switches = {} archiveLFN = "/vo/tars/myTar.tar" op.Arguments = DEncode.encode({ "SourceSE": switches.get("SourceSE", "SOURCE-SE"), "TarballSE": switches.get("TarballSE", "TARBALL-SE"), "RegisterDescendent": False, "ArchiveLFN": archiveLFN, }) op.Type = "ArchiveFiles" for index, lfn in enumerate(listOfLFNs): oFile = File() oFile.LFN = lfn oFile.Size = index oFile.Checksum = "01130a%0d" % index oFile.ChecksumType = "adler32" op.addFile(oFile) req.addOperation(op) return req, op
def archiveRequestAndOp(listOfLFNs): """Return a tuple of the request and operation.""" req = Request() req.RequestName = 'MyRequest' op = Operation() switches = {} archiveLFN = '/vo/tars/myTar.tar' op.Arguments = DEncode.encode({ 'SourceSE': switches.get('SourceSE', 'SOURCE-SE'), 'TarballSE': switches.get('TarballSE', 'TARBALL-SE'), 'RegisterDescendent': False, 'ArchiveLFN': archiveLFN }) op.Type = 'ArchiveFiles' for index, lfn in enumerate(listOfLFNs): oFile = File() oFile.LFN = lfn oFile.Size = index oFile.Checksum = '01130a%0d' % index oFile.ChecksumType = 'adler32' op.addFile(oFile) req.addOperation(op) return req, op
def __deleteSandboxFromExternalBackend( self, SEName, SEPFN ): if self.getCSOption( "DelayedExternalDeletion", True ): gLogger.info( "Setting deletion request" ) try: request = Request() request.RequestName = "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time() ) physicalRemoval = Operation() physicalRemoval.Type = "PhysicalRemoval" physicalRemoval.TargetSE = SEName fileToRemove = File() fileToRemove.PFN = SEPFN physicalRemoval.addFile( fileToRemove ) request.addOperation( physicalRemoval ) return ReqClient().putRequest( request ) except Exception as e: gLogger.exception( "Exception while setting deletion request" ) return S_ERROR( "Cannot set deletion request: %s" % str( e ) ) else: gLogger.info( "Deleting external Sandbox" ) try: return StorageElement( SEName ).removeFile( SEPFN ) except Exception as e: gLogger.exception( "RM raised an exception while trying to delete a remote sandbox" ) return S_ERROR( "RM raised an exception while trying to delete a remote sandbox" )
def setUp(self): """ test case set up """ gLogger.setLevel('NOTICE') self.file = File() self.file.LFN = "/lhcb/user/c/cibak/testFile" self.file.Checksum = "123456" self.file.ChecksumType = "ADLER32" self.file2 = File() self.file2.LFN = "/lhcb/user/f/fstagni/testFile" self.file2.Checksum = "654321" self.file2.ChecksumType = "ADLER32" self.operation = Operation() self.operation.Type = "ReplicateAndRegister" self.operation.TargetSE = "CERN-USER" self.operation.addFile(self.file) self.operation.addFile(self.file2) self.request = Request() self.request.RequestName = "RequestManagerHandlerTests" self.request.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" self.request.OwnerGroup = "dirac_user" self.request.JobID = 123 self.request.addOperation(self.operation) # # JSON representation of a whole request self.jsonStr = self.request.toJSON()['Value'] # # request client self.requestClient = ReqClient() self.stressRequests = 1000 self.bulkRequest = 1000
def __deleteSandboxFromExternalBackend(self, SEName, SEPFN): if self.getCSOption("DelayedExternalDeletion", True): gLogger.info("Setting deletion request") try: request = Request() request.RequestName = "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time()) physicalRemoval = Operation() physicalRemoval.Type = "PhysicalRemoval" physicalRemoval.TargetSE = SEName fileToRemove = File() fileToRemove.PFN = SEPFN physicalRemoval.addFile(fileToRemove) request.addOperation(physicalRemoval) return ReqClient().putRequest(request) except Exception as e: gLogger.exception("Exception while setting deletion request") return S_ERROR("Cannot set deletion request: %s" % str(e)) else: gLogger.info("Deleting external Sandbox") try: return StorageElement(SEName).removeFile(SEPFN) except Exception as e: gLogger.exception( "RM raised an exception while trying to delete a remote sandbox" ) return S_ERROR( "RM raised an exception while trying to delete a remote sandbox" )
class OperationHandlerBaseTests( unittest.TestCase ): """ .. class:: OperationHandlerBaseTests """ def setUp( self ): """ test set up """ self.req = Request() self.req.RequestName = "testRequest" self.op = Operation( {"Type" : "ForwardDISET", "Arguments" : "foobar" } ) self.req.addOperation( self.op ) self.baseOp = OperationHandlerBase() def tearDown( self ): """ test tear down """ del self.baseOp del self.op del self.req def testOperationHandlerBase( self ): """ base op test """ self.baseOp.setOperation( self.op ) # # log is there self.assertEqual( "log" in dir( self.baseOp ), True, "log missing" ) # # operation is there self.assertEqual( "operation" in dir( self.baseOp ), True, "operation is missing" ) # # request is there self.assertEqual( "request" in dir( self.baseOp ), True, "request is missing" ) # # __call__ not implemented self.assertRaises( NotImplementedError, self.baseOp ) # # replica manager self.assertEqual( isinstance( self.baseOp.dm, DataManager ), True, "DataManager is missing" )
def setUp(self): """ test case setup """ self.request = Request({"RequestName": "test1", "JobID": 1}) self.operation1 = Operation({ "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER" }) self.file = File({ "LFN": "/a/b/c", "ChecksumType": "ADLER32", "Checksum": "123456" }) self.request.addOperation(self.operation1) self.operation1.addFile(self.file) self.operation2 = Operation() self.operation2.Type = "RemoveFile" self.operation2.addFile(File({"LFN": "/c/d/e"})) self.request.addOperation(self.operation2) # ## set some defaults gConfig.setOptionValue('DIRAC/Setup', 'Test') gConfig.setOptionValue('/DIRAC/Setups/Test/RequestManagement', 'Test') gConfig.setOptionValue( '/Systems/RequestManagement/Test/Databases/ReqDB/Host', 'localhost') gConfig.setOptionValue( '/Systems/RequestManagement/Test/Databases/ReqDB/DBName', 'ReqDB') gConfig.setOptionValue( '/Systems/RequestManagement/Test/Databases/ReqDB/User', 'Dirac') self.i = 1000
def setUp(self): """ test case set up """ gLogger.setLevel('INFO') self.file = File() self.file.LFN = "/lhcb/user/c/cibak/testFile" self.file.Checksum = "123456" self.file.ChecksumType = "ADLER32" self.file2 = File() self.file2.LFN = "/lhcb/user/f/fstagni/testFile" self.file2.Checksum = "654321" self.file2.ChecksumType = "ADLER32" self.operation = Operation() self.operation.Type = "ReplicateAndRegister" self.operation.TargetSE = "CERN-USER" self.operation.addFile(self.file) self.operation.addFile(self.file2) proxyInfo = getProxyInfo()['Value'] self.request = Request() self.request.RequestName = "RequestManagerHandlerTests" self.request.OwnerDN = proxyInfo['identity'] self.request.OwnerGroup = proxyInfo['group'] self.request.JobID = 123 self.request.addOperation(self.operation) # # JSON representation of a whole request self.jsonStr = self.request.toJSON()['Value'] # # request client self.requestClient = ReqClient()
class RequestTaskTests( unittest.TestCase ): """ .. class:: RequestTaskTests """ def setUp( self ): """ test case set up """ self.handlersDict = { "ForwardDISET" : "DIRAC/RequestManagementSystem/private/ForwardDISET" } self.req = Request() self.req.RequestName = "foobarbaz" self.req.OwnerGroup = "dirac_user" self.req.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" self.op = Operation( { "Type": "ForwardDISET", "Arguments" : "tts10:helloWorldee" } ) self.req.addOperation( self.op ) self.task = None def tearDown( self ): """ test case tear down """ del self.req del self.op del self.task def testAPI( self ): """ test API """ self.task = RequestTask( self.req.toXML()["Value"], self.handlersDict ) self.task.requestClient = Mock( return_value = Mock(spec=RequestClient) ) self.task.requestClient().updateRequest = Mock() self.task.requestClient().updateRequest.return_value = { "OK" : True, "Value" : None } ret = self.task() self.assertEqual( ret["OK"], True , "call failed")
def export_putRequest(self, requestJSON): """put a new request into RequestDB :param cls: class ref :param str requestJSON: request serialized to JSON format """ requestDict = json.loads(requestJSON) requestName = requestDict.get("RequestID", requestDict.get("RequestName", "***UNKNOWN***")) request = Request(requestDict) # Check whether the credentials in the Requests are correct and allowed to be set isAuthorized = RequestValidator.setAndCheckRequestOwner(request, self.getRemoteCredentials()) if not isAuthorized: return S_ERROR(DErrno.ENOAUTH, "Credentials in the requests are not allowed") optimized = request.optimize() if optimized.get("Value", False): gLogger.debug("putRequest: request was optimized") else: gLogger.debug("putRequest: request unchanged", optimized.get("Message", "Nothing could be optimized")) valid = self.validate(request) if not valid["OK"]: gLogger.error("putRequest: request %s not valid: %s" % (requestName, valid["Message"])) return valid # If NotBefore is not set or user defined, we calculate its value now = datetime.datetime.utcnow().replace(microsecond=0) extraDelay = datetime.timedelta(0) if request.Status not in Request.FINAL_STATES and (not request.NotBefore or request.NotBefore < now): # We don't delay if it is the first insertion if getattr(request, "RequestID", 0): # If it is a constant delay, just set it if self.constantRequestDelay: extraDelay = datetime.timedelta(minutes=self.constantRequestDelay) else: # If there is a waiting Operation with Files op = request.getWaiting().get("Value") if op and len(op): attemptList = [opFile.Attempt for opFile in op if opFile.Status == "Waiting"] if attemptList: maxWaitingAttempt = max([opFile.Attempt for opFile in op if opFile.Status == "Waiting"]) # In case it is the first attempt, extraDelay is 0 # maxWaitingAttempt can be None if the operation has no File, like the ForwardDiset extraDelay = datetime.timedelta( minutes=2 * math.log(maxWaitingAttempt) if maxWaitingAttempt else 0 ) request.NotBefore = now + extraDelay gLogger.info( "putRequest: request %s not before %s (extra delay %s)" % (request.RequestName, request.NotBefore, extraDelay) ) requestName = request.RequestName gLogger.info("putRequest: Attempting to set request '%s'" % requestName) return self.__requestDB.putRequest(request)
def export_putRequest(cls, requestJSON): """ put a new request into RequestDB :param cls: class ref :param str requestJSON: request serialized to JSON format """ requestName = requestJSON.get("RequestName", "***UNKNOWN***") request = Request(requestJSON) requestID = request.RequestID optimized = request.optimize() if optimized.get("Value", False): if request.RequestID == 0 and requestID != 0: # A new request has been created, delete the old one delete = cls.__requestDB.deleteRequest(request.RequestName) if not delete['OK']: return delete gLogger.debug( "putRequest: request was optimized and removed for a new insertion" ) else: gLogger.debug("putRequest: request was optimized") else: gLogger.debug( "putRequest: request unchanged", optimized.get("Message", "Nothing could be optimized")) valid = cls.validate(request) if not valid["OK"]: gLogger.error("putRequest: request %s not valid: %s" % (requestName, valid["Message"])) return valid requestName = request.RequestName gLogger.info("putRequest: Attempting to set request '%s'" % requestName) return cls.__requestDB.putRequest(request)
def setUp(self): """ test set up """ self.req = Request() self.req.RequestName = "testRequest" self.op = Operation({"Type": "ForwardDISET", "Arguments": "foobar"}) self.req.addOperation(self.op) self.baseOp = OperationHandlerBase()
def setUp(self): """ test case set up """ self.handlersDict = { "ForwardDISET": "DIRAC/RequestManagementSystem/private/ForwardDISET" } self.req = Request() self.req.RequestName = "foobarbaz" self.req.OwnerGroup = "lhcb_user" self.req.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=chaen/CN=705305/CN=Christophe Haen" self.op = Operation({ "Type": "ForwardDISET", "Arguments": "tts10:helloWorldee" }) self.req.addOperation(self.op) self.task = None self.mockRC = MagicMock() self.mockObjectOps = MagicMock() self.mockObjectOps.getSections.return_value = { 'OK': True, 'Value': ['DataProcessing', 'DataManager'] } self.mockObjectOps.getOptionsDict.return_value = { 'OK': True, 'Value': { 'Group': 'lhcb_user', 'User': '******' } } self.mockOps = MagicMock() self.mockOps.return_value = self.mockObjectOps
def test01CtorSerilization(self): """ c'tor and serialization """ # # empty c'tor req = Request() self.assertEqual(isinstance(req, Request), True) self.assertEqual(req.JobID, 0) self.assertEqual(req.Status, "Waiting") req = Request(self.fromDict) self.assertEqual(isinstance(req, Request), True) self.assertEqual(req.RequestName, "test") self.assertEqual(req.JobID, 12345) self.assertEqual(req.Status, "Waiting") toJSON = req.toJSON() self.assertEqual(toJSON["OK"], True, "JSON serialization failed") fromJSON = toJSON["Value"] req = Request(fromJSON) toSQL = req.toSQL() self.assertEqual(toSQL["OK"], True) toSQL = toSQL["Value"] self.assertEqual(toSQL.startswith("INSERT"), True) req.RequestID = 1 toSQL = req.toSQL() self.assertEqual(toSQL["OK"], True) toSQL = toSQL["Value"] self.assertEqual(toSQL.startswith("UPDATE"), True)
def export_putRequest(cls, requestJSON): """ put a new request into RequestDB :param cls: class ref :param str requestJSON: request serialized to JSON format """ requestName = requestJSON.get("RequestName", "***UNKNOWN***") request = Request(requestJSON) optimized = request.optimize() if optimized.get("Value", False): gLogger.debug("putRequest: request was optimized") else: gLogger.debug( "putRequest: request unchanged", optimized.get("Message", "Nothing could be optimize")) valid = cls.validate(request) if not valid["OK"]: gLogger.error("putRequest: request %s not valid: %s" % (requestName, valid["Message"])) return valid requestName = request.RequestName gLogger.info("putRequest: Attempting to set request '%s'" % requestName) return cls.__requestDB.putRequest(request)
class MoveReplicaSuccess( ReqOpsTestCase ): def setUp(self): self.op = Operation() self.op.Type = "MoveFile" self.op.SourceSE = "%s,%s" % ( "sourceSE1", "sourceSE2" ) self.op.TargetSE = "%s,%s" % ( "targetSE1", "targetSE2" ) self.File = File() self.File.LFN = '/cta/file1' self.File.Size = 2L self.File.Checksum = '011300a2' self.File.ChecksumType = "adler32" self.op.addFile( self.File ) self.req = Request() self.req.addOperation( self.op ) self.mr = MoveReplica( self.op ) self.mr.dm = MagicMock() self.mr.fc = MagicMock() # This test needs to be fixed. It currently fails because StorageElement is not mocked '''def test__dmTransfer( self ): successful = {} for sourceSE in self.op.sourceSEList: successful[sourceSE] = 'dips://' + sourceSE.lower() + ':9148/DataManagement/StorageElement' + self.File.LFN res = {'OK': True, 'Value': {'Successful': {self.File.LFN : successful}, 'Failed': {}}} self.mr.dm.getActiveReplicas.return_value = res res = {'OK': True, 'Value': {'Successful': {self.File.LFN : {'register': 0.1228799819946289, 'replicate': 9.872732877731323}}, 'Failed': {}}} self.mr.dm.replicateAndRegister.return_value = res res = self.mr.dmTransfer( self.File ) self.assertTrue( res['OK'] ) self.assertEqual( self.mr.operation.__files__[0].Status, 'Waiting' ) self.assertEqual( self.mr.operation.Status, 'Waiting' ) self.assertEqual( self.mr.request.Status, 'Waiting' )''' def test__dmRemoval( self ): res = {'OK': True, 'Value': {'Successful': { self.File.LFN : {'DIRACFileCatalog': True}}, 'Failed': {}}} self.mr.dm.removeReplica.return_value = res toRemoveDict = {self.File.LFN: self.File} targetSEs = self.op.sourceSEList res = self.mr.dmRemoval( toRemoveDict, targetSEs ) self.assertTrue( res['OK'] ) resvalue = dict( [ ( targetSE, '' ) for targetSE in targetSEs ] ) self.assertEqual( res['Value'], {self.File.LFN: resvalue} ) self.assertEqual( self.mr.operation.__files__[0].Status, 'Done' ) self.assertEqual( self.mr.operation.Status, 'Done' ) self.assertEqual( self.mr.request.Status, 'Done' )
def createRequest(requestID, opType, opStatus, fileStatus, lfnError=" ", lfn="/ilc/fake/lfn"): """Create a request.""" req = Request({"RequestID": requestID}) op = Operation({"Type": opType, "Status": opStatus}) op.addFile(File({"LFN": lfn, "Status": fileStatus, "Error": lfnError})) req.addOperation(op) return req
def export_putRequest(cls, requestJSON): """ put a new request into RequestDB :param cls: class ref :param str requestJSON: request serialized to JSON format """ requestDict = json.loads(requestJSON) requestName = requestDict.get( "RequestID", requestDict.get('RequestName', "***UNKNOWN***")) request = Request(requestDict) optimized = request.optimize() if optimized.get("Value", False): gLogger.debug("putRequest: request was optimized") else: gLogger.debug( "putRequest: request unchanged", optimized.get("Message", "Nothing could be optimized")) valid = cls.validate(request) if not valid["OK"]: gLogger.error("putRequest: request %s not valid: %s" % (requestName, valid["Message"])) return valid # If NotBefore is not set or user defined, we calculate its value now = datetime.datetime.utcnow().replace(microsecond=0) extraDelay = datetime.timedelta(0) if request.Status not in Request.FINAL_STATES and ( not request.NotBefore or request.NotBefore < now): op = request.getWaiting().get('Value') # If there is a waiting Operation with Files if op and len(op): attemptList = [ opFile.Attempt for opFile in op if opFile.Status == "Waiting" ] if attemptList: maxWaitingAttempt = max([ opFile.Attempt for opFile in op if opFile.Status == "Waiting" ]) # In case it is the first attempt, extraDelay is 0 # maxWaitingAttempt can be None if the operation has no File, like the ForwardDiset extraDelay = datetime.timedelta( minutes=2 * math.log(maxWaitingAttempt) if maxWaitingAttempt else 0 ) request.NotBefore = now + extraDelay gLogger.info("putRequest: request %s not before %s (extra delay %s)" % (request.RequestName, request.NotBefore, extraDelay)) requestName = request.RequestName gLogger.info("putRequest: Attempting to set request '%s'" % requestName) return cls.__requestDB.putRequest(request)
def __init__( self, requestJSON, handlersDict, csPath, agentName, standalone=False, requestClient=None, rmsMonitoring=False): """c'tor :param self: self reference :param str requestJSON: request serialized to JSON :param dict opHandlers: operation handlers """ self.request = Request(requestJSON) # # csPath self.csPath = csPath # # agent name self.agentName = agentName # # standalone flag self.standalone = standalone # # handlers dict self.handlersDict = handlersDict # # handlers class def self.handlers = {} # # own sublogger self.log = gLogger.getSubLogger("pid_%s/%s" % (os.getpid(), self.request.RequestName)) # # get shifters info self.__managersDict = {} shifterProxies = self.__setupManagerProxies() if not shifterProxies["OK"]: self.log.error("Cannot setup shifter proxies", shifterProxies["Message"]) # This flag which is set and sent from the RequestExecutingAgent and is False by default. self.rmsMonitoring = rmsMonitoring if self.rmsMonitoring: self.rmsMonitoringReporter = MonitoringReporter(monitoringType="RMSMonitoring") else: # # initialize gMonitor gMonitor.setComponentType(gMonitor.COMPONENT_AGENT) gMonitor.setComponentName(self.agentName) gMonitor.initialize() # # own gMonitor activities gMonitor.registerActivity("RequestAtt", "Requests processed", "RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM) gMonitor.registerActivity("RequestFail", "Requests failed", "RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM) gMonitor.registerActivity("RequestOK", "Requests done", "RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM) if requestClient is None: self.requestClient = ReqClient() else: self.requestClient = requestClient
def test06Dirty( self ): """ dirty records """ db = RequestDB() r = Request() r.RequestName = "dirty" op1 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} ) op1 += File( {"LFN": "/a/b/c/1", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } ) op2 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} ) op2 += File( {"LFN": "/a/b/c/2", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } ) op3 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} ) op3 += File( {"LFN": "/a/b/c/3", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } ) r += op1 r += op2 r += op3 put = db.putRequest( r ) self.assertEqual( put["OK"], True, "1. putRequest failed: %s" % put.get( "Message", "" ) ) reqID = put['Value'] r = db.getRequest( reqID ) self.assertEqual( r["OK"], True, "1. getRequest failed: %s" % r.get( "Message", "" ) ) r = r["Value"] del r[0] self.assertEqual( len( r ), 2, "1. len wrong" ) put = db.putRequest( r ) self.assertEqual( put["OK"], True, "2. putRequest failed: %s" % put.get( "Message", "" ) ) reqID = put['Value'] r = db.getRequest( reqID ) self.assertEqual( r["OK"], True, "2. getRequest failed: %s" % r.get( "Message", "" ) ) r = r["Value"] self.assertEqual( len( r ), 2, "2. len wrong" ) op4 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} ) op4 += File( {"LFN": "/a/b/c/4", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } ) r[0] = op4 put = db.putRequest( r ) self.assertEqual( put["OK"], True, "3. putRequest failed: %s" % put.get( "Message", "" ) ) reqID = put['Value'] r = db.getRequest( reqID ) self.assertEqual( r["OK"], True, "3. getRequest failed: %s" % r.get( "Message", "" ) ) r = r["Value"] self.assertEqual( len( r ), 2, "3. len wrong" ) delete = db.deleteRequest( reqID ) self.assertEqual( delete["OK"], True, delete['Message'] if 'Message' in delete else 'OK' )
def __init__(self,request=None,init=True): # A common set of attributes that define requests. self.requestAttributes = ['SubRequestID','TargetSE','Status','Operation','SourceSE','Catalogue','SpaceToken','RequestType'] # Possible keys to define the files in the request. self.fileAttributes = ['LFN','Size','PFN','GUID','Md5','Addler','Status','Attempt','FileID'] # Possible keys to define the dataset in the request. self.datasetAttributes = ['Handle'] Request.__init__(self,request,init)
def setUp( self ): """ test case set up """ self.handlersDict = { "ForwardDISET" : "DIRAC/RequestManagementSystem/private/ForwardDISET" } self.req = Request() self.req.RequestName = "foobarbaz" self.req.OwnerGroup = "dirac_user" self.req.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" self.op = Operation( { "Type": "ForwardDISET", "Arguments" : "tts10:helloWorldee" } ) self.req.addOperation( self.op ) self.task = None
def taskToRequest(self, taskID, task, transID): """Convert a task into an RMS with a single ForwardDISET Operation whose attribute is the number of files in the task """ req = Request() op = Operation() op.Type = "ForwardDISET" op.Arguments = json.dumps(len(task["InputData"]) * self.factor) req.addOperation(op) return req
def createRequest(self, requestName, lfnChunk): """Create the Request.""" request = Request() request.RequestName = requestName replicate = Operation() replicate.Type = 'ReplicateAndRegister' replicate.TargetSE = self.switches.get('TargetSE') self.addLFNs(replicate, lfnChunk, addPFN=True) request.addOperation(replicate) if self.switches.get('CheckMigration'): checkMigration = Operation() checkMigration.Type = 'CheckMigration' checkMigration.TargetSE = self.switches.get('TargetSE') self.addLFNs(checkMigration, lfnChunk, addPFN=True) request.addOperation(checkMigration) removeReplicas = Operation() removeReplicas.Type = 'RemoveReplica' removeReplicas.TargetSE = ','.join(self.switches.get('SourceSE', [])) self.addLFNs(removeReplicas, lfnChunk) request.addOperation(removeReplicas) return request
def createRequest(self, requestName, lfnChunk): """Create the Request.""" request = Request() request.RequestName = requestName replicate = Operation() replicate.Type = "ReplicateAndRegister" replicate.TargetSE = self.switches.get("TargetSE") self.addLFNs(replicate, lfnChunk, addPFN=True) request.addOperation(replicate) if self.switches.get("CheckMigration"): checkMigration = Operation() checkMigration.Type = "CheckMigration" checkMigration.TargetSE = self.switches.get("TargetSE") self.addLFNs(checkMigration, lfnChunk, addPFN=True) request.addOperation(checkMigration) removeReplicas = Operation() removeReplicas.Type = "RemoveReplica" removeReplicas.TargetSE = ",".join(self.switches.get("SourceSE", [])) self.addLFNs(removeReplicas, lfnChunk) request.addOperation(removeReplicas) return request
def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup): """ deal with a Request that has multiple operations :param transJson: list of lists of string and dictionaries, e.g.: .. code :: python body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }), ( "RemoveReplica", { "TargetSE":"FOO-SRM" } ), ] :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ failedTasks = [] for taskID, task in taskDict.items(): transID = task['TransformationID'] if not task.get('InputData'): self._logError("Error creating request for task", "%s, No input data" % taskID, transID=transID) taskDict.pop(taskID) continue files = [] oRequest = Request() if isinstance(task['InputData'], list): files = task['InputData'] elif isinstance(task['InputData'], basestring): files = task['InputData'].split(';') # create the operations from the json structure for operationTuple in transJson: op = Operation() op.Type = operationTuple[0] for parameter, value in operationTuple[1].iteritems(): setattr(op, parameter, value) for lfn in files: opFile = File() opFile.LFN = lfn op.addFile(opFile) oRequest.addOperation(op) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result['OK']: failedTasks.append(taskID) # Remove failed tasks for taskID in failedTasks: taskDict.pop(taskID)
class RequestTaskTests(unittest.TestCase): """ .. class:: RequestTaskTests """ def setUp(self): """test case set up""" self.handlersDict = {"ForwardDISET": "DIRAC/RequestManagementSystem/private/ForwardDISET"} self.req = Request() self.req.RequestName = "foobarbaz" self.req.OwnerGroup = "lhcb_user" self.req.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=chaen/CN=705305/CN=Christophe Haen" self.op = Operation({"Type": "ForwardDISET", "Arguments": "tts10:helloWorldee"}) self.req.addOperation(self.op) self.task = None self.mockRC = MagicMock() self.mockObjectOps = MagicMock() self.mockObjectOps.getSections.return_value = {"OK": True, "Value": ["DataProcessing", "DataManager"]} self.mockObjectOps.getOptionsDict.return_value = { "OK": True, "Value": {"Group": "lhcb_user", "User": "******"}, } self.mockOps = MagicMock() self.mockOps.return_value = self.mockObjectOps def tearDown(self): """test case tear down""" del self.req del self.op del self.task def testAPI(self): """test API""" rt = importlib.import_module("DIRAC.RequestManagementSystem.private.RequestTask") rt.gMonitor = MagicMock() rt.Operations = self.mockOps rt.CS = MagicMock() self.task = RequestTask( self.req.toJSON()["Value"], self.handlersDict, "csPath", "RequestManagement/RequestExecutingAgent", requestClient=self.mockRC, ) self.task.requestClient = Mock(return_value=Mock(spec=ReqClient)) self.task.requestClient().updateRequest = Mock() self.task.requestClient().updateRequest.return_value = {"OK": True, "Value": None} ret = self.task() self.assertEqual(ret["OK"], True, "call failed") ret = self.task.setupProxy() print(ret)
def _sendToFailover( rpcStub ): """ Create a ForwardDISET operation for failover """ request = Request() request.RequestName = "Accounting.DataStore.%s.%s" % ( time.time(), random.random() ) forwardDISETOp = Operation() forwardDISETOp.Type = "ForwardDISET" forwardDISETOp.Arguments = DEncode.encode( rpcStub ) request.addOperation( forwardDISETOp ) return ReqClient().putRequest( request )
def export_putRequest(cls, requestJSON): """ put a new request into RequestDB :param cls: class ref :param str requestJSON: request serialized to JSON format """ requestDict = json.loads(requestJSON) requestName = requestDict.get("RequestID", requestDict.get("RequestName", "***UNKNOWN***")) request = Request(requestDict) optimized = request.optimize() if optimized.get("Value", False): gLogger.debug("putRequest: request was optimized") else: gLogger.debug("putRequest: request unchanged", optimized.get("Message", "Nothing could be optimized")) valid = cls.validate(request) if not valid["OK"]: gLogger.error("putRequest: request %s not valid: %s" % (requestName, valid["Message"])) return valid # If NotBefore is not set or user defined, we calculate its value now = datetime.datetime.utcnow().replace(microsecond=0) extraDelay = datetime.timedelta(0) if request.Status not in Request.FINAL_STATES and (not request.NotBefore or request.NotBefore < now): # We don't delay if it is the first insertion if getattr(request, "RequestID", 0): # If it is a constant delay, just set it if cls.constantRequestDelay: extraDelay = datetime.timedelta(minutes=cls.constantRequestDelay) else: # If there is a waiting Operation with Files op = request.getWaiting().get("Value") if op and len(op): attemptList = [opFile.Attempt for opFile in op if opFile.Status == "Waiting"] if attemptList: maxWaitingAttempt = max([opFile.Attempt for opFile in op if opFile.Status == "Waiting"]) # In case it is the first attempt, extraDelay is 0 # maxWaitingAttempt can be None if the operation has no File, like the ForwardDiset extraDelay = datetime.timedelta( minutes=2 * math.log(maxWaitingAttempt) if maxWaitingAttempt else 0 ) request.NotBefore = now + extraDelay gLogger.info( "putRequest: request %s not before %s (extra delay %s)" % (request.RequestName, request.NotBefore, extraDelay) ) requestName = request.RequestName gLogger.info("putRequest: Attempting to set request '%s'" % requestName) return cls.__requestDB.putRequest(request)
def test07Authorization(self): """ Test whether request sets on behalf of others are rejected""" request = Request({"RequestName": "unauthorized"}) request.OwnerDN = 'NotMe' request.OwnerDN = 'AnotherGroup' op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"}) op += File({"LFN": "/lhcb/user/c/cibak/foo"}) request += op res = self.requestClient.putRequest(request) self.assertFalse(res['OK'], res)
def __init__( self, requestObject = None ): """ Constructor function, can specify request object to instantiate FailoverTransfer or a new request object is created. """ self.log = gLogger.getSubLogger( "FailoverTransfer" ) self.replicaMgr = ReplicaManager() self.request = requestObject if not self.request: self.request = Request() self.request.RequestName = 'default_request.xml' self.request.SourceComponent = 'FailoverTransfer'
def test_setGet(self): res = self.transformation.setTransformationName('TestTName') self.assertTrue(res['OK']) description = 'Test transformation description' res = self.transformation.setDescription(description) longDescription = 'Test transformation long description' res = self.transformation.setLongDescription(longDescription) self.assertTrue(res['OK']) res = self.transformation.setType('MCSimulation') self.assertTrue(res['OK']) res = self.transformation.setPlugin('aPlugin') self.assertTrue(res['OK']) # # Test DataOperation Body res = self.transformation.setBody("") self.assertTrue(res['OK']) self.assertEqual(self.transformation.paramValues["Body"], "") res = self.transformation.setBody("_requestType;RemoveReplica") self.assertTrue(res['OK']) self.assertEqual(self.transformation.paramValues["Body"], "_requestType;RemoveReplica") # #Json will turn tuples to lists and strings to unicode transBody = [[u"ReplicateAndRegister", {u"SourceSE": u"FOO-SRM", u"TargetSE": u"BAR-SRM"}], [u"RemoveReplica", {u"TargetSE": u"FOO-SRM"}], ] res = self.transformation.setBody(transBody) self.assertTrue(res['OK']) self.assertEqual(self.transformation.paramValues["Body"], json.dumps(transBody)) # # This is not true if any of the keys or values are not strings, e.g., integers self.assertEqual(json.loads(self.transformation.paramValues["Body"]), transBody) with self.assertRaisesRegexp(TypeError, "Expected list"): self.transformation.setBody({"ReplicateAndRegister": {"foo": "bar"}}) with self.assertRaisesRegexp(TypeError, "Expected tuple"): self.transformation.setBody(["ReplicateAndRegister", "RemoveReplica"]) with self.assertRaisesRegexp(TypeError, "Expected 2-tuple"): self.transformation.setBody([("ReplicateAndRegister", "RemoveReplica", "LogUpload")]) with self.assertRaisesRegexp(TypeError, "Expected string"): self.transformation.setBody([(123, "Parameter:Value")]) with self.assertRaisesRegexp(TypeError, "Expected dictionary"): self.transformation.setBody([("ReplicateAndRegister", "parameter=foo")]) with self.assertRaisesRegexp(TypeError, "Expected string"): self.transformation.setBody([("ReplicateAndRegister", {123: "foo"})]) with self.assertRaisesRegexp(ValueError, "Unknown attribute"): self.transformation.setBody([("ReplicateAndRegister", {"Request": Request()})]) with self.assertRaisesRegexp(TypeError, "Cannot encode"): self.transformation.setBody([("ReplicateAndRegister", {"Arguments": Request()})])
class RequestTaskTests( unittest.TestCase ): """ .. class:: RequestTaskTests """ def setUp( self ): """ test case set up """ self.handlersDict = { "ForwardDISET" : "DIRAC/RequestManagementSystem/private/ForwardDISET" } self.req = Request() self.req.RequestName = "foobarbaz" self.req.OwnerGroup = "lhcb_user" self.req.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=chaen/CN=705305/CN=Christophe Haen" self.op = Operation( { "Type": "ForwardDISET", "Arguments" : "tts10:helloWorldee" } ) self.req.addOperation( self.op ) self.task = None self.mockRC = MagicMock() self.mockObjectOps = MagicMock() self.mockObjectOps.getSections.return_value = {'OK': True, 'Value': ['DataProcessing', 'DataManager']} self.mockObjectOps.getOptionsDict.return_value = {'OK': True, 'Value': {'Group': 'lhcb_user', 'User': '******'}} self.mockOps = MagicMock() self.mockOps.return_value = self.mockObjectOps def tearDown( self ): """ test case tear down """ del self.req del self.op del self.task def testAPI( self ): """ test API """ rt = importlib.import_module( 'DIRAC.RequestManagementSystem.private.RequestTask' ) rt.gMonitor = MagicMock() rt.Operations = self.mockOps rt.CS = MagicMock() self.task = RequestTask( self.req.toJSON()["Value"], self.handlersDict, 'csPath', 'RequestManagement/RequestExecutingAgent', requestClient = self.mockRC ) self.task.requestClient = Mock( return_value = Mock( spec = ReqClient ) ) self.task.requestClient().updateRequest = Mock() self.task.requestClient().updateRequest.return_value = { "OK" : True, "Value" : None } ret = self.task() self.assertEqual( ret["OK"], True , "call failed" ) ret = self.task.setupProxy() print ret
class RequestManagerHandlerTests( unittest.TestCase ): """ .. class:: RequestManagerHandlerTests """ def setUp( self ): """ test setup :param self: self reference """ self.request = Request() self.request.RequestName = "RequestManagerHandlerTests" self.request.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" self.request.OwnerGroup = "dirac_user" self.operation = Operation() self.operation.Type = "ReplicateAndRegister" self.operation.TargetSE = "CERN-USER" self.file = File() self.file.LFN = "/lhcb/user/c/cibak/testFile" self.file.Checksum = "123456" self.file.ChecksumType = "ADLER32" self.request.addOperation( self.operation ) self.operation.addFile( self.file ) # # xml representation of a whole request self.xmlStr = self.request.toXML( True )["Value"] # # request client self.requestClient = RequestClient() def tearDown( self ): """ test case tear down """ del self.request del self.operation del self.file del self.xmlStr def test01PutRequest( self ): """ test set request """ put = self.requestClient.putRequest( self.request ) self.assertEqual( put["OK"], True, "put failed" ) def test02GetRequest( self ): """ test get request """ get = self.requestClient.getRequest( self.request.RequestName ) self.assertEqual( get["OK"], True, "get failed" ) def test03DeleteRequest( self ): """ test delete request """ delete = self.requestClient.deleteRequest( "test" ) self.assertEqual( delete["OK"], True, "delete failed" )
def buildRequest(self, owner, group, sourceSE, targetSE1, targetSE2): files = self.files(owner, group) putAndRegister = Operation() putAndRegister.Type = "PutAndRegister" putAndRegister.TargetSE = sourceSE for fname, lfn, size, checksum, guid in files: putFile = File() putFile.LFN = lfn putFile.PFN = fname putFile.Checksum = checksum putFile.ChecksumType = "adler32" putFile.Size = size putFile.GUID = guid putAndRegister.addFile(putFile) replicateAndRegister = Operation() replicateAndRegister.Type = "ReplicateAndRegister" replicateAndRegister.TargetSE = "%s,%s" % (targetSE1, targetSE2) for fname, lfn, size, checksum, guid in files: repFile = File() repFile.LFN = lfn repFile.Size = size repFile.Checksum = checksum repFile.ChecksumType = "adler32" replicateAndRegister.addFile(repFile) removeReplica = Operation() removeReplica.Type = "RemoveReplica" removeReplica.TargetSE = sourceSE for fname, lfn, size, checksum, guid in files: removeReplica.addFile(File({"LFN": lfn})) removeFile = Operation() removeFile.Type = "RemoveFile" for fname, lfn, size, checksum, guid in files: removeFile.addFile(File({"LFN": lfn})) removeFileInit = Operation() removeFileInit.Type = "RemoveFile" for fname, lfn, size, checksum, guid in files: removeFileInit.addFile(File({"LFN": lfn})) req = Request() req.addOperation(removeFileInit) req.addOperation(putAndRegister) req.addOperation(replicateAndRegister) req.addOperation(removeReplica) req.addOperation(removeFile) return req
def checkRequestAndOp(listOfLFNs): req = Request() req.RequestName = 'MyRequest' op = Operation() op.Type = 'CheckMigration' for index, lfn in enumerate(listOfLFNs): oFile = File() oFile.LFN = lfn oFile.Size = index oFile.Checksum = '01130a%0d' % index oFile.ChecksumType = 'adler32' op.addFile(oFile) req.addOperation(op) return req, op
class ReqClientTestCase(unittest.TestCase): """ .. class:: ReqClientTestCase """ def setUp(self): """ test case set up """ gLogger.setLevel('INFO') self.file = File() self.file.LFN = "/lhcb/user/c/cibak/testFile" self.file.Checksum = "123456" self.file.ChecksumType = "ADLER32" self.file2 = File() self.file2.LFN = "/lhcb/user/f/fstagni/testFile" self.file2.Checksum = "654321" self.file2.ChecksumType = "ADLER32" self.operation = Operation() self.operation.Type = "ReplicateAndRegister" self.operation.TargetSE = "CERN-USER" self.operation.addFile(self.file) self.operation.addFile(self.file2) proxyInfo = getProxyInfo()['Value'] self.request = Request() self.request.RequestName = "RequestManagerHandlerTests" self.request.OwnerDN = proxyInfo['identity'] self.request.OwnerGroup = proxyInfo['group'] self.request.JobID = 123 self.request.addOperation(self.operation) # # JSON representation of a whole request self.jsonStr = self.request.toJSON()['Value'] # # request client self.requestClient = ReqClient() self.stressRequests = 1000 self.bulkRequest = 1000 def tearDown(self): """ clean up """ del self.request del self.operation del self.file del self.jsonStr
class ReqClientTestCase( unittest.TestCase ): """ .. class:: ReqClientTestCase """ def setUp( self ): """ test case set up """ gLogger.setLevel( 'NOTICE' ) self.file = File() self.file.LFN = "/lhcb/user/c/cibak/testFile" self.file.Checksum = "123456" self.file.ChecksumType = "ADLER32" self.file2 = File() self.file2.LFN = "/lhcb/user/f/fstagni/testFile" self.file2.Checksum = "654321" self.file2.ChecksumType = "ADLER32" self.operation = Operation() self.operation.Type = "ReplicateAndRegister" self.operation.TargetSE = "CERN-USER" self.operation.addFile( self.file ) self.operation.addFile( self.file2 ) self.request = Request() self.request.RequestName = "RequestManagerHandlerTests" self.request.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" self.request.OwnerGroup = "dirac_user" self.request.JobID = 123 self.request.addOperation( self.operation ) # # JSON representation of a whole request self.jsonStr = self.request.toJSON()['Value'] # # request client self.requestClient = ReqClient() self.stressRequests = 1000 self.bulkRequest = 1000 def tearDown( self ): """ clean up """ del self.request del self.operation del self.file del self.jsonStr
def test03sql( self ): """ sql insert or update """ operation = Operation() operation.Type = "ReplicateAndRegister" request = Request() request.RequestName = "testRequest" request.RequestID = 1 # # no parent request set try: operation.toSQL() except Exception, error: self.assertEqual( isinstance( error, AttributeError ), True, "wrong exc raised" ) self.assertEqual( str( error ), "RequestID not set", "wrong exc reason" )
def _sendToFailover( rpcStub ): """ Create a ForwardDISET operation for failover """ try: request = Request() request.RequestName = "Accounting.DataStore.%s.%s" % ( time.time(), random.random() ) forwardDISETOp = Operation() forwardDISETOp.Type = "ForwardDISET" forwardDISETOp.Arguments = DEncode.encode( rpcStub ) request.addOperation( forwardDISETOp ) return ReqClient().putRequest( request ) # We catch all the exceptions, because it should never crash except Exception as e: # pylint: disable=broad-except return S_ERROR( ERMSUKN, "Exception sending accounting failover request: %s" % repr( e ) )
def setUp( self ): """ test set up """ self.req = Request() self.req.RequestName = "testRequest" self.op = Operation( {"Type" : "ForwardDISET", "Arguments" : "foobar" } ) self.req.addOperation( self.op ) self.baseOp = OperationHandlerBase()
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup): """ deal with a Request that has just one operation, as it was sofar :param transBody: string, can be an empty string :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split(';') except AttributeError: pass failedTasks = [] # Do not remove sorted, we might pop elements in the loop for taskID, task in taskDict.iteritems(): transID = task['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = task['TargetSE'] # If there are input files if task.get('InputData'): if isinstance(task['InputData'], list): files = task['InputData'] elif isinstance(task['InputData'], basestring): files = task['InputData'].split(';') for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile(trFile) oRequest.addOperation(transfer) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result['OK']: failedTasks.append(taskID) # Remove failed tasks for taskID in failedTasks: taskDict.pop(taskID)
def buildRequest( self, owner, group, sourceSE, targetSE1, targetSE2 ): files = self.files( owner, group ) putAndRegister = Operation() putAndRegister.Type = "PutAndRegister" putAndRegister.TargetSE = sourceSE for fname, lfn, size, checksum, guid in files: putFile = File() putFile.LFN = lfn putFile.PFN = fname putFile.Checksum = checksum putFile.ChecksumType = "adler32" putFile.Size = size putFile.GUID = guid putAndRegister.addFile( putFile ) replicateAndRegister = Operation() replicateAndRegister.Type = "ReplicateAndRegister" replicateAndRegister.TargetSE = "%s,%s" % ( targetSE1, targetSE2 ) for fname, lfn, size, checksum, guid in files: repFile = File() repFile.LFN = lfn repFile.Size = size repFile.Checksum = checksum repFile.ChecksumType = "adler32" replicateAndRegister.addFile( repFile ) removeReplica = Operation() removeReplica.Type = "RemoveReplica" removeReplica.TargetSE = sourceSE for fname, lfn, size, checksum, guid in files: removeReplica.addFile( File( {"LFN": lfn } ) ) removeFile = Operation() removeFile.Type = "RemoveFile" for fname, lfn, size, checksum, guid in files: removeFile.addFile( File( {"LFN": lfn } ) ) removeFileInit = Operation() removeFileInit.Type = "RemoveFile" for fname, lfn, size, checksum, guid in files: removeFileInit.addFile( File( {"LFN": lfn } ) ) req = Request() req.addOperation( removeFileInit ) req.addOperation( putAndRegister ) req.addOperation( replicateAndRegister ) req.addOperation( removeReplica ) req.addOperation( removeFile ) return req
def test01CtorSerilization( self ): """ c'tor and serialization """ # # empty c'tor req = Request() self.assertEqual( isinstance( req, Request ), True ) self.assertEqual( req.JobID, 0 ) self.assertEqual( req.Status, "Waiting" ) req = Request( self.fromDict ) self.assertEqual( isinstance( req, Request ), True ) self.assertEqual( req.RequestName, "test" ) self.assertEqual( req.JobID, 12345 ) self.assertEqual( req.Status, "Waiting" ) toJSON = req.toJSON() self.assertEqual( toJSON["OK"], True, "JSON serialization failed" ) fromJSON = toJSON["Value"] req = Request( fromJSON ) toSQL = req.toSQL() self.assertEqual( toSQL["OK"], True ) toSQL = toSQL["Value"] self.assertEqual( toSQL.startswith( "INSERT" ), True ) req.RequestID = 1 toSQL = req.toSQL() self.assertEqual( toSQL["OK"], True ) toSQL = toSQL["Value"] self.assertEqual( toSQL.startswith( "UPDATE" ), True )
def test01TableDesc(self): """ table description """ tableDict = RequestDB.getTableMeta() self.assertEqual("Request" in tableDict, True) self.assertEqual("Operation" in tableDict, True) self.assertEqual("File" in tableDict, True) self.assertEqual(tableDict["Request"], Request.tableDesc()) self.assertEqual(tableDict["Operation"], Operation.tableDesc()) self.assertEqual(tableDict["File"], File.tableDesc())
def test07List( self ): """ setitem, delitem, getitem and dirty """ r = Request() ops = [ Operation() for i in range( 5 ) ] for op in ops: r.addOperation( op ) for i, op in enumerate( ops ): self.assertEqual( op, r[i], "__getitem__ failed" ) op = Operation() r[0] = op self.assertEqual( op, r[0], "__setitem__ failed" ) del r[0] self.assertEqual( len( r ), 4, "__delitem__ failed" )
def export_putRequest( cls, requestJSON ): """ put a new request into RequestDB :param cls: class ref :param str requestJSON: request serialized to JSON format """ requestName = requestJSON.get( "RequestName", "***UNKNOWN***" ) request = Request( requestJSON ) optimized = request.optimize() if optimized.get("Value", False): gLogger.debug( "putRequest: request was optimized" ) else: gLogger.debug( "putRequest: request unchanged", optimized.get( "Message", "Nothing could be optimize" ) ) valid = cls.validate( request ) if not valid["OK"]: gLogger.error( "putRequest: request %s not valid: %s" % ( requestName, valid["Message"] ) ) return valid requestName = request.RequestName gLogger.info( "putRequest: Attempting to set request '%s'" % requestName ) return cls.__requestDB.putRequest( request )
def test07Authorization(self): """ Test whether request sets on behalf of others are rejected, unless done with Delegation properties This test is kind of stupid though, since we do the same thing than the server... not a real test ! """ request = Request({"RequestName": "unauthorized"}) request.OwnerDN = 'NotMe' request.OwnerDN = 'AnotherGroup' op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"}) op += File({"LFN": "/lhcb/user/c/cibak/foo"}) request += op res = self.requestClient.putRequest(request) credProperties = getProxyInfo()['Value']['groupProperties'] # If the proxy with which we test has delegation, it should work if FULL_DELEGATION in credProperties or LIMITED_DELEGATION in credProperties: self.assertTrue(res['OK'], res) self.requestClient.deleteRequest(res['Value']) # otherwise no else: self.assertFalse(res['OK'], res)
def myRequest(): """Create a request and put it to the db""" request = Request() request.RequestName = 'myAwesomeRemovalRequest.xml' request.JobID = 0 request.SourceComponent = "myScript" remove = Operation() remove.Type = "RemoveFile" lfn = "/ilc/user/s/sailer/test.txt" rmFile = File() rmFile.LFN = lfn remove.addFile( rmFile ) request.addOperation( remove ) isValid = RequestValidator().validate( request ) if not isValid['OK']: raise RuntimeError( "Failover request is not valid: %s" % isValid['Message'] ) else: print("It is a GOGOGO") requestClient = ReqClient() result = requestClient.putRequest( request ) print(result)
def __init__( self, requestJSON, handlersDict, csPath, agentName, standalone=False, requestClient=None): """c'tor :param self: self reference :param str requestJSON: request serialized to JSON :param dict opHandlers: operation handlers """ self.request = Request(requestJSON) # # csPath self.csPath = csPath # # agent name self.agentName = agentName # # standalone flag self.standalone = standalone # # handlers dict self.handlersDict = handlersDict # # handlers class def self.handlers = {} # # own sublogger self.log = gLogger.getSubLogger("pid_%s/%s" % (os.getpid(), self.request.RequestName)) # # get shifters info self.__managersDict = {} shifterProxies = self.__setupManagerProxies() if not shifterProxies["OK"]: self.log.error(shifterProxies["Message"]) # # initialize gMonitor gMonitor.setComponentType(gMonitor.COMPONENT_AGENT) gMonitor.setComponentName(self.agentName) gMonitor.initialize() # # own gMonitor activities gMonitor.registerActivity("RequestAtt", "Requests processed", "RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM) gMonitor.registerActivity("RequestFail", "Requests failed", "RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM) gMonitor.registerActivity("RequestOK", "Requests done", "RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM) if requestClient is None: self.requestClient = ReqClient() else: self.requestClient = requestClient
def __init__( self, requestObject = None, log = None, defaultChecksumType = 'ADLER32' ): """ Constructor function, can specify request object to instantiate FailoverTransfer or a new request object is created. """ self.log = log if not self.log: self.log = gLogger.getSubLogger( "FailoverTransfer" ) self.request = requestObject if not self.request: self.request = Request() self.request.RequestName = 'noname_request' self.request.SourceComponent = 'FailoverTransfer' self.defaultChecksumType = defaultChecksumType
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if ( not owner ) or ( not ownerGroup ): res = getProxyInfo( False, False ) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername( owner ) if not res['OK']: return res ownerDN = res['Value'][0] requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass for taskID in sorted( taskDict ): paramDict = taskDict[taskID] if paramDict['InputData']: transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] if isinstance( paramDict['InputData'], list ): files = paramDict['InputData'] elif isinstance( paramDict['InputData'], basestring ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = _requestName( transID, taskID ) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate( oRequest ) if not isValid['OK']: return isValid taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict )