def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if ( not owner ) or ( not ownerGroup ): res = getProxyInfo( False, False ) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername( owner ) if not res['OK']: return res ownerDN = res['Value'][0] requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass for taskID in sorted( taskDict ): paramDict = taskDict[taskID] if paramDict['InputData']: transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] if isinstance( paramDict['InputData'], list ): files = paramDict['InputData'] elif isinstance( paramDict['InputData'], basestring ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = _requestName( transID, taskID ) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate( oRequest ) if not isValid['OK']: return isValid taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if ( not owner ) or ( not ownerGroup ): res = getProxyInfo( False, False ) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername( owner ) if not res['OK']: return res ownerDN = res['Value'][0] requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass for taskID in sorted( taskDict ): paramDict = taskDict[taskID] if paramDict['InputData']: transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] if isinstance( paramDict['InputData'], list ): files = paramDict['InputData'] elif isinstance( paramDict['InputData'], basestring ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = _requestName( transID, taskID ) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate( oRequest ) if not isValid['OK']: return isValid taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict )
def export_putRequest(self, requestJSON): """ put a new request into RequestDB """ requestDict = json.loads(requestJSON) request = Request(requestDict) operation = Operation() # # create new operation operation.Type = "WMSSecureOutputData" request.insertBefore(operation, request[0]) userDN, userGroup, _ = self.__getOwnerGroupDN('ProductionManager') request.OwnerDN = userDN request.OwnerGroup = userGroup return ReqClient().putRequest(request)
def test07Authorization(self): """ Test whether request sets on behalf of others are rejected""" request = Request({"RequestName": "unauthorized"}) request.OwnerDN = 'NotMe' request.OwnerDN = 'AnotherGroup' op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"}) op += File({"LFN": "/lhcb/user/c/cibak/foo"}) request += op res = self.requestClient.putRequest(request) self.assertFalse(res['OK'], res)
def export_putRequest(self, requestJSON): """ put a new request into RequestDB """ requestDict = json.loads(requestJSON) requestName = requestDict.get("RequestID", requestDict.get('RequestName', "***UNKNOWN***")) #pylint: disable=unused-variable request = Request(requestDict) operation = Operation() # # create new operation operation.Type = "WMSSecureOutputData" request.insertBefore(operation, request[0]) userDN, userGroup, userName = self.__getOwnerGroupDN( 'ProductionManager') #pylint: disable=unused-variable request.OwnerDN = userDN request.OwnerGroup = userGroup return ReqClient().putRequest(request)
def __deleteSandboxFromExternalBackend(self, SEName, SEPFN): if self.getCSOption("DelayedExternalDeletion", True): gLogger.info("Setting deletion request") try: # We need the hostDN used in order to pass these credentials to the # SandboxStoreDB.. hostCertLocation, _ = Locations.getHostCertificateAndKeyLocation( ) hostCert = X509Certificate.X509Certificate() hostCert.loadFromFile(hostCertLocation) hostDN = hostCert.getSubjectDN().get("Value") # use the host authentication to fetch the data result = self.sandboxDB.getSandboxOwner( SEName, SEPFN, hostDN, "hosts") if not result["OK"]: return result _owner, ownerDN, ownerGroup = result["Value"] request = Request() request.RequestName = "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time()) request.OwnerDN = ownerDN request.OwnerGroup = ownerGroup physicalRemoval = Operation() physicalRemoval.Type = "PhysicalRemoval" physicalRemoval.TargetSE = SEName fileToRemove = File() fileToRemove.PFN = SEPFN physicalRemoval.addFile(fileToRemove) request.addOperation(physicalRemoval) return ReqClient().putRequest(request) except Exception as e: gLogger.exception("Exception while setting deletion request") return S_ERROR(f"Cannot set deletion request: {e}") else: gLogger.info("Deleting external Sandbox") try: return StorageElement(SEName).removeFile(SEPFN) except Exception: gLogger.exception( "RM raised an exception while trying to delete a remote sandbox" ) return S_ERROR( "RM raised an exception while trying to delete a remote sandbox" )
def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ): """ Set removal request with the given credentials """ oRequest = Request() oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup oRequest.RequestName = os.path.basename( lfn ).strip() + '_removal_request.xml' oRequest.SourceComponent = 'JobCleaningAgent' removeFile = Operation() removeFile.Type = 'RemoveFile' removedFile = File() removedFile.LFN = lfn removeFile.addFile( removedFile ) oRequest.addOperation( removeFile ) return ReqClient().putRequest( oRequest )
def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ): """ Set removal request with the given credentials """ oRequest = Request() oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup oRequest.RequestName = os.path.basename( lfn ).strip() + '_removal_request.xml' oRequest.SourceComponent = 'JobCleaningAgent' removeFile = Operation() removeFile.Type = 'RemoveFile' removedFile = File() removedFile.LFN = lfn removeFile.addFile( removedFile ) oRequest.addOperation( removeFile ) return ReqClient().putRequest( oRequest )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass for taskID in sorted( taskDict ): paramDict = taskDict[taskID] if paramDict['InputData']: transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] if type( paramDict['InputData'] ) == type( [] ): files = paramDict['InputData'] elif type( paramDict['InputData'] ) == type( '' ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 ) oRequest.OwnerDN = owner oRequest.OwnerGroup = ownerGroup isValid = gRequestValidator.validate( oRequest ) if not isValid['OK']: return isValid taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict )
def test07Authorization(self): """ Test whether request sets on behalf of others are rejected, unless done with Delegation properties This test is kind of stupid though, since we do the same thing than the server... not a real test ! """ request = Request({"RequestName": "unauthorized"}) request.OwnerDN = 'NotMe' request.OwnerDN = 'AnotherGroup' op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"}) op += File({"LFN": "/lhcb/user/c/cibak/foo"}) request += op res = self.requestClient.putRequest(request) credProperties = getProxyInfo()['Value']['groupProperties'] # If the proxy with which we test has delegation, it should work if FULL_DELEGATION in credProperties or LIMITED_DELEGATION in credProperties: self.assertTrue(res['OK'], res) self.requestClient.deleteRequest(res['Value']) # otherwise no else: self.assertFalse(res['OK'], res)
def test02Authorization(self): """ Test whether request sets on behalf of others are rejected, unless done with Delegation properties This test is kind of stupid though, since we do the same thing than the server... not a real test ! """ request = Request({"RequestName": "unauthorized"}) request.OwnerDN = 'NotMe' request.OwnerDN = 'AnotherGroup' op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"}) op += File({"LFN": "/lhcb/user/c/cibak/foo"}) request += op res = self.requestClient.putRequest(request) credProperties = getProxyInfo()['Value']['groupProperties'] # If the proxy with which we test has delegation, it should work if FULL_DELEGATION in credProperties or LIMITED_DELEGATION in credProperties: self.assertTrue(res['OK'], res) self.requestClient.deleteRequest(res['Value']) # otherwise no else: self.assertFalse(res['OK'], res)
def __setRemovalRequest(self, lfn, ownerDN, ownerGroup): """Set removal request with the given credentials""" oRequest = Request() oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup oRequest.RequestName = os.path.basename( lfn).strip() + "_removal_request.xml" oRequest.SourceComponent = "JobCleaningAgent" removeFile = Operation() removeFile.Type = "RemoveFile" removedFile = File() removedFile.LFN = lfn removeFile.addFile(removedFile) oRequest.addOperation(removeFile) # put the request with the owner certificate to make sure it's still a valid DN return ReqClient(useCertificates=True, delegatedDN=ownerDN, delegatedGroup=ownerGroup).putRequest(oRequest)
ar_mock.commit.return_value = {'OK': True, 'Value': ''} jr_mock = MagicMock() jr_mock.setApplicationStatus.return_value = {'OK': True, 'Value': ''} jr_mock.generateForwardDISET.return_value = {'OK': True, 'Value': Operation()} jr_mock.setJobParameter.return_value = {'OK': True, 'Value': 'pippo'} fr_mock = MagicMock() fr_mock.getFiles.return_value = {} fr_mock.setFileStatus.return_value = {'OK': True, 'Value': ''} fr_mock.commit.return_value = {'OK': True, 'Value': ''} fr_mock.generateForwardDISET.return_value = {'OK': True, 'Value': Operation()} rc_mock = Request() rc_mock.RequestName = 'aRequestName' rc_mock.OwnerDN = 'pippo' rc_mock.OwnerGroup = 'pippoGRP' rOp = Operation() rOp.Type = 'PutAndRegister' rOp.TargetSE = 'anSE' f = File() f.LFN = '/foo/bar.py' f.PFN = '/foo/bar.py' rOp.addFile(f) rc_mock.addOperation(rOp) wf_commons = [{'PRODUCTION_ID': prod_id, 'JOB_ID': prod_job_id, 'eventType': '123456789', 'jobType': 'merge', 'configName': 'aConfigName', 'configVersion': 'aConfigVersion', 'outputDataFileMask': '', 'BookkeepingLFNs': 'aa', 'ProductionOutputData': 'ProductionOutputData', 'numberOfEvents': '100', 'JobReport': jr_mock, 'Request': rc_mock, 'AccountingReport': ar_mock, 'FileReport': fr_mock,
def test01fullChain( self ): put = self.requestClient.putRequest( self.request ) self.assert_( put['OK'] ) self.assertEqual( type( put['Value'] ), long ) reqID = put['Value'] # # summary ret = RequestDB().getDBSummary() self.assertEqual( ret, { 'OK': True, 'Value': { 'Operation': { 'ReplicateAndRegister': { 'Waiting': 1L } }, 'Request': { 'Waiting': 1L }, 'File': { 'Waiting': 2L} } } ) get = self.requestClient.getRequest( reqID ) self.assert_( get['OK'] ) self.assertEqual( isinstance( get['Value'], Request ), True ) # # summary - the request became "Assigned" res = RequestDB().getDBSummary() self.assertEqual( res, { 'OK': True, 'Value': { 'Operation': { 'ReplicateAndRegister': { 'Waiting': 1L } }, 'Request': { 'Assigned': 1L }, 'File': { 'Waiting': 2L} } } ) res = self.requestClient.getRequestInfo( reqID ) self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' ) res = self.requestClient.getRequestFileStatus( reqID, self.file.LFN ) self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' ) res = self.requestClient.getRequestFileStatus( reqID, [self.file.LFN] ) self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' ) res = self.requestClient.getDigest( reqID ) self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' ) res = self.requestClient.readRequestsForJobs( [123] ) self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' ) self.assert_( isinstance( res['Value']['Successful'][123], Request ) ) # Adding new request request2 = Request() request2.RequestName = "RequestManagerHandlerTests-2" request2.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" request2.OwnerGroup = "dirac_user" request2.JobID = 456 request2.addOperation( self.operation ) # # update res = self.requestClient.putRequest( request2 ) self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' ) reqID2 = res['Value'] # # get summary again ret = RequestDB().getDBSummary() self.assertEqual( ret, { 'OK': True, 'Value': { 'Operation': { 'ReplicateAndRegister': {'Waiting': 2L } }, 'Request': { 'Waiting': 1L, 'Assigned': 1L }, 'File': { 'Waiting': 4L} } } )
res = self.requestClient.getRequestNamesForJobs( [123] ) self.assert_( res['OK'] ) self.assertEqual( res['Value'], {'Successful': {123L:self.request.RequestName}, 'Failed': {}} ) res = self.requestClient.getRequestNamesList() self.assert_( res['OK'] ) res = self.requestClient.readRequestsForJobs( [123] ) self.assert_( res['OK'] ) self.assert_( isinstance( res['Value']['Successful'][123], Request ) ) # Adding new request request2 = Request() request2.RequestName = "RequestManagerHandlerTests-2" request2.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" request2.OwnerGroup = "dirac_user" request2.JobID = 456 request2.addOperation( self.operation ) # # update res = self.requestClient.putRequest( request2 ) self.assert_( res['OK'] ) # # get summary again ret = RequestDB().getDBSummary() self.assertEqual( ret, { 'OK': True, 'Value': { 'Operation': { 'ReplicateAndRegister': {'Waiting': 2L } }, 'Request': { 'Waiting': 1L, 'Assigned': 1L }, 'File': { 'Waiting': 4L} } } )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if not taskDict: return S_OK({}) if ( not owner ) or ( not ownerGroup ): res = getProxyInfo( False, False ) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername( owner ) if not res['OK']: return res ownerDN = res['Value'][0] requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass # Do not remove sorted, we might pop elements in the loop for taskID in sorted( taskDict ): paramDict = taskDict[taskID] transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] # If there are input files if paramDict['InputData']: if isinstance( paramDict['InputData'], list ): files = paramDict['InputData'] elif isinstance( paramDict['InputData'], basestring ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = _requestName( transID, taskID ) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate( oRequest ) if not isValid['OK']: self.log.error( "Error creating request for task", "%s %s" % ( taskID, isValid ) ) # This works because we loop over a copy of the keys ! taskDict.pop( taskID ) continue taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict )
def test01fullChain( self ): put = self.requestClient.putRequest( self.request ) self.assert_( put['OK'] ) self.assertEqual( type( put['Value'] ), long ) reqID = put['Value'] # # summary ret = RequestDB().getDBSummary() self.assertEqual( ret, { 'OK': True, 'Value': { 'Operation': { 'ReplicateAndRegister': { 'Waiting': 1L } }, 'Request': { 'Waiting': 1L }, 'File': { 'Waiting': 2L} } } ) get = self.requestClient.getRequest( reqID ) self.assert_( get['OK'] ) self.assertEqual( isinstance( get['Value'], Request ), True ) # # summary - the request became "Assigned" res = RequestDB().getDBSummary() self.assertEqual( res, { 'OK': True, 'Value': { 'Operation': { 'ReplicateAndRegister': { 'Waiting': 1L } }, 'Request': { 'Assigned': 1L }, 'File': { 'Waiting': 2L} } } ) res = self.requestClient.getRequestInfo( reqID ) self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' ) res = self.requestClient.getRequestFileStatus( reqID, self.file.LFN ) self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' ) res = self.requestClient.getRequestFileStatus( reqID, [self.file.LFN] ) self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' ) res = self.requestClient.getDigest( reqID ) self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' ) res = self.requestClient.readRequestsForJobs( [123] ) self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' ) self.assert_( isinstance( res['Value']['Successful'][123], Request ) ) # Adding new request request2 = Request() request2.RequestName = "RequestManagerHandlerTests-2" request2.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" request2.OwnerGroup = "dirac_user" request2.JobID = 456 request2.addOperation( self.operation ) # # update res = self.requestClient.putRequest( request2 ) self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' ) reqID2 = res['Value'] # # get summary again ret = RequestDB().getDBSummary() self.assertEqual( ret, { 'OK': True, 'Value': { 'Operation': { 'ReplicateAndRegister': {'Waiting': 2L } }, 'Request': { 'Waiting': 1L, 'Assigned': 1L }, 'File': { 'Waiting': 4L} } } )
123L: self.request.RequestName }, 'Failed': {} }) res = self.requestClient.getRequestNamesList() self.assert_(res['OK']) res = self.requestClient.readRequestsForJobs([123]) self.assert_(res['OK']) self.assert_(isinstance(res['Value']['Successful'][123], Request)) # Adding new request request2 = Request() request2.RequestName = "RequestManagerHandlerTests-2" request2.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" request2.OwnerGroup = "dirac_user" request2.JobID = 456 request2.addOperation(self.operation) # # update res = self.requestClient.putRequest(request2) self.assert_(res['OK']) # # get summary again ret = RequestDB().getDBSummary() self.assertEqual( ret, { 'OK': True, 'Value': { 'Operation': {
def prepareTransformationTasks(self, transBody, taskDict, owner='', ownerGroup='', ownerDN=''): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if not taskDict: return S_OK({}) if (not owner) or (not ownerGroup): res = getProxyInfo(False, False) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername(owner) if not res['OK']: return res ownerDN = res['Value'][0] requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split(';') except AttributeError: pass # Do not remove sorted, we might pop elements in the loop for taskID in sorted(taskDict): paramDict = taskDict[taskID] transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] # If there are input files if paramDict['InputData']: if isinstance(paramDict['InputData'], list): files = paramDict['InputData'] elif isinstance(paramDict['InputData'], basestring): files = paramDict['InputData'].split(';') for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile(trFile) oRequest.addOperation(transfer) oRequest.RequestName = _requestName(transID, taskID) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate(oRequest) if not isValid['OK']: self.log.error("Error creating request for task", "%s %s" % (taskID, isValid)) # This works because we loop over a copy of the keys ! taskDict.pop(taskID) continue taskDict[taskID]['TaskObject'] = oRequest return S_OK(taskDict)