def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if ( not owner ) or ( not ownerGroup ): res = getProxyInfo( False, False ) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername( owner ) if not res['OK']: return res ownerDN = res['Value'][0] requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass for taskID in sorted( taskDict ): paramDict = taskDict[taskID] if paramDict['InputData']: transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] if isinstance( paramDict['InputData'], list ): files = paramDict['InputData'] elif isinstance( paramDict['InputData'], basestring ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = _requestName( transID, taskID ) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate( oRequest ) if not isValid['OK']: return isValid taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if ( not owner ) or ( not ownerGroup ): res = getProxyInfo( False, False ) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername( owner ) if not res['OK']: return res ownerDN = res['Value'][0] requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass for taskID in sorted( taskDict ): paramDict = taskDict[taskID] if paramDict['InputData']: transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] if isinstance( paramDict['InputData'], list ): files = paramDict['InputData'] elif isinstance( paramDict['InputData'], basestring ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = _requestName( transID, taskID ) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate( oRequest ) if not isValid['OK']: return isValid taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict )
def export_putRequest(self, requestJSON): """ put a new request into RequestDB """ requestDict = json.loads(requestJSON) request = Request(requestDict) operation = Operation() # # create new operation operation.Type = "WMSSecureOutputData" request.insertBefore(operation, request[0]) userDN, userGroup, _ = self.__getOwnerGroupDN('ProductionManager') request.OwnerDN = userDN request.OwnerGroup = userGroup return ReqClient().putRequest(request)
def export_putRequest(self, requestJSON): """ put a new request into RequestDB """ requestDict = json.loads(requestJSON) requestName = requestDict.get("RequestID", requestDict.get('RequestName', "***UNKNOWN***")) #pylint: disable=unused-variable request = Request(requestDict) operation = Operation() # # create new operation operation.Type = "WMSSecureOutputData" request.insertBefore(operation, request[0]) userDN, userGroup, userName = self.__getOwnerGroupDN( 'ProductionManager') #pylint: disable=unused-variable request.OwnerDN = userDN request.OwnerGroup = userGroup return ReqClient().putRequest(request)
def __deleteSandboxFromExternalBackend(self, SEName, SEPFN): if self.getCSOption("DelayedExternalDeletion", True): gLogger.info("Setting deletion request") try: # We need the hostDN used in order to pass these credentials to the # SandboxStoreDB.. hostCertLocation, _ = Locations.getHostCertificateAndKeyLocation( ) hostCert = X509Certificate.X509Certificate() hostCert.loadFromFile(hostCertLocation) hostDN = hostCert.getSubjectDN().get("Value") # use the host authentication to fetch the data result = self.sandboxDB.getSandboxOwner( SEName, SEPFN, hostDN, "hosts") if not result["OK"]: return result _owner, ownerDN, ownerGroup = result["Value"] request = Request() request.RequestName = "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time()) request.OwnerDN = ownerDN request.OwnerGroup = ownerGroup physicalRemoval = Operation() physicalRemoval.Type = "PhysicalRemoval" physicalRemoval.TargetSE = SEName fileToRemove = File() fileToRemove.PFN = SEPFN physicalRemoval.addFile(fileToRemove) request.addOperation(physicalRemoval) return ReqClient().putRequest(request) except Exception as e: gLogger.exception("Exception while setting deletion request") return S_ERROR(f"Cannot set deletion request: {e}") else: gLogger.info("Deleting external Sandbox") try: return StorageElement(SEName).removeFile(SEPFN) except Exception: gLogger.exception( "RM raised an exception while trying to delete a remote sandbox" ) return S_ERROR( "RM raised an exception while trying to delete a remote sandbox" )
def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ): """ Set removal request with the given credentials """ oRequest = Request() oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup oRequest.RequestName = os.path.basename( lfn ).strip() + '_removal_request.xml' oRequest.SourceComponent = 'JobCleaningAgent' removeFile = Operation() removeFile.Type = 'RemoveFile' removedFile = File() removedFile.LFN = lfn removeFile.addFile( removedFile ) oRequest.addOperation( removeFile ) return ReqClient().putRequest( oRequest )
def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ): """ Set removal request with the given credentials """ oRequest = Request() oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup oRequest.RequestName = os.path.basename( lfn ).strip() + '_removal_request.xml' oRequest.SourceComponent = 'JobCleaningAgent' removeFile = Operation() removeFile.Type = 'RemoveFile' removedFile = File() removedFile.LFN = lfn removeFile.addFile( removedFile ) oRequest.addOperation( removeFile ) return ReqClient().putRequest( oRequest )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass for taskID in sorted( taskDict ): paramDict = taskDict[taskID] if paramDict['InputData']: transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] if type( paramDict['InputData'] ) == type( [] ): files = paramDict['InputData'] elif type( paramDict['InputData'] ) == type( '' ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 ) oRequest.OwnerDN = owner oRequest.OwnerGroup = ownerGroup isValid = gRequestValidator.validate( oRequest ) if not isValid['OK']: return isValid taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict )
def __setRemovalRequest(self, lfn, ownerDN, ownerGroup): """Set removal request with the given credentials""" oRequest = Request() oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup oRequest.RequestName = os.path.basename( lfn).strip() + "_removal_request.xml" oRequest.SourceComponent = "JobCleaningAgent" removeFile = Operation() removeFile.Type = "RemoveFile" removedFile = File() removedFile.LFN = lfn removeFile.addFile(removedFile) oRequest.addOperation(removeFile) # put the request with the owner certificate to make sure it's still a valid DN return ReqClient(useCertificates=True, delegatedDN=ownerDN, delegatedGroup=ownerGroup).putRequest(oRequest)
res = self.requestClient.getRequestNamesForJobs( [123] ) self.assert_( res['OK'] ) self.assertEqual( res['Value'], {'Successful': {123L:self.request.RequestName}, 'Failed': {}} ) res = self.requestClient.getRequestNamesList() self.assert_( res['OK'] ) res = self.requestClient.readRequestsForJobs( [123] ) self.assert_( res['OK'] ) self.assert_( isinstance( res['Value']['Successful'][123], Request ) ) # Adding new request request2 = Request() request2.RequestName = "RequestManagerHandlerTests-2" request2.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" request2.OwnerGroup = "dirac_user" request2.JobID = 456 request2.addOperation( self.operation ) # # update res = self.requestClient.putRequest( request2 ) self.assert_( res['OK'] ) # # get summary again ret = RequestDB().getDBSummary() self.assertEqual( ret, { 'OK': True, 'Value': { 'Operation': { 'ReplicateAndRegister': {'Waiting': 2L } }, 'Request': { 'Waiting': 1L, 'Assigned': 1L }, 'File': { 'Waiting': 4L} } } )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if not taskDict: return S_OK({}) if ( not owner ) or ( not ownerGroup ): res = getProxyInfo( False, False ) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername( owner ) if not res['OK']: return res ownerDN = res['Value'][0] requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass # Do not remove sorted, we might pop elements in the loop for taskID in sorted( taskDict ): paramDict = taskDict[taskID] transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] # If there are input files if paramDict['InputData']: if isinstance( paramDict['InputData'], list ): files = paramDict['InputData'] elif isinstance( paramDict['InputData'], basestring ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = _requestName( transID, taskID ) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate( oRequest ) if not isValid['OK']: self.log.error( "Error creating request for task", "%s %s" % ( taskID, isValid ) ) # This works because we loop over a copy of the keys ! taskDict.pop( taskID ) continue taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict )
}, 'Failed': {} }) res = self.requestClient.getRequestNamesList() self.assert_(res['OK']) res = self.requestClient.readRequestsForJobs([123]) self.assert_(res['OK']) self.assert_(isinstance(res['Value']['Successful'][123], Request)) # Adding new request request2 = Request() request2.RequestName = "RequestManagerHandlerTests-2" request2.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" request2.OwnerGroup = "dirac_user" request2.JobID = 456 request2.addOperation(self.operation) # # update res = self.requestClient.putRequest(request2) self.assert_(res['OK']) # # get summary again ret = RequestDB().getDBSummary() self.assertEqual( ret, { 'OK': True, 'Value': { 'Operation': { 'ReplicateAndRegister': {
def prepareTransformationTasks(self, transBody, taskDict, owner='', ownerGroup='', ownerDN=''): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if not taskDict: return S_OK({}) if (not owner) or (not ownerGroup): res = getProxyInfo(False, False) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername(owner) if not res['OK']: return res ownerDN = res['Value'][0] requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split(';') except AttributeError: pass # Do not remove sorted, we might pop elements in the loop for taskID in sorted(taskDict): paramDict = taskDict[taskID] transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] # If there are input files if paramDict['InputData']: if isinstance(paramDict['InputData'], list): files = paramDict['InputData'] elif isinstance(paramDict['InputData'], basestring): files = paramDict['InputData'].split(';') for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile(trFile) oRequest.addOperation(transfer) oRequest.RequestName = _requestName(transID, taskID) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate(oRequest) if not isValid['OK']: self.log.error("Error creating request for task", "%s %s" % (taskID, isValid)) # This works because we loop over a copy of the keys ! taskDict.pop(taskID) continue taskDict[taskID]['TaskObject'] = oRequest return S_OK(taskDict)
def test01fullChain( self ): put = self.requestClient.putRequest( self.request ) self.assert_( put['OK'] ) self.assertEqual( type( put['Value'] ), long ) reqID = put['Value'] # # summary ret = RequestDB().getDBSummary() self.assertEqual( ret, { 'OK': True, 'Value': { 'Operation': { 'ReplicateAndRegister': { 'Waiting': 1L } }, 'Request': { 'Waiting': 1L }, 'File': { 'Waiting': 2L} } } ) get = self.requestClient.getRequest( reqID ) self.assert_( get['OK'] ) self.assertEqual( isinstance( get['Value'], Request ), True ) # # summary - the request became "Assigned" res = RequestDB().getDBSummary() self.assertEqual( res, { 'OK': True, 'Value': { 'Operation': { 'ReplicateAndRegister': { 'Waiting': 1L } }, 'Request': { 'Assigned': 1L }, 'File': { 'Waiting': 2L} } } ) res = self.requestClient.getRequestInfo( reqID ) self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' ) res = self.requestClient.getRequestFileStatus( reqID, self.file.LFN ) self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' ) res = self.requestClient.getRequestFileStatus( reqID, [self.file.LFN] ) self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' ) res = self.requestClient.getDigest( reqID ) self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' ) res = self.requestClient.readRequestsForJobs( [123] ) self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' ) self.assert_( isinstance( res['Value']['Successful'][123], Request ) ) # Adding new request request2 = Request() request2.RequestName = "RequestManagerHandlerTests-2" request2.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" request2.OwnerGroup = "dirac_user" request2.JobID = 456 request2.addOperation( self.operation ) # # update res = self.requestClient.putRequest( request2 ) self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' ) reqID2 = res['Value'] # # get summary again ret = RequestDB().getDBSummary() self.assertEqual( ret, { 'OK': True, 'Value': { 'Operation': { 'ReplicateAndRegister': {'Waiting': 2L } }, 'Request': { 'Waiting': 1L, 'Assigned': 1L }, 'File': { 'Waiting': 4L} } } )
def test01fullChain( self ): put = self.requestClient.putRequest( self.request ) self.assert_( put['OK'] ) self.assertEqual( type( put['Value'] ), long ) reqID = put['Value'] # # summary ret = RequestDB().getDBSummary() self.assertEqual( ret, { 'OK': True, 'Value': { 'Operation': { 'ReplicateAndRegister': { 'Waiting': 1L } }, 'Request': { 'Waiting': 1L }, 'File': { 'Waiting': 2L} } } ) get = self.requestClient.getRequest( reqID ) self.assert_( get['OK'] ) self.assertEqual( isinstance( get['Value'], Request ), True ) # # summary - the request became "Assigned" res = RequestDB().getDBSummary() self.assertEqual( res, { 'OK': True, 'Value': { 'Operation': { 'ReplicateAndRegister': { 'Waiting': 1L } }, 'Request': { 'Assigned': 1L }, 'File': { 'Waiting': 2L} } } ) res = self.requestClient.getRequestInfo( reqID ) self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' ) res = self.requestClient.getRequestFileStatus( reqID, self.file.LFN ) self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' ) res = self.requestClient.getRequestFileStatus( reqID, [self.file.LFN] ) self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' ) res = self.requestClient.getDigest( reqID ) self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' ) res = self.requestClient.readRequestsForJobs( [123] ) self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' ) self.assert_( isinstance( res['Value']['Successful'][123], Request ) ) # Adding new request request2 = Request() request2.RequestName = "RequestManagerHandlerTests-2" request2.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" request2.OwnerGroup = "dirac_user" request2.JobID = 456 request2.addOperation( self.operation ) # # update res = self.requestClient.putRequest( request2 ) self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' ) reqID2 = res['Value'] # # get summary again ret = RequestDB().getDBSummary() self.assertEqual( ret, { 'OK': True, 'Value': { 'Operation': { 'ReplicateAndRegister': {'Waiting': 2L } }, 'Request': { 'Waiting': 1L, 'Assigned': 1L }, 'File': { 'Waiting': 4L} } } )
jr_mock = MagicMock() jr_mock.setApplicationStatus.return_value = {'OK': True, 'Value': ''} jr_mock.generateForwardDISET.return_value = {'OK': True, 'Value': Operation()} jr_mock.setJobParameter.return_value = {'OK': True, 'Value': 'pippo'} fr_mock = MagicMock() fr_mock.getFiles.return_value = {} fr_mock.setFileStatus.return_value = {'OK': True, 'Value': ''} fr_mock.commit.return_value = {'OK': True, 'Value': ''} fr_mock.generateForwardDISET.return_value = {'OK': True, 'Value': Operation()} rc_mock = Request() rc_mock.RequestName = 'aRequestName' rc_mock.OwnerDN = 'pippo' rc_mock.OwnerGroup = 'pippoGRP' rOp = Operation() rOp.Type = 'PutAndRegister' rOp.TargetSE = 'anSE' f = File() f.LFN = '/foo/bar.py' f.PFN = '/foo/bar.py' rOp.addFile(f) rc_mock.addOperation(rOp) wf_commons = [{'PRODUCTION_ID': prod_id, 'JOB_ID': prod_job_id, 'eventType': '123456789', 'jobType': 'merge', 'configName': 'aConfigName', 'configVersion': 'aConfigVersion', 'outputDataFileMask': '', 'BookkeepingLFNs': 'aa', 'ProductionOutputData': 'ProductionOutputData', 'numberOfEvents': '100', 'JobReport': jr_mock, 'Request': rc_mock, 'AccountingReport': ar_mock, 'FileReport': fr_mock, 'SystemConfig': 'sys_config', 'runNumber': 'Unknown', 'gaudiSteps': ['someApp_1'],