def __init__(self, *args, **kwargs): """ c'tor """ TaskManagerAgentBase.__init__(self, *args, **kwargs) self.transType = [] self.taskManager = None
def test__fillTheQueue(mocker, operationsOnTransformationsDict, expected): mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule', side_effect=mockAM) mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport', side_effect=MagicMock()) tmab = TaskManagerAgentBase() tmab.log = gLogger tmab.am_getOption = mockAM tmab._fillTheQueue(operationsOnTransformationsDict) assert tmab.transInQueue == expected[0] assert tmab.transQueue.qsize() == expected[1]
def test_updateTaskStatusSuccess(mocker, tcMockReturnValue, tmMockGetSubmittedTaskStatusReturnvalue, expected): mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule', side_effect=mockAM) mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport', side_effect=MagicMock()) mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.TaskManagerAgentBase.am_getOption', side_effect=mockAM) tmab = TaskManagerAgentBase() tc_mock.getTransformationTasks.return_value = tcMockReturnValue tm_mock.getSubmittedTaskStatus.return_value = tmMockGetSubmittedTaskStatusReturnvalue res = tmab.updateTaskStatus(transIDOPBody, clients) assert res['OK'] == expected
def __init__( self, *args, **kwargs ): ''' c'tor ''' TaskManagerAgentBase.__init__( self, *args, **kwargs ) self.taskManager = WorkflowTasks( transClient = self.transClient ) self.shifterProxy = 'ProductionManager' agentTSTypes = self.am_getOption( 'TransType', [] ) if agentTSTypes: self.transType = agentTSTypes else: self.transType = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge'] )
def __init__(self, *args, **kwargs): """ c'tor """ TaskManagerAgentBase.__init__(self, *args, **kwargs) self.taskManager = RequestTasks(transClient=self.transClient) self.shifterProxy = "ProductionManager" agentTSTypes = self.am_getOption("TransType", []) if agentTSTypes: self.transType = agentTSTypes else: self.transType = Operations().getValue("Transformations/DataManipulation", ["Replication", "Removal"])
def initialize(self): """ Sets defaults """ TaskManagerAgentBase.initialize(self) RequestTasks.__init__(self) self.transType = ['Replication', 'Removal'] # This sets the Default Proxy to used as that defined under # /Operations/Shifter/ProductionManager # the shifterProxy option in the Configuration can be used to change this default. self.am_setOption('shifterProxy', 'ProductionManager') return S_OK()
def __init__( self, *args, **kwargs ): ''' c'tor ''' TaskManagerAgentBase.__init__( self, *args, **kwargs ) self.taskManager = RequestTasks( transClient = self.transClient ) self.shifterProxy = 'ProductionManager' agentTSTypes = self.am_getOption( 'TransType', [] ) if agentTSTypes: self.transType = agentTSTypes else: self.transType = Operations().getValue( 'Transformations/DataManipulation', ['Replication', 'Removal'] )
def initialize( self ): """ Sets defaults """ TaskManagerAgentBase.initialize( self ) WorkflowTasks.__init__( self ) self.transType = self.am_getOption( "TransType", ['MCSimulation', 'DataReconstruction', 'DataStripping', 'MCStripping', 'Merge'] ) # This sets the Default Proxy to used as that defined under # /Operations/Shifter/ProductionManager # the shifterProxy option in the Configuration can be used to change this default. self.am_setOption( 'shifterProxy', 'ProductionManager' ) return S_OK()
def initialize( self ): """ Sets defaults """ TaskManagerAgentBase.initialize( self ) RequestTasks.__init__( self ) self.transType = ['Replication', 'Removal'] # This sets the Default Proxy to used as that defined under # /Operations/Shifter/ProductionManager # the shifterProxy option in the Configuration can be used to change this default. self.am_setOption( 'shifterProxy', 'ProductionManager' ) return S_OK()
def __init__(self, *args, **kwargs): ''' c'tor ''' TaskManagerAgentBase.__init__(self, *args, **kwargs) self.taskManager = RequestTasks(transClient=self.transClient) self.shifterProxy = 'ProductionManager' agentTSTypes = self.am_getOption('TransType', []) if agentTSTypes: self.transType = agentTSTypes else: self.transType = Operations().getValue( 'Transformations/DataManipulation', ['Replication', 'Removal'])
def test_checkReservedTasks(mocker, tcMockGetTransformationTasksReturnValue, tmMockUpdateTransformationReservedTasksReturnValue, tcMockSetTaskStatusAndWmsIDReturnValue, expected): mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule', side_effect=mockAM) mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport', side_effect=MagicMock()) tmab = TaskManagerAgentBase() tc_mock.getTransformationTasks.return_value = tcMockGetTransformationTasksReturnValue tm_mock.updateTransformationReservedTasks.return_value = tmMockUpdateTransformationReservedTasksReturnValue tc_mock.setTaskStatusAndWmsID.return_value = tcMockSetTaskStatusAndWmsIDReturnValue res = tmab.checkReservedTasks(transIDOPBody, clients) assert res['OK'] == expected
def test_updateFileStatusSuccess(mocker, tcMockGetTransformationFilesReturnValue, tmMockGetSubmittedFileStatusReturnValue, expected): mocker.patch( "DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule", side_effect=mockAM) mocker.patch( "DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport", side_effect=MagicMock()) tmab = TaskManagerAgentBase() tc_mock.getTransformationFiles.return_value = tcMockGetTransformationFilesReturnValue tm_mock.getSubmittedFileStatus.return_value = tmMockGetSubmittedFileStatusReturnValue res = tmab.updateFileStatus(transDict, clients) assert res["OK"] == expected
def initialize(self): """ Sets defaults """ TaskManagerAgentBase.initialize(self) WorkflowTasks.__init__(self) self.transType = self.am_getOption("TransType", [ 'MCSimulation', 'DataReconstruction', 'DataStripping', 'MCStripping', 'Merge' ]) # This sets the Default Proxy to used as that defined under # /Operations/Shifter/ProductionManager # the shifterProxy option in the Configuration can be used to change this default. self.am_setOption('shifterProxy', 'ProductionManager') return S_OK()
def test_submitTasks(mocker, tcMockGetTasksToSubmitReturnValue, tmMockPrepareTransformationTasksReturnValue, tmMockSubmitTransformationTasksReturnValue, tmMockUpdateDBAfterTaskSubmissionReturnValue, expected): mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule', side_effect=mockAM) mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport', side_effect=MagicMock()) tmab = TaskManagerAgentBase() tc_mock.getTasksToSubmit.return_value = tcMockGetTasksToSubmitReturnValue tm_mock.prepareTransformationTasks.return_value = tmMockPrepareTransformationTasksReturnValue tm_mock.submitTransformationTasks.return_value = tmMockSubmitTransformationTasksReturnValue tm_mock.updateDBAfterTaskSubmission.return_value = tmMockUpdateDBAfterTaskSubmissionReturnValue res = tmab.submitTasks(transIDOPBody, clients) assert res['OK'] == expected
def test_checkReservedTasks(mocker, tcMockGetTransformationTasksReturnValue, tmMockUpdateTransformationReservedTasksReturnValue, tcMockSetTaskStatusAndWmsIDReturnValue, expected): mocker.patch( 'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule', side_effect=mockAM) mocker.patch( 'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport', side_effect=MagicMock()) tmab = TaskManagerAgentBase() tc_mock.getTransformationTasks.return_value = tcMockGetTransformationTasksReturnValue tm_mock.updateTransformationReservedTasks.return_value = tmMockUpdateTransformationReservedTasksReturnValue tc_mock.setTaskStatusAndWmsID.return_value = tcMockSetTaskStatusAndWmsIDReturnValue res = tmab.checkReservedTasks(transIDOPBody, clients) assert res['OK'] == expected
def __init__(self, *args, **kwargs): ''' c'tor ''' TaskManagerAgentBase.__init__(self, *args, **kwargs) self.submissionClient = WMSClient() self.taskManager = WorkflowTasks( transClient=self.transClient, submissionClient=self.submissionClient) self.shifterProxy = 'ProductionManager' agentTSTypes = self.am_getOption('TransType', []) if agentTSTypes: self.transType = agentTSTypes else: self.transType = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge'])
def _getClients( self ): """ Here the taskManager becomes a RequestTasks object """ res = TaskManagerAgentBase._getClients( self ) threadTaskManager = RequestTasks() res.update( {'TaskManager': threadTaskManager} ) return res
def initialize(self): """Standard initialize method""" res = TaskManagerAgentBase.initialize(self) if not res["OK"]: return res objLoader = ObjectLoader() _class = objLoader.loadObject( "TransformationSystem.Client.RequestTasks", "RequestTasks") if not _class["OK"]: raise Exception(_class["Message"]) self.requestTasksCls = _class["Value"] # clients self.taskManager = self.requestTasksCls(transClient=self.transClient) agentTSTypes = self.am_getOption("TransType", []) if agentTSTypes: self.transType = agentTSTypes else: self.transType = Operations().getValue( "Transformations/DataManipulation", ["Replication", "Removal"]) return S_OK()
def _getClients(self): """ Here the taskManager becomes a RequestTasks object """ res = TaskManagerAgentBase._getClients(self) threadTaskManager = RequestTasks() res.update({'TaskManager': threadTaskManager}) return res
def test_submitTasks(mocker, tcMockGetTasksToSubmitReturnValue, tmMockPrepareTransformationTasksReturnValue, tmMockSubmitTransformationTasksReturnValue, tmMockUpdateDBAfterTaskSubmissionReturnValue, expected): mocker.patch( 'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule', side_effect=mockAM) mocker.patch( 'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport', side_effect=MagicMock()) tmab = TaskManagerAgentBase() tc_mock.getTasksToSubmit.return_value = tcMockGetTasksToSubmitReturnValue tm_mock.prepareTransformationTasks.return_value = tmMockPrepareTransformationTasksReturnValue tm_mock.submitTransformationTasks.return_value = tmMockSubmitTransformationTasksReturnValue tm_mock.updateDBAfterTaskSubmission.return_value = tmMockUpdateDBAfterTaskSubmissionReturnValue res = tmab.submitTasks(transIDOPBody, clients) assert res['OK'] == expected
def test_updateTaskStatusSuccess(mocker, tcMockReturnValue, tmMockGetSubmittedTaskStatusReturnvalue, expected): mocker.patch( 'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule', side_effect=mockAM) mocker.patch( 'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport', side_effect=MagicMock()) mocker.patch( 'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.TaskManagerAgentBase.am_getOption', side_effect=mockAM) tmab = TaskManagerAgentBase() tc_mock.getTransformationTasks.return_value = tcMockReturnValue tm_mock.getSubmittedTaskStatus.return_value = tmMockGetSubmittedTaskStatusReturnvalue res = tmab.updateTaskStatus(transIDOPBody, clients) assert res['OK'] == expected
def test_submitTasks( mocker, tcMockGetTasksToSubmitReturnValue, tmMockPrepareTransformationTasksReturnValue, tmMockSubmitTransformationTasksReturnValue, tmMockUpdateDBAfterTaskSubmissionReturnValue, expected, ): mocker.patch( "DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule", side_effect=mockAM) mocker.patch( "DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport", side_effect=MagicMock()) tmab = TaskManagerAgentBase() tc_mock.getTasksToSubmit.return_value = tcMockGetTasksToSubmitReturnValue tm_mock.prepareTransformationTasks.return_value = tmMockPrepareTransformationTasksReturnValue tm_mock.submitTransformationTasks.return_value = tmMockSubmitTransformationTasksReturnValue tm_mock.updateDBAfterTaskSubmission.return_value = tmMockUpdateDBAfterTaskSubmissionReturnValue res = tmab.submitTasks(transDict, clients) assert res["OK"] == expected tmab.maxParametricJobs = 10 tmab.bulkSubmissionFlag = True res = tmab.submitTasks(transDict, clients) assert res["OK"] == expected
def initialize(self): """Standard initialize method""" res = TaskManagerAgentBase.initialize(self) if not res["OK"]: return res agentTSTypes = self.am_getOption("TransType", []) if agentTSTypes: self.transType = agentTSTypes else: self.transType = Operations().getValue("Transformations/DataProcessing", ["MCSimulation", "Merge"]) return S_OK()
def initialize(self): """ Standard initialize method """ res = TaskManagerAgentBase.initialize(self) if not res['OK']: return res agentTSTypes = self.am_getOption('TransType', []) if agentTSTypes: self.transType = agentTSTypes else: self.transType = Operations().getValue('Transformations/DataProcessing', ['MCSimulation', 'Merge']) return S_OK()
def initialize( self ): """ Standard initialize method """ res = TaskManagerAgentBase.initialize( self ) if not res['OK']: return res # clients self.taskManager = RequestTasks( transClient = self.transClient ) agentTSTypes = self.am_getOption( 'TransType', [] ) if agentTSTypes: self.transType = agentTSTypes else: self.transType = Operations().getValue( 'Transformations/DataManipulation', ['Replication', 'Removal'] ) return S_OK()
def initialize(self): """ Standard initialize method """ res = TaskManagerAgentBase.initialize(self) if not res['OK']: return res # clients self.taskManager = RequestTasks(transClient=self.transClient) agentTSTypes = self.am_getOption('TransType', []) if agentTSTypes: self.transType = agentTSTypes else: self.transType = Operations().getValue( 'Transformations/DataManipulation', ['Replication', 'Removal']) return S_OK()
def test__fillTheQueue(mocker, operationsOnTransformationsDict, expected): mocker.patch( 'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule', side_effect=mockAM) mocker.patch( 'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport', side_effect=MagicMock()) tmab = TaskManagerAgentBase() tmab.log = gLogger tmab.am_getOption = mockAM tmab._fillTheQueue(operationsOnTransformationsDict) assert tmab.transInQueue == expected[0] assert tmab.transQueue.qsize() == expected[1]
def setUp( self ): self.mockAM = MagicMock() self.tmab_m = importlib.import_module( 'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase' ) self.tmab_m.AgentModule = self.mockAM self.tmab_m.FileReport = MagicMock() self.tmab = TaskManagerAgentBase() self.tmab.log = gLogger self.tmab.am_getOption = self.mockAM self.tmab.log.setLevel( 'DEBUG' ) self.ta_m = importlib.import_module( 'DIRAC.TransformationSystem.Agent.TransformationAgent' ) self.ta_m.AgentModule = self.mockAM self.ta = TransformationAgent() self.ta.log = gLogger self.ta.am_getOption = self.mockAM self.tmab.log.setLevel( 'DEBUG' ) self.tc_mock = MagicMock() self.tm_mock = MagicMock()
import pytest from mock import MagicMock from DIRAC import gLogger # sut from DIRAC.TransformationSystem.Agent.TaskManagerAgentBase import TaskManagerAgentBase from DIRAC.TransformationSystem.Agent.TransformationAgent import TransformationAgent gLogger.setLevel('DEBUG') mockAM = MagicMock() tmab_m = importlib.import_module( 'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase') tmab_m.AgentModule = mockAM tmab_m.FileReport = MagicMock() tmab = TaskManagerAgentBase() tmab.log = gLogger tmab.am_getOption = mockAM tmab.log.setLevel('DEBUG') @pytest.mark.parametrize("operationsOnTransformationsDict, expected", [({ 1: { 'Operations': ['op1', 'op2'], 'Body': 'veryBigBody' } }, ([1], 1)), ({ 2: { 'Operations': ['op3', 'op2'],