def __init__(self, *args, **kwargs):
""" c'tor
"""
TaskManagerAgentBase.__init__(self, *args, **kwargs)
self.transType = []
self.taskManager = None
def __init__(self, *args, **kwargs):
""" c'tor
"""
TaskManagerAgentBase.__init__(self, *args, **kwargs)
self.transType = []
self.taskManager = None
def test__fillTheQueue(mocker, operationsOnTransformationsDict, expected):
mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule', side_effect=mockAM)
mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport', side_effect=MagicMock())
tmab = TaskManagerAgentBase()
tmab.log = gLogger
tmab.am_getOption = mockAM
tmab._fillTheQueue(operationsOnTransformationsDict)
assert tmab.transInQueue == expected[0]
assert tmab.transQueue.qsize() == expected[1]
def test_updateTaskStatusSuccess(mocker, tcMockReturnValue, tmMockGetSubmittedTaskStatusReturnvalue, expected):
mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule', side_effect=mockAM)
mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport', side_effect=MagicMock())
mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.TaskManagerAgentBase.am_getOption',
side_effect=mockAM)
tmab = TaskManagerAgentBase()
tc_mock.getTransformationTasks.return_value = tcMockReturnValue
tm_mock.getSubmittedTaskStatus.return_value = tmMockGetSubmittedTaskStatusReturnvalue
res = tmab.updateTaskStatus(transIDOPBody, clients)
assert res['OK'] == expected
def __init__( self, *args, **kwargs ):
''' c'tor
'''
TaskManagerAgentBase.__init__( self, *args, **kwargs )
self.taskManager = WorkflowTasks( transClient = self.transClient )
self.shifterProxy = 'ProductionManager'
agentTSTypes = self.am_getOption( 'TransType', [] )
if agentTSTypes:
self.transType = agentTSTypes
else:
self.transType = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge'] )
def __init__(self, *args, **kwargs):
""" c'tor
"""
TaskManagerAgentBase.__init__(self, *args, **kwargs)
self.taskManager = RequestTasks(transClient=self.transClient)
self.shifterProxy = "ProductionManager"
agentTSTypes = self.am_getOption("TransType", [])
if agentTSTypes:
self.transType = agentTSTypes
else:
self.transType = Operations().getValue("Transformations/DataManipulation", ["Replication", "Removal"])
def initialize(self):
""" Sets defaults """
TaskManagerAgentBase.initialize(self)
RequestTasks.__init__(self)
self.transType = ['Replication', 'Removal']
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/ProductionManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'ProductionManager')
return S_OK()
def __init__( self, *args, **kwargs ):
''' c'tor
'''
TaskManagerAgentBase.__init__( self, *args, **kwargs )
self.taskManager = RequestTasks( transClient = self.transClient )
self.shifterProxy = 'ProductionManager'
agentTSTypes = self.am_getOption( 'TransType', [] )
if agentTSTypes:
self.transType = agentTSTypes
else:
self.transType = Operations().getValue( 'Transformations/DataManipulation', ['Replication', 'Removal'] )
def initialize( self ):
""" Sets defaults """
TaskManagerAgentBase.initialize( self )
WorkflowTasks.__init__( self )
self.transType = self.am_getOption( "TransType", ['MCSimulation', 'DataReconstruction', 'DataStripping', 'MCStripping', 'Merge'] )
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/ProductionManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'ProductionManager' )
return S_OK()
def initialize( self ):
""" Sets defaults """
TaskManagerAgentBase.initialize( self )
RequestTasks.__init__( self )
self.transType = ['Replication', 'Removal']
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/ProductionManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'ProductionManager' )
return S_OK()
def __init__(self, *args, **kwargs):
''' c'tor
'''
TaskManagerAgentBase.__init__(self, *args, **kwargs)
self.taskManager = RequestTasks(transClient=self.transClient)
self.shifterProxy = 'ProductionManager'
agentTSTypes = self.am_getOption('TransType', [])
if agentTSTypes:
self.transType = agentTSTypes
else:
self.transType = Operations().getValue(
'Transformations/DataManipulation', ['Replication', 'Removal'])
def test_checkReservedTasks(mocker,
tcMockGetTransformationTasksReturnValue,
tmMockUpdateTransformationReservedTasksReturnValue,
tcMockSetTaskStatusAndWmsIDReturnValue,
expected):
mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule', side_effect=mockAM)
mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport', side_effect=MagicMock())
tmab = TaskManagerAgentBase()
tc_mock.getTransformationTasks.return_value = tcMockGetTransformationTasksReturnValue
tm_mock.updateTransformationReservedTasks.return_value = tmMockUpdateTransformationReservedTasksReturnValue
tc_mock.setTaskStatusAndWmsID.return_value = tcMockSetTaskStatusAndWmsIDReturnValue
res = tmab.checkReservedTasks(transIDOPBody, clients)
assert res['OK'] == expected
def test_updateFileStatusSuccess(mocker,
tcMockGetTransformationFilesReturnValue,
tmMockGetSubmittedFileStatusReturnValue,
expected):
mocker.patch(
"DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule",
side_effect=mockAM)
mocker.patch(
"DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport",
side_effect=MagicMock())
tmab = TaskManagerAgentBase()
tc_mock.getTransformationFiles.return_value = tcMockGetTransformationFilesReturnValue
tm_mock.getSubmittedFileStatus.return_value = tmMockGetSubmittedFileStatusReturnValue
res = tmab.updateFileStatus(transDict, clients)
assert res["OK"] == expected
def initialize(self):
""" Sets defaults """
TaskManagerAgentBase.initialize(self)
WorkflowTasks.__init__(self)
self.transType = self.am_getOption("TransType", [
'MCSimulation', 'DataReconstruction', 'DataStripping',
'MCStripping', 'Merge'
])
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/ProductionManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'ProductionManager')
return S_OK()
def test_submitTasks(mocker,
tcMockGetTasksToSubmitReturnValue,
tmMockPrepareTransformationTasksReturnValue,
tmMockSubmitTransformationTasksReturnValue,
tmMockUpdateDBAfterTaskSubmissionReturnValue,
expected):
mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule', side_effect=mockAM)
mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport', side_effect=MagicMock())
tmab = TaskManagerAgentBase()
tc_mock.getTasksToSubmit.return_value = tcMockGetTasksToSubmitReturnValue
tm_mock.prepareTransformationTasks.return_value = tmMockPrepareTransformationTasksReturnValue
tm_mock.submitTransformationTasks.return_value = tmMockSubmitTransformationTasksReturnValue
tm_mock.updateDBAfterTaskSubmission.return_value = tmMockUpdateDBAfterTaskSubmissionReturnValue
res = tmab.submitTasks(transIDOPBody, clients)
assert res['OK'] == expected
def test_checkReservedTasks(mocker, tcMockGetTransformationTasksReturnValue,
tmMockUpdateTransformationReservedTasksReturnValue,
tcMockSetTaskStatusAndWmsIDReturnValue, expected):
mocker.patch(
'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule',
side_effect=mockAM)
mocker.patch(
'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport',
side_effect=MagicMock())
tmab = TaskManagerAgentBase()
tc_mock.getTransformationTasks.return_value = tcMockGetTransformationTasksReturnValue
tm_mock.updateTransformationReservedTasks.return_value = tmMockUpdateTransformationReservedTasksReturnValue
tc_mock.setTaskStatusAndWmsID.return_value = tcMockSetTaskStatusAndWmsIDReturnValue
res = tmab.checkReservedTasks(transIDOPBody, clients)
assert res['OK'] == expected
def __init__(self, *args, **kwargs):
''' c'tor
'''
TaskManagerAgentBase.__init__(self, *args, **kwargs)
self.submissionClient = WMSClient()
self.taskManager = WorkflowTasks(
transClient=self.transClient,
submissionClient=self.submissionClient)
self.shifterProxy = 'ProductionManager'
agentTSTypes = self.am_getOption('TransType', [])
if agentTSTypes:
self.transType = agentTSTypes
else:
self.transType = Operations().getValue(
'Transformations/DataProcessing', ['MCSimulation', 'Merge'])
def _getClients( self ):
""" Here the taskManager becomes a RequestTasks object
"""
res = TaskManagerAgentBase._getClients( self )
threadTaskManager = RequestTasks()
res.update( {'TaskManager': threadTaskManager} )
return res
def initialize(self):
"""Standard initialize method"""
res = TaskManagerAgentBase.initialize(self)
if not res["OK"]:
return res
objLoader = ObjectLoader()
_class = objLoader.loadObject(
"TransformationSystem.Client.RequestTasks", "RequestTasks")
if not _class["OK"]:
raise Exception(_class["Message"])
self.requestTasksCls = _class["Value"]
# clients
self.taskManager = self.requestTasksCls(transClient=self.transClient)
agentTSTypes = self.am_getOption("TransType", [])
if agentTSTypes:
self.transType = agentTSTypes
else:
self.transType = Operations().getValue(
"Transformations/DataManipulation", ["Replication", "Removal"])
return S_OK()
def _getClients(self):
""" Here the taskManager becomes a RequestTasks object
"""
res = TaskManagerAgentBase._getClients(self)
threadTaskManager = RequestTasks()
res.update({'TaskManager': threadTaskManager})
return res
def test_submitTasks(mocker, tcMockGetTasksToSubmitReturnValue,
tmMockPrepareTransformationTasksReturnValue,
tmMockSubmitTransformationTasksReturnValue,
tmMockUpdateDBAfterTaskSubmissionReturnValue, expected):
mocker.patch(
'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule',
side_effect=mockAM)
mocker.patch(
'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport',
side_effect=MagicMock())
tmab = TaskManagerAgentBase()
tc_mock.getTasksToSubmit.return_value = tcMockGetTasksToSubmitReturnValue
tm_mock.prepareTransformationTasks.return_value = tmMockPrepareTransformationTasksReturnValue
tm_mock.submitTransformationTasks.return_value = tmMockSubmitTransformationTasksReturnValue
tm_mock.updateDBAfterTaskSubmission.return_value = tmMockUpdateDBAfterTaskSubmissionReturnValue
res = tmab.submitTasks(transIDOPBody, clients)
assert res['OK'] == expected
def test_updateTaskStatusSuccess(mocker, tcMockReturnValue,
tmMockGetSubmittedTaskStatusReturnvalue,
expected):
mocker.patch(
'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule',
side_effect=mockAM)
mocker.patch(
'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport',
side_effect=MagicMock())
mocker.patch(
'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.TaskManagerAgentBase.am_getOption',
side_effect=mockAM)
tmab = TaskManagerAgentBase()
tc_mock.getTransformationTasks.return_value = tcMockReturnValue
tm_mock.getSubmittedTaskStatus.return_value = tmMockGetSubmittedTaskStatusReturnvalue
res = tmab.updateTaskStatus(transIDOPBody, clients)
assert res['OK'] == expected
def test_submitTasks(
mocker,
tcMockGetTasksToSubmitReturnValue,
tmMockPrepareTransformationTasksReturnValue,
tmMockSubmitTransformationTasksReturnValue,
tmMockUpdateDBAfterTaskSubmissionReturnValue,
expected,
):
mocker.patch(
"DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule",
side_effect=mockAM)
mocker.patch(
"DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport",
side_effect=MagicMock())
tmab = TaskManagerAgentBase()
tc_mock.getTasksToSubmit.return_value = tcMockGetTasksToSubmitReturnValue
tm_mock.prepareTransformationTasks.return_value = tmMockPrepareTransformationTasksReturnValue
tm_mock.submitTransformationTasks.return_value = tmMockSubmitTransformationTasksReturnValue
tm_mock.updateDBAfterTaskSubmission.return_value = tmMockUpdateDBAfterTaskSubmissionReturnValue
res = tmab.submitTasks(transDict, clients)
assert res["OK"] == expected
tmab.maxParametricJobs = 10
tmab.bulkSubmissionFlag = True
res = tmab.submitTasks(transDict, clients)
assert res["OK"] == expected
def initialize(self):
"""Standard initialize method"""
res = TaskManagerAgentBase.initialize(self)
if not res["OK"]:
return res
agentTSTypes = self.am_getOption("TransType", [])
if agentTSTypes:
self.transType = agentTSTypes
else:
self.transType = Operations().getValue("Transformations/DataProcessing", ["MCSimulation", "Merge"])
return S_OK()
def initialize(self):
""" Standard initialize method
"""
res = TaskManagerAgentBase.initialize(self)
if not res['OK']:
return res
agentTSTypes = self.am_getOption('TransType', [])
if agentTSTypes:
self.transType = agentTSTypes
else:
self.transType = Operations().getValue('Transformations/DataProcessing', ['MCSimulation', 'Merge'])
return S_OK()
def initialize(self):
""" Standard initialize method
"""
res = TaskManagerAgentBase.initialize(self)
if not res['OK']:
return res
agentTSTypes = self.am_getOption('TransType', [])
if agentTSTypes:
self.transType = agentTSTypes
else:
self.transType = Operations().getValue('Transformations/DataProcessing', ['MCSimulation', 'Merge'])
return S_OK()
def initialize( self ):
""" Standard initialize method
"""
res = TaskManagerAgentBase.initialize( self )
if not res['OK']:
return res
# clients
self.taskManager = RequestTasks( transClient = self.transClient )
agentTSTypes = self.am_getOption( 'TransType', [] )
if agentTSTypes:
self.transType = agentTSTypes
else:
self.transType = Operations().getValue( 'Transformations/DataManipulation', ['Replication', 'Removal'] )
return S_OK()
def initialize(self):
""" Standard initialize method
"""
res = TaskManagerAgentBase.initialize(self)
if not res['OK']:
return res
# clients
self.taskManager = RequestTasks(transClient=self.transClient)
agentTSTypes = self.am_getOption('TransType', [])
if agentTSTypes:
self.transType = agentTSTypes
else:
self.transType = Operations().getValue(
'Transformations/DataManipulation', ['Replication', 'Removal'])
return S_OK()
def test__fillTheQueue(mocker, operationsOnTransformationsDict, expected):
mocker.patch(
'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule',
side_effect=mockAM)
mocker.patch(
'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport',
side_effect=MagicMock())
tmab = TaskManagerAgentBase()
tmab.log = gLogger
tmab.am_getOption = mockAM
tmab._fillTheQueue(operationsOnTransformationsDict)
assert tmab.transInQueue == expected[0]
assert tmab.transQueue.qsize() == expected[1]
def setUp( self ):
self.mockAM = MagicMock()
self.tmab_m = importlib.import_module( 'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase' )
self.tmab_m.AgentModule = self.mockAM
self.tmab_m.FileReport = MagicMock()
self.tmab = TaskManagerAgentBase()
self.tmab.log = gLogger
self.tmab.am_getOption = self.mockAM
self.tmab.log.setLevel( 'DEBUG' )
self.ta_m = importlib.import_module( 'DIRAC.TransformationSystem.Agent.TransformationAgent' )
self.ta_m.AgentModule = self.mockAM
self.ta = TransformationAgent()
self.ta.log = gLogger
self.ta.am_getOption = self.mockAM
self.tmab.log.setLevel( 'DEBUG' )
self.tc_mock = MagicMock()
self.tm_mock = MagicMock()
import pytest
from mock import MagicMock
from DIRAC import gLogger
# sut
from DIRAC.TransformationSystem.Agent.TaskManagerAgentBase import TaskManagerAgentBase
from DIRAC.TransformationSystem.Agent.TransformationAgent import TransformationAgent
gLogger.setLevel('DEBUG')
mockAM = MagicMock()
tmab_m = importlib.import_module(
'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase')
tmab_m.AgentModule = mockAM
tmab_m.FileReport = MagicMock()
tmab = TaskManagerAgentBase()
tmab.log = gLogger
tmab.am_getOption = mockAM
tmab.log.setLevel('DEBUG')
@pytest.mark.parametrize("operationsOnTransformationsDict, expected",
[({
1: {
'Operations': ['op1', 'op2'],
'Body': 'veryBigBody'
}
}, ([1], 1)),
({
2: {
'Operations': ['op3', 'op2'],