Example #1
0
  def __init__(self, *args, **kwargs):
    """ c'tor
    """
    TaskManagerAgentBase.__init__(self, *args, **kwargs)

    self.transType = []
    self.taskManager = None
Example #2
0
    def __init__(self, *args, **kwargs):
        """ c'tor
    """
        TaskManagerAgentBase.__init__(self, *args, **kwargs)

        self.transType = []
        self.taskManager = None
def test__fillTheQueue(mocker, operationsOnTransformationsDict, expected):
  mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule', side_effect=mockAM)
  mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport', side_effect=MagicMock())
  tmab = TaskManagerAgentBase()
  tmab.log = gLogger
  tmab.am_getOption = mockAM
  tmab._fillTheQueue(operationsOnTransformationsDict)
  assert tmab.transInQueue == expected[0]
  assert tmab.transQueue.qsize() == expected[1]
def test_updateTaskStatusSuccess(mocker, tcMockReturnValue, tmMockGetSubmittedTaskStatusReturnvalue, expected):
  mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule', side_effect=mockAM)
  mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport', side_effect=MagicMock())
  mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.TaskManagerAgentBase.am_getOption',
               side_effect=mockAM)
  tmab = TaskManagerAgentBase()
  tc_mock.getTransformationTasks.return_value = tcMockReturnValue
  tm_mock.getSubmittedTaskStatus.return_value = tmMockGetSubmittedTaskStatusReturnvalue
  res = tmab.updateTaskStatus(transIDOPBody, clients)
  assert res['OK'] == expected
Example #5
0
  def __init__( self, *args, **kwargs ):
    ''' c'tor
    '''
    TaskManagerAgentBase.__init__( self, *args, **kwargs )

    self.taskManager = WorkflowTasks( transClient = self.transClient )
    self.shifterProxy = 'ProductionManager'
    agentTSTypes = self.am_getOption( 'TransType', [] )
    if agentTSTypes:
      self.transType = agentTSTypes
    else:
      self.transType = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge'] )
Example #6
0
    def __init__(self, *args, **kwargs):
        """ c'tor
    """
        TaskManagerAgentBase.__init__(self, *args, **kwargs)

        self.taskManager = RequestTasks(transClient=self.transClient)
        self.shifterProxy = "ProductionManager"
        agentTSTypes = self.am_getOption("TransType", [])
        if agentTSTypes:
            self.transType = agentTSTypes
        else:
            self.transType = Operations().getValue("Transformations/DataManipulation", ["Replication", "Removal"])
Example #7
0
    def initialize(self):
        """ Sets defaults """
        TaskManagerAgentBase.initialize(self)
        RequestTasks.__init__(self)
        self.transType = ['Replication', 'Removal']

        # This sets the Default Proxy to used as that defined under
        # /Operations/Shifter/ProductionManager
        # the shifterProxy option in the Configuration can be used to change this default.
        self.am_setOption('shifterProxy', 'ProductionManager')

        return S_OK()
Example #8
0
  def __init__( self, *args, **kwargs ):
    ''' c'tor
    '''
    TaskManagerAgentBase.__init__( self, *args, **kwargs )

    self.taskManager = RequestTasks( transClient = self.transClient )
    self.shifterProxy = 'ProductionManager'
    agentTSTypes = self.am_getOption( 'TransType', [] )
    if agentTSTypes:
      self.transType = agentTSTypes
    else:
      self.transType = Operations().getValue( 'Transformations/DataManipulation', ['Replication', 'Removal'] )
Example #9
0
  def initialize( self ):
    """ Sets defaults """
    TaskManagerAgentBase.initialize( self )
    WorkflowTasks.__init__( self )
    self.transType = self.am_getOption( "TransType", ['MCSimulation', 'DataReconstruction', 'DataStripping', 'MCStripping', 'Merge'] )

    # This sets the Default Proxy to used as that defined under 
    # /Operations/Shifter/ProductionManager
    # the shifterProxy option in the Configuration can be used to change this default.
    self.am_setOption( 'shifterProxy', 'ProductionManager' )

    return S_OK()
Example #10
0
  def initialize( self ):
    """ Sets defaults """
    TaskManagerAgentBase.initialize( self )
    RequestTasks.__init__( self )
    self.transType = ['Replication', 'Removal']

    # This sets the Default Proxy to used as that defined under 
    # /Operations/Shifter/ProductionManager
    # the shifterProxy option in the Configuration can be used to change this default.
    self.am_setOption( 'shifterProxy', 'ProductionManager' )

    return S_OK()
Example #11
0
    def __init__(self, *args, **kwargs):
        ''' c'tor
    '''
        TaskManagerAgentBase.__init__(self, *args, **kwargs)

        self.taskManager = RequestTasks(transClient=self.transClient)
        self.shifterProxy = 'ProductionManager'
        agentTSTypes = self.am_getOption('TransType', [])
        if agentTSTypes:
            self.transType = agentTSTypes
        else:
            self.transType = Operations().getValue(
                'Transformations/DataManipulation', ['Replication', 'Removal'])
def test_checkReservedTasks(mocker,
                            tcMockGetTransformationTasksReturnValue,
                            tmMockUpdateTransformationReservedTasksReturnValue,
                            tcMockSetTaskStatusAndWmsIDReturnValue,
                            expected):
  mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule', side_effect=mockAM)
  mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport', side_effect=MagicMock())
  tmab = TaskManagerAgentBase()
  tc_mock.getTransformationTasks.return_value = tcMockGetTransformationTasksReturnValue
  tm_mock.updateTransformationReservedTasks.return_value = tmMockUpdateTransformationReservedTasksReturnValue
  tc_mock.setTaskStatusAndWmsID.return_value = tcMockSetTaskStatusAndWmsIDReturnValue
  res = tmab.checkReservedTasks(transIDOPBody, clients)
  assert res['OK'] == expected
Example #13
0
def test_updateFileStatusSuccess(mocker,
                                 tcMockGetTransformationFilesReturnValue,
                                 tmMockGetSubmittedFileStatusReturnValue,
                                 expected):
    mocker.patch(
        "DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule",
        side_effect=mockAM)
    mocker.patch(
        "DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport",
        side_effect=MagicMock())
    tmab = TaskManagerAgentBase()
    tc_mock.getTransformationFiles.return_value = tcMockGetTransformationFilesReturnValue
    tm_mock.getSubmittedFileStatus.return_value = tmMockGetSubmittedFileStatusReturnValue
    res = tmab.updateFileStatus(transDict, clients)
    assert res["OK"] == expected
Example #14
0
    def initialize(self):
        """ Sets defaults """
        TaskManagerAgentBase.initialize(self)
        WorkflowTasks.__init__(self)
        self.transType = self.am_getOption("TransType", [
            'MCSimulation', 'DataReconstruction', 'DataStripping',
            'MCStripping', 'Merge'
        ])

        # This sets the Default Proxy to used as that defined under
        # /Operations/Shifter/ProductionManager
        # the shifterProxy option in the Configuration can be used to change this default.
        self.am_setOption('shifterProxy', 'ProductionManager')

        return S_OK()
def test_submitTasks(mocker,
                     tcMockGetTasksToSubmitReturnValue,
                     tmMockPrepareTransformationTasksReturnValue,
                     tmMockSubmitTransformationTasksReturnValue,
                     tmMockUpdateDBAfterTaskSubmissionReturnValue,
                     expected):
  mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule', side_effect=mockAM)
  mocker.patch('DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport', side_effect=MagicMock())
  tmab = TaskManagerAgentBase()
  tc_mock.getTasksToSubmit.return_value = tcMockGetTasksToSubmitReturnValue
  tm_mock.prepareTransformationTasks.return_value = tmMockPrepareTransformationTasksReturnValue
  tm_mock.submitTransformationTasks.return_value = tmMockSubmitTransformationTasksReturnValue
  tm_mock.updateDBAfterTaskSubmission.return_value = tmMockUpdateDBAfterTaskSubmissionReturnValue
  res = tmab.submitTasks(transIDOPBody, clients)
  assert res['OK'] == expected
Example #16
0
def test_checkReservedTasks(mocker, tcMockGetTransformationTasksReturnValue,
                            tmMockUpdateTransformationReservedTasksReturnValue,
                            tcMockSetTaskStatusAndWmsIDReturnValue, expected):
    mocker.patch(
        'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule',
        side_effect=mockAM)
    mocker.patch(
        'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport',
        side_effect=MagicMock())
    tmab = TaskManagerAgentBase()
    tc_mock.getTransformationTasks.return_value = tcMockGetTransformationTasksReturnValue
    tm_mock.updateTransformationReservedTasks.return_value = tmMockUpdateTransformationReservedTasksReturnValue
    tc_mock.setTaskStatusAndWmsID.return_value = tcMockSetTaskStatusAndWmsIDReturnValue
    res = tmab.checkReservedTasks(transIDOPBody, clients)
    assert res['OK'] == expected
Example #17
0
    def __init__(self, *args, **kwargs):
        ''' c'tor
    '''
        TaskManagerAgentBase.__init__(self, *args, **kwargs)

        self.submissionClient = WMSClient()
        self.taskManager = WorkflowTasks(
            transClient=self.transClient,
            submissionClient=self.submissionClient)
        self.shifterProxy = 'ProductionManager'
        agentTSTypes = self.am_getOption('TransType', [])
        if agentTSTypes:
            self.transType = agentTSTypes
        else:
            self.transType = Operations().getValue(
                'Transformations/DataProcessing', ['MCSimulation', 'Merge'])
Example #18
0
 def _getClients( self ):
   """ Here the taskManager becomes a RequestTasks object
   """
   res = TaskManagerAgentBase._getClients( self )
   threadTaskManager = RequestTasks()
   res.update( {'TaskManager': threadTaskManager} )
   return res
Example #19
0
    def initialize(self):
        """Standard initialize method"""
        res = TaskManagerAgentBase.initialize(self)
        if not res["OK"]:
            return res

        objLoader = ObjectLoader()
        _class = objLoader.loadObject(
            "TransformationSystem.Client.RequestTasks", "RequestTasks")

        if not _class["OK"]:
            raise Exception(_class["Message"])

        self.requestTasksCls = _class["Value"]

        # clients
        self.taskManager = self.requestTasksCls(transClient=self.transClient)

        agentTSTypes = self.am_getOption("TransType", [])
        if agentTSTypes:
            self.transType = agentTSTypes
        else:
            self.transType = Operations().getValue(
                "Transformations/DataManipulation", ["Replication", "Removal"])

        return S_OK()
Example #20
0
 def _getClients(self):
     """ Here the taskManager becomes a RequestTasks object
 """
     res = TaskManagerAgentBase._getClients(self)
     threadTaskManager = RequestTasks()
     res.update({'TaskManager': threadTaskManager})
     return res
def test_submitTasks(mocker, tcMockGetTasksToSubmitReturnValue,
                     tmMockPrepareTransformationTasksReturnValue,
                     tmMockSubmitTransformationTasksReturnValue,
                     tmMockUpdateDBAfterTaskSubmissionReturnValue, expected):
    mocker.patch(
        'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule',
        side_effect=mockAM)
    mocker.patch(
        'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport',
        side_effect=MagicMock())
    tmab = TaskManagerAgentBase()
    tc_mock.getTasksToSubmit.return_value = tcMockGetTasksToSubmitReturnValue
    tm_mock.prepareTransformationTasks.return_value = tmMockPrepareTransformationTasksReturnValue
    tm_mock.submitTransformationTasks.return_value = tmMockSubmitTransformationTasksReturnValue
    tm_mock.updateDBAfterTaskSubmission.return_value = tmMockUpdateDBAfterTaskSubmissionReturnValue
    res = tmab.submitTasks(transIDOPBody, clients)
    assert res['OK'] == expected
Example #22
0
def test_updateTaskStatusSuccess(mocker, tcMockReturnValue,
                                 tmMockGetSubmittedTaskStatusReturnvalue,
                                 expected):
    mocker.patch(
        'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule',
        side_effect=mockAM)
    mocker.patch(
        'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport',
        side_effect=MagicMock())
    mocker.patch(
        'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.TaskManagerAgentBase.am_getOption',
        side_effect=mockAM)
    tmab = TaskManagerAgentBase()
    tc_mock.getTransformationTasks.return_value = tcMockReturnValue
    tm_mock.getSubmittedTaskStatus.return_value = tmMockGetSubmittedTaskStatusReturnvalue
    res = tmab.updateTaskStatus(transIDOPBody, clients)
    assert res['OK'] == expected
Example #23
0
def test_submitTasks(
    mocker,
    tcMockGetTasksToSubmitReturnValue,
    tmMockPrepareTransformationTasksReturnValue,
    tmMockSubmitTransformationTasksReturnValue,
    tmMockUpdateDBAfterTaskSubmissionReturnValue,
    expected,
):
    mocker.patch(
        "DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule",
        side_effect=mockAM)
    mocker.patch(
        "DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport",
        side_effect=MagicMock())
    tmab = TaskManagerAgentBase()
    tc_mock.getTasksToSubmit.return_value = tcMockGetTasksToSubmitReturnValue
    tm_mock.prepareTransformationTasks.return_value = tmMockPrepareTransformationTasksReturnValue
    tm_mock.submitTransformationTasks.return_value = tmMockSubmitTransformationTasksReturnValue
    tm_mock.updateDBAfterTaskSubmission.return_value = tmMockUpdateDBAfterTaskSubmissionReturnValue
    res = tmab.submitTasks(transDict, clients)
    assert res["OK"] == expected
    tmab.maxParametricJobs = 10
    tmab.bulkSubmissionFlag = True
    res = tmab.submitTasks(transDict, clients)
    assert res["OK"] == expected
Example #24
0
    def initialize(self):
        """Standard initialize method"""
        res = TaskManagerAgentBase.initialize(self)
        if not res["OK"]:
            return res

        agentTSTypes = self.am_getOption("TransType", [])
        if agentTSTypes:
            self.transType = agentTSTypes
        else:
            self.transType = Operations().getValue("Transformations/DataProcessing", ["MCSimulation", "Merge"])

        return S_OK()
Example #25
0
  def initialize(self):
    """ Standard initialize method
    """
    res = TaskManagerAgentBase.initialize(self)
    if not res['OK']:
      return res

    agentTSTypes = self.am_getOption('TransType', [])
    if agentTSTypes:
      self.transType = agentTSTypes
    else:
      self.transType = Operations().getValue('Transformations/DataProcessing', ['MCSimulation', 'Merge'])

    return S_OK()
Example #26
0
  def initialize(self):
    """ Standard initialize method
    """
    res = TaskManagerAgentBase.initialize(self)
    if not res['OK']:
      return res

    agentTSTypes = self.am_getOption('TransType', [])
    if agentTSTypes:
      self.transType = agentTSTypes
    else:
      self.transType = Operations().getValue('Transformations/DataProcessing', ['MCSimulation', 'Merge'])

    return S_OK()
Example #27
0
  def initialize( self ):
    """ Standard initialize method
    """
    res = TaskManagerAgentBase.initialize( self )
    if not res['OK']:
      return res

    # clients
    self.taskManager = RequestTasks( transClient = self.transClient )

    agentTSTypes = self.am_getOption( 'TransType', [] )
    if agentTSTypes:
      self.transType = agentTSTypes
    else:
      self.transType = Operations().getValue( 'Transformations/DataManipulation', ['Replication', 'Removal'] )

    return S_OK()
Example #28
0
    def initialize(self):
        """ Standard initialize method
    """
        res = TaskManagerAgentBase.initialize(self)
        if not res['OK']:
            return res

        # clients
        self.taskManager = RequestTasks(transClient=self.transClient)

        agentTSTypes = self.am_getOption('TransType', [])
        if agentTSTypes:
            self.transType = agentTSTypes
        else:
            self.transType = Operations().getValue(
                'Transformations/DataManipulation', ['Replication', 'Removal'])

        return S_OK()
Example #29
0
def test__fillTheQueue(mocker, operationsOnTransformationsDict, expected):
    mocker.patch(
        'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.AgentModule',
        side_effect=mockAM)
    mocker.patch(
        'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase.FileReport',
        side_effect=MagicMock())
    tmab = TaskManagerAgentBase()
    tmab.log = gLogger
    tmab.am_getOption = mockAM
    tmab._fillTheQueue(operationsOnTransformationsDict)
    assert tmab.transInQueue == expected[0]
    assert tmab.transQueue.qsize() == expected[1]
Example #30
0
  def setUp( self ):
    self.mockAM = MagicMock()
    self.tmab_m = importlib.import_module( 'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase' )
    self.tmab_m.AgentModule = self.mockAM
    self.tmab_m.FileReport = MagicMock()
    self.tmab = TaskManagerAgentBase()
    self.tmab.log = gLogger
    self.tmab.am_getOption = self.mockAM
    self.tmab.log.setLevel( 'DEBUG' )

    self.ta_m = importlib.import_module( 'DIRAC.TransformationSystem.Agent.TransformationAgent' )
    self.ta_m.AgentModule = self.mockAM
    self.ta = TransformationAgent()
    self.ta.log = gLogger
    self.ta.am_getOption = self.mockAM
    self.tmab.log.setLevel( 'DEBUG' )

    self.tc_mock = MagicMock()
    self.tm_mock = MagicMock()
Example #31
0
import pytest
from mock import MagicMock

from DIRAC import gLogger
# sut
from DIRAC.TransformationSystem.Agent.TaskManagerAgentBase import TaskManagerAgentBase
from DIRAC.TransformationSystem.Agent.TransformationAgent import TransformationAgent

gLogger.setLevel('DEBUG')

mockAM = MagicMock()
tmab_m = importlib.import_module(
    'DIRAC.TransformationSystem.Agent.TaskManagerAgentBase')
tmab_m.AgentModule = mockAM
tmab_m.FileReport = MagicMock()
tmab = TaskManagerAgentBase()
tmab.log = gLogger
tmab.am_getOption = mockAM
tmab.log.setLevel('DEBUG')


@pytest.mark.parametrize("operationsOnTransformationsDict, expected",
                         [({
                             1: {
                                 'Operations': ['op1', 'op2'],
                                 'Body': 'veryBigBody'
                             }
                         }, ([1], 1)),
                          ({
                              2: {
                                  'Operations': ['op3', 'op2'],