Example #1
0
    def setUp(self):
        """
        _setUp_

        """
        self.transitions = Transitions()
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setupCouch("changestate_t/jobs", "JobDump")
        self.testInit.setupCouch("changestate_t/fwjrs", "FWJRDump")
        self.testInit.setupCouch("job_summary", "WMStats")

        self.testInit.setSchema(customModules = ["WMCore.WMBS"],
                                useDefault = False)

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)

        couchurl = os.getenv("COUCHURL")
        self.couchServer = CouchServer(dburl = couchurl)
        self.config = self.testInit.getConfiguration()
        self.taskName = "/TestWorkflow/ReReco1"
        self.specGen = WMSpecGenerator()
        self.specUrl = self.specGen.createProcessingSpec("TestWorkflow", returnType="file")
        return
Example #2
0
    def setUp(self):
        """
        If we dont have a wmspec file create one
        """

        WorkQueueTestCase.setUp(self)

        self.cacheDir = tempfile.mkdtemp()
        self.specGenerator = WMSpecGenerator(self.cacheDir)
        self.specs = self.createReRecoSpec(1, "file")
        # setup rucio parameters for global/local queue
        self.queueParams = {}
        self.queueParams['log_reporter'] = "lq_profile_test"
        self.queueParams['rucioAccount'] = "wma_test"
        self.queueParams['rucioAuthUrl'] = "http://cmsrucio-int.cern.ch"
        self.queueParams['rucioUrl'] = "https://cmsrucio-auth-int.cern.ch"

        # Create queues
        self.localQueue = localQueue(DbName=self.queueDB,
                                     InboxDbName=self.queueInboxDB,
                                     NegotiationTimeout=0,
                                     QueueURL='global.example.com',
                                     CacheDir=self.cacheDir,
                                     central_logdb_url="%s/%s" %
                                     (self.couchURL, self.logDBName),
                                     **self.queueParams)
Example #3
0
    def setUp(self):
        """
        _setUp_
        """
        super(WorkQueueTest, self).setUp()

        self.specGenerator = WMSpecGenerator("WMSpecs")
        # self.configFile = EmulatorSetup.setupWMAgentConfig()
        self.schema = []
        self.couchApps = ["WorkQueue"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules=self.schema, useDefault=False)
        self.testInit.setupCouch('workqueue_t', *self.couchApps)
        self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps)
        self.testInit.setupCouch('local_workqueue_t', *self.couchApps)
        self.testInit.setupCouch('local_workqueue_t_inbox', *self.couchApps)
        self.testInit.generateWorkDir()

        # setup rucio parameters for global/local queue
        self.queueParams = {}
        self.queueParams['log_reporter'] = "Services_WorkQueue_Unittest"
        self.queueParams['rucioAccount'] = "wma_test"
        self.queueParams['rucioAuthUrl'] = "http://cmsrucio-int.cern.ch"
        self.queueParams['rucioUrl'] = "https://cmsrucio-auth-int.cern.ch"

        if PY3:
            self.assertItemsEqual = self.assertCountEqual
Example #4
0
class WorkQueueProfileTest(WorkQueueTestCase):
    """
    _WorkQueueTest_

    """

    def setUp(self):
        """
        If we dont have a wmspec file create one

        Warning: For the real profiling test including
        spec generation. need to use real spec instead of
        using emulator generated spec which doesn't include
        couchDB access and cmssw access
        """
        EmulatorHelper.setEmulators(phedex=True, dbs=True, siteDB=True, requestMgr=True)
        WorkQueueTestCase.setUp(self)

        self.cacheDir = tempfile.mkdtemp()
        self.specGenerator = WMSpecGenerator(self.cacheDir)
        self.specNamePrefix = "TestReReco_"
        self.specs = self.createReRecoSpec(5, "file")
        # Create queues
        self.globalQueue = globalQueue(DbName=self.globalQDB, InboxDbName=self.globalQInboxDB, NegotiationTimeout=0)

    def tearDown(self):
        """tearDown"""
        WorkQueueTestCase.tearDown(self)
        try:
            self.specGenerator.removeSpecs()
        except:
            pass
        EmulatorHelper.resetEmulators()

    def createReRecoSpec(self, numOfSpec, type="spec"):
        specs = []
        for i in range(numOfSpec):
            specName = "%s%s" % (self.specNamePrefix, (i + 1))
            specs.append(self.specGenerator.createReRecoSpec(specName, type))
        return specs

    def createProfile(self, name, function):
        file = name
        prof = cProfile.Profile()
        prof.runcall(function)
        prof.dump_stats(file)
        p = pstats.Stats(file)
        p.strip_dirs().sort_stats("cumulative").print_stats(0.1)
        p.strip_dirs().sort_stats("time").print_stats(0.1)
        p.strip_dirs().sort_stats("calls").print_stats(0.1)
        # p.strip_dirs().sort_stats('name').print_stats(10)

    def testQueueElementProfile(self):
        self.createProfile("queueElementProfile.prof", self.multipleQueueWorkCall)

    def multipleQueueWorkCall(self):
        i = 0
        for wmspec in self.specs:
            i += 1
            self.globalQueue.queueWork(wmspec, self.specNamePrefix + str(i), "test_team")
Example #5
0
class WorkQueueTest(unittest.TestCase):
    """
    Test WorkQueue Service client
    It will start WorkQueue RESTService
    Server DB sets from environment variable.
    Client DB sets from environment variable. 

    This checks whether DS call makes without error and return the results.
    Not the correctness of functions. That will be tested in different module.
    """
    def setUp(self):
        """
        _setUp_
        """
        EmulatorHelper.setEmulators(phedex = True, dbs = True, 
                                    siteDB = True, requestMgr = True)

        self.specGenerator = WMSpecGenerator("WMSpecs")
        #self.configFile = EmulatorSetup.setupWMAgentConfig()
        self.schema = []
        self.couchApps = ["WorkQueue"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        self.testInit.setupCouch('workqueue_t', *self.couchApps)
        self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps)
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()
        #EmulatorSetup.deleteConfig(self.configFile)
        self.specGenerator.removeSpecs()
        

    def testWorkQueueService(self):
        # test getWork
        specName = "RerecoSpec"
        specUrl = self.specGenerator.createReRecoSpec(specName, "file")
        globalQ = globalQueue(DbName = 'workqueue_t',
                              QueueURL = self.testInit.couchUrl)
        self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)

        wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
        #This only checks minimum client call not exactly correctness of return
        # values.
        self.assertEqual(wqApi.getTopLevelJobsByRequest(),
                         [{'total_jobs': 2, 'request_name': specName}])
        self.assertEqual(wqApi.getChildQueues(), [])
        self.assertEqual(wqApi.getJobStatusByRequest(),
            [{'status': 'Available', 'jobs': 2, 'request_name': specName}])
        self.assertEqual(wqApi.getChildQueuesByRequest(), [])
        self.assertEqual(wqApi.getWMBSUrl(), [])
        self.assertEqual(wqApi.getWMBSUrlByRequest(), [])
Example #6
0
    def createWorkload(self):
        """
        Create a workload in order to test things

        """
        generator = WMSpecGenerator()
        workload = generator.createReRecoSpec("Tier1ReReco")
        return workload
Example #7
0
    def setUp(self):
        """
        _setUp_

        Setup some reasonable defaults for the ReReco workflow.
        """
        self.unmergedLFNBase = "/store/backfill/2/unmerged"
        self.mergedLFNBase = "/store/backfill/2"
        self.processingVersion = "v1"
        self.cmsswVersion = "CMSSW_3_4_2_patch1"
        self.acquisitionEra = "WMAgentCommissioining10"
        self.primaryDataset = "MinimumBias"

        self.workload = WMSpecGenerator().createReRecoSpec("Tier1ReReco")
        print self.workload.data
        return
Example #8
0
class RequestManager(object):
    
    def __init__(self, *args, **kwargs):
        print "Using RequestManager Emulator ..."
        self.specGenerator = WMSpecGenerator()
        self.count = 0
        self.maxWmSpec = 1
    
    def getAssignment(self, teamName=None, request=None):
        if self.count < self.maxWmSpec:
            #specName = "FakeProductionSpec_%s" % self.count
            #specUrl =self.specGenerator.createProductionSpec(specName, "file")
            specName = "FakeProcessingSpec_%s" % self.count
            specUrl =self.specGenerator.createProcessingSpec(specName, "file")
        
            #specName = "ReRecoTest_v%sEmulator" % self.count
            #specUrl =self.specGenerator.create ReRecoSpec(specName, "file")
            self.count += 1
            return {specName:specUrl}
        else:
            return {}
    
        
        
    def postAssignment(self, requestName, prodAgentUrl=None):
        # do not thing or return success of fail massage 
        return 
Example #9
0
    def setUp(self):
        """
        If we dont have a wmspec file create one
        """

        EmulatorHelper.setEmulators(phedex=False,
                                    dbs=False,
                                    siteDB=True,
                                    requestMgr=True)
        WorkQueueTestCase.setUp(self)

        self.cacheDir = tempfile.mkdtemp()
        self.specGenerator = WMSpecGenerator(self.cacheDir)
        self.specs = self.createReRecoSpec(1, "file")

        # Create queues
        self.localQueue = localQueue(DbName=self.queueDB, InboxDbName=self.queueInboxDB, \
                NegotiationTimeout=0, QueueURL='global.example.com', CacheDir=self.cacheDir, central_logdb_url="%s/%s" %(self.couchURL, self.logDBName), log_reporter='lq_profile_test')
Example #10
0
    def setUp(self):
        """
        If we dont have a wmspec file create one
        """

        EmulatorHelper.setEmulators(phedex = True, dbs = True,
                                    siteDB = True, requestMgr = True)
        WorkQueueTestCase.setUp(self)

        self.cacheDir = tempfile.mkdtemp()
        self.specGenerator = WMSpecGenerator(self.cacheDir)
        self.specs = self.createReRecoSpec(1, "file")

        # Create queues
        self.localQueue = localQueue(DbName = self.queueDB,
                                     InboxDbName = self.queueInboxDB,
                                     NegotiationTimeout = 0,
                                     QueueURL = 'global.example.com',
                                     CacheDir = self.cacheDir)
Example #11
0
    def setUp(self):
        """
        If we dont have a wmspec file create one

        Warning: For the real profiling test including
        spec generation. need to use real spec instead of
        using emulator generated spec which doesn't include
        couchDB access and cmssw access
        """
        WorkQueueTestCase.setUp(self)

        self.cacheDir = tempfile.mkdtemp()
        self.specGenerator = WMSpecGenerator(self.cacheDir)
        self.specNamePrefix = "TestReReco_"
        self.specs = self.createReRecoSpec(5, "file")
        # Create queues
        self.globalQueue = globalQueue(DbName=self.globalQDB,
                                       InboxDbName=self.globalQInboxDB,
                                       NegotiationTimeout=0)
Example #12
0
 def __init__(self, *args, **kwargs):
     """
     all the private valuable is defined for test values
     """
     self.specGenerator = WMSpecGenerator()
     self.count = 0
     self.maxWmSpec = kwargs.setdefault('numOfSpecs', 1)
     self.type = kwargs.setdefault("type", 'ReReco')
     if self.type not in ['ReReco', 'MonteCarlo']:
         raise TypeError, 'unknown request type %s' % self.type
     self.splitter = kwargs.setdefault('splitter', 'DatasetBlock')
     self.inputDataset = kwargs.setdefault('inputDataset', None)
     self.dbsUrl = kwargs.setdefault('dbsUrl', None)
     self.status = {}
     self.progress = {}
     self.msg = {}
     self.names = []
     import logging
     self['logger'] = logging
Example #13
0
    def setUp(self):
        """
        _setUp_
        """
        super(WorkQueueTest, self).setUp()

        self.specGenerator = WMSpecGenerator("WMSpecs")
        # self.configFile = EmulatorSetup.setupWMAgentConfig()
        self.schema = []
        self.couchApps = ["WorkQueue"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules=self.schema, useDefault=False)
        self.testInit.setupCouch('workqueue_t', *self.couchApps)
        self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps)
        self.testInit.setupCouch('local_workqueue_t', *self.couchApps)
        self.testInit.setupCouch('local_workqueue_t_inbox', *self.couchApps)
        self.testInit.generateWorkDir()
        return
Example #14
0
    def setUp(self):
        """
        _setUp_
        """
        EmulatorHelper.setEmulators(phedex=True,
                                    dbs=True,
                                    siteDB=True,
                                    requestMgr=True)

        self.specGenerator = WMSpecGenerator("WMSpecs")
        #self.configFile = EmulatorSetup.setupWMAgentConfig()
        self.schema = []
        self.couchApps = ["WorkQueue"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules=self.schema, useDefault=False)
        self.testInit.setupCouch('workqueue_t', *self.couchApps)
        self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps)
        return
Example #15
0
    def setUp(self):
        """
        If we dont have a wmspec file create one
        """

        EmulatorHelper.setEmulators(phedex=False, dbs=False, siteDB=True, requestMgr=True)
        WorkQueueTestCase.setUp(self)

        self.cacheDir = tempfile.mkdtemp()
        self.specGenerator = WMSpecGenerator(self.cacheDir)
        self.specs = self.createReRecoSpec(1, "file")

        # Create queues
        self.localQueue = localQueue(DbName=self.queueDB, InboxDbName=self.queueInboxDB, \
                NegotiationTimeout=0, QueueURL='global.example.com', CacheDir=self.cacheDir, central_logdb_url="%s/%s" %(self.couchURL, self.logDBName), log_reporter='lq_profile_test')
Example #16
0
    def setUp(self):
        """
        _setUp_

        Setup some reasonable defaults for the ReReco workflow.
        """
        self.unmergedLFNBase = "/store/backfill/2/unmerged"
        self.mergedLFNBase = "/store/backfill/2"
        self.processingVersion = "v1"
        self.cmsswVersion = "CMSSW_3_4_2_patch1"
        self.acquisitionEra = "WMAgentCommissioining10"
        self.primaryDataset = "MinimumBias"

        self.workload = WMSpecGenerator().createReRecoSpec("Tier1ReReco")
        print self.workload.data
        return
Example #17
0
    def setUp(self):
        """
        If we dont have a wmspec file create one

        Warning: For the real profiling test including
        spec generation. need to use real spec instead of
        using emulator generated spec which doesn't include
        couchDB access and cmssw access
        """
        EmulatorHelper.setEmulators(phedex=True, dbs=True, siteDB=True, requestMgr=True)
        WorkQueueTestCase.setUp(self)

        self.cacheDir = tempfile.mkdtemp()
        self.specGenerator = WMSpecGenerator(self.cacheDir)
        self.specNamePrefix = "TestReReco_"
        self.specs = self.createReRecoSpec(5, "file")
        # Create queues
        self.globalQueue = globalQueue(DbName=self.globalQDB, InboxDbName=self.globalQInboxDB, NegotiationTimeout=0)
Example #18
0
    def setUp(self):
        """
        If we dont have a wmspec file create one
        """

        EmulatorHelper.setEmulators(phedex = True, dbs = True,
                                    siteDB = True, requestMgr = True)
        WorkQueueTestCase.setUp(self)

        self.cacheDir = tempfile.mkdtemp()
        self.specGenerator = WMSpecGenerator(self.cacheDir)
        self.specs = self.createReRecoSpec(1, "file")

        # Create queues
        self.localQueue = localQueue(DbName = self.queueDB,
                                     InboxDbName = self.queueInboxDB,
                                     NegotiationTimeout = 0,
                                     QueueURL = 'global.example.com',
                                     CacheDir = self.cacheDir)
Example #19
0
    def setUp(self):
        """
        _setUp_
        """
        EmulatorHelper.setEmulators(phedex = True, dbs = True, 
                                    siteDB = True, requestMgr = True)

        self.specGenerator = WMSpecGenerator("WMSpecs")
        #self.configFile = EmulatorSetup.setupWMAgentConfig()
        self.schema = []
        self.couchApps = ["WorkQueue"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        self.testInit.setupCouch('workqueue_t', *self.couchApps)
        self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps)
        return
Example #20
0
 def __init__(self, *args, **kwargs):
     """
     all the private valuable is defined for test values
     """
     self.specGenerator = WMSpecGenerator()
     self.count = 0
     self.maxWmSpec = kwargs.setdefault('numOfSpecs', 1)
     self.type = kwargs.setdefault("type", 'ReReco')
     if self.type not in ['ReReco', 'MonteCarlo']:
         raise TypeError, 'unknown request type %s' % self.type
     self.splitter = kwargs.setdefault('splitter', 'DatasetBlock')
     self.inputDataset = kwargs.setdefault('inputDataset', None)
     self.dbsUrl = kwargs.setdefault('dbsUrl', None)
     self.status = {}
     self.progress = {}
     self.msg = {}
     self.names = []
     import logging
     self['logger'] = logging
Example #21
0
    def setUp(self):
        """
        _setUp_
        """
        super(WorkQueueTest, self).setUp()

        self.specGenerator = WMSpecGenerator("WMSpecs")
        # self.configFile = EmulatorSetup.setupWMAgentConfig()
        self.schema = []
        self.couchApps = ["WorkQueue"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules=self.schema,
                                useDefault=False)
        self.testInit.setupCouch('workqueue_t', *self.couchApps)
        self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps)
        self.testInit.setupCouch('local_workqueue_t', *self.couchApps)
        self.testInit.setupCouch('local_workqueue_t_inbox', *self.couchApps)
        self.testInit.generateWorkDir()
        return
Example #22
0
class LocalWorkQueueProfileTest(WorkQueueTestCase):
    """
    _WorkQueueTest_

    """

    def setUp(self):
        """
        If we dont have a wmspec file create one
        """

        WorkQueueTestCase.setUp(self)

        self.cacheDir = tempfile.mkdtemp()
        self.specGenerator = WMSpecGenerator(self.cacheDir)
        self.specs = self.createReRecoSpec(1, "file")

        # Create queues
        self.localQueue = localQueue(DbName=self.queueDB, InboxDbName=self.queueInboxDB,
                                     NegotiationTimeout=0, QueueURL='global.example.com', CacheDir=self.cacheDir,
                                     central_logdb_url="%s/%s" % (self.couchURL, self.logDBName),
                                     log_reporter='lq_profile_test')


    def tearDown(self):
        """tearDown"""
        WorkQueueTestCase.tearDown(self)
        try:
            shutil.rmtree(self.cacheDir)
            self.specGenerator.removeSpecs()
        except Exception:
            pass

    def createReRecoSpec(self, numOfSpec, kind="spec"):
        specs = []
        for i in range(numOfSpec):
            specName = "MinBiasProcessingSpec_Test_%s" % (i+1)
            specs.append(self.specGenerator.createReRecoSpec(specName, kind))
        return specs

    def createProfile(self, name, function):
        fileName = name
        prof = cProfile.Profile()
        prof.runcall(function)
        prof.dump_stats(fileName)
        p = pstats.Stats(fileName)
        p.strip_dirs().sort_stats('cumulative').print_stats(0.1)
        p.strip_dirs().sort_stats('time').print_stats(0.1)
        p.strip_dirs().sort_stats('calls').print_stats(0.1)
        p.strip_dirs().sort_stats('name').print_stats(10)

    def testGetWorkLocalQueue(self):
        i = 0
        for spec in self.specs:
            i += 1
            specName = "MinBiasProcessingSpec_Test_%s" % i
            self.localQueue.queueWork(spec, specName, team="A-team")
        self.localQueue.updateLocationInfo()
        self.createProfile('getWorkProfile.prof', self.localQueueGetWork)

    def localQueueGetWork(self):
        siteJobs = {}
        for site in DUMMY_SITES:
            siteJobs[site] = 100000
        self.localQueue.getWork(siteJobs, {})
Example #23
0
class WorkQueueTest(EmulatedUnitTestCase):
    """
    Test WorkQueue Service client
    It will start WorkQueue RESTService
    Server DB sets from environment variable.
    Client DB sets from environment variable.

    This checks whether DS call makes without error and return the results.
    Not the correctness of functions. That will be tested in different module.
    """
    def setUp(self):
        """
        _setUp_
        """
        super(WorkQueueTest, self).setUp()

        self.specGenerator = WMSpecGenerator("WMSpecs")
        # self.configFile = EmulatorSetup.setupWMAgentConfig()
        self.schema = []
        self.couchApps = ["WorkQueue"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules=self.schema, useDefault=False)
        self.testInit.setupCouch('workqueue_t', *self.couchApps)
        self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps)
        self.testInit.setupCouch('local_workqueue_t', *self.couchApps)
        self.testInit.setupCouch('local_workqueue_t_inbox', *self.couchApps)
        self.testInit.generateWorkDir()
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()
        self.specGenerator.removeSpecs()
        super(WorkQueueTest, self).tearDown()

    def testWorkQueueService(self):
        # test getWork
        specName = "RerecoSpec"
        specUrl = self.specGenerator.createReRecoSpec(
            specName, "file", assignKwargs={'SiteWhitelist': ['T2_XX_SiteA']})
        globalQ = globalQueue(DbName='workqueue_t',
                              QueueURL=self.testInit.couchUrl,
                              UnittestFlag=True)
        self.assertTrue(globalQ.queueWork(specUrl, specName, "teamA") > 0)

        wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
        # overwrite default - can't test with stale view
        wqApi.defaultOptions = {'reduce': True, 'group': True}
        # This only checks minimum client call not exactly correctness of return
        # values.
        self.assertEqual(wqApi.getTopLevelJobsByRequest(),
                         [{
                             'total_jobs': 339,
                             'request_name': specName
                         }])
        # work still available, so no childQueue
        results = wqApi.getChildQueuesAndStatus()
        self.assertItemsEqual(set([item['agent_name'] for item in results]),
                              ["AgentNotDefined"])
        result = wqApi.getElementsCountAndJobsByWorkflow()
        self.assertEqual(len(result), 1)
        self.assertEqual(result[specName]['Available']['Jobs'], 339)

        results = wqApi.getChildQueuesAndPriority()
        resultsPrio = set([
            item['priority'] for item in results
            if item['agent_name'] == "AgentNotDefined"
        ])
        self.assertItemsEqual(resultsPrio, [8000])
        self.assertEqual(wqApi.getWMBSUrl(), [])
        self.assertEqual(wqApi.getWMBSUrlByRequest(), [])

    def testUpdatePriorityService(self):
        """
        _testUpdatePriorityService_

        Check that we can update the priority correctly also
        check the available workflows feature
        """
        specName = "RerecoSpec"
        specUrl = self.specGenerator.createReRecoSpec(
            specName, "file", assignKwargs={'SiteWhitelist': ["T2_XX_SiteA"]})
        globalQ = globalQueue(DbName='workqueue_t',
                              QueueURL=self.testInit.couchUrl,
                              UnittestFlag=True)
        localQ = localQueue(DbName='local_workqueue_t',
                            QueueURL=self.testInit.couchUrl,
                            CacheDir=self.testInit.testDir,
                            ParentQueueCouchUrl='%s/workqueue_t' %
                            self.testInit.couchUrl,
                            ParentQueueInboxCouchDBName='workqueue_t_inbox')
        # Try a full chain of priority update and propagation
        self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)
        globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
        # overwrite default - can't test with stale view
        globalApi.defaultOptions = {'reduce': True, 'group': True}
        globalApi.updatePriority(specName, 100)
        self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100)
        storedElements = globalQ.backend.getElementsForWorkflow(specName)
        for element in storedElements:
            self.assertEqual(element['Priority'], 100)
        numWorks = localQ.pullWork({'T2_XX_SiteA': 10})
        self.assertTrue(numWorks > 0)
        # replicate from GQ to LQ manually
        localQ.backend.pullFromParent(continuous=False)
        # wait until replication is done
        time.sleep(2)

        localQ.processInboundWork(continuous=False)
        storedElements = localQ.backend.getElementsForWorkflow(specName)
        for element in storedElements:
            self.assertEqual(element['Priority'], 100)
        localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t')
        # overwrite default - can't test with stale view
        localApi.defaultOptions = {'reduce': True, 'group': True}
        localApi.updatePriority(specName, 500)
        self.assertEqual(localQ.backend.getWMSpec(specName).priority(), 500)
        storedElements = localQ.backend.getElementsForWorkflow(specName)
        for element in storedElements:
            self.assertEqual(element['Priority'], 500)
        availableWF = localApi.getAvailableWorkflows()
        self.assertEqual(availableWF, set([(specName, 500)]))
        # Attempt to update an inexistent workflow in the queue
        try:
            globalApi.updatePriority('NotExistent', 2)
        except Exception as ex:
            self.fail('No exception should be raised.: %s' % str(ex))

    def testCompletedWorkflow(self):
        # test getWork
        specName = "RerecoSpec"
        specUrl = self.specGenerator.createReRecoSpec(
            specName, "file", assignKwargs={'SiteWhitelist': ['T2_XX_SiteA']})

        globalQ = globalQueue(DbName='workqueue_t',
                              QueueURL=self.testInit.couchUrl,
                              UnittestFlag=True)
        self.assertTrue(globalQ.queueWork(specUrl, specName, "teamA") > 0)

        wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
        # overwrite default - can't test with stale view
        wqApi.defaultOptions = {'reduce': True, 'group': True}
        # This only checks minimum client call not exactly correctness of return
        # values.
        self.assertEqual(wqApi.getTopLevelJobsByRequest(),
                         [{
                             'total_jobs': 339,
                             'request_name': specName
                         }])

        results = wqApi.getJobsByStatus()
        self.assertEqual(results['Available']['sum_jobs'], 339)
        results = wqApi.getJobsByStatusAndPriority()
        resultsPrio = set(
            [item['priority'] for item in results.get('Available')])
        self.assertItemsEqual(resultsPrio, [8000])
        resultsJobs = sum([
            item['sum_jobs'] for item in results.get('Available')
            if item['priority'] == 8000
        ])
        self.assertTrue(resultsJobs, 339)
        result = wqApi.getElementsCountAndJobsByWorkflow()
        self.assertEqual(len(result), 1)
        self.assertEqual(result[specName]['Available']['Jobs'], 339)
        data = wqApi.db.loadView('WorkQueue',
                                 'elementsDetailByWorkflowAndStatus', {
                                     'startkey': [specName],
                                     'endkey': [specName, {}],
                                     'reduce': False
                                 })
        elements = [x['id'] for x in data.get('rows', [])]
        wqApi.updateElements(*elements, Status='Canceled')
        # load this view once again to make sure it will be updated in the next assert..
        data = wqApi.db.loadView('WorkQueue',
                                 'elementsDetailByWorkflowAndStatus', {
                                     'startkey': [specName],
                                     'endkey': [specName, {}],
                                     'reduce': False
                                 })
        self.assertEqual(len(wqApi.getCompletedWorkflow(stale=False)), 1)
        results = wqApi.getJobsByStatusAndPriority()
        resultsPrio = set(
            [item['priority'] for item in results.get('Canceled')])
        self.assertItemsEqual(resultsPrio, [8000])

    def testConvertWQElementsStatusToWFStatus(self):
        """
        _testConvertWQElementsStatusToWFStatus_

        Check that a set of all the workqueue element status in a request
        properly maps to a single state to trigger the ReqMgr request transition.
        """
        # workflows acquired by global_workqueue (nothing acquired by agents so far)
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Available"])),
                         "acquired")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(set(["Negotiating"])),
            "acquired")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(set(["Available",
                                                   "Negotiating"])),
            "acquired")

        # workflows not completely acquired yet by the agents
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Acquired"])),
                         "running-open")

        self.assertEqual(
            convertWQElementsStatusToWFStatus(
                set(["Available", "Negotiating", "Acquired"])), "running-open")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(
                set(["Available", "Negotiating", "Acquired", "Running"])),
            "running-open")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(
                set([
                    "Available", "Negotiating", "Acquired", "Running", "Done"
                ])), "running-open")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(
                set([
                    "Available", "Negotiating", "Acquired", "Running", "Done",
                    "CancelRequested"
                ])), "running-open")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(
                set([
                    "Available", "Negotiating", "Acquired", "Running", "Done",
                    "CancelRequested", "Canceled"
                ])), "running-open")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(set(["Negotiating",
                                                   "Acquired"])),
            "running-open")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(
                set(["Negotiating", "Acquired", "Running"])), "running-open")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(
                set(["Negotiating", "Acquired", "Running", "Done"])),
            "running-open")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(
                set([
                    "Negotiating", "Acquired", "Running", "Done",
                    "CancelRequested"
                ])), "running-open")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(
                set([
                    "Negotiating", "Acquired", "Running", "Done",
                    "CancelRequested", "Canceled"
                ])), "running-open")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(set(["Acquired", "Running"])),
            "running-open")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(
                set(["Acquired", "Running", "Done"])), "running-open")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(
                set(["Acquired", "Running", "Done", "CancelRequested"])),
            "running-open")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(
                set([
                    "Acquired", "Running", "Done", "CancelRequested",
                    "Canceled"
                ])), "running-open")

        # workflows completely acquired by the agents
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Running"])),
                         "running-closed")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(set(["Running", "Done"])),
            "running-closed")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(
                set(["Running", "Done", "CancelRequested"])), "running-closed")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(
                set(["Running", "Done", "CancelRequested", "Canceled"])),
            "running-closed")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(
                set(["Running", "Done", "Canceled"])), "running-closed")

        # workflows completed/aborted/force-completed, thus existent elements
        #  but no more active workqueue elements in the system
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Done"])),
                         "completed")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Canceled"])),
                         "completed")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(set(["Done", "Canceled"])),
            "completed")

        # workflows failed
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Failed"])),
                         "failed")

        # non-failed workflows but with Failed elements
        self.assertEqual(
            convertWQElementsStatusToWFStatus(
                set(["Available", "Negotiating", "Acquired", "Failed"])),
            "running-open")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(
                set([
                    "Available", "Negotiating", "Acquired", "Running", "Done",
                    "Failed"
                ])), "running-open")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(
                set([
                    "Negotiating", "Acquired", "Running", "Done", "Canceled",
                    "Failed"
                ])), "running-open")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(set(["Running", "Failed"])),
            "running-closed")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(
                set(["Running", "Done", "Canceled", "Failed"])),
            "running-closed")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(set(["Done", "Failed"])),
            "completed")
        self.assertEqual(
            convertWQElementsStatusToWFStatus(set(["Canceled", "Failed"])),
            "completed")

        # workflows in a temporary state, nothing to do with them yet
        self.assertIsNone(
            convertWQElementsStatusToWFStatus(set(["Done",
                                                   "CancelRequested"])))
        self.assertIsNone(
            convertWQElementsStatusToWFStatus(set(["CancelRequested"])))
Example #24
0
class WorkQueueTest(unittest.TestCase):
    """
    Test WorkQueue Service client
    It will start WorkQueue RESTService
    Server DB sets from environment variable.
    Client DB sets from environment variable.

    This checks whether DS call makes without error and return the results.
    Not the correctness of functions. That will be tested in different module.
    """
    def setUp(self):
        """
        _setUp_
        """
        EmulatorHelper.setEmulators(phedex=True,
                                    dbs=True,
                                    siteDB=True,
                                    requestMgr=True)

        self.specGenerator = WMSpecGenerator("WMSpecs")
        #self.configFile = EmulatorSetup.setupWMAgentConfig()
        self.schema = []
        self.couchApps = ["WorkQueue"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules=self.schema, useDefault=False)
        self.testInit.setupCouch('workqueue_t', *self.couchApps)
        self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps)
        self.testInit.setupCouch('local_workqueue_t', *self.couchApps)
        self.testInit.setupCouch('local_workqueue_t_inbox', *self.couchApps)
        self.testInit.generateWorkDir()
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()
        EmulatorHelper.resetEmulators()
        self.specGenerator.removeSpecs()

    def testWorkQueueService(self):
        # test getWork
        specName = "RerecoSpec"
        specUrl = self.specGenerator.createReRecoSpec(specName, "file")
        globalQ = globalQueue(DbName='workqueue_t',
                              QueueURL=self.testInit.couchUrl)
        self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)

        wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
        #overwrite default - can't test with stale view
        wqApi.defaultOptions = {'reduce': True, 'group': True}
        #This only checks minimum client call not exactly correctness of return
        # values.
        self.assertEqual(wqApi.getTopLevelJobsByRequest(),
                         [{
                             'total_jobs': 10,
                             'request_name': specName
                         }])
        self.assertEqual(wqApi.getChildQueues(), [])
        self.assertEqual(wqApi.getJobStatusByRequest(),
                         [{
                             'status': 'Available',
                             'jobs': 10,
                             'request_name': specName
                         }])
        self.assertEqual(wqApi.getChildQueuesByRequest(), [])
        self.assertEqual(wqApi.getWMBSUrl(), [])
        self.assertEqual(wqApi.getWMBSUrlByRequest(), [])

    def testUpdatePriorityService(self):
        """
        _testUpdatePriorityService_

        Check that we can update the priority correctly also
        check the available workflows feature
        """
        specName = "RerecoSpec"
        specUrl = self.specGenerator.createReRecoSpec(specName, "file")
        globalQ = globalQueue(DbName='workqueue_t',
                              QueueURL=self.testInit.couchUrl)
        localQ = localQueue(DbName='local_workqueue_t',
                            QueueURL=self.testInit.couchUrl,
                            CacheDir=self.testInit.testDir,
                            ParentQueueCouchUrl='%s/workqueue_t' %
                            self.testInit.couchUrl,
                            ParentQueueInboxCouchDBName='workqueue_t_inbox')
        # Try a full chain of priority update and propagation
        self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)
        globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
        #overwrite default - can't test with stale view
        globalApi.defaultOptions = {'reduce': True, 'group': True}
        globalApi.updatePriority(specName, 100)
        self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100)
        storedElements = globalQ.backend.getElementsForWorkflow(specName)
        for element in storedElements:
            self.assertEqual(element['Priority'], 100)
        self.assertTrue(localQ.pullWork({'T2_XX_SiteA': 10}) > 0)
        localQ.processInboundWork(continuous=False)
        storedElements = localQ.backend.getElementsForWorkflow(specName)
        for element in storedElements:
            self.assertEqual(element['Priority'], 100)
        localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t')
        #overwrite default - can't test with stale view
        localApi.defaultOptions = {'reduce': True, 'group': True}
        localApi.updatePriority(specName, 500)
        self.assertEqual(localQ.backend.getWMSpec(specName).priority(), 500)
        storedElements = localQ.backend.getElementsForWorkflow(specName)
        for element in storedElements:
            self.assertEqual(element['Priority'], 500)
        self.assertEqual(localApi.getAvailableWorkflows(),
                         set([(specName, 500)]))
        # Attempt to update an inexistent workflow in the queue
        try:
            globalApi.updatePriority('NotExistent', 2)
        except:
            self.fail('No exception should be raised.')
Example #25
0
class RequestManager(dict):
    
    def __init__(self, *args, **kwargs):
        """
        all the private valuable is defined for test values
        """
        self.specGenerator = WMSpecGenerator()
        self.count = 0
        self.maxWmSpec = kwargs.setdefault('numOfSpecs', 1)
        self.type = kwargs.setdefault("type", 'ReReco')
        if self.type not in ['ReReco', 'MonteCarlo']:
            raise TypeError, 'unknown request type %s' % self.type
        self.splitter = kwargs.setdefault('splitter', 'DatasetBlock')
        self.inputDataset = kwargs.setdefault('inputDataset', None)
        self.dbsUrl = kwargs.setdefault('dbsUrl', None)
        self.status = {}
        self.progress = {}
        self.msg = {}
        self.names = []
        import logging
        self['logger'] = logging
    
    def getAssignment(self, teamName=None, request=None):
        if self.count < self.maxWmSpec:
            if self.type == 'ReReco':
                specName = "ReRecoTest_v%sEmulator" % self.count
                specUrl =self.specGenerator.createReRecoSpec(specName, "file",
                                                             self.splitter,
                                                             self.inputDataset,
                                                             self.dbsUrl)
            elif self.type == 'MonteCarlo':
                specName = "MCTest_v%sEmulator" % self.count
                specUrl =self.specGenerator.createMCSpec(specName, "file",
                                                         self.splitter)
            self.names.append(specName)
            self.status[specName] = 'assigned'
            #specName = "FakeProductionSpec_%s" % self.count
            #specUrl =self.specGenerator.createProductionSpec(specName, "file")
            #specName = "FakeProcessingSpec_%s" % self.count
            #specUrl =self.specGenerator.createProcessingSpec(specName, "file")
        
            self.count += 1
            # returns list of list(rquest name, spec url)
            return [[specName, specUrl],]
        else:
            return []
    
    def putWorkQueue(self, reqName, prodAgentUrl=None):
        self.status[reqName] = 'acquired'

    def reportRequestStatus(self, name, status):
        if status not in NextStatus[self.status[name]]:
            raise RuntimeError, "Invalid status move: %s" % status
        self.status[name] = status

    def reportRequestProgress(self, name, **args):
        self.progress.setdefault(name, {})
        self.progress[name].update(args)

    def sendMessage(self, request, msg):
        self.msg[request] = msg
        
    def _removeSpecs(self):
        """
        This is just for clean up not part of emulated function
        """
        self.specGenerator.removeSpecs()
Example #26
0
class LocalWorkQueueProfileTest(WorkQueueTestCase):
    """
    _WorkQueueTest_

    """
    def setUp(self):
        """
        If we dont have a wmspec file create one
        """

        WorkQueueTestCase.setUp(self)

        self.cacheDir = tempfile.mkdtemp()
        self.specGenerator = WMSpecGenerator(self.cacheDir)
        self.specs = self.createReRecoSpec(1, "file")
        # setup rucio parameters for global/local queue
        self.queueParams = {}
        self.queueParams['log_reporter'] = "lq_profile_test"
        self.queueParams['rucioAccount'] = "wma_test"
        self.queueParams['rucioAuthUrl'] = "http://cmsrucio-int.cern.ch"
        self.queueParams['rucioUrl'] = "https://cmsrucio-auth-int.cern.ch"

        # Create queues
        self.localQueue = localQueue(DbName=self.queueDB,
                                     InboxDbName=self.queueInboxDB,
                                     NegotiationTimeout=0,
                                     QueueURL='global.example.com',
                                     CacheDir=self.cacheDir,
                                     central_logdb_url="%s/%s" %
                                     (self.couchURL, self.logDBName),
                                     **self.queueParams)

    def tearDown(self):
        """tearDown"""
        WorkQueueTestCase.tearDown(self)
        try:
            shutil.rmtree(self.cacheDir)
            self.specGenerator.removeSpecs()
        except Exception:
            pass

    def createReRecoSpec(self, numOfSpec, kind="spec"):
        specs = []
        for i in range(numOfSpec):
            specName = "MinBiasProcessingSpec_Test_%s" % (i + 1)
            specs.append(
                self.specGenerator.createReRecoSpec(
                    specName,
                    kind,
                    assignKwargs={'SiteWhitelist': ['T2_XX_SiteA']}))
        return specs

    def createProfile(self, name, function):
        fileName = name
        prof = cProfile.Profile()
        prof.runcall(function)
        prof.dump_stats(fileName)
        p = pstats.Stats(fileName)
        p.strip_dirs().sort_stats('cumulative').print_stats(0.1)
        p.strip_dirs().sort_stats('time').print_stats(0.1)
        p.strip_dirs().sort_stats('calls').print_stats(0.1)
        p.strip_dirs().sort_stats('name').print_stats(10)

    def testGetWorkLocalQueue(self):
        i = 0
        for spec in self.specs:
            i += 1
            specName = "MinBiasProcessingSpec_Test_%s" % i
            self.localQueue.queueWork(spec, specName, team="A-team")
        self.localQueue.updateLocationInfo()
        self.createProfile('getWorkProfile.prof', self.localQueueGetWork)

    def localQueueGetWork(self):
        siteJobs = {}
        for site in DUMMY_SITES:
            siteJobs[site] = 100000
        self.localQueue.getWork(siteJobs, {})
Example #27
0
class ReportEmuTest(unittest.TestCase):
    """
    _ReportEmuTest_

    """
    def setUp(self):
        """
        _setUp_

        Setup some reasonable defaults for the ReReco workflow.
        """
        self.unmergedLFNBase = "/store/backfill/2/unmerged"
        self.mergedLFNBase = "/store/backfill/2"
        self.processingVersion = "v1"
        self.cmsswVersion = "CMSSW_3_4_2_patch1"
        self.acquisitionEra = "WMAgentCommissioining10"
        self.primaryDataset = "MinimumBias"

        self.workload = WMSpecGenerator().createReRecoSpec("Tier1ReReco")
        print self.workload.data
        return

    def verifyOutputMetaData(self, outputFile, job):
        """
        _verifyOutputMetaData_

        Verify that metadata that in an emulated FWJR.  Most of the meta data
        should be the same as the input file.
        """
        goldenRuns = []

        for inputFile in job["input_files"]:
            for run in inputFile["runs"]:
                goldenRuns.append(Run(run.run, *run.lumis))

        assert len(outputFile["runs"]) == len(goldenRuns), \
                   "Error: Wrong number of runs in output file."

        for outputRun in outputFile["runs"]:
            for goldenRun in goldenRuns:
                if outputRun.run == goldenRun.run:
                    goldenRun.lumis.sort()
                    outputRun.lumis.sort()

                    if goldenRun.lumis == outputRun.lumis:
                        goldenRuns.remove(goldenRun)
                        break

        assert len(goldenRuns) == 0, \
               "Error: Run information wrong on output file."

        assert len(outputFile["locations"]) == 1,  \
               "Error: Wrong number of locations."

        assert list(outputFile["locations"])[0] == job["location"], \
               "Error: Output file at the wrong location."

        assert outputFile["merged"] == False, \
               "Error: Output should be unmerged."

        assert "adler32" in outputFile["checksums"].keys(), \
               "Error: Adler32 checksum missing."
        assert "cksum" in outputFile["checksums"].keys(), \
               "Error: CKSum checksum missing."

        return

    def testProcessing(self):
        """
        _testProcessing_

        Setup a processing workflow and job and verify that the FWJR produced
        by the emulator is reasonable.
        """
        rerecoTask = self.workload.getTask("DataProcessing")
        cmsRunStep = rerecoTask.getStep("cmsRun1")

        inputFile = File(lfn = "/path/to/test/lfn", size = 1048576, events = 1000, merged = True)
        inputFile.addRun(Run(1, *[1, 2, 3, 4, 5]))
        inputFile.addRun(Run(2, *[1, 2, 3, 4, 5, 6]))

        processingJob = Job(name = "ProcessingJob", files = [inputFile])
        processingJob["task"] = "/Tier1ReReco/ReReco"
        processingJob["mask"].setMaxAndSkipEvents(500, 0)
        processingJob["id"] = 1
        processingJob["location"] = "cmssrm.fnal.gov"

        emu = ReportEmu(WMStep = cmsRunStep.getTypeHelper(), Job = processingJob)
        report = emu()

        reportInputFiles = report.getInputFilesFromStep("cmsRun1")

        assert len(reportInputFiles) == 1, \
               "Error: Wrong number of input files for the job."
        assert reportInputFiles[0]["lfn"] == inputFile["lfn"], \
               "Error: Input LFNs do not match: %s" % reportInputFiles[0]["lfn"]
        assert reportInputFiles[0]["size"] == inputFile["size"], \
               "Error: Input file sizes do not match."
        assert reportInputFiles[0]["events"] == inputFile["events"], \
               "Error: Input file events do not match."

        goldenRuns = [Run(1, *[1, 2, 3, 4, 5]), Run(2, *[1, 2, 3, 4, 5, 6])]

        assert len(reportInputFiles[0]["runs"]) == len(goldenRuns), \
                   "Error: Wrong number of runs in input file."

        for inputRun in reportInputFiles[0]["runs"]:
            for goldenRun in goldenRuns:
                if inputRun.run == goldenRun.run:
                    goldenRun.lumis.sort()
                    inputRun.lumis.sort()

                    if goldenRun.lumis == inputRun.lumis:
                        goldenRuns.remove(goldenRun)
                        break

        assert len(goldenRuns) == 0, \
               "Error: Run information wrong on input file."

        recoOutputFiles = report.getFilesFromOutputModule("cmsRun1", "RECOoutput")
        alcaOutputFiles = report.getFilesFromOutputModule("cmsRun1", "ALCARECOoutput")

        assert len(recoOutputFiles) == 1, \
               "Error: There should only be one RECO output file."
        assert len(alcaOutputFiles) == 1, \
               "Error: There should only be one ALCA output file."

        assert recoOutputFiles[0]["module_label"] == "RECOoutput", \
               "Error: RECO file has wrong output module."
        assert alcaOutputFiles[0]["module_label"] == "ALCARECOoutput", \
               "Error: ALCA file has wrong output module."

        self.verifyOutputMetaData(recoOutputFiles[0], processingJob)
        self.verifyOutputMetaData(alcaOutputFiles[0], processingJob)

        dataTierMap = {"RECOoutput": "RECO", "ALCARECOoutput": "ALCARECO"}
        for outputFile in [recoOutputFiles[0], alcaOutputFiles[0]]:
            assert outputFile["dataset"]["applicationName"] == "cmsRun", \
                   "Error: Application name is incorrect."
            assert outputFile["dataset"]["primaryDataset"] == self.primaryDataset, \
                   "Error: Primary dataset is incorrect."
            assert outputFile["dataset"]["dataTier"] == dataTierMap[outputFile["module_label"]], \
                   "Error: Data tier is incorrect."

        return

    def testMerge(self):
        """
        _testMerge_

        Setup a merge workflow and job and verify that the FWJR produced by the
        emulator is reasonable.
        """
        #emu = ReportEmu(WMStep = self.cmssw, Job = self.job)
        #report = emu()
        return
Example #28
0
class WorkQueueTest(EmulatedUnitTestCase):
    """
    Test WorkQueue Service client
    It will start WorkQueue RESTService
    Server DB sets from environment variable.
    Client DB sets from environment variable.

    This checks whether DS call makes without error and return the results.
    Not the correctness of functions. That will be tested in different module.
    """

    def setUp(self):
        """
        _setUp_
        """
        super(WorkQueueTest, self).setUp()

        self.specGenerator = WMSpecGenerator("WMSpecs")
        # self.configFile = EmulatorSetup.setupWMAgentConfig()
        self.schema = []
        self.couchApps = ["WorkQueue"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules=self.schema,
                                useDefault=False)
        self.testInit.setupCouch('workqueue_t', *self.couchApps)
        self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps)
        self.testInit.setupCouch('local_workqueue_t', *self.couchApps)
        self.testInit.setupCouch('local_workqueue_t_inbox', *self.couchApps)
        self.testInit.generateWorkDir()
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()
        self.specGenerator.removeSpecs()
        super(WorkQueueTest, self).tearDown()

    def testWorkQueueService(self):
        # test getWork
        specName = "RerecoSpec"
        specUrl = self.specGenerator.createReRecoSpec(specName, "file",
                                                      assignKwargs={'SiteWhitelist': ['T2_XX_SiteA']})
        globalQ = globalQueue(DbName='workqueue_t',
                              QueueURL=self.testInit.couchUrl,
                              UnittestFlag=True)
        self.assertTrue(globalQ.queueWork(specUrl, specName, "teamA") > 0)

        wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
        # overwrite default - can't test with stale view
        wqApi.defaultOptions = {'reduce': True, 'group': True}
        # This only checks minimum client call not exactly correctness of return
        # values.
        self.assertEqual(wqApi.getTopLevelJobsByRequest(),
                         [{'total_jobs': 339, 'request_name': specName}])
        # work still available, so no childQueue
        results = wqApi.getChildQueuesAndStatus()
        self.assertItemsEqual(set([item['agent_name'] for item in results]), ["AgentNotDefined"])
        result = wqApi.getElementsCountAndJobsByWorkflow()
        self.assertEqual(len(result), 1)
        self.assertEqual(result[specName]['Available']['Jobs'], 339)

        results = wqApi.getChildQueuesAndPriority()
        resultsPrio = set([item['priority'] for item in results if item['agent_name'] == "AgentNotDefined"])
        self.assertItemsEqual(resultsPrio, [8000])
        self.assertEqual(wqApi.getWMBSUrl(), [])
        self.assertEqual(wqApi.getWMBSUrlByRequest(), [])

    def testUpdatePriorityService(self):
        """
        _testUpdatePriorityService_

        Check that we can update the priority correctly also
        check the available workflows feature
        """
        specName = "RerecoSpec"
        specUrl = self.specGenerator.createReRecoSpec(specName, "file",
                                                      assignKwargs={'SiteWhitelist':["T2_XX_SiteA"]})
        globalQ = globalQueue(DbName='workqueue_t',
                              QueueURL=self.testInit.couchUrl,
                              UnittestFlag=True)
        localQ = localQueue(DbName='local_workqueue_t',
                            QueueURL=self.testInit.couchUrl,
                            CacheDir=self.testInit.testDir,
                            ParentQueueCouchUrl='%s/workqueue_t' % self.testInit.couchUrl,
                            ParentQueueInboxCouchDBName='workqueue_t_inbox'
                            )
        # Try a full chain of priority update and propagation
        self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)
        globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
        # overwrite default - can't test with stale view
        globalApi.defaultOptions = {'reduce': True, 'group': True}
        globalApi.updatePriority(specName, 100)
        self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100)
        storedElements = globalQ.backend.getElementsForWorkflow(specName)
        for element in storedElements:
            self.assertEqual(element['Priority'], 100)
        numWorks = localQ.pullWork({'T2_XX_SiteA': 10})
        self.assertTrue(numWorks > 0)
        # replicate from GQ to LQ manually
        localQ.backend.pullFromParent(continuous=False)
        # wait until replication is done
        time.sleep(2)

        localQ.processInboundWork(continuous=False)
        storedElements = localQ.backend.getElementsForWorkflow(specName)
        for element in storedElements:
            self.assertEqual(element['Priority'], 100)
        localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t')
        # overwrite default - can't test with stale view
        localApi.defaultOptions = {'reduce': True, 'group': True}
        localApi.updatePriority(specName, 500)
        self.assertEqual(localQ.backend.getWMSpec(specName).priority(), 500)
        storedElements = localQ.backend.getElementsForWorkflow(specName)
        for element in storedElements:
            self.assertEqual(element['Priority'], 500)
        availableWF = localApi.getAvailableWorkflows()
        self.assertEqual(availableWF, set([(specName, 500)]))
        # Attempt to update an inexistent workflow in the queue
        try:
            globalApi.updatePriority('NotExistent', 2)
        except Exception as ex:
            self.fail('No exception should be raised.: %s' % str(ex))

    def testCompletedWorkflow(self):
        # test getWork
        specName = "RerecoSpec"
        specUrl = self.specGenerator.createReRecoSpec(specName, "file",
                                                      assignKwargs={'SiteWhitelist':['T2_XX_SiteA']})

        globalQ = globalQueue(DbName='workqueue_t',
                              QueueURL=self.testInit.couchUrl,
                              UnittestFlag=True)
        self.assertTrue(globalQ.queueWork(specUrl, specName, "teamA") > 0)

        wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
        # overwrite default - can't test with stale view
        wqApi.defaultOptions = {'reduce': True, 'group': True}
        # This only checks minimum client call not exactly correctness of return
        # values.
        self.assertEqual(wqApi.getTopLevelJobsByRequest(),
                         [{'total_jobs': 339, 'request_name': specName}])

        results = wqApi.getJobsByStatus()
        self.assertEqual(results['Available']['sum_jobs'], 339)
        results = wqApi.getJobsByStatusAndPriority()
        resultsPrio = set([item['priority'] for item in results.get('Available')])
        self.assertItemsEqual(resultsPrio, [8000])
        resultsJobs = sum([item['sum_jobs'] for item in results.get('Available') if item['priority'] == 8000])
        self.assertTrue(resultsJobs, 339)
        result = wqApi.getElementsCountAndJobsByWorkflow()
        self.assertEqual(len(result), 1)
        self.assertEqual(result[specName]['Available']['Jobs'], 339)
        data = wqApi.db.loadView('WorkQueue', 'elementsDetailByWorkflowAndStatus',
                                 {'startkey': [specName], 'endkey': [specName, {}],
                                  'reduce': False})
        elements = [x['id'] for x in data.get('rows', [])]
        wqApi.updateElements(*elements, Status='Canceled')
        # load this view once again to make sure it will be updated in the next assert..
        data = wqApi.db.loadView('WorkQueue', 'elementsDetailByWorkflowAndStatus',
                                 {'startkey': [specName], 'endkey': [specName, {}],
                                  'reduce': False})
        self.assertEqual(len(wqApi.getCompletedWorkflow(stale=False)), 1)
        results = wqApi.getJobsByStatusAndPriority()
        resultsPrio = set([item['priority'] for item in results.get('Canceled')])
        self.assertItemsEqual(resultsPrio, [8000])

    def testConvertWQElementsStatusToWFStatus(self):
        """
        _testConvertWQElementsStatusToWFStatus_

        Check that a set of all the workqueue element status in a request
        properly maps to a single state to trigger the ReqMgr request transition.
        """
        # workflows acquired by global_workqueue (nothing acquired by agents so far)
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Available"])), "acquired")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Negotiating"])), "acquired")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Available", "Negotiating"])), "acquired")

        # workflows not completely acquired yet by the agents
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Acquired"])), "running-open")

        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Available", "Negotiating", "Acquired"])), "running-open")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Available", "Negotiating", "Acquired", "Running"])), "running-open")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Available", "Negotiating", "Acquired", "Running", "Done"])), "running-open")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Available", "Negotiating", "Acquired", "Running", "Done", "CancelRequested"])), "running-open")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Available", "Negotiating", "Acquired", "Running", "Done", "CancelRequested", "Canceled"])), "running-open")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Negotiating", "Acquired"])), "running-open")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Negotiating", "Acquired", "Running"])), "running-open")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Negotiating", "Acquired", "Running", "Done"])), "running-open")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Negotiating", "Acquired", "Running", "Done", "CancelRequested"])), "running-open")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Negotiating", "Acquired", "Running", "Done", "CancelRequested", "Canceled"])), "running-open")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Acquired", "Running"])), "running-open")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Acquired", "Running", "Done"])), "running-open")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Acquired", "Running", "Done", "CancelRequested"])), "running-open")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Acquired", "Running", "Done", "CancelRequested", "Canceled"])), "running-open")

        # workflows completely acquired by the agents
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Running"])), "running-closed")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Running", "Done"])), "running-closed")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Running", "Done", "CancelRequested"])), "running-closed")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Running", "Done", "CancelRequested", "Canceled"])), "running-closed")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Running", "Done", "Canceled"])), "running-closed")

        # workflows completed/aborted/force-completed, thus existent elements
        #  but no more active workqueue elements in the system
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Done"])), "completed")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Canceled"])), "completed")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Done", "Canceled"])), "completed")

        # workflows failed
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Failed"])), "failed")

        # non-failed workflows but with Failed elements
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Available", "Negotiating", "Acquired", "Failed"])), "running-open")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Available", "Negotiating", "Acquired", "Running", "Done", "Failed"])), "running-open")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Negotiating", "Acquired", "Running", "Done", "Canceled", "Failed"])), "running-open")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Running", "Failed"])), "running-closed")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Running", "Done", "Canceled", "Failed"])), "running-closed")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Done", "Failed"])), "completed")
        self.assertEqual(convertWQElementsStatusToWFStatus(set(["Canceled", "Failed"])), "completed")

        # workflows in a temporary state, nothing to do with them yet
        self.assertIsNone(convertWQElementsStatusToWFStatus(set(["Done", "CancelRequested"])))
        self.assertIsNone(convertWQElementsStatusToWFStatus(set(["CancelRequested"])))
Example #29
0
class ReqMgr(dict):

    def __init__(self, *args, **kwargs):
        """
        all the private valuable is defined for test values
        """
        self.specGenerator = WMSpecGenerator()
        self.count = 0
        self.maxWmSpec = kwargs.setdefault('numOfSpecs', 1)
        self.type = kwargs.setdefault("type", 'ReReco')
        if self.type not in ['ReReco', 'MonteCarlo']:
            raise TypeError('unknown request type %s' % self.type)
        self.splitter = kwargs.setdefault('splitter', 'DatasetBlock')
        self.inputDataset = kwargs.setdefault('inputDataset', None)
        self.dbsUrl = kwargs.setdefault('dbsUrl', None)
        self.status = {}
        self.progress = {}
        self.msg = {}
        self.names = []
        self.openRunningTimeout = kwargs.setdefault('openRunningTimeout', 0)
        import logging
        self['logger'] = logging

    def getAssignment(self, teamName=None, request=None):
        if self.count < self.maxWmSpec:
            if self.type == 'ReReco':
                specName = "ReRecoTest_v%sEmulator" % self.count
                specUrl =self.specGenerator.createReRecoSpec(specName, "file",
                                                             self.splitter,
                                                             self.inputDataset,
                                                             self.dbsUrl,
                                                             OpenRunningTimeout = self.openRunningTimeout)
            elif self.type == 'MonteCarlo':
                specName = "MCTest_v%sEmulator" % self.count
                specUrl =self.specGenerator.createMCSpec(specName, "file",
                                                         self.splitter)
            self.names.append(specName)
            self.status[specName] = 'assigned'
            #specName = "FakeProductionSpec_%s" % self.count
            #specUrl =self.specGenerator.createProductionSpec(specName, "file")
            #specName = "FakeProcessingSpec_%s" % self.count
            #specUrl =self.specGenerator.createProcessingSpec(specName, "file")

            self.count += 1
            # returns list of list(rquest name, spec url)
            return [[specName, specUrl],]
        else:
            return []

    def getRunningOpen(self, teamName):
        """Returns a list of request names of requests in running-open state"""
        result = []
        for request in self.status:
            if self.status[request] == 'running-open':
                result.append(request)
        return result

    def getRequest(self, requestName):
        """Get request info"""
        if requestName not in self.names:
            raise RuntimeError("unknown request %s" % requestName)

        request = {'RequestName' : requestName,
                   'RequestStatus' : self.status[requestName],
                   'RequestPriority' : 100}
        if requestName in self.progress:
            request.update(self.progress[requestName])
        request.setdefault('percent_complete', 0)
        request.setdefault('percent_success', 0)
        return request

    def putWorkQueue(self, reqName, prodAgentUrl=None):
        self.status[reqName] = 'acquired'

    def reportRequestStatus(self, name, status):
        if not check_allowed_transition(self.status[name], status):
            raise RuntimeError("Invalid status move: %s" % status)
        self.status[name] = status

    def reportRequestProgress(self, name, **args):
        self.progress.setdefault(name, {})
        self.progress[name].update(args)

    def sendMessage(self, request, msg):
        self.msg[request] = msg

    def _removeSpecs(self):
        """
        This is just for clean up not part of emulated function
        """
        self.specGenerator.removeSpecs()

    def getTeam(self):
        """Give a fake team name"""
        return ["The A-Team", "some other bloke"]
Example #30
0
class ReportEmuTest(unittest.TestCase):
    """
    _ReportEmuTest_

    """
    def setUp(self):
        """
        _setUp_

        Setup some reasonable defaults for the ReReco workflow.
        """
        self.unmergedLFNBase = "/store/backfill/2/unmerged"
        self.mergedLFNBase = "/store/backfill/2"
        self.processingVersion = "v1"
        self.cmsswVersion = "CMSSW_3_4_2_patch1"
        self.acquisitionEra = "WMAgentCommissioining10"
        self.primaryDataset = "MinimumBias"

        self.workload = WMSpecGenerator().createReRecoSpec("Tier1ReReco")
        print self.workload.data
        return

    def verifyOutputMetaData(self, outputFile, job):
        """
        _verifyOutputMetaData_

        Verify that metadata that in an emulated FWJR.  Most of the meta data
        should be the same as the input file.
        """
        goldenRuns = []

        for inputFile in job["input_files"]:
            for run in inputFile["runs"]:
                goldenRuns.append(Run(run.run, *run.lumis))

        assert len(outputFile["runs"]) == len(goldenRuns), \
                   "Error: Wrong number of runs in output file."

        for outputRun in outputFile["runs"]:
            for goldenRun in goldenRuns:
                if outputRun.run == goldenRun.run:
                    goldenRun.lumis.sort()
                    outputRun.lumis.sort()

                    if goldenRun.lumis == outputRun.lumis:
                        goldenRuns.remove(goldenRun)
                        break

        assert len(goldenRuns) == 0, \
               "Error: Run information wrong on output file."

        assert len(outputFile["locations"]) == 1,  \
               "Error: Wrong number of locations."

        assert list(outputFile["locations"])[0] == job["location"], \
               "Error: Output file at the wrong location."

        assert outputFile["merged"] == False, \
               "Error: Output should be unmerged."

        assert "adler32" in outputFile["checksums"].keys(), \
               "Error: Adler32 checksum missing."
        assert "cksum" in outputFile["checksums"].keys(), \
               "Error: CKSum checksum missing."

        return

    def testProcessing(self):
        """
        _testProcessing_

        Setup a processing workflow and job and verify that the FWJR produced
        by the emulator is reasonable.
        """
        rerecoTask = self.workload.getTask("DataProcessing")
        cmsRunStep = rerecoTask.getStep("cmsRun1")

        inputFile = File(lfn="/path/to/test/lfn",
                         size=1048576,
                         events=1000,
                         merged=True)
        inputFile.addRun(Run(1, *[1, 2, 3, 4, 5]))
        inputFile.addRun(Run(2, *[1, 2, 3, 4, 5, 6]))

        processingJob = Job(name="ProcessingJob", files=[inputFile])
        processingJob["task"] = "/Tier1ReReco/ReReco"
        processingJob["mask"].setMaxAndSkipEvents(500, 0)
        processingJob["id"] = 1
        processingJob["location"] = "cmssrm.fnal.gov"

        emu = ReportEmu(WMStep=cmsRunStep.getTypeHelper(), Job=processingJob)
        report = emu()

        reportInputFiles = report.getInputFilesFromStep("cmsRun1")

        assert len(reportInputFiles) == 1, \
               "Error: Wrong number of input files for the job."
        assert reportInputFiles[0]["lfn"] == inputFile["lfn"], \
               "Error: Input LFNs do not match: %s" % reportInputFiles[0]["lfn"]
        assert reportInputFiles[0]["size"] == inputFile["size"], \
               "Error: Input file sizes do not match."
        assert reportInputFiles[0]["events"] == inputFile["events"], \
               "Error: Input file events do not match."

        goldenRuns = [Run(1, *[1, 2, 3, 4, 5]), Run(2, *[1, 2, 3, 4, 5, 6])]

        assert len(reportInputFiles[0]["runs"]) == len(goldenRuns), \
                   "Error: Wrong number of runs in input file."

        for inputRun in reportInputFiles[0]["runs"]:
            for goldenRun in goldenRuns:
                if inputRun.run == goldenRun.run:
                    goldenRun.lumis.sort()
                    inputRun.lumis.sort()

                    if goldenRun.lumis == inputRun.lumis:
                        goldenRuns.remove(goldenRun)
                        break

        assert len(goldenRuns) == 0, \
               "Error: Run information wrong on input file."

        recoOutputFiles = report.getFilesFromOutputModule(
            "cmsRun1", "RECOoutput")
        alcaOutputFiles = report.getFilesFromOutputModule(
            "cmsRun1", "ALCARECOoutput")

        assert len(recoOutputFiles) == 1, \
               "Error: There should only be one RECO output file."
        assert len(alcaOutputFiles) == 1, \
               "Error: There should only be one ALCA output file."

        assert recoOutputFiles[0]["module_label"] == "RECOoutput", \
               "Error: RECO file has wrong output module."
        assert alcaOutputFiles[0]["module_label"] == "ALCARECOoutput", \
               "Error: ALCA file has wrong output module."

        self.verifyOutputMetaData(recoOutputFiles[0], processingJob)
        self.verifyOutputMetaData(alcaOutputFiles[0], processingJob)

        dataTierMap = {"RECOoutput": "RECO", "ALCARECOoutput": "ALCARECO"}
        for outputFile in [recoOutputFiles[0], alcaOutputFiles[0]]:
            assert outputFile["dataset"]["applicationName"] == "cmsRun", \
                   "Error: Application name is incorrect."
            assert outputFile["dataset"]["primaryDataset"] == self.primaryDataset, \
                   "Error: Primary dataset is incorrect."
            assert outputFile["dataset"]["dataTier"] == dataTierMap[outputFile["module_label"]], \
                   "Error: Data tier is incorrect."

        return

    def testMerge(self):
        """
        _testMerge_

        Setup a merge workflow and job and verify that the FWJR produced by the
        emulator is reasonable.
        """
        #emu = ReportEmu(WMStep = self.cmssw, Job = self.job)
        #report = emu()
        return
Example #31
0
class WorkQueueTest(unittest.TestCase):
    """
    Test WorkQueue Service client
    It will start WorkQueue RESTService
    Server DB sets from environment variable.
    Client DB sets from environment variable.

    This checks whether DS call makes without error and return the results.
    Not the correctness of functions. That will be tested in different module.
    """
    def setUp(self):
        """
        _setUp_
        """
        EmulatorHelper.setEmulators(phedex = True, dbs = True,
                                    siteDB = True, requestMgr = True)

        self.specGenerator = WMSpecGenerator("WMSpecs")
        #self.configFile = EmulatorSetup.setupWMAgentConfig()
        self.schema = []
        self.couchApps = ["WorkQueue"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        self.testInit.setupCouch('workqueue_t', *self.couchApps)
        self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps)
        self.testInit.setupCouch('local_workqueue_t', *self.couchApps)
        self.testInit.setupCouch('local_workqueue_t_inbox', *self.couchApps)
        self.testInit.generateWorkDir()
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()
        EmulatorHelper.resetEmulators()
        self.specGenerator.removeSpecs()


    def testWorkQueueService(self):
        # test getWork
        specName = "RerecoSpec"
        specUrl = self.specGenerator.createReRecoSpec(specName, "file")
        globalQ = globalQueue(DbName = 'workqueue_t',
                              QueueURL = self.testInit.couchUrl)
        self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)

        wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
        #overwrite default - can't test with stale view
        wqApi.defaultOptions =  {'reduce' : True, 'group' : True}
        #This only checks minimum client call not exactly correctness of return
        # values.
        self.assertEqual(wqApi.getTopLevelJobsByRequest(),
                         [{'total_jobs': 10, 'request_name': specName}])
        self.assertEqual(wqApi.getChildQueues(), [])
        self.assertEqual(wqApi.getJobStatusByRequest(),
            [{'status': 'Available', 'jobs': 10, 'request_name': specName}])
        self.assertEqual(wqApi.getChildQueuesByRequest(), [])
        self.assertEqual(wqApi.getWMBSUrl(), [])
        self.assertEqual(wqApi.getWMBSUrlByRequest(), [])

    def testUpdatePriorityService(self):
        """
        _testUpdatePriorityService_

        Check that we can update the priority correctly also
        check the available workflows feature
        """
        specName = "RerecoSpec"
        specUrl = self.specGenerator.createReRecoSpec(specName, "file")
        globalQ = globalQueue(DbName = 'workqueue_t',
                              QueueURL = self.testInit.couchUrl)
        localQ = localQueue(DbName = 'local_workqueue_t',
                            QueueURL = self.testInit.couchUrl,
                            CacheDir = self.testInit.testDir,
                            ParentQueueCouchUrl = '%s/workqueue_t' % self.testInit.couchUrl,
                            ParentQueueInboxCouchDBName = 'workqueue_t_inbox'
                            )
        # Try a full chain of priority update and propagation
        self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)
        globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
        #overwrite default - can't test with stale view
        globalApi.defaultOptions =  {'reduce' : True, 'group' : True}
        globalApi.updatePriority(specName, 100)
        self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100)
        storedElements = globalQ.backend.getElementsForWorkflow(specName)
        for element in storedElements:
            self.assertEqual(element['Priority'], 100)
        self.assertTrue(localQ.pullWork({'T2_XX_SiteA' : 10}) > 0)
        localQ.processInboundWork(continuous = False)
        storedElements = localQ.backend.getElementsForWorkflow(specName)
        for element in storedElements:
            self.assertEqual(element['Priority'], 100)
        localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t')
        #overwrite default - can't test with stale view
        localApi.defaultOptions =  {'reduce' : True, 'group' : True}
        localApi.updatePriority(specName, 500)
        self.assertEqual(localQ.backend.getWMSpec(specName).priority(), 500)
        storedElements = localQ.backend.getElementsForWorkflow(specName)
        for element in storedElements:
            self.assertEqual(element['Priority'], 500)
        self.assertEqual(localApi.getAvailableWorkflows(), set([(specName, 500)]))
        # Attempt to update an inexistent workflow in the queue
        try:
            globalApi.updatePriority('NotExistent', 2)
        except:
            self.fail('No exception should be raised.')
Example #32
0
class TestChangeState(unittest.TestCase):
    def setUp(self):
        """
        _setUp_

        """
        self.transitions = Transitions()
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setupCouch("changestate_t/jobs", "JobDump")
        self.testInit.setupCouch("changestate_t/fwjrs", "FWJRDump")
        self.testInit.setupCouch("job_summary", "WMStats")

        self.testInit.setSchema(customModules = ["WMCore.WMBS"],
                                useDefault = False)

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)

        couchurl = os.getenv("COUCHURL")
        self.couchServer = CouchServer(dburl = couchurl)
        self.config = self.testInit.getConfiguration()
        self.taskName = "/TestWorkflow/ReReco1"
        self.specGen = WMSpecGenerator()
        self.specUrl = self.specGen.createProcessingSpec("TestWorkflow", returnType="file")
        return

    def tearDown(self):
        """
        _tearDown_

        Cleanup the databases.
        """
        self.testInit.clearDatabase()
        self.testInit.tearDownCouch()
        self.specGen.removeSpecs()
        return

    def testCheck(self):
        """
        This is the test class for function Check from module ChangeState
        """
        change = ChangeState(self.config, "changestate_t")

        # Run through all good state transitions and assert that they work
        for state in self.transitions.keys():
            for dest in self.transitions[state]:
                change.check(dest, state)
        dummystates = ['dummy1', 'dummy2', 'dummy3', 'dummy4']

        # Then run through some bad state transistions and assertRaises(AssertionError)
        for state in self.transitions.keys():
            for dest in dummystates:
                self.assertRaises(AssertionError, change.check, dest, state)
        return

    def testRecordInCouch(self):
        """
        _testRecordInCouch_

        Verify that jobs, state transitions and fwjrs are recorded correctly.
        """
        change = ChangeState(self.config, "changestate_t")

        locationAction = self.daoFactory(classname="Locations.New")
        locationAction.execute("site1", pnn="T2_CH_CERN")

        testWorkflow = Workflow(spec=self.specUrl, owner="Steve",
                                name="wf001", task=self.taskName)
        testWorkflow.create()
        testFileset = Fileset(name="TestFileset")
        testFileset.create()
        testSubscription = Subscription(fileset=testFileset,
                                        workflow=testWorkflow,
                                        split_algo="FileBased")
        testSubscription.create()

        testFileA = File(lfn="SomeLFNA", events=1024, size=2048,
                         locations=set(["T2_CH_CERN"]))
        testFileB = File(lfn="SomeLFNB", events=1025, size=2049,
                         locations=set(["T2_CH_CERN"]))
        testFileA.create()
        testFileB.create()

        testFileset.addFile(testFileA)
        testFileset.addFile(testFileB)
        testFileset.commit()

        splitter = SplitterFactory()
        jobFactory = splitter(package="WMCore.WMBS",
                              subscription=testSubscription)
        jobGroup = jobFactory(files_per_job=1)[0]

        assert len(jobGroup.jobs) == 2, \
               "Error: Splitting should have created two jobs."

        testJobA = jobGroup.jobs[0]
        testJobA["user"] = "******"
        testJobA["group"] = "DMWM"
        testJobA["taskType"] = "Merge"
        testJobB = jobGroup.jobs[1]
        testJobB["user"] = "******"
        testJobB["group"] = "DMWM"
        testJobB["taskType"] = "Processing"

        change.propagate([testJobA, testJobB], "new", "none")
        change.propagate([testJobA, testJobB], "created", "new")
        change.propagate([testJobA, testJobB], "executing", "created")

        testJobADoc = change.jobsdatabase.document(testJobA["couch_record"])

        for transition in testJobADoc["states"].itervalues():
            self.assertTrue(isinstance(transition["timestamp"], int) or 
                            isinstance(transition["timestamp"], long))

        self.assertEqual(testJobADoc["jobid"] , testJobA["id"], "Error: ID parameter is incorrect.")
        assert testJobADoc["name"] == testJobA["name"], \
               "Error: Name parameter is incorrect."
        assert testJobADoc["jobgroup"] == testJobA["jobgroup"], \
               "Error: Jobgroup parameter is incorrect."
        assert testJobADoc["workflow"] == testJobA["workflow"], \
               "Error: Workflow parameter is incorrect."
        assert testJobADoc["task"] == testJobA["task"], \
               "Error: Task parameter is incorrect."
        assert testJobADoc["owner"] == testJobA["owner"], \
               "Error: Owner parameter is incorrect."

        assert testJobADoc["mask"]["FirstEvent"] == testJobA["mask"]["FirstEvent"], \
               "Error: First event in mask is incorrect."
        assert testJobADoc["mask"]["LastEvent"] == testJobA["mask"]["LastEvent"], \
               "Error: Last event in mask is incorrect."
        assert testJobADoc["mask"]["FirstLumi"] == testJobA["mask"]["FirstLumi"], \
               "Error: First lumi in mask is incorrect."
        assert testJobADoc["mask"]["LastLumi"] == testJobA["mask"]["LastLumi"], \
               "Error: First lumi in mask is incorrect."
        assert testJobADoc["mask"]["FirstRun"] == testJobA["mask"]["FirstRun"], \
               "Error: First run in mask is incorrect."
        assert testJobADoc["mask"]["LastEvent"] == testJobA["mask"]["LastRun"], \
               "Error: First event in mask is incorrect."

        assert len(testJobADoc["inputfiles"]) == 1, \
               "Error: Input files parameter is incorrect."

        testJobBDoc = change.jobsdatabase.document(testJobB["couch_record"])

        assert testJobBDoc["jobid"] == testJobB["id"], \
               "Error: ID parameter is incorrect."
        assert testJobBDoc["name"] == testJobB["name"], \
               "Error: Name parameter is incorrect."
        assert testJobBDoc["jobgroup"] == testJobB["jobgroup"], \
               "Error: Jobgroup parameter is incorrect."

        assert testJobBDoc["mask"]["FirstEvent"] == testJobB["mask"]["FirstEvent"], \
               "Error: First event in mask is incorrect."
        assert testJobBDoc["mask"]["LastEvent"] == testJobB["mask"]["LastEvent"], \
               "Error: Last event in mask is incorrect."
        assert testJobBDoc["mask"]["FirstLumi"] == testJobB["mask"]["FirstLumi"], \
               "Error: First lumi in mask is incorrect."
        assert testJobBDoc["mask"]["LastLumi"] == testJobB["mask"]["LastLumi"], \
               "Error: First lumi in mask is incorrect."
        assert testJobBDoc["mask"]["FirstRun"] == testJobB["mask"]["FirstRun"], \
               "Error: First run in mask is incorrect."
        assert testJobBDoc["mask"]["LastEvent"] == testJobB["mask"]["LastRun"], \
               "Error: First event in mask is incorrect."

        assert len(testJobBDoc["inputfiles"]) == 1, \
               "Error: Input files parameter is incorrect."

        changeStateDB = self.couchServer.connectDatabase(dbname="changestate_t/jobs")
        allDocs = changeStateDB.document("_all_docs")

        self.assertEqual(len(allDocs["rows"]), 3,
                         "Error: Wrong number of documents.")

        couchJobDoc = changeStateDB.document("1")

        assert couchJobDoc["name"] == testJobA["name"], \
               "Error: Name is wrong"
        assert len(couchJobDoc["inputfiles"]) == 1, \
               "Error: Wrong number of input files."

        result = changeStateDB.loadView("JobDump", "jobsByWorkflowName")

        self.assertEqual(len(result["rows"]), 2,
                         "Error: Wrong number of rows.")
        for row in result["rows"]:
            couchJobDoc = changeStateDB.document(row["value"]["id"])
            self.assertEqual(couchJobDoc["_rev"], row["value"]["rev"],
                             "Error: Rev is wrong.")

        return

    def testUpdateFailedDoc(self):
        """
        _testUpdateFailedDoc_

        Verify that the update function will work correctly and not throw a 500
        error if the doc didn't make it into the database for some reason.
        """
        change = ChangeState(self.config, "changestate_t")

        locationAction = self.daoFactory(classname="Locations.New")
        locationAction.execute("site1", pnn="T2_CH_CERN")

        testWorkflow = Workflow(spec=self.specUrl, owner="Steve",
                                name="wf001", task=self.taskName)
        testWorkflow.create()
        testFileset = Fileset(name = "TestFileset")
        testFileset.create()
        testSubscription = Subscription(fileset=testFileset,
                                        workflow=testWorkflow,
                                        split_algo="FileBased")
        testSubscription.create()

        testFileA = File(lfn="SomeLFNA", events=1024, size=2048,
                         locations=set(["T2_CH_CERN"]))
        testFileA.create()
        testFileset.addFile(testFileA)
        testFileset.commit()

        splitter = SplitterFactory()
        jobFactory = splitter(package="WMCore.WMBS",
                              subscription=testSubscription)
        jobGroup = jobFactory(files_per_job=1)[0]

        testJobA = jobGroup.jobs[0]
        testJobA["user"] = "******"
        testJobA["group"] = "DMWM"
        testJobA["taskType"] = "Merge"
        testJobA["couch_record"] = str(testJobA["id"])

        change.propagate([testJobA], "new", "none")
        testJobADoc = change.jobsdatabase.document(testJobA["couch_record"])

        self.assertTrue("states" in testJobADoc)
        self.assertTrue("1" in testJobADoc["states"])
        return

    def testPersist(self):
        """
        _testPersist_

        This is the test class for function Propagate from module ChangeState
        """
        change = ChangeState(self.config, "changestate_t")

        locationAction = self.daoFactory(classname="Locations.New")
        locationAction.execute("site1", pnn="T2_CH_CERN")

        testWorkflow = Workflow(spec=self.specUrl, owner="Steve",
                                name="wf001", task=self.taskName)
        testWorkflow.create()
        testFileset = Fileset(name="TestFileset")
        testFileset.create()

        for i in range(4):
            newFile = File(lfn="File%s" % i, locations=set(["T2_CH_CERN"]))
            newFile.create()
            testFileset.addFile(newFile)

        testFileset.commit()
        testSubscription = Subscription(fileset=testFileset,
                                        workflow=testWorkflow,
                                        split_algo="FileBased")
        testSubscription.create()

        splitter = SplitterFactory()
        jobFactory = splitter(package="WMCore.WMBS",
                              subscription=testSubscription)
        jobGroup = jobFactory(files_per_job=1)[0]

        assert len(jobGroup.jobs) == 4, \
               "Error: Splitting should have created four jobs."

        testJobA = jobGroup.jobs[0]
        testJobA["user"] = "******"
        testJobA["group"] = "DMWM"
        testJobA["taskType"] = "Processing"
        testJobB = jobGroup.jobs[1]
        testJobB["user"] = "******"
        testJobB["group"] = "DMWM"
        testJobB["taskType"] = "Processing"
        testJobC = jobGroup.jobs[2]
        testJobC["user"] = "******"
        testJobC["group"] = "DMWM"
        testJobC["taskType"] = "Processing"
        testJobD = jobGroup.jobs[3]
        testJobD["user"] = "******"
        testJobD["group"] = "DMWM"
        testJobD["taskType"] = "Processing"

        change.persist([testJobA, testJobB], "created", "new")
        change.persist([testJobC, testJobD], "new", "none")

        stateDAO = self.daoFactory(classname="Jobs.GetState")

        jobAState = stateDAO.execute(id=testJobA["id"])
        jobBState = stateDAO.execute(id=testJobB["id"])
        jobCState = stateDAO.execute(id=testJobC["id"])
        jobDState = stateDAO.execute(id=testJobD["id"])

        assert jobAState == "created" and jobBState =="created" and \
               jobCState == "new" and jobDState == "new", \
               "Error: Jobs didn't change state correctly."

        return

    def testRetryCount(self):
        """
        _testRetryCount_

        Verify that the retry count is incremented when we move out of the
        submitcooloff or jobcooloff state.
        """
        change = ChangeState(self.config, "changestate_t")

        locationAction = self.daoFactory(classname = "Locations.New")
        locationAction.execute("site1", pnn = "T2_CH_CERN")

        testWorkflow = Workflow(spec=self.specUrl, owner="Steve",
                                name="wf001", task=self.taskName)
        testWorkflow.create()
        testFileset = Fileset(name="TestFileset")
        testFileset.create()

        for i in range(4):
            newFile = File(lfn="File%s" % i, locations=set(["T2_CH_CERN"]))
            newFile.create()
            testFileset.addFile(newFile)

        testFileset.commit()
        testSubscription = Subscription(fileset=testFileset,
                                        workflow=testWorkflow,
                                        split_algo="FileBased")
        testSubscription.create()

        splitter = SplitterFactory()
        jobFactory = splitter(package="WMCore.WMBS",
                              subscription=testSubscription)
        jobGroup = jobFactory(files_per_job=1)[0]

        assert len(jobGroup.jobs) == 4, \
               "Error: Splitting should have created four jobs."

        testJobA = jobGroup.jobs[0]
        testJobA["user"] = "******"
        testJobA["group"] = "DMWM"
        testJobA["taskType"] = "Processing"
        testJobB = jobGroup.jobs[1]
        testJobB["user"] = "******"
        testJobB["group"] = "DMWM"
        testJobB["taskType"] = "Processing"
        testJobC = jobGroup.jobs[2]
        testJobC["user"] = "******"
        testJobC["group"] = "DMWM"
        testJobC["taskType"] = "Processing"
        testJobD = jobGroup.jobs[3]
        testJobD["user"] = "******"
        testJobD["group"] = "DMWM"
        testJobD["taskType"] = "Processing"

        change.persist([testJobA], "created", "submitcooloff")
        change.persist([testJobB], "created", "jobcooloff")
        change.persist([testJobC, testJobD], "new", "none")

        testJobA.load()
        testJobB.load()
        testJobC.load()
        testJobD.load()

        assert testJobA["retry_count"] == 1, \
               "Error: Retry count is wrong."
        assert testJobB["retry_count"] == 1, \
               "Error: Retry count is wrong."
        assert testJobC["retry_count"] == 0, \
               "Error: Retry count is wrong."
        assert testJobD["retry_count"] == 0, \
               "Error: Retry count is wrong."

        return

    def testJobSerialization(self):
        """
        _testJobSerialization_

        Verify that serialization of a job works when adding a FWJR.
        """
        change = ChangeState(self.config, "changestate_t")

        locationAction = self.daoFactory(classname="Locations.New")
        locationAction.execute("site1", pnn="T2_CH_CERN")

        testWorkflow = Workflow(spec=self.specUrl, owner="Steve",
                                name="wf001", task=self.taskName)
        testWorkflow.create()
        testFileset = Fileset(name="TestFileset")
        testFileset.create()

        testFile = File(lfn="SomeLFNC", locations=set(["T2_CH_CERN"]))
        testFile.create()
        testFileset.addFile(testFile)
        testFileset.commit()

        testSubscription = Subscription(fileset=testFileset,
                                        workflow=testWorkflow)
        testSubscription.create()

        splitter = SplitterFactory()
        jobFactory = splitter(package="WMCore.WMBS",
                              subscription=testSubscription)
        jobGroup = jobFactory(files_per_job=1)[0]

        assert len(jobGroup.jobs) == 1, \
               "Error: Splitting should have created one job."

        testJobA = jobGroup.jobs[0]
        testJobA["user"] = "******"
        testJobA["group"] = "DMWM"
        testJobA["taskType"] = "Processing"

        change.propagate([testJobA], 'created', 'new')
        myReport = Report()
        reportPath = os.path.join(getTestBase(),
                                  "WMCore_t/JobStateMachine_t/Report.pkl")
        myReport.unpersist(reportPath)
        testJobA["fwjr"] = myReport

        change.propagate([testJobA], 'executing', 'created')

        changeStateDB = self.couchServer.connectDatabase(dbname = "changestate_t/fwjrs")
        allDocs = changeStateDB.document("_all_docs")

        self.assertEqual(len(allDocs["rows"]), 2,
                         "Error: Wrong number of documents")

        result = changeStateDB.loadView("FWJRDump", "fwjrsByWorkflowName")
        self.assertEqual(len(result["rows"]), 1,
                         "Error: Wrong number of rows.")
        for row in result["rows"]:
            couchJobDoc = changeStateDB.document(row["value"]["id"])
            self.assertEqual(couchJobDoc["_rev"], row["value"]["rev"],
                             "Error: Rev is wrong.")

        for resultRow in allDocs["rows"]:
            if resultRow["id"] != "_design/FWJRDump":
                fwjrDoc = changeStateDB.document(resultRow["id"])
                break

        assert fwjrDoc["retrycount"] == 0, \
               "Error: Retry count is wrong."

        assert len(fwjrDoc["fwjr"]["steps"].keys()) == 2, \
               "Error: Wrong number of steps in FWJR."
        assert "cmsRun1" in fwjrDoc["fwjr"]["steps"].keys(), \
               "Error: cmsRun1 step is missing from FWJR."
        assert "stageOut1" in fwjrDoc["fwjr"]["steps"].keys(), \
               "Error: stageOut1 step is missing from FWJR."

        return

    def testDuplicateJobReports(self):
        """
        _testDuplicateJobReports_

        Verify that everything works correctly if a job report is added to the
        database more than once.
        """
        change = ChangeState(self.config, "changestate_t")

        locationAction = self.daoFactory(classname="Locations.New")
        locationAction.execute("site1", pnn="T2_CH_CERN")

        testWorkflow = Workflow(spec=self.specUrl, owner="Steve",
                                name="wf001", task=self.taskName)
        testWorkflow.create()
        testFileset = Fileset(name="TestFileset")
        testFileset.create()

        testFile = File(lfn="SomeLFNC", locations=set(["T2_CH_CERN"]))
        testFile.create()
        testFileset.addFile(testFile)
        testFileset.commit()

        testSubscription = Subscription(fileset=testFileset,
                                        workflow=testWorkflow)
        testSubscription.create()

        splitter = SplitterFactory()
        jobFactory = splitter(package="WMCore.WMBS",
                              subscription=testSubscription)
        jobGroup = jobFactory(files_per_job=1)[0]

        assert len(jobGroup.jobs) == 1, \
               "Error: Splitting should have created one job."

        testJobA = jobGroup.jobs[0]
        testJobA["user"] = "******"
        testJobA["group"] = "DMWM"
        testJobA["taskType"] = "Processing"

        change.propagate([testJobA], 'created', 'new')
        myReport = Report()
        reportPath = os.path.join(getTestBase(),
                                  "WMCore_t/JobStateMachine_t/Report.pkl")
        myReport.unpersist(reportPath)
        testJobA["fwjr"] = myReport

        change.propagate([testJobA], 'executing', 'created')
        change.propagate([testJobA], 'executing', 'created')

        changeStateDB = self.couchServer.connectDatabase(dbname="changestate_t/fwjrs")
        allDocs = changeStateDB.document("_all_docs")

        self.assertEqual(len(allDocs["rows"]), 2,
                         "Error: Wrong number of documents")

        for resultRow in allDocs["rows"]:
            if resultRow["id"] != "_design/FWJRDump":
                changeStateDB.document(resultRow["id"])
                break

        return


    def testJobKilling(self):
        """
        _testJobKilling_

        Test that we can successfully set jobs to the killed state
        """
        change = ChangeState(self.config, "changestate_t")

        locationAction = self.daoFactory(classname="Locations.New")
        locationAction.execute("site1", pnn="T2_CH_CERN")

        testWorkflow = Workflow(spec=self.specUrl, owner="Steve",
                                name="wf001", task=self.taskName)
        testWorkflow.create()
        testFileset = Fileset(name="TestFileset")
        testFileset.create()

        for i in range(4):
            newFile = File(lfn="File%s" % i, locations=set(["T2_CH_CERN"]))
            newFile.create()
            testFileset.addFile(newFile)

        testFileset.commit()
        testSubscription = Subscription(fileset=testFileset,
                                        workflow=testWorkflow,
                                        split_algo="FileBased")
        testSubscription.create()

        splitter = SplitterFactory()
        jobFactory = splitter(package="WMCore.WMBS",
                              subscription=testSubscription)
        jobGroup = jobFactory(files_per_job=1)[0]

        assert len(jobGroup.jobs) == 4, \
               "Error: Splitting should have created four jobs."

        testJobA = jobGroup.jobs[0]
        testJobA["user"] = "******"
        testJobA["group"] = "DMWM"
        testJobA["taskType"] = "Processing"
        testJobB = jobGroup.jobs[1]
        testJobB["user"] = "******"
        testJobB["group"] = "DMWM"
        testJobB["taskType"] = "Processing"
        testJobC = jobGroup.jobs[2]
        testJobC["user"] = "******"
        testJobC["group"] = "DMWM"
        testJobC["taskType"] = "Processing"
        testJobD = jobGroup.jobs[3]
        testJobD["user"] = "******"
        testJobD["group"] = "DMWM"
        testJobD["taskType"] = "Processing"

        change.persist([testJobA], "created", "new")
        change.persist([testJobB], "jobfailed", "executing")
        change.persist([testJobC, testJobD], "executing", "created")

        change.persist([testJobA], "killed", "created")
        change.persist([testJobB], "killed", "jobfailed")
        change.persist([testJobC, testJobD], "killed", "executing")

        for job in [testJobA, testJobB, testJobC, testJobD]:
            job.load()
            self.assertEqual(job['retry_count'], 99999)
            self.assertEqual(job['state'], 'killed')

        return

    def testFWJRInputFileTruncation(self):
        """
        _testFWJRInputFileTruncation_

        Test and see whether the ChangeState code can
        be used to automatically truncate the number of input files
        in a FWJR

        Code stolen from the serialization test
        """

        self.config.JobStateMachine.maxFWJRInputFiles = 0
        change = ChangeState(self.config, "changestate_t")

        locationAction = self.daoFactory(classname="Locations.New")
        locationAction.execute("site1", pnn="T2_CH_CERN")

        testWorkflow = Workflow(spec=self.specUrl, owner="Steve",
                                name="wf001", task=self.taskName)
        testWorkflow.create()
        testFileset = Fileset(name="TestFileset")
        testFileset.create()

        testFile = File(lfn="SomeLFNC", locations=set(["T2_CH_CERN"]))
        testFile.create()
        testFileset.addFile(testFile)
        testFileset.commit()

        testSubscription = Subscription(fileset=testFileset,
                                        workflow=testWorkflow)
        testSubscription.create()

        splitter = SplitterFactory()
        jobFactory = splitter(package="WMCore.WMBS",
                              subscription=testSubscription)
        jobGroup = jobFactory(files_per_job=1)[0]

        self.assertEqual(len(jobGroup.jobs), 1,
                         "Error: Splitting should have created one job.")

        testJobA = jobGroup.jobs[0]
        testJobA["user"] = "******"
        testJobA["group"] = "DMWM"
        testJobA["taskType"] = "Processing"

        change.propagate([testJobA], 'created', 'new')
        myReport = Report()
        reportPath = os.path.join(getTestBase(),
                                  "WMCore_t/JobStateMachine_t/Report.pkl")
        myReport.unpersist(reportPath)

        testJobA["fwjr"] = myReport

        change.propagate([testJobA], 'executing', 'created')

        changeStateDB = self.couchServer.connectDatabase(dbname="changestate_t/fwjrs")
        allDocs = changeStateDB.document("_all_docs")

        self.assertEqual(len(allDocs["rows"]), 2,
                         "Error: Wrong number of documents")

        result = changeStateDB.loadView("FWJRDump", "fwjrsByWorkflowName")
        self.assertEqual(len(result["rows"]), 1,
                         "Error: Wrong number of rows.")
        for row in result["rows"]:
            couchJobDoc = changeStateDB.document(row["value"]["id"])
            self.assertEqual(couchJobDoc["_rev"], row["value"]["rev"],
                             "Error: Rev is wrong.")

        for resultRow in allDocs["rows"]:
            if resultRow["id"] != "_design/FWJRDump":
                fwjrDoc = changeStateDB.document(resultRow["id"])
                break

        self.assertEqual(fwjrDoc["fwjr"]["steps"]['cmsRun1']['input']['source'], [])

        return


    def testJobSummary(self):
        """
        _testJobSummary_

        verify that job summary for jobs with fwjr are correctly created
        and that status is updated when updatesummary flag is enabled
        """
        change = ChangeState(self.config, "changestate_t")

        locationAction = self.daoFactory(classname="Locations.New")
        locationAction.execute("site1", pnn="T2_CH_CERN")

        testWorkflow = Workflow(spec=self.specUrl, owner="Steve",
                                name="wf001", task=self.taskName)
        testWorkflow.create()
        testFileset = Fileset(name="TestFileset")
        testFileset.create()

        testFile = File(lfn="SomeLFNC", locations=set(["T2_CH_CERN"]))
        testFile.create()
        testFileset.addFile(testFile)
        testFileset.commit()

        testSubscription = Subscription(fileset=testFileset,
                                        workflow=testWorkflow)
        testSubscription.create()

        splitter = SplitterFactory()
        jobFactory = splitter(package="WMCore.WMBS",
                              subscription=testSubscription)
        jobGroup = jobFactory(files_per_job=1)[0]

        assert len(jobGroup.jobs) == 1, \
               "Error: Splitting should have created one job."

        testJobA = jobGroup.jobs[0]
        testJobA["user"] = "******"
        testJobA["group"] = "DMWM"
        testJobA["taskType"] = "Production"

        change.propagate([testJobA], 'created', 'new')
        myReport = Report()
        reportPath = os.path.join(getTestBase(),
                                  "WMCore_t/JobStateMachine_t/Report.pkl")
        myReport.unpersist(reportPath)

        change.propagate([testJobA], 'executing', 'created')
        testJobA["fwjr"] = myReport
        change.propagate([testJobA], 'jobfailed', 'executing')

        changeStateDB = self.couchServer.connectDatabase(dbname=self.config.JobStateMachine.jobSummaryDBName)
        allDocs = changeStateDB.document("_all_docs")

        self.assertEqual(len(allDocs["rows"]), 2,
                         "Error: Wrong number of documents")

        fwjrDoc = {'state': None}
        for resultRow in allDocs["rows"]:
            if resultRow["id"] != "_design/WMStats":
                fwjrDoc = changeStateDB.document(resultRow["id"])
                break

        self.assertEqual(fwjrDoc['state'], 'jobfailed',
                         "Error: summary doesn't have the expected job state")

        del testJobA["fwjr"]

        change.propagate([testJobA], 'jobcooloff', 'jobfailed', updatesummary=True)
        return


    def testIndexConflict(self):
        """
        _testIndexConflict_

        Verify that in case of conflict in the job index
        we discard the old document and replace with a new
        one
        """
        change = ChangeState(self.config, "changestate_t")

        locationAction = self.daoFactory(classname="Locations.New")
        locationAction.execute("site1", pnn="T2_CH_CERN")

        testWorkflow = Workflow(spec=self.specUrl, owner="Steve",
                                name="wf001", task=self.taskName)
        testWorkflow.create()
        testFileset = Fileset(name="TestFileset")
        testFileset.create()

        testFile = File(lfn="SomeLFNC", locations=set(["T2_CH_CERN"]))
        testFile.create()
        testFileset.addFile(testFile)
        testFileset.commit()

        testSubscription = Subscription(fileset=testFileset,
                                        workflow=testWorkflow)
        testSubscription.create()

        splitter = SplitterFactory()
        jobFactory = splitter(package="WMCore.WMBS",
                              subscription=testSubscription)
        jobGroup = jobFactory(files_per_job=1)[0]

        assert len(jobGroup.jobs) == 1, \
               "Error: Splitting should have created one job."

        testJobA = jobGroup.jobs[0]
        testJobA["user"] = "******"
        testJobA["group"] = "CompOps"
        testJobA["taskType"] = "Processing"

        myReport = Report()
        reportPath = os.path.join(getTestBase(),
                                  "WMCore_t/JobStateMachine_t/Report.pkl")
        myReport.unpersist(reportPath)

        testJobA["fwjr"] = myReport
        change.propagate([testJobA], 'created', 'new')

        jobdatabase = self.couchServer.connectDatabase('changestate_t/jobs', False)
        fwjrdatabase = self.couchServer.connectDatabase('changestate_t/fwjrs', False)
        jobDoc = jobdatabase.document("1")
        fwjrDoc = fwjrdatabase.document("1-0")
        self.assertEqual(jobDoc["workflow"], "wf001", "Wrong workflow in couch job document")
        self.assertEqual(fwjrDoc["fwjr"]["task"], self.taskName, "Wrong task in fwjr couch document")

        testJobA.delete()

        myThread = threading.currentThread()
        myThread.dbi.processData("ALTER TABLE wmbs_job AUTO_INCREMENT = 1")

        testWorkflow = Workflow(spec=self.specUrl, owner="Steve",
                                name="wf002", task="/TestWorkflow/Test2")
        testWorkflow.create()
        testFileset = Fileset(name="TestFilesetB")
        testFileset.create()

        testFile = File(lfn="SomeLFNB", locations=set(["T2_CH_CERN"]))
        testFile.create()
        testFileset.addFile(testFile)
        testFileset.commit()

        testSubscription = Subscription(fileset=testFileset,
                                        workflow=testWorkflow)
        testSubscription.create()

        splitter = SplitterFactory()
        jobFactory = splitter(package="WMCore.WMBS",
                              subscription=testSubscription)
        jobGroup = jobFactory(files_per_job=1)[0]

        testJobB = jobGroup.jobs[0]
        testJobB["user"] = "******"
        testJobB["group"] = "CompOps"
        testJobB["taskType"] = "Processing"
        testJobB["fwjr"] = myReport

        change.propagate([testJobB], 'created', 'new')
        jobDoc = jobdatabase.document("1")
        fwjrDoc = fwjrdatabase.document("1-0")
        self.assertEqual(jobDoc["workflow"], "wf002", "Job document was not overwritten")
        self.assertEqual(fwjrDoc["fwjr"]["task"], "/TestWorkflow/Test2", "FWJR document was not overwritten")

        return

    def testUpdateLocation(self):
        """
        _testUpdateLocation_

        Check that we can update the location of a job through
        the state machine.
        """
        change = ChangeState(self.config, "changestate_t")

        locationAction = self.daoFactory(classname="Locations.New")
        locationAction.execute("site1", pnn="T2_CH_CERN")
        locationAction.execute("site2", pnn="T1_US_FNAL_Disk")

        testWorkflow = Workflow(spec=self.specUrl, owner="Steve",
                                name="wf001", task=self.taskName)
        testWorkflow.create()
        testFileset = Fileset(name="TestFileset")
        testFileset.create()
        testSubscription = Subscription(fileset=testFileset,
                                        workflow=testWorkflow,
                                        split_algo="FileBased")
        testSubscription.create()

        testFileA = File(lfn="SomeLFNA", events=1024, size=2048,
                         locations=set(["T2_CH_CERN", "T1_US_FNAL_Disk"]))
        testFileB = File(lfn="SomeLFNB", events=1025, size=2049,
                         locations=set(["T2_CH_CERN", "T1_US_FNAL_Disk"]))
        testFileA.create()
        testFileB.create()

        testFileset.addFile(testFileA)
        testFileset.addFile(testFileB)
        testFileset.commit()

        splitter = SplitterFactory()
        jobFactory = splitter(package="WMCore.WMBS",
                              subscription=testSubscription)
        jobGroup = jobFactory(files_per_job=1)[0]

        assert len(jobGroup.jobs) == 2, \
               "Error: Splitting should have created two jobs."

        testJobA = jobGroup.jobs[0]
        testJobA["user"] = "******"
        testJobA["group"] = "DMWM"
        testJobA["taskType"] = "Merge"
        testJobA["site_cms_name"] = "site1"
        testJobB = jobGroup.jobs[1]
        testJobB["user"] = "******"
        testJobB["group"] = "DMWM"
        testJobB["taskType"] = "Processing"
        testJobB["site_cms_name"] = "site2"

        change.propagate([testJobA, testJobB], "new", "none")
        change.propagate([testJobA, testJobB], "created", "new")
        change.propagate([testJobA, testJobB], "executing", "created")

        testJobADoc = change.jobsdatabase.document(testJobA["couch_record"])

        maxKey = max(testJobADoc["states"].keys())
        transition = testJobADoc["states"][maxKey]
        self.assertEqual(transition["location"], "site1")

        testJobBDoc = change.jobsdatabase.document(testJobB["couch_record"])

        maxKey = max(testJobBDoc["states"].keys())
        transition = testJobBDoc["states"][maxKey]
        self.assertEqual(transition["location"], "site2")

        jobs = [{'jobid' : 1, 'location' : 'site2'}]

        change.recordLocationChange(jobs)

        testJobADoc = change.jobsdatabase.document(testJobA["couch_record"])

        maxKey = max(testJobADoc["states"].keys())
        transition = testJobADoc["states"][maxKey]
        self.assertEqual(transition["location"], "site2")

        listJobsDAO = self.daoFactory(classname="Jobs.GetLocation")
        jobid = [{'jobid' : 1}, {'jobid' : 2}]
        jobsLocation = listJobsDAO.execute(jobid)
        for job in jobsLocation:
            self.assertEqual(job['site_name'], 'site2')

        return
Example #33
0
 def __init__(self, *args, **kwargs):
     print "Using RequestManager Emulator ..."
     self.specGenerator = WMSpecGenerator()
     self.count = 0
     self.maxWmSpec = 1
Example #34
0
class LocalWorkQueueProfileTest(WorkQueueTestCase):
    """
    _WorkQueueTest_

    """
    def setUp(self):
        """
        If we dont have a wmspec file create one
        """

        EmulatorHelper.setEmulators(phedex=True,
                                    dbs=True,
                                    siteDB=True,
                                    requestMgr=True)
        WorkQueueTestCase.setUp(self)

        self.cacheDir = tempfile.mkdtemp()
        self.specGenerator = WMSpecGenerator(self.cacheDir)
        self.specs = self.createReRecoSpec(1, "file")

        # Create queues
        self.localQueue = localQueue(DbName=self.queueDB,
                                     InboxDbName=self.queueInboxDB,
                                     NegotiationTimeout=0,
                                     QueueURL='global.example.com',
                                     CacheDir=self.cacheDir)

    def tearDown(self):
        """tearDown"""
        WorkQueueTestCase.tearDown(self)
        try:
            shutil.rmtree(self.cacheDir)
            self.specGenerator.removeSpecs()
        except:
            pass
        EmulatorHelper.resetEmulators()

    def createReRecoSpec(self, numOfSpec, type="spec"):
        specs = []
        for i in range(numOfSpec):
            specName = "MinBiasProcessingSpec_Test_%s" % (i + 1)
            specs.append(self.specGenerator.createReRecoSpec(specName, type))
        return specs

    def createProfile(self, name, function):
        file = name
        prof = cProfile.Profile()
        prof.runcall(function)
        prof.dump_stats(file)
        p = pstats.Stats(file)
        p.strip_dirs().sort_stats('cumulative').print_stats(0.1)
        p.strip_dirs().sort_stats('time').print_stats(0.1)
        p.strip_dirs().sort_stats('calls').print_stats(0.1)
        #p.strip_dirs().sort_stats('name').print_stats(10)

    def testGetWorkLocalQueue(self):
        i = 0
        for spec in self.specs:
            i += 1
            specName = "MinBiasProcessingSpec_Test_%s" % i
            self.localQueue.queueWork(spec, specName, team="A-team")
        self.localQueue.updateLocationInfo()
        self.createProfile('getWorkProfile.prof', self.localQueueGetWork)

    def localQueueGetWork(self):
        siteJobs = {}
        for site in Globals.SITES:
            siteJobs[site] = 100000
        self.localQueue.getWork(siteJobs, {})
Example #35
0
class ReqMgr(dict):
    def __init__(self, *args, **kwargs):
        """
        all the private valuable is defined for test values
        """
        self.specGenerator = WMSpecGenerator()
        self.count = 0
        self.maxWmSpec = kwargs.setdefault('numOfSpecs', 1)
        self.type = kwargs.setdefault("type", 'ReReco')
        if self.type not in ['ReReco', 'MonteCarlo']:
            raise TypeError('unknown request type %s' % self.type)
        self.splitter = kwargs.setdefault('splitter', 'DatasetBlock')
        self.inputDataset = kwargs.setdefault('inputDataset', None)
        self.dbsUrl = kwargs.setdefault('dbsUrl', None)
        self.status = {}
        self.progress = {}
        self.msg = {}
        self.names = []
        self.openRunningTimeout = kwargs.setdefault('openRunningTimeout', 0)
        import logging
        self['logger'] = logging

    def getAssignment(self, teamName=None, request=None):
        if self.count < self.maxWmSpec:
            if self.type == 'ReReco':
                specName = "ReRecoTest_v%sEmulator" % self.count
                specUrl = self.specGenerator.createReRecoSpec(
                    specName,
                    "file",
                    self.splitter,
                    InputDataset=self.inputDataset,
                    DbsUrl=self.dbsUrl,
                    OpenRunningTimeout=self.openRunningTimeout)
            elif self.type == 'MonteCarlo':
                specName = "MCTest_v%sEmulator" % self.count
                specUrl = self.specGenerator.createMCSpec(
                    specName, "file", self.splitter)
            self.names.append(specName)
            self.status[specName] = 'assigned'
            #specName = "FakeProductionSpec_%s" % self.count
            #specUrl =self.specGenerator.createProductionSpec(specName, "file")
            #specName = "FakeProcessingSpec_%s" % self.count
            #specUrl =self.specGenerator.createProcessingSpec(specName, "file")

            self.count += 1
            # returns list of list(rquest name, spec url)
            return [
                [specName, specUrl],
            ]
        else:
            return []

    def getRunningOpen(self, teamName):
        """Returns a list of request names of requests in running-open state"""
        result = []
        for request in self.status:
            if self.status[request] == 'running-open':
                result.append(request)
        return result

    def getRequest(self, requestName):
        """Get request info"""
        if requestName not in self.names:
            raise RuntimeError("unknown request %s" % requestName)

        request = {
            'RequestName': requestName,
            'RequestStatus': self.status[requestName],
            'RequestPriority': 100
        }
        if requestName in self.progress:
            request.update(self.progress[requestName])
        request.setdefault('percent_complete', 0)
        request.setdefault('percent_success', 0)
        return request

    def putWorkQueue(self, reqName, prodAgentUrl=None):
        self.status[reqName] = 'acquired'

    def reportRequestStatus(self, name, status):
        if not check_allowed_transition(self.status[name], status):
            raise RuntimeError("Invalid status move: %s" % status)
        self.status[name] = status

    def reportRequestProgress(self, name, **args):
        self.progress.setdefault(name, {})
        self.progress[name].update(args)

    def sendMessage(self, request, msg):
        self.msg[request] = msg

    def _removeSpecs(self):
        """
        This is just for clean up not part of emulated function
        """
        self.specGenerator.removeSpecs()

    def getTeam(self):
        """Give a fake team name"""
        return ["The A-Team", "some other bloke"]
Example #36
0
class WorkQueueTest(unittest.TestCase):
    """
    Test WorkQueue Service client
    It will start WorkQueue RESTService
    Server DB sets from environment variable.
    Client DB sets from environment variable.

    This checks whether DS call makes without error and return the results.
    Not the correctness of functions. That will be tested in different module.
    """
    def setUp(self):
        """
        _setUp_
        """
        EmulatorHelper.setEmulators(phedex=True,
                                    dbs=True,
                                    siteDB=True,
                                    requestMgr=True)

        self.specGenerator = WMSpecGenerator("WMSpecs")
        #self.configFile = EmulatorSetup.setupWMAgentConfig()
        self.schema = []
        self.couchApps = ["WorkQueue"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules=self.schema, useDefault=False)
        self.testInit.setupCouch('workqueue_t', *self.couchApps)
        self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps)
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()
        #EmulatorSetup.deleteConfig(self.configFile)
        EmulatorHelper.resetEmulators()
        self.specGenerator.removeSpecs()

    def testWorkQueueService(self):
        # test getWork
        specName = "RerecoSpec"
        specUrl = self.specGenerator.createReRecoSpec(specName, "file")
        globalQ = globalQueue(DbName='workqueue_t',
                              QueueURL=self.testInit.couchUrl)
        self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)

        wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
        #This only checks minimum client call not exactly correctness of return
        # values.
        self.assertEqual(wqApi.getTopLevelJobsByRequest(),
                         [{
                             'total_jobs': 2,
                             'request_name': specName
                         }])
        self.assertEqual(wqApi.getChildQueues(), [])
        self.assertEqual(wqApi.getJobStatusByRequest(),
                         [{
                             'status': 'Available',
                             'jobs': 2,
                             'request_name': specName
                         }])
        self.assertEqual(wqApi.getChildQueuesByRequest(), [])
        self.assertEqual(wqApi.getWMBSUrl(), [])
        self.assertEqual(wqApi.getWMBSUrlByRequest(), [])
Example #37
0
class WorkQueueProfileTest(WorkQueueTestCase):
    """
    _WorkQueueTest_

    """
    def setUp(self):
        """
        If we dont have a wmspec file create one

        Warning: For the real profiling test including
        spec generation. need to use real spec instead of
        using emulator generated spec which doesn't include
        couchDB access and cmssw access
        """
        EmulatorHelper.setEmulators(phedex=True,
                                    dbs=True,
                                    siteDB=True,
                                    requestMgr=True)
        WorkQueueTestCase.setUp(self)

        self.cacheDir = tempfile.mkdtemp()
        self.specGenerator = WMSpecGenerator(self.cacheDir)
        self.specNamePrefix = "TestReReco_"
        self.specs = self.createReRecoSpec(5, "file")
        # Create queues
        self.globalQueue = globalQueue(DbName=self.globalQDB,
                                       InboxDbName=self.globalQInboxDB,
                                       NegotiationTimeout=0)

    def tearDown(self):
        """tearDown"""
        WorkQueueTestCase.tearDown(self)
        try:
            self.specGenerator.removeSpecs()
        except:
            pass
        EmulatorHelper.resetEmulators()

    def createReRecoSpec(self, numOfSpec, type="spec"):
        specs = []
        for i in range(numOfSpec):
            specName = "%s%s" % (self.specNamePrefix, (i + 1))
            specs.append(self.specGenerator.createReRecoSpec(specName, type))
        return specs

    def createProfile(self, name, function):
        file = name
        prof = cProfile.Profile()
        prof.runcall(function)
        prof.dump_stats(file)
        p = pstats.Stats(file)
        p.strip_dirs().sort_stats('cumulative').print_stats(0.1)
        p.strip_dirs().sort_stats('time').print_stats(0.1)
        p.strip_dirs().sort_stats('calls').print_stats(0.1)
        #p.strip_dirs().sort_stats('name').print_stats(10)

    def testQueueElementProfile(self):
        self.createProfile('queueElementProfile.prof',
                           self.multipleQueueWorkCall)

    def multipleQueueWorkCall(self):
        i = 0
        for wmspec in self.specs:
            i += 1
            self.globalQueue.queueWork(wmspec, self.specNamePrefix + str(i),
                                       'test_team')
class LocalWorkQueueProfileTest(WorkQueueTestCase):
    """
    _WorkQueueTest_

    """

    def setUp(self):
        """
        If we dont have a wmspec file create one
        """

        EmulatorHelper.setEmulators(phedex=True, dbs=True, siteDB=True, requestMgr=True)
        WorkQueueTestCase.setUp(self)

        self.cacheDir = tempfile.mkdtemp()
        self.specGenerator = WMSpecGenerator(self.cacheDir)
        self.specs = self.createReRecoSpec(1, "file")

        # Create queues
        self.localQueue = localQueue(
            DbName=self.queueDB,
            InboxDbName=self.queueInboxDB,
            NegotiationTimeout=0,
            QueueURL="global.example.com",
            CacheDir=self.cacheDir,
        )

    def tearDown(self):
        """tearDown"""
        WorkQueueTestCase.tearDown(self)
        try:
            shutil.rmtree(self.cacheDir)
            self.specGenerator.removeSpecs()
        except:
            pass
        EmulatorHelper.resetEmulators()

    def createReRecoSpec(self, numOfSpec, type="spec"):
        specs = []
        for i in range(numOfSpec):
            specName = "MinBiasProcessingSpec_Test_%s" % (i + 1)
            specs.append(self.specGenerator.createReRecoSpec(specName, type))
        return specs

    def createProfile(self, name, function):
        file = name
        prof = cProfile.Profile()
        prof.runcall(function)
        prof.dump_stats(file)
        p = pstats.Stats(file)
        p.strip_dirs().sort_stats("cumulative").print_stats(0.1)
        p.strip_dirs().sort_stats("time").print_stats(0.1)
        p.strip_dirs().sort_stats("calls").print_stats(0.1)
        # p.strip_dirs().sort_stats('name').print_stats(10)

    def testGetWorkLocalQueue(self):
        i = 0
        for spec in self.specs:
            i += 1
            specName = "MinBiasProcessingSpec_Test_%s" % i
            self.localQueue.queueWork(spec, specName, team="A-team")
        self.localQueue.updateLocationInfo()
        self.createProfile("getWorkProfile.prof", self.localQueueGetWork)

    def localQueueGetWork(self):
        siteJobs = {}
        for site in Globals.SITES:
            siteJobs[site] = 100000
        self.localQueue.getWork(siteJobs, {})