コード例 #1
0
class AlertProcessorTest(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInit(__file__)
        self.testInit.setLogging(logLevel = logging.DEBUG)
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = ["WMCore.WMBS",'WMCore.Agent.Database',
                                                 "WMCore.ResourceControl"],
                                 useDefault = False)
        self.testDir = self.testInit.generateWorkDir()

        self.config = Configuration()
        self.config.section_("Agent")
        self.config.Agent.useMsgService = False
        self.config.Agent.useTrigger = False
        self.config.component_("AlertProcessor")
        self.config.AlertProcessor.componentDir = self.testDir
        self.config.AlertProcessor.address = "tcp://127.0.0.1:5557"
        self.config.AlertProcessor.controlAddr = "tcp://127.0.0.1:5559"
        self.config.section_("CoreDatabase")

        self.config.CoreDatabase.socket = os.environ.get("DBSOCK")
        self.config.CoreDatabase.connectUrl = os.environ.get("DATABASE")

        self.config.AlertProcessor.section_("critical")
        self.config.AlertProcessor.section_("soft")

        self.config.AlertProcessor.critical.level = 5
        self.config.AlertProcessor.soft.level = 0
        self.config.AlertProcessor.soft.bufferSize = 3

        self.config.AlertProcessor.critical.section_("sinks")
        self.config.AlertProcessor.soft.section_("sinks")


    def tearDown(self):
        self.testInit.clearDatabase()
        self.testInit.delWorkDir()


    def testAlertProcessorBasic(self):
        alertProcessor = AlertProcessor(self.config)
        try:
            # alertProcessor.startComponent() causes the flow to stop, Harness.py
            # the method just calls prepareToStart() and waits for ever
            # alertProcessor.startDaemon() no good for this either ... puts everything
            # on background
            alertProcessor.prepareToStart()
        except Exception, ex:
            print ex
            self.fail(str(ex))

        logging.debug("AlertProcessor and its sub-components should be running now ...")
        logging.debug("Going to stop the component ...")

        # stop via component method
        try:
            alertProcessor.stopAlertProcessor()
        except Exception, ex:
            print ex
            self.fail(str(ex))
コード例 #2
0
ファイル: Couch_t.py プロジェクト: ticoann/WMCore
class CouchTest(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInit(__file__)
        self.testInit.setLogging(logLevel=logging.DEBUG)
        self.testDir = self.testInit.generateWorkDir()
        self.config = getConfig(self.testDir)
        # mock generator instance to communicate some configuration values
        self.generator = utils.AlertGeneratorMock(self.config)
        self.testName = self.id().split('.')[-1]

    def tearDown(self):
        self.testInit.delWorkDir()
        self.generator = None

    def testAlertGeneratorCouchDbSizePollerBasic(self):
        config = getConfig("/tmp")
        try:
            poller = CouchDbSizePoller(config.AlertGenerator.couchDbSizePoller,
                                       self.generator)
        except Exception, ex:
            self.fail("%s: exception: %s" % (self.testName, ex))
        poller.check()  # -> on real system dir may result in permission denied
        poller._dbDirectory = "/dev"
        poller.check()  # -> OK

        # test failing during set up
        poller = CouchDbSizePoller(config.AlertGenerator.couchDbSizePoller,
                                   self.generator)
        poller._query = "nonsense query"
        # this will fail on the above query
        self.assertRaises(Exception, poller._getDbDir)
        poller.check()
コード例 #3
0
ファイル: Couch_t.py プロジェクト: zhiwenuil/WMCore
class CouchTest(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInit(__file__)
        self.testInit.setLogging(logLevel = logging.DEBUG)
        self.testDir = self.testInit.generateWorkDir()
        self.config = getConfig(self.testDir)
        # mock generator instance to communicate some configuration values
        self.generator = utils.AlertGeneratorMock(self.config)        
        self.testProcesses = []
        self.testName = self.id().split('.')[-1]
         
        
    def tearDown(self):       
        self.testInit.delWorkDir()
        self.generator = None
        utils.terminateProcesses(self.testProcesses)


    def testAlertGeneratorCouchDbSizePollerBasic(self):
        config = getConfig("/tmp")
        try:
            poller = CouchDbSizePoller(config.AlertGenerator.couchDbSizePoller, self.generator)
        except Exception, ex:
            self.fail("%s: exception: %s" % (self.testName, ex))
        poller.check() # -> on real system dir may result in permission denied
        poller._dbDirectory = "/dev"
        poller.check() # -> OK
        
        # test failing during set up
        poller = CouchDbSizePoller(config.AlertGenerator.couchDbSizePoller, self.generator)
        poller._query = "nonsense query"
        poller._dbDirectory = poller._getDbDir()
        poller.check()
        self.assertEquals(poller._dbDirectory, None)        
コード例 #4
0
class AlertProcessorTest(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInit(__file__)
        self.testInit.setLogging(logLevel=logging.DEBUG)
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules=[
            "WMCore.WMBS", 'WMCore.Agent.Database', "WMCore.ResourceControl"
        ],
                                useDefault=False)
        self.testDir = self.testInit.generateWorkDir()

        self.config = Configuration()
        self.config.section_("Agent")
        self.config.Agent.useMsgService = False
        self.config.Agent.useTrigger = False
        self.config.component_("AlertProcessor")
        self.config.AlertProcessor.componentDir = self.testDir
        self.config.AlertProcessor.address = "tcp://127.0.0.1:5557"
        self.config.AlertProcessor.controlAddr = "tcp://127.0.0.1:5559"
        self.config.section_("CoreDatabase")

        self.config.CoreDatabase.socket = os.environ.get("DBSOCK")
        self.config.CoreDatabase.connectUrl = os.environ.get("DATABASE")

        self.config.AlertProcessor.section_("critical")
        self.config.AlertProcessor.section_("soft")

        self.config.AlertProcessor.critical.level = 5
        self.config.AlertProcessor.soft.level = 0
        self.config.AlertProcessor.soft.bufferSize = 3

        self.config.AlertProcessor.critical.section_("sinks")
        self.config.AlertProcessor.soft.section_("sinks")

    def tearDown(self):
        self.testInit.clearDatabase()
        self.testInit.delWorkDir()

    def testAlertProcessorBasic(self):
        alertProcessor = AlertProcessor(self.config)
        try:
            # alertProcessor.startComponent() causes the flow to stop, Harness.py
            # the method just calls prepareToStart() and waits for ever
            # alertProcessor.startDaemon() no good for this either ... puts everything
            # on background
            alertProcessor.prepareToStart()
        except Exception as ex:
            print ex
            self.fail(str(ex))

        logging.debug(
            "AlertProcessor and its sub-components should be running now ...")
        logging.debug("Going to stop the component ...")

        # stop via component method
        try:
            alertProcessor.stopAlertProcessor()
        except Exception as ex:
            print ex
            self.fail(str(ex))
コード例 #5
0
ファイル: MockPlugin_t.py プロジェクト: AndrewLevin/WMCore
class MockPluginTest(unittest.TestCase):
    def setUp(self):
        self.testinit = TestInit(__file__)
        self.workdir = self.testinit.generateWorkDir()
        jobList[0]['cache_dir'] = self.workdir

    def tearDown(self):
        self.testinit.delWorkDir()

    def testInit(self):
        wrongconfig = Configuration()
        wrongconfig.section_('BossAir')
        self.assertRaises( BossAirPluginException, MockPlugin, wrongconfig )

        wrongconfig.BossAir.section_('MockPlugin')
        self.assertRaises( BossAirPluginException, MockPlugin, wrongconfig )
        #The config does not contain fakeReport parameter
        self.assertRaises( BossAirPluginException, MockPlugin, wrongconfig )

        #The fakeReport does not exist
        wrongconfig.BossAir.MockPlugin.fakeReport = 'asdf'
        self.assertRaises( BossAirPluginException, MockPlugin, wrongconfig )

    def testTrack(self):
        mp = MockPlugin(config)

        #Check that the job has been scheduled
        self.assertEquals({}, mp.jobsScheduledEnd)

        # Don't be racy
        currentTime = datetime.now()
        #id is the only required parameter in the job dictionary
        res = mp.track( jobList, currentTime )
        self.assertTrue( mp.jobsScheduledEnd.has_key(1L) )
        #check scheduled end (N.B. this includes 20% of random time)
        scheduledEnd = mp.jobsScheduledEnd[1L]
        timeTillJob = scheduledEnd - currentTime
        self.assertTrue( timeTillJob >= timedelta(minutes = TEST_JOB_LEN - 1), \
                         "Time till Job %s !>= Delta %s" % (timeTillJob, \
                         timedelta(minutes = TEST_JOB_LEN - 1)))
        self.assertTrue( timeTillJob <= timedelta(minutes = TEST_JOB_LEN*120/100 + 1), \
                         "Time till Job %s !<= Delta %s" % (timeTillJob, \
                         timedelta(minutes = TEST_JOB_LEN * 120/100 + 1)) )
        #the job is running
        self.assertEquals( 'Running', res[0][0]['status'])
        self.assertEquals( 'Running', res[1][0]['status'])
        self.assertEquals( [], res[2])

        #the job is not running anymore
        mp.jobsScheduledEnd[1L] = datetime(1900,1,1)
        res = mp.track( jobList )
        self.assertEquals( [], res[0])
        self.assertEquals( 'Done', res[1][0]['status'])
        self.assertEquals( 'Done', res[2][0]['status'])

        del mp
コード例 #6
0
class SetupCMSSWPsetTest(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testDir = self.testInit.generateWorkDir()
        sys.path.insert(
            0,
            os.path.join(WMCore.WMBase.getTestBase(),
                         "WMCore_t/WMRuntime_t/Scripts_t"))

    def tearDown(self):
        sys.path.remove(
            os.path.join(WMCore.WMBase.getTestBase(),
                         "WMCore_t/WMRuntime_t/Scripts_t"))
        del sys.modules["WMTaskSpace"]
        self.testInit.delWorkDir()

    def createTestStep(self):
        """
        _createTestStep_

        Create a test step that can be passed to the setup script.

        """
        newStep = WMStep("cmsRun1")
        newStepHelper = CMSSWStepHelper(newStep)
        newStepHelper.setStepType("CMSSW")
        newStepHelper.setGlobalTag("SomeGlobalTag")
        stepTemplate = StepFactory.getStepTemplate("CMSSW")
        stepTemplate(newStep)
        newStep.application.command.configuration = "PSet.py"
        newStep.application.multicore.numberOfCores = "auto"
        return newStepHelper

    def loadProcessFromPSet(self):
        """
        _loadProcessFromPSet_

        This requires changing the working directory,
        do so in a safe manner to encapsulate the change to this method only
        """

        currentPath = os.getcwd()
        loadedProcess = None
        try:
            if not os.path.isdir(self.testDir):
                raise
            os.chdir(self.testDir)
            testFile = "PSet.py"
            pset = imp.load_source('process', testFile)
            loadedProcess = pset.process
        except Exception, ex:
            self.fail(
                "An exception was caught while trying to load the PSet, %s" %
                str(ex))
        finally:
コード例 #7
0
class MockPluginTest(unittest.TestCase):
    def setUp(self):
        self.testinit = TestInit(__file__)
        self.workdir = self.testinit.generateWorkDir()
        jobList[0]['cache_dir'] = self.workdir

    def tearDown(self):
        self.testinit.delWorkDir()

    def testInit(self):
        wrongconfig = Configuration()
        wrongconfig.section_('BossAir')
        self.assertRaises( BossAirPluginException, MockPlugin, wrongconfig )

        wrongconfig.BossAir.section_('MockPlugin')
        self.assertRaises( BossAirPluginException, MockPlugin, wrongconfig )
        #The config does not contain fakeReport parameter
        self.assertRaises( BossAirPluginException, MockPlugin, wrongconfig )

        #The fakeReport does not exist
        wrongconfig.BossAir.MockPlugin.fakeReport = 'asdf'
        self.assertRaises( BossAirPluginException, MockPlugin, wrongconfig )

    def testTrack(self):
        mp = MockPlugin(config)

        #Check that the job has been scheduled
        self.assertEquals({}, mp.jobsScheduledEnd)

        # Don't be racy
        currentTime = datetime.now()
        #id is the only required parameter in the job dictionary
        res = mp.track( jobList, currentTime )
        self.assertTrue( 1 in mp.jobsScheduledEnd )
        #check scheduled end (N.B. this includes 20% of random time)
        scheduledEnd = mp.jobsScheduledEnd[1]
        timeTillJob = scheduledEnd - currentTime
        self.assertTrue( timeTillJob >= timedelta(minutes = TEST_JOB_LEN - 1), \
                         "Time till Job %s !>= Delta %s" % (timeTillJob, \
                         timedelta(minutes = TEST_JOB_LEN - 1)))
        self.assertTrue( timeTillJob <= timedelta(minutes = TEST_JOB_LEN*120/100 + 1), \
                         "Time till Job %s !<= Delta %s" % (timeTillJob, \
                         timedelta(minutes = TEST_JOB_LEN * 120/100 + 1)) )
        #the job is running
        self.assertEquals( 'Running', res[0][0]['status'])
        self.assertEquals( 'Running', res[1][0]['status'])
        self.assertEquals( [], res[2])

        #the job is not running anymore
        mp.jobsScheduledEnd[1] = datetime(1900,1,1)
        res = mp.track( jobList )
        self.assertEquals( [], res[0])
        self.assertEquals( 'Done', res[1][0]['status'])
        self.assertEquals( 'Done', res[2][0]['status'])

        del mp
コード例 #8
0
ファイル: WorkQueueTestCase.py プロジェクト: tsarangi/WMCore
class WorkQueueTestCase(unittest.TestCase):

    def setSchema(self):
        "this can be override if the schema setting is different"
        self.schema = ["WMCore.WMBS","WMComponent.DBS3Buffer","WMCore.BossAir"]
        self.couchApps = ["WorkQueue"]

    def setUp(self):
        """
        _setUp_

        Setup the database and logging connection.  Try to create all of the
        WMBS tables.  Also add some dummy locations.
        """
        self.queueDB = 'workqueue_t'
        self.queueInboxDB = 'workqueue_t_inbox'
        self.globalQDB = 'workqueue_t_global'
        self.globalQInboxDB = 'workqueue_t_global_inbox'
        self.localQDB = 'workqueue_t_local'
        self.localQInboxDB = 'workqueue_t_local_inbox'
        self.localQDB2 = 'workqueue_t_local2'
        self.localQInboxDB2 = 'workqueue_t_local2_inbox'
        self.configCacheDB = 'workqueue_t_config_cache'

        self.setSchema()
        self.testInit = TestInit('WorkQueueTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        self.testInit.setupCouch(self.queueDB, *self.couchApps)
        self.testInit.setupCouch(self.queueInboxDB, *self.couchApps)
        self.testInit.setupCouch(self.globalQDB, *self.couchApps)
        self.testInit.setupCouch(self.globalQInboxDB , *self.couchApps)
        self.testInit.setupCouch(self.localQDB, *self.couchApps)
        self.testInit.setupCouch(self.localQInboxDB, *self.couchApps)
        self.testInit.setupCouch(self.localQDB2, *self.couchApps)
        self.testInit.setupCouch(self.localQInboxDB2, *self.couchApps)
        self.testInit.setupCouch(self.configCacheDB, 'ConfigCache')

        couchServer = CouchServer(os.environ.get("COUCHURL"))
        self.configCacheDBInstance = couchServer.connectDatabase(self.configCacheDB)

        self.workDir = self.testInit.generateWorkDir()
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        #self.testInit.tearDownCouch()
        self.testInit.clearDatabase()
        self.testInit.delWorkDir()
コード例 #9
0
class SetupCMSSWPsetTest(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testDir = self.testInit.generateWorkDir()
        sys.path.insert(0, os.path.join(WMCore.WMBase.getTestBase(),
                                        "WMCore_t/WMRuntime_t/Scripts_t"))

    def tearDown(self):
        sys.path.remove(os.path.join(WMCore.WMBase.getTestBase(),
                                     "WMCore_t/WMRuntime_t/Scripts_t"))
        del sys.modules["WMTaskSpace"]
        self.testInit.delWorkDir()

    def createTestStep(self):
        """
        _createTestStep_

        Create a test step that can be passed to the setup script.

        """
        newStep = WMStep("cmsRun1")
        newStepHelper = CMSSWStepHelper(newStep)
        newStepHelper.setStepType("CMSSW")
        newStepHelper.setGlobalTag("SomeGlobalTag")
        stepTemplate = StepFactory.getStepTemplate("CMSSW")
        stepTemplate(newStep)
        newStep.application.command.configuration = "PSet.py"
        newStep.application.multicore.numberOfCores = "auto"
        return newStepHelper

    def loadProcessFromPSet(self):
        """
        _loadProcessFromPSet_

        This requires changing the working directory,
        do so in a safe manner to encapsulate the change to this method only
        """

        currentPath = os.getcwd()
        loadedProcess = None
        try:
            if not os.path.isdir(self.testDir):
                raise
            os.chdir(self.testDir)
            testFile = "PSet.py"
            pset = imp.load_source('process', testFile)
            loadedProcess = pset.process
        except Exception, ex:
            self.fail("An exception was caught while trying to load the PSet, %s" % str(ex))
        finally:
コード例 #10
0
ファイル: AlertGenerator_t.py プロジェクト: zhiwenuil/WMCore
class AlertGeneratorTest(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInit(__file__)
        self.testInit.setLogging(logLevel = logging.DEBUG)
        self.testInit.clearDatabase()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = ["WMCore.WMBS",'WMCore.Agent.Database',
                                                 "WMCore.ResourceControl"],
                                useDefault = False)
        self.testDir = self.testInit.generateWorkDir()
        # AlertGenerator instance
        self.generator = None
        
        self.config = getConfig(self.testDir)
         
        self.config.section_("CoreDatabase")
        self.config.CoreDatabase.socket = os.environ.get("DBSOCK")
        self.config.CoreDatabase.connectUrl = os.environ.get("DATABASE")
        
        self.testProcesses = []
        self.testComponentDaemonXml = "/tmp/TestComponent/Daemon.xml" 
                

    def tearDown(self):
        self.testInit.clearDatabase()       
        self.testInit.delWorkDir()
        self.generator = None
        utils.terminateProcesses(self.testProcesses)
        # if the directory and file "/tmp/TestComponent/Daemon.xml" after
        # ComponentsPoller test exist, then delete it
        d = os.path.dirname(self.testComponentDaemonXml)
        if os.path.exists(d):
            shutil.rmtree(d)

        
    def _startComponent(self):
        self.generator = AlertGenerator(self.config)
        try:
            # self.proc.startComponent() causes the flow to stop, Harness.py
            # the method just calls prepareToStart() and waits for ever
            # self.proc.startDaemon() no good for this either ... puts everything
            # on background
            self.generator.prepareToStart()
        except Exception, ex:
            print ex
            self.fail(str(ex))
        print "AlertGenerator and its sub-components should be running now ..."
コード例 #11
0
ファイル: FileSink_t.py プロジェクト: tsarangi/WMCore
class FileSinkTest(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInit(__file__)
        self.testInit.setLogging(logLevel=logging.DEBUG)
        self.testDir = self.testInit.generateWorkDir()
        self.config = ConfigSection("file")
        self.config.outputfile = os.path.join(self.testDir,
                                              "FileSinkTestNew.json")

    def tearDown(self):
        self.testInit.delWorkDir()

    def testFileSinkBasic(self):
        sink = FileSink(self.config)
        alerts = []
        nAlerts = 10
        for i in range(nAlerts):
            a = Alert(Source=__file__,
                      Level=i,
                      Timestamp=time.time(),
                      Type="Test")
            alerts.append(a)
        sink.send(alerts)
        # test by reading back
        loadAlerts = sink.load()
        self.assertEqual(len(loadAlerts), nAlerts)

        # Since FileSink implementation depends on line-separated JSONs of
        # Alert instance, test handling new lines in the payload
        alerts = []
        testMsg = "addtional \n message"
        for i in range(10, 20):
            a = Alert(Source=__file__,
                      Level=i,
                      Timestamp=time.time(),
                      Type="Test",
                      Details={"message": testMsg})
            alerts.append(a)
        self.failUnless(os.path.exists(self.config.outputfile))
        sink.send(alerts)
        # test by reading back
        loadAlerts = sink.load()
        self.assertEqual(len(loadAlerts), 20)
        for a in loadAlerts[10:]:
            self.assertEqual(a["Details"]["message"], testMsg)
コード例 #12
0
ファイル: MySQL_t.py プロジェクト: zhiwenuil/WMCore
class MySQLTest(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInit(__file__)
        self.testInit.setLogging(logLevel = logging.DEBUG)
        self.testInit.setDatabaseConnection()
        self.testDir = self.testInit.generateWorkDir()
        self.config = getConfig(self.testDir)
        # mock generator instance to communicate some configuration values
        self.generator = utils.AlertGeneratorMock(self.config)        
        self.testProcesses = []
        self.testName = self.id().split('.')[-1]
         
        
    def tearDown(self):       
        self.testInit.delWorkDir()
        self.generator = None
        utils.terminateProcesses(self.testProcesses)
        

    def testMySQLPollerBasic(self):
        config = getConfig("/tmp")
        generator = utils.AlertGeneratorMock(config)
        # take for instance mysqlCPUPoller configuration here, just need
        # appropriate attributes set
        try:
            poller = MySQLPoller(config.AlertGenerator.mysqlCPUPoller, generator)
        except Exception, ex:
            self.fail("%s: exception: %s" % (self.testName, ex))                                
        # this class would not have defined polling sample function, give it one
        poller.sample = lambda proc: float(12)
        self.assertEqual(len(poller._measurements), 0)
        poller.check()
        self.assertEqual(len(poller._measurements), 1)
        self.assertEqual(poller._measurements[0], 12)
        
        # test handling of a non-existing process
        MySQLPoller._getProcessPID = lambda inst: 1212121212
        poller = MySQLPoller(config.AlertGenerator.mysqlCPUPoller, generator)
        # polling should not even happen so don't have to define sample function
        poller.check()
        self.assertEqual(poller._measurements, None)
        self.assertEqual(poller._dbProcessDetail, None)
コード例 #13
0
class ConfigurationExTest(unittest.TestCase):
    """
    test case for Configuration object

    """
    def setUp(self):
        """set up"""
        self.testInit = TestInit(__file__)
        self.testDir  = self.testInit.generateWorkDir()
        self.functionSave = "%s/WMCore_Agent_Configuration_t_function.py" % self.testDir


    def tearDown(self):
        """clean up"""
        self.testInit.delWorkDir()


    def testCallableConfigParams(self):
        """ctor"""
        def f():
            return True

        config = Configuration()
        config.section_("SectionF")
        #creating field for the following test
        config.SectionF.aFunction = ''
        #Cannot set a function for plain Configuration objects
        #config.SectionF.__setattr__('aFunction', f)
        self.assertRaises(RuntimeError, config.SectionF.__setattr__, config.SectionF.aFunction, f)

        config = ConfigurationEx()
        config.section_("SectionF")
        #No failures with configurationEx
        config.SectionF.aFunction = f

        #However ConfigurationEx instances cannot be saved
        self.assertRaises(RuntimeError, saveConfigurationFile, config, self.functionSave)
コード例 #14
0
ファイル: FileSink_t.py プロジェクト: pietverwilligen/WMCore
class FileSinkTest(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInit(__file__)
        self.testInit.setLogging(logLevel=logging.DEBUG)
        self.testDir = self.testInit.generateWorkDir()
        self.config = ConfigSection("file")
        self.config.outputfile = os.path.join(self.testDir, "FileSinkTestNew.json")

    def tearDown(self):
        self.testInit.delWorkDir()

    def testFileSinkBasic(self):
        sink = FileSink(self.config)
        alerts = []
        nAlerts = 10
        for i in range(nAlerts):
            a = Alert(Source=__file__, Level=i, Timestamp=time.time(), Type="Test")
            alerts.append(a)
        sink.send(alerts)
        # test by reading back
        loadAlerts = sink.load()
        self.assertEqual(len(loadAlerts), nAlerts)

        # Since FileSink implementation depends on line-separated JSONs of
        # Alert instance, test handling new lines in the payload
        alerts = []
        testMsg = "addtional \n message"
        for i in range(10, 20):
            a = Alert(Source=__file__, Level=i, Timestamp=time.time(), Type="Test", Details={"message": testMsg})
            alerts.append(a)
        self.failUnless(os.path.exists(self.config.outputfile))
        sink.send(alerts)
        # test by reading back
        loadAlerts = sink.load()
        self.assertEqual(len(loadAlerts), 20)
        for a in loadAlerts[10:]:
            self.assertEqual(a["Details"]["message"], testMsg)
コード例 #15
0
ファイル: Configuration_t.py プロジェクト: vytjan/WMCore
class ConfigurationExTest(unittest.TestCase):
    """
    test case for Configuration object

    """
    def setUp(self):
        """set up"""
        self.testInit = TestInit(__file__)
        self.testDir = self.testInit.generateWorkDir()
        self.functionSave = "%s/WMCore_Agent_Configuration_t_function.py" % self.testDir

    def tearDown(self):
        """clean up"""
        self.testInit.delWorkDir()

    def testCallableConfigParams(self):
        """ctor"""
        def f():
            return True

        config = Configuration()
        config.section_("SectionF")
        #creating field for the following test
        config.SectionF.aFunction = ''
        #Cannot set a function for plain Configuration objects
        #config.SectionF.__setattr__('aFunction', f)
        self.assertRaises(RuntimeError, config.SectionF.__setattr__,
                          config.SectionF.aFunction, f)

        config = ConfigurationEx()
        config.section_("SectionF")
        #No failures with configurationEx
        config.SectionF.aFunction = f

        #However ConfigurationEx instances cannot be saved
        self.assertRaises(RuntimeError, saveConfigurationFile, config,
                          self.functionSave)
コード例 #16
0
class MySQLTest(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInit(__file__)
        self.testInit.setLogging(logLevel = logging.DEBUG)
        self.testInit.setDatabaseConnection()
        self.testDir = self.testInit.generateWorkDir()
        self.config = getConfig(self.testDir)
        # mock generator instance to communicate some configuration values
        self.generator = utils.AlertGeneratorMock(self.config)
        self.testName = self.id().split('.')[-1]


    def tearDown(self):
        self.testInit.delWorkDir()
        self.generator = None


    def testMySQLPollerBasic(self):
        config = getConfig("/tmp")
        generator = utils.AlertGeneratorMock(config)
        # take for instance mysqlCPUPoller configuration here, just need
        # appropriate attributes set
        try:
            poller = MySQLPoller(config.AlertGenerator.mysqlCPUPoller, generator)
        except Exception, ex:
            self.fail("%s: exception: %s" % (self.testName, ex))
        # this class would not have defined polling sample function, give it one
        poller.sample = lambda proc: float(12)
        self.assertEqual(len(poller._measurements), 0)
        poller.check()
        self.assertEqual(len(poller._measurements), 1)
        self.assertEqual(poller._measurements[0], 12)

        # test handling of a non-existing process
        MySQLPoller._getProcessPID = lambda inst: 1212121212
        self.assertRaises(Exception, MySQLPoller,
                          config.AlertGenerator.mysqlCPUPoller, generator)
コード例 #17
0
ファイル: Runtime_t.py プロジェクト: zhiwenuil/WMCore
class RuntimeTest(unittest.TestCase):
    """
    _RuntimeTest_

    A unittest to test the WMRuntime/WMSpec/Storage/etc tree
    """


    # This is an integration test
    __integration__ = "Any old bollocks"


    def setUp(self):
        """
        Basic setUp

        """

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()


        self.testDir = self.testInit.generateWorkDir()

        # Random variables
        self.workloadDir = None
        self.unpackDir   = None
        self.initialDir  = os.getcwd()
        self.origPath    = sys.path


        # Create some dirs
        os.makedirs(os.path.join(self.testDir, 'packages'))

        return


    def tearDown(self):
        """
        _tearDown_
        
        Remove any references you put directly into the modules
        """

        self.testInit.delWorkDir()

        # Clean up imports
        if 'WMSandbox' in sys.modules.keys():
                del sys.modules['WMSandbox']
        if 'WMSandbox.JobIndex' in sys.modules.keys():
                del sys.modules['WMSandbox.JobIndex']
        

        return


    def createTestWorkload(self, workloadName = 'Test', emulator = True):
        """
        _createTestWorkload_

        Creates a test workload for us to run on, hold the basic necessities.
        """

        workloadDir = os.path.join(self.testDir, workloadName)

        #arguments = getTestArguments()

        #workload = rerecoWorkload("Tier1ReReco", arguments)
        #rereco = workload.getTask("ReReco")

        workload = testWorkload(emulation = emulator)
        rereco = workload.getTask("ReReco")

        # Set environment and site-local-config
        siteConfigPath = os.path.join(workloadDir, 'SITECONF/local/JobConfig/')
        if not os.path.exists(siteConfigPath):
            os.makedirs(siteConfigPath)
        shutil.copy('site-local-config.xml', siteConfigPath)
        environment = rereco.data.section_('environment')
        environment.CMS_PATH = workloadDir

        taskMaker = TaskMaker(workload, workloadDir)
        taskMaker.skipSubscription = True
        taskMaker.processWorkload()

        workload.save(workloadName)

        return workload



    def unpackComponents(self, workload):
        """
        Run the unpacker to build the directories
        IMPORTANT NOTE:
          This is not how we do things on the worker node
          On the worker node we do not run multiple tasks
          So here we create multiple tasks in different directories
          To mimic running on multiple systems

        """

        listOfTasks = getListOfTasks(workload = workload)

        self.unpackDir = os.path.join(self.testDir, 'unpack')

        if not os.path.exists(self.unpackDir):
            os.makedirs(self.unpackDir)

        os.chdir(self.unpackDir)

        sandbox  = workload.data.sandbox

        for task in listOfTasks:
            # We have to create a directory, unpack in it, and then get out
            taskName = task.name()
            taskDir = os.path.join(self.unpackDir, taskName)
            if not os.path.exists(taskDir):
                # Well then we have to make it
                os.makedirs(taskDir)
            os.chdir(taskDir)
            # Now that we're here, run the unpacker

            package  = os.path.join(self.testDir, 'packages', '%sJobPackage.pkl' % (taskName))
            jobIndex = 1

            RunUnpacker(sandbox = sandbox, package = package,
                        jobIndex = jobIndex, jobname = taskName)

            # And go back to where we started
            os.chdir(self.unpackDir)


        os.chdir(self.initialDir)

        return


    def createWMBSComponents(self, workload):
        """
        Create the WMBS Components for this job

        """

        listOfTasks = []
        listOfSubs  = []

        rerecoTask  = None
        
        for primeTask in workload.taskIterator():
            # There should only be one prime task, and it should be the rerecoTask
            rerecoTask = primeTask
            for task in primeTask.taskIterator():
                listOfTasks.append(task)

        for task in listOfTasks:
            fileset = self.getFileset()
            sub = self.createSubscriptions(task = task,
                                           fileset = fileset)
            #listOfSubs.append(sub)

        

        return



    def createSubscriptions(self, task, fileset):
        """
        Create a subscription based on a task


        """
        type = task.taskType()
        work = task.makeWorkflow()
        
        sub = Subscription(fileset = fileset,
                           workflow = work,
                           split_algo = "FileBased",
                           type = type)

        package = self.createWMBSJobs(subscription = sub,
                                      task = task)        

        packName = os.path.join(self.testDir, 'packages',
                                '%sJobPackage.pkl' %(task.name()))
        package.save(packName)

        return sub




    def createWMBSJobs(self, subscription, task):
        """
        Create the jobs for WMBS Components
        Send a subscription/task, get back a package.

        """

        splitter = SplitterFactory()
        jobfactory = splitter(subscription = subscription,
                              package = "WMCore.DataStructs",
                              generators = makeGenerators(task))
        params = task.jobSplittingParameters()
        jobGroups = jobfactory(**params)

        jobID = 1
        package = JobPackage()
        for group in jobGroups:
            for job in group.jobs:
                job['id'] = jobID
                jobID += 1
                package[job['id']] = job

        return package


    

    def getFileset(self):
        """
        Get a fileset based on the task

        """

        fileset = Fileset(name = 'Merge%s' %(type))


        for i in range(0, random.randint(15,25)):
            # Use the testDir to generate a random lfn
            inpFile = File(lfn = "%s/%s.root" %(self.testDir, makeUUID()),
                           size = random.randint(200000, 1000000),
                           events = random.randint(1000,2000) )
            inpFile.setLocation('Megiddo')
            fileset.addFile(inpFile)


        return fileset



    def runJobs(self, workload):
        """
        This might actually run the job.  Who knows?


        """
        listOfTasks = []

        for primeTask in workload.taskIterator():
            listOfTasks.append(primeTask)
            # Only run primeTasks for now


        for task in listOfTasks:
            jobName = task.name()
            taskDir = os.path.join(self.unpackDir, jobName, 'job')
            os.chdir(taskDir)
            sys.path.append(taskDir)

            # Scream, run around in panic, blow up machine
            print "About to run jobs"
            print taskDir
            miniStartup(dir = taskDir)
            

            # When exiting, go back to where you started
            os.chdir(self.initialDir)
            sys.path.remove(taskDir)
            
        return





    def testA_CreateWorkload(self):
        """
        _CreateWorkload_
        
        Create a workload
        Unpack the workload
        Check for consistency
        """

        workloadName = 'basicWorkload'
        workload     = self.createTestWorkload(workloadName = workloadName)

        self.createWMBSComponents(workload = workload)

        taskNames = []
        for task in getListOfTasks(workload = workload):
            taskNames.append(task.name())

        workloadPath  = os.path.join(self.testDir, workloadName, "TestWorkload")
        siteConfigDir = os.path.join(self.testDir, workloadName, 'SITECONF/local/JobConfig/')


        # Pre-run checks

        # Does it have the right directories?
        dirList = os.listdir(workloadPath)
        self.assertEqual(dirList, ['WMSandbox', 'TestWorkload-Sandbox.tar.bz2'])
        dirList = os.listdir(os.path.join(workloadPath, 'WMSandbox'))
        for taskName in taskNames:
            self.assertTrue(taskName in dirList)

        # Do we have job packages
        for task in taskNames:
            self.assertTrue('%sJobPackage.pkl' % (task) in os.listdir(os.path.join(self.testDir, 'packages')))


        # Does it have the SITECONF?
        self.assertTrue('site-local-config.xml' in os.listdir(siteConfigDir))



        # Now actually see if you can unpack it.
        self.unpackComponents(workload = workload)


        # Check for proper unpacking
        # Check the the task has the right directories,
        # and that the PSetTweaks and WMSandbox directories
        # have the right contents
        taskContents = ['WMSandbox', 'WMCore', 'PSetTweaks']
        PSetContents = ['PSetTweak.pyc', 'CVS', 'PSetTweak.py',
                        '__init__.pyc', 'WMTweak.py', '__init__.py']
        taskSandbox  = ['JobPackage.pcl', 'JobIndex.py', '__init__.py', 'WMWorkload.pkl']
        taskSandbox.extend(taskNames)  # Should have a directory for each task

        for task in taskNames:
            self.assertTrue(task in os.listdir(os.path.join(self.testDir, 'unpack')))
            taskDir = os.path.join(self.testDir, 'unpack', task, 'job')
            self.assertTrue(os.path.isdir(taskDir))
            self.assertEqual(os.listdir(taskDir).sort(), taskContents.sort())
            self.assertEqual(os.listdir(os.path.join(taskDir, 'WMSandbox')).sort(),
                             taskSandbox.sort())
            self.assertEqual(os.listdir(os.path.join(taskDir, 'PSetTweaks')).sort(),
                             PSetContents.sort())


        # And we're done.
        # Assume if we got this far everything is good
        

        # At the end, copy the directory
        #if os.path.exists('tmpDir'):
        #    shutil.rmtree('tmpDir')
        #shutil.copytree(self.testDir, 'tmpDir')

        return


    def testB_EmulatorTest(self):
        """
        _EmulatorTest_
        
        This is where things get scary.  We need to not only unpack the job,
        but also ascertain whether it can run locally in emulator mode.

        This requires...uh...emulator emulation.
        """


        # Assume all this works, because we tested it in testA
        workloadName = 'basicWorkload'
        workload     = self.createTestWorkload(workloadName = workloadName)

        self.createWMBSComponents(workload = workload)

        self.unpackComponents(workload = workload)


        self.runJobs(workload = workload)

        # Check the report
        taskDir = os.path.join(self.testDir, 'unpack/ReReco/job/WMTaskSpace')
        report = Report()
        report.load(os.path.join(taskDir, 'Report.0.pkl'))
        cmsReport = report.data.cmsRun1



        # Now validate the report
        self.assertEqual(report.data.ceName, socket.gethostname())
        self.assertEqual(report.data.seName, 'cmssrm.fnal.gov')
        self.assertEqual(report.data.siteName, 'T1_US_FNAL')
        self.assertEqual(report.data.hostName, socket.gethostname())
        self.assertTrue(report.data.completed)

        # Should have status 0 (emulator job)
        self.assertEqual(cmsReport.status, 0)

        # Should have one output module
        self.assertEqual(cmsReport.outputModules, ['TestOutputModule'])

        # It should have one file for input and output
        self.assertEqual(cmsReport.input.PoolSource.files.fileCount, 1)
        self.assertEqual(cmsReport.output.TestOutputModule.files.fileCount, 1)

        # So, um, I guess we're done


        # At the end, copy the directory
        #if os.path.exists('tmpDir'):
        #    shutil.rmtree('tmpDir')
        #shutil.copytree(self.testDir, 'tmpDir')

        return
コード例 #18
0
ファイル: LogArch_t.py プロジェクト: huohuo21/WMCore
class otherLogArchiveTexst:#(unittest.TestCase):

    def setUp(self):
        # stolen from CMSSWExecutor_t. thanks, dave

        # first, delete all the sandboxen and taskspaces
        #    because of caching, this leaks from other tests in other files
        #    this sucks because the other tests are using sandboxen that
        #    are deleted after the test is over, which causes theses tests
        #    to break
        modsToDelete = []
        # not sure what happens if you delete from
        # an arrey you're iterating over. doing it in
        # two steps
        for modname in sys.modules.keys():
            # need to blow away things in sys.modules, otherwise
            # they are cached and we look at old taskspaces
            if modname.startswith('WMTaskSpace'):
                modsToDelete.append(modname)
            if modname.startswith('WMSandbox'):
                modsToDelete.append(modname)
        for modname in modsToDelete:
            try:
                reload(sys.modules[modname])
            except:
                pass
            del sys.modules[modname]

        self.oldpath = sys.path[:]
        self.testInit = TestInit(__file__)


        self.testDir = self.testInit.generateWorkDir()
        self.job = Job(name = "/UnitTests/DeleterTask/DeleteTest-test-job")
        shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))

        self.workload = newWorkload("UnitTests")
        self.task = self.workload.newTask("DeleterTask")

        cmsswHelper = self.task.makeStep("cmsRun1")
        cmsswHelper.setStepType('CMSSW')
        stepHelper = cmsswHelper.addStep("DeleteTest")
        stepHelper.setStepType('LogArchive')

        self.cmsswstep = cmsswHelper.data
        self.cmsswHelper = cmsswHelper


        self.stepdata = stepHelper.data
        self.stephelp = LogArchiveTemplate.LogArchiveStepHelper(stepHelper.data)
        self.task.applyTemplates()

        self.executor = StepFactory.getStepExecutor(self.stephelp.stepType())
        taskMaker = TaskMaker(self.workload, os.path.join(self.testDir))
        taskMaker.skipSubscription = True
        taskMaker.processWorkload()


        self.task.build(os.path.join(self.testDir, 'UnitTests'))

        sys.path.insert(0, self.testDir)
        sys.path.insert(0, os.path.join(self.testDir, 'UnitTests'))


#        binDir = inspect.getsourcefile(ModuleLocator)
#        binDir = binDir.replace("__init__.py", "bin")
#
#        if not binDir in os.environ['PATH']:
#            os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
        open( os.path.join( self.testDir, 'UnitTests', '__init__.py'),'w').close()
        shutil.copyfile( os.path.join( os.path.dirname( __file__ ), 'MergeSuccess.pkl'),
                         os.path.join( self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))

    def tearDown(self):
        sys.path = self.oldpath[:]
        self.testInit.delWorkDir()


        # making double sure WMTaskSpace and WMSandbox are gone
        modsToDelete = []
        # not sure what happens if you delete from
        # an arrey you're iterating over. doing it in
        # two steps
        for modname in sys.modules.keys():
            # need to blow away things in sys.modules, otherwise
            # they are cached and we look at old taskspaces
            if modname.startswith('WMTaskSpace'):
                modsToDelete.append(modname)
            if modname.startswith('WMSandbox'):
                modsToDelete.append(modname)
        for modname in modsToDelete:
            try:
                reload(sys.modules[modname])
            except:
                pass
            del sys.modules[modname]
        myThread = threading.currentThread()
        if hasattr(myThread, "factory"):
            myThread.factory = {}

    @attr('integration')
    def testCPBackendLogArchiveAgainstReportNew(self):
        myReport = Report()
        myReport.unpersist(os.path.join( self.testDir, 'UnitTests','WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
        myReport.data.cmsRun1.status = 0
        myReport.persist(os.path.join( self.testDir,'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
        executor = LogArchiveExecutor.LogArchive()
        executor.initialise( self.stepdata, self.job)
        self.setLocalOverride(self.stepdata)
        self.stepdata.override.newLogArchive = True
        executor.step = self.stepdata
        executor.execute( )
        self.assertTrue( os.path.exists( os.path.join( self.testDir, 'hosts' )))
        self.assertTrue( os.path.exists( os.path.join( self.testDir, 'test1', 'hosts')))

    @attr('integration')
    def testCPBackendLogArchiveAgainstReportFailedStepNew(self):
        myReport = Report()
        myReport.unpersist(os.path.join( self.testDir, 'UnitTests','WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
        myReport.data.cmsRun1.status = 1
        myReport.persist(os.path.join( self.testDir,'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
        executor = LogArchiveExecutor.LogArchive()
        executor.initialise( self.stepdata, self.job)
        self.setLocalOverride(self.stepdata)
        self.stepdata.override.newLogArchive = True
        executor.step = self.stepdata
        executor.execute( )
        self.assertFalse( os.path.exists( os.path.join( self.testDir, 'hosts' )))
        self.assertFalse( os.path.exists( os.path.join( self.testDir, 'test1', 'hosts')))
        return

    @attr('integration')
    def testCPBackendLogArchiveAgainstReportOld(self):

        myReport = Report()
        myReport.unpersist(os.path.join( self.testDir,'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
        myReport.data.cmsRun1.status = 0
        myReport.persist(os.path.join( self.testDir,'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
        executor = LogArchiveExecutor.LogArchive()
        executor.initialise( self.stepdata, self.job)
        self.setLocalOverride(self.stepdata)
        executor.step = self.stepdata
        executor.execute( )
        self.assertTrue( os.path.exists( os.path.join( self.testDir, 'hosts' )))
        self.assertTrue( os.path.exists( os.path.join( self.testDir, 'test1', 'hosts')))
        return

    @attr('integration')
    def testCPBackendLogArchiveAgainstReportFailedStepOld(self):
        myReport = Report()
        myReport.unpersist(os.path.join( self.testDir,'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
        myReport.data.cmsRun1.status = 1
        myReport.persist(os.path.join( self.testDir, 'UnitTests','WMTaskSpace', 'cmsRun1' , 'Report.pkl'))

        executor = LogArchiveExecutor.LogArchive()
        executor.initialise( self.stepdata, self.job)
        self.setLocalOverride(self.stepdata)
        executor.step = self.stepdata
        executor.execute( )
        self.assertFalse( os.path.exists( os.path.join( self.testDir, 'hosts' )))
        self.assertFalse( os.path.exists( os.path.join( self.testDir, 'test1', 'hosts')))
        return

    @attr('workerNodeTest')
    def testOnWorkerNodes(self):
        raise RuntimeError
        # Stage a file out, stage it back in, check it, delete it
        myReport = Report()
        myReport.unpersist(os.path.join( self.testDir,'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
        myReport.data.cmsRun1.status = 1
        del myReport.data.cmsRun1.output
        myReport.data.cmsRun1.section_('output')
        myReport.data.cmsRun1.output.section_('stagingTestOutput')
        myReport.data.cmsRun1.output.stagingTestOutput.section_('files')
        myReport.data.cmsRun1.output.stagingTestOutput.fileCount = 0
        targetFiles = [ '/store/temp/WMAgent/storetest-%s' % time.time(),
                        '/store/unmerged/WMAgent/storetest-%s' % time.time()]

        for file in targetFiles:
            print("Adding file for LogArchive %s" % file)
            self.addLogArchiveFile(myReport, file )

        myReport.persist(os.path.join( self.testDir, 'UnitTests','WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
        executor = LogArchiveExecutor.LogArchive()

        executor.initialise( self.stepdata, self.job)
        executor.step = self.stepdata
        print("beginning stageout")
        executor.execute( )
        print("stageout done")

        # pull in the report with the stage out info
        myReport = Report()
        myReport.unpersist(os.path.join( self.testDir,'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
        print("Got the stage out data back")
        print(myReport.data)


        # now, transfer them back
        # TODO make a stagein step in the task - Melo
        import WMCore.Storage.FileManager as FileManagerModule
        fileManager = FileManagerModule.FileManager( numberOfRetries = 10, retryPauseTime = 1)
        for file in targetFiles:
            print("Staging in %s" % file)

            fileManager.stageOut( fileToStage = { 'LFN' : file,
                                    'PFN' : '%s/%s' % (self.testDir, file) },
                                    stageOut = False)
            self.assertTrue( os.path.exists( '%s/%s' % (self.testDir, file)))
            self.assertEqual( os.path.getsize('/etc/hosts', '%s/%s' % (self.testDir, file)))

        # now, should delete the files we made
        for file in targetFiles:
            print("deleting %s" % file)
            fileManager.deleteLFN(file)

        # try staging in again to make sure teh files are gone
        for file in targetFiles:
            print("Staging in (should fail) %s" % file)
            self.assertRaises( LogArchiveError, \
                               FileManagerModule.FileManager.stageOut, \
                               fileManager,fileToStage = { 'LFN' : file,
                                    'PFN' : '%s/%s' % (self.testDir, file) },
                                    stageOut = False )



        # need to make sure files didn't show up
        self.assertFalse( os.path.exists( os.path.join( self.testDir, 'hosts' )))
        self.assertFalse( os.path.exists( os.path.join( self.testDir, 'test1', 'hosts')))

    def addLogArchiveFile(self, myReport, lfn):
        myId = myReport.data.cmsRun1.output.stagingTestOutput.fileCount
        mySection = myReport.data.cmsRun1.output.stagingTestOutput.section_('file%s' % myId)
        mySection.section_('runs')
        setattr(mySection.runs,'114475', [33])
        mySection.section_('branches')
        mySection.lfn = lfn
        mySection.dataset = {'applicationName': 'cmsRun', 'primaryDataset': 'Calo', 'processedDataset': 'Commissioning09-PromptReco-v8', 'dataTier': 'ALCARECO', 'applicationVersion': 'CMSSW_3_2_7'}
        mySection.module_label = 'ALCARECOStreamCombined'
        mySection.parents = []
        mySection.location = 'srm-cms.cern.ch'
        mySection.checksums = {'adler32': 'bbcf2215', 'cksum': '2297542074'}
        mySection.pfn = '/etc/hosts'
        mySection.events = 20000
        mySection.merged = False
        mySection.size = 37556367
        myReport.data.cmsRun1.output.stagingTestOutput.fileCount = myId + 1





    def setLocalOverride(self, step):
        step.section_('override')
        step.override.command    = 'cp'
        step.override.option     = ''
        step.override.__setattr__('lfn-prefix', self.testDir +"/")
        step.override.__setattr__('se-name','DUMMYSE')
        step.override.__setattr__('phedex-node','DUMMYPNN')
コード例 #19
0
class RuntimeTest(unittest.TestCase):
    """
    _RuntimeTest_

    A unittest to test the WMRuntime/WMSpec/Storage/etc tree
    """


    # This is an integration test
    __integration__ = "Any old bollocks"


    def setUp(self):
        """
        Basic setUp

        """

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()


        self.testDir = self.testInit.generateWorkDir()

        # Random variables
        self.workloadDir = None
        self.unpackDir   = None
        self.initialDir  = os.getcwd()
        self.origPath    = sys.path


        # Create some dirs
        os.makedirs(os.path.join(self.testDir, 'packages'))

        return


    def tearDown(self):
        """
        _tearDown_

        Remove any references you put directly into the modules
        """

        self.testInit.delWorkDir()

        # Clean up imports
        if 'WMSandbox' in sys.modules.keys():
            del sys.modules['WMSandbox']
        if 'WMSandbox.JobIndex' in sys.modules.keys():
            del sys.modules['WMSandbox.JobIndex']


        return


    def createTestWorkload(self, workloadName = 'Test', emulator = True):
        """
        _createTestWorkload_

        Creates a test workload for us to run on, hold the basic necessities.
        """

        workloadDir = os.path.join(self.testDir, workloadName)

        #arguments = getTestArguments()

        #workload = rerecoWorkload("Tier1ReReco", arguments)
        #rereco = workload.getTask("ReReco")

        workload = testWorkload(emulation = emulator)
        rereco = workload.getTask("ReReco")

        # Set environment and site-local-config
        siteConfigPath = os.path.join(workloadDir, 'SITECONF/local/JobConfig/')
        if not os.path.exists(siteConfigPath):
            os.makedirs(siteConfigPath)
        shutil.copy('site-local-config.xml', siteConfigPath)
        environment = rereco.data.section_('environment')
        environment.CMS_PATH = workloadDir

        taskMaker = TaskMaker(workload, workloadDir)
        taskMaker.skipSubscription = True
        taskMaker.processWorkload()

        workload.save(workloadName)

        return workload



    def unpackComponents(self, workload):
        """
        Run the unpacker to build the directories
        IMPORTANT NOTE:
          This is not how we do things on the worker node
          On the worker node we do not run multiple tasks
          So here we create multiple tasks in different directories
          To mimic running on multiple systems

        """

        listOfTasks = getListOfTasks(workload = workload)

        self.unpackDir = os.path.join(self.testDir, 'unpack')

        if not os.path.exists(self.unpackDir):
            os.makedirs(self.unpackDir)

        os.chdir(self.unpackDir)

        sandbox  = workload.data.sandbox

        for task in listOfTasks:
            # We have to create a directory, unpack in it, and then get out
            taskName = task.name()
            taskDir = os.path.join(self.unpackDir, taskName)
            if not os.path.exists(taskDir):
            # Well then we have to make it
                os.makedirs(taskDir)
            os.chdir(taskDir)
            # Now that we're here, run the unpacker

            package  = os.path.join(self.testDir, 'packages', '%sJobPackage.pkl' % (taskName))
            jobIndex = 1

            RunUnpacker(sandbox = sandbox, package = package,
                        jobIndex = jobIndex, jobname = taskName)

            # And go back to where we started
            os.chdir(self.unpackDir)


        os.chdir(self.initialDir)

        return


    def createWMBSComponents(self, workload):
        """
        Create the WMBS Components for this job

        """

        listOfTasks = []
        listOfSubs  = []

        rerecoTask  = None

        for primeTask in workload.taskIterator():
            # There should only be one prime task, and it should be the rerecoTask
            rerecoTask = primeTask
            for task in primeTask.taskIterator():
                listOfTasks.append(task)

        for task in listOfTasks:
            fileset = self.getFileset()
            sub = self.createSubscriptions(task = task,
                                           fileset = fileset)
            #listOfSubs.append(sub)



        return



    def createSubscriptions(self, task, fileset):
        """
        Create a subscription based on a task


        """
        type = task.taskType()
        work = task.makeWorkflow()

        sub = Subscription(fileset = fileset,
                           workflow = work,
                           split_algo = "FileBased",
                           type = type)

        package = self.createWMBSJobs(subscription = sub,
                                      task = task)

        packName = os.path.join(self.testDir, 'packages',
                                '%sJobPackage.pkl' %(task.name()))
        package.save(packName)

        return sub




    def createWMBSJobs(self, subscription, task):
        """
        Create the jobs for WMBS Components
        Send a subscription/task, get back a package.

        """

        splitter = SplitterFactory()
        geneFac  = GeneratorFactory()
        jobfactory = splitter(subscription = subscription,
                              package = "WMCore.DataStructs",
                              generators = geneFac.makeGenerators(task))
        params = task.jobSplittingParameters()
        jobGroups = jobfactory(**params)

        jobID = 1
        package = JobPackage()
        for group in jobGroups:
            for job in group.jobs:
                job['id'] = jobID
                jobID += 1
                package[job['id']] = job

        return package




    def getFileset(self):
        """
        Get a fileset based on the task

        """

        fileset = Fileset(name = 'Merge%s' %(type))


        for i in range(0, random.randint(15,25)):
            # Use the testDir to generate a random lfn
            inpFile = File(lfn = "%s/%s.root" %(self.testDir, makeUUID()),
                           size = random.randint(200000, 1000000),
                           events = random.randint(1000,2000) )
            inpFile.setLocation('Megiddo')
            fileset.addFile(inpFile)


        return fileset



    def runJobs(self, workload):
        """
        This might actually run the job.  Who knows?


        """
        listOfTasks = []

        for primeTask in workload.taskIterator():
            listOfTasks.append(primeTask)
            # Only run primeTasks for now


        for task in listOfTasks:
            jobName = task.name()
            taskDir = os.path.join(self.unpackDir, jobName, 'job')
            os.chdir(taskDir)
            sys.path.append(taskDir)

            # Scream, run around in panic, blow up machine
            print "About to run jobs"
            print taskDir
            miniStartup(dir = taskDir)


            # When exiting, go back to where you started
            os.chdir(self.initialDir)
            sys.path.remove(taskDir)

        return




    @attr('integration')
    def testA_CreateWorkload(self):
        """
        _CreateWorkload_

        Create a workload
        Unpack the workload
        Check for consistency
        """

        workloadName = 'basicWorkload'
        workload     = self.createTestWorkload(workloadName = workloadName)

        self.createWMBSComponents(workload = workload)

        taskNames = []
        for task in getListOfTasks(workload = workload):
            taskNames.append(task.name())

        workloadPath  = os.path.join(self.testDir, workloadName, "TestWorkload")
        siteConfigDir = os.path.join(self.testDir, workloadName, 'SITECONF/local/JobConfig/')


        # Pre-run checks

        # Does it have the right directories?
        dirList = os.listdir(workloadPath)
        self.assertEqual(dirList, ['WMSandbox', 'TestWorkload-Sandbox.tar.bz2'])
        dirList = os.listdir(os.path.join(workloadPath, 'WMSandbox'))
        for taskName in taskNames:
            self.assertTrue(taskName in dirList)

        # Do we have job packages
        for task in taskNames:
            self.assertTrue('%sJobPackage.pkl' % (task) in os.listdir(os.path.join(self.testDir, 'packages')))


        # Does it have the SITECONF?
        self.assertTrue('site-local-config.xml' in os.listdir(siteConfigDir))



        # Now actually see if you can unpack it.
        self.unpackComponents(workload = workload)


        # Check for proper unpacking
        # Check the the task has the right directories,
        # and that the PSetTweaks and WMSandbox directories
        # have the right contents
        taskContents = ['WMSandbox', 'WMCore', 'PSetTweaks']
        PSetContents = ['PSetTweak.pyc', 'CVS', 'PSetTweak.py',
                        '__init__.pyc', 'WMTweak.py', '__init__.py']
        taskSandbox  = ['JobPackage.pcl', 'JobIndex.py', '__init__.py', 'WMWorkload.pkl']
        taskSandbox.extend(taskNames)  # Should have a directory for each task

        for task in taskNames:
            self.assertTrue(task in os.listdir(os.path.join(self.testDir, 'unpack')))
            taskDir = os.path.join(self.testDir, 'unpack', task, 'job')
            self.assertTrue(os.path.isdir(taskDir))
            self.assertEqual(os.listdir(taskDir).sort(), taskContents.sort())
            self.assertEqual(os.listdir(os.path.join(taskDir, 'WMSandbox')).sort(),
                             taskSandbox.sort())
            self.assertEqual(os.listdir(os.path.join(taskDir, 'PSetTweaks')).sort(),
                             PSetContents.sort())


        # And we're done.
        # Assume if we got this far everything is good


        # At the end, copy the directory
        #if os.path.exists('tmpDir'):
        #    shutil.rmtree('tmpDir')
        #shutil.copytree(self.testDir, 'tmpDir')

        return

    @attr('integration')
    def testB_EmulatorTest(self):
        """
        _EmulatorTest_

        This is where things get scary.  We need to not only unpack the job,
        but also ascertain whether it can run locally in emulator mode.

        This requires...uh...emulator emulation.
        """


        # Assume all this works, because we tested it in testA
        workloadName = 'basicWorkload'
        workload     = self.createTestWorkload(workloadName = workloadName)

        self.createWMBSComponents(workload = workload)

        self.unpackComponents(workload = workload)


        self.runJobs(workload = workload)

        # Check the report
        taskDir = os.path.join(self.testDir, 'unpack/ReReco/job/WMTaskSpace')
        report = Report()
        report.load(os.path.join(taskDir, 'Report.0.pkl'))
        cmsReport = report.data.cmsRun1



        # Now validate the report
        self.assertEqual(report.data.ceName, socket.gethostname())
        self.assertEqual(report.data.seName, 'cmssrm.fnal.gov')
        self.assertEqual(report.data.siteName, 'T1_US_FNAL')
        self.assertEqual(report.data.hostName, socket.gethostname())
        self.assertTrue(report.data.completed)

        # Should have status 0 (emulator job)
        self.assertEqual(cmsReport.status, 0)

        # Should have one output module
        self.assertEqual(cmsReport.outputModules, ['TestOutputModule'])

        # It should have one file for input and output
        self.assertEqual(cmsReport.input.PoolSource.files.fileCount, 1)
        self.assertEqual(cmsReport.output.TestOutputModule.files.fileCount, 1)

        # So, um, I guess we're done


        # At the end, copy the directory
        #if os.path.exists('tmpDir'):
        #    shutil.rmtree('tmpDir')
        #shutil.copytree(self.testDir, 'tmpDir')

        return
コード例 #20
0
ファイル: ResourceControl_t.py プロジェクト: vkuznet/WMCore
class ResourceControlTest(EmulatedUnitTestCase):
    def setUp(self):
        """
        _setUp_

        Install schema and create a DAO factory for WMBS.
        """
        super(ResourceControlTest, self).setUp()
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules=["WMCore.WMBS",
                                               "WMCore.ResourceControl",
                                               "WMCore.BossAir"],
                                useDefault=False)

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package="WMCore.WMBS",
                                     logger=myThread.logger,
                                     dbinterface=myThread.dbi)

        self.baDaoFactory = DAOFactory(package="WMCore.BossAir",
                                       logger=myThread.logger,
                                       dbinterface=myThread.dbi)

        self.insertRunJob = self.baDaoFactory(classname="NewJobs")
        self.insertState = self.baDaoFactory(classname="NewState")
        states = ['PEND', 'RUN', 'Idle', 'Running']
        self.insertState.execute(states)

        self.tempDir = self.testInit.generateWorkDir()
        return

    def tearDown(self):
        """
        _tearDown_

        Clear the schema.
        """
        super(ResourceControlTest, self).tearDown()
        self.testInit.clearDatabase()
        return

    def createJobs(self):
        """
        _createJobs_

        Create test jobs in WMBS and BossAir
        """
        testWorkflow = Workflow(spec=makeUUID(), owner="tapas",
                                name=makeUUID(), task="Test")
        testWorkflow.create()

        testFilesetA = Fileset(name="TestFilesetA")
        testFilesetA.create()
        testFilesetB = Fileset(name="TestFilesetB")
        testFilesetB.create()
        testFilesetC = Fileset(name="TestFilesetC")
        testFilesetC.create()

        testFileA = File(lfn="testFileA", locations=set(["testSE1", "testSE2"]))
        testFileA.create()
        testFilesetA.addFile(testFileA)
        testFilesetA.commit()
        testFilesetB.addFile(testFileA)
        testFilesetB.commit()
        testFilesetC.addFile(testFileA)
        testFilesetC.commit()

        testSubscriptionA = Subscription(fileset=testFilesetA,
                                         workflow=testWorkflow,
                                         type="Processing")
        testSubscriptionA.create()
        testSubscriptionA.addWhiteBlackList([{"site_name": "testSite1", "valid": True}])
        testSubscriptionB = Subscription(fileset=testFilesetB,
                                         workflow=testWorkflow,
                                         type="Processing")
        testSubscriptionB.create()
        testSubscriptionB.addWhiteBlackList([{"site_name": "testSite1", "valid": False}])
        testSubscriptionC = Subscription(fileset=testFilesetC,
                                         workflow=testWorkflow,
                                         type="Merge")
        testSubscriptionC.create()

        testJobGroupA = JobGroup(subscription=testSubscriptionA)
        testJobGroupA.create()
        testJobGroupB = JobGroup(subscription=testSubscriptionB)
        testJobGroupB.create()
        testJobGroupC = JobGroup(subscription=testSubscriptionC)
        testJobGroupC.create()

        # Site1, Has been assigned a location and is complete.
        testJobA = Job(name="testJobA", files=[testFileA])
        testJobA["couch_record"] = makeUUID()
        testJobA.create(group=testJobGroupA)
        testJobA["state"] = "success"

        # Site 1, Has been assigned a location and is incomplete.
        testJobB = Job(name="testJobB", files=[testFileA])
        testJobB["couch_record"] = makeUUID()
        testJobB["cache_dir"] = self.tempDir
        testJobB.create(group=testJobGroupA)
        testJobB["state"] = "executing"
        runJobB = RunJob()
        runJobB.buildFromJob(testJobB)
        runJobB["status"] = "PEND"

        # Does not have a location, white listed to site 1
        testJobC = Job(name="testJobC", files=[testFileA])
        testJobC["couch_record"] = makeUUID()
        testJobC.create(group=testJobGroupA)
        testJobC["state"] = "new"

        # Site 2, Has been assigned a location and is complete.
        testJobD = Job(name="testJobD", files=[testFileA])
        testJobD["couch_record"] = makeUUID()
        testJobD.create(group=testJobGroupB)
        testJobD["state"] = "success"

        # Site 2, Has been assigned a location and is incomplete.
        testJobE = Job(name="testJobE", files=[testFileA])
        testJobE["couch_record"] = makeUUID()
        testJobE.create(group=testJobGroupB)
        testJobE["state"] = "executing"
        runJobE = RunJob()
        runJobE.buildFromJob(testJobE)
        runJobE["status"] = "RUN"

        # Does not have a location, site 1 is blacklisted.
        testJobF = Job(name="testJobF", files=[testFileA])
        testJobF["couch_record"] = makeUUID()
        testJobF.create(group=testJobGroupB)
        testJobF["state"] = "new"

        # Site 3, Has been assigned a location and is complete.
        testJobG = Job(name="testJobG", files=[testFileA])
        testJobG["couch_record"] = makeUUID()
        testJobG.create(group=testJobGroupC)
        testJobG["state"] = "cleanout"

        # Site 3, Has been assigned a location and is incomplete.
        testJobH = Job(name="testJobH", files=[testFileA])
        testJobH["couch_record"] = makeUUID()
        testJobH.create(group=testJobGroupC)
        testJobH["state"] = "new"

        # Site 3, Does not have a location.
        testJobI = Job(name="testJobI", files=[testFileA])
        testJobI["couch_record"] = makeUUID()
        testJobI.create(group=testJobGroupC)
        testJobI["state"] = "new"

        # Site 3, Does not have a location and is in cleanout.
        testJobJ = Job(name="testJobJ", files=[testFileA])
        testJobJ["couch_record"] = makeUUID()
        testJobJ.create(group=testJobGroupC)
        testJobJ["state"] = "cleanout"

        changeStateAction = self.daoFactory(classname="Jobs.ChangeState")
        changeStateAction.execute(jobs=[testJobA, testJobB, testJobC, testJobD,
                                        testJobE, testJobF, testJobG, testJobH,
                                        testJobI, testJobJ])

        self.insertRunJob.execute([runJobB, runJobE])

        setLocationAction = self.daoFactory(classname="Jobs.SetLocation")
        setLocationAction.execute(testJobA["id"], "testSite1")
        setLocationAction.execute(testJobB["id"], "testSite1")
        setLocationAction.execute(testJobD["id"], "testSite1")
        setLocationAction.execute(testJobE["id"], "testSite2")
        setLocationAction.execute(testJobG["id"], "testSite1")
        setLocationAction.execute(testJobH["id"], "testSite1")

        return

    def testInsert(self):
        """
        _testInsert_

        Verify that inserting sites and thresholds works correctly, even if the
        site or threshold already exists.
        """
        myResourceControl = ResourceControl()
        myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1")
        myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1")
        myResourceControl.insertSite("testSite2", 100, 200, "testSE2", "testCE2")

        myResourceControl.insertThreshold("testSite1", "Processing", 20, 10)
        myResourceControl.insertThreshold("testSite1", "Merge", 200, 100)
        myResourceControl.insertThreshold("testSite1", "Merge", 250, 150)
        myResourceControl.insertThreshold("testSite2", "Processing", 50, 30)
        myResourceControl.insertThreshold("testSite2", "Merge", 135, 100)

        createThresholds = myResourceControl.listThresholdsForCreate()

        self.assertEqual(len(createThresholds.keys()), 2,
                         "Error: Wrong number of site in Resource Control DB")

        self.assertTrue("testSite1" in createThresholds.keys(),
                        "Error: Test Site 1 missing from thresholds.")

        self.assertTrue("testSite2" in createThresholds.keys(),
                        "Error: Test Site 2 missing from thresholds.")

        self.assertEqual(createThresholds["testSite1"]["total_slots"], 10,
                         "Error: Wrong number of total slots.")

        self.assertEqual(createThresholds["testSite1"]["pending_jobs"], {0: 0},
                         "Error: Wrong number of running jobs: %s" %
                         createThresholds["testSite1"]["pending_jobs"])

        self.assertEqual(createThresholds["testSite2"]["total_slots"], 100,
                         "Error: Wrong number of total slots.")

        self.assertEqual(createThresholds["testSite2"]["pending_jobs"], {0: 0},
                         "Error: Wrong number of running jobs.")

        thresholds = myResourceControl.listThresholdsForSubmit()

        self.assertEqual(len(thresholds.keys()), 2,
                         "Error: Wrong number of sites in Resource Control DB")

        self.assertTrue("testSite1" in thresholds.keys(),
                        "Error: testSite1 missing from thresholds.")

        self.assertTrue("testSite2" in thresholds.keys(),
                        "Error: testSite2 missing from thresholds.")

        site1Info = thresholds["testSite1"]
        site2Info = thresholds["testSite2"]
        site1Thresholds = site1Info["thresholds"]
        site2Thresholds = site2Info["thresholds"]

        procThreshold1 = None
        procThreshold2 = None
        mergeThreshold1 = None
        mergeThreshold2 = None
        for taskType, threshold in site1Thresholds.items():
            if taskType == "Merge":
                mergeThreshold1 = threshold
            elif taskType == "Processing":
                procThreshold1 = threshold
        for taskType, threshold in site2Thresholds.items():
            if taskType == "Merge":
                mergeThreshold2 = threshold
            elif taskType == "Processing":
                procThreshold2 = threshold

        self.assertEqual(len(site1Thresholds), 2,
                         "Error: Wrong number of task types.")

        self.assertEqual(len(site2Thresholds), 2,
                         "Error: Wrong number of task types.")

        self.assertNotEqual(procThreshold1, None)
        self.assertNotEqual(procThreshold2, None)
        self.assertNotEqual(mergeThreshold1, None)
        self.assertNotEqual(mergeThreshold2, None)

        self.assertEqual(site1Info["total_pending_slots"], 10,
                         "Error: Site thresholds wrong")

        self.assertEqual(site1Info["total_running_slots"], 20,
                         "Error: Site thresholds wrong")

        self.assertEqual(site1Info["total_running_jobs"], 0,
                         "Error: Site thresholds wrong")

        self.assertEqual(site1Info["total_pending_jobs"], 0,
                         "Error: Site thresholds wrong")

        self.assertEqual(procThreshold1["task_running_jobs"], 0,
                         "Error: Site thresholds wrong")

        self.assertEqual(procThreshold1["task_pending_jobs"], 0,
                         "Error: Site thresholds wrong")

        self.assertEqual(procThreshold1["max_slots"], 20,
                         "Error: Site thresholds wrong")

        self.assertEqual(procThreshold1["pending_slots"], 10,
                         "Error: Site thresholds wrong")

        self.assertEqual(mergeThreshold1["task_running_jobs"], 0,
                         "Error: Site thresholds wrong")

        self.assertEqual(mergeThreshold1["task_pending_jobs"], 0,
                         "Error: Site thresholds wrong")

        self.assertEqual(mergeThreshold1["max_slots"], 250,
                         "Error: Site thresholds wrong")

        self.assertEqual(mergeThreshold1["pending_slots"], 150,
                         "Error: Site thresholds wrong")

        self.assertEqual(site2Info["total_pending_slots"], 100,
                         "Error: Site thresholds wrong")

        self.assertEqual(site2Info["total_running_slots"], 200,
                         "Error: Site thresholds wrong")

        self.assertEqual(site2Info["total_running_jobs"], 0,
                         "Error: Site thresholds wrong")

        self.assertEqual(site2Info["total_pending_jobs"], 0,
                         "Error: Site thresholds wrong")

        self.assertEqual(procThreshold2["task_running_jobs"], 0,
                         "Error: Site thresholds wrong")

        self.assertEqual(procThreshold2["task_pending_jobs"], 0,
                         "Error: Site thresholds wrong")

        self.assertEqual(procThreshold2["max_slots"], 50,
                         "Error: Site thresholds wrong")

        self.assertEqual(procThreshold2["pending_slots"], 30,
                         "Error: Site thresholds wrong")

        self.assertEqual(mergeThreshold2["task_running_jobs"], 0,
                         "Error: Site thresholds wrong")

        self.assertEqual(mergeThreshold2["task_pending_jobs"], 0,
                         "Error: Site thresholds wrong")

        self.assertEqual(mergeThreshold2["max_slots"], 135,
                         "Error: Site thresholds wrong")

        self.assertEqual(mergeThreshold2["pending_slots"], 100,
                         "Error: Site thresholds wrong")

    def testList(self):
        """
        _testList_

        Test the functions that list thresholds for creating jobs and submitting
        jobs.
        """
        myResourceControl = ResourceControl()
        myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1", "T1_US_FNAL", "LsfPlugin")
        myResourceControl.insertSite("testSite2", 20, 40, "testSE2", "testCE2", "T3_US_FNAL", "LsfPlugin")

        myResourceControl.insertThreshold("testSite1", "Processing", 20, 10)
        myResourceControl.insertThreshold("testSite1", "Merge", 200, 100)
        myResourceControl.insertThreshold("testSite2", "Processing", 50, 25)
        myResourceControl.insertThreshold("testSite2", "Merge", 135, 65)

        self.createJobs()

        createThresholds = myResourceControl.listThresholdsForCreate()
        submitThresholds = myResourceControl.listThresholdsForSubmit()

        self.assertEqual(len(createThresholds.keys()), 2,
                         "Error: Wrong number of sites in create thresholds")

        self.assertEqual(createThresholds["testSite1"]["total_slots"], 10,
                         "Error: Wrong number of slots for site 1")

        self.assertEqual(createThresholds["testSite2"]["total_slots"], 20,
                         "Error: Wrong number of slots for site 2")

        # We should have two running jobs with locations at site one,
        # two running jobs without locations at site two, and one running
        # job without a location at site one and two.
        self.assertEqual(createThresholds["testSite1"]["pending_jobs"], {0: 4},
                         "Error: Wrong number of pending jobs for site 1")

        # We should have one running job with a location at site 2 and
        # another running job without a location.
        self.assertEqual(createThresholds["testSite2"]["pending_jobs"], {0: 2},
                         "Error: Wrong number of pending jobs for site 2")

        # We should also have a phedex_name
        self.assertEqual(createThresholds["testSite1"]["cms_name"], "T1_US_FNAL")
        self.assertEqual(createThresholds["testSite2"]["cms_name"], "T3_US_FNAL")

        mergeThreshold1 = None
        mergeThreshold2 = None
        procThreshold1 = None
        procThreshold2 = None
        self.assertEqual(set(submitThresholds.keys()), set(["testSite1", "testSite2"]))
        for taskType, threshold in submitThresholds["testSite1"]["thresholds"].items():
            if taskType == "Merge":
                mergeThreshold1 = threshold
            elif taskType == "Processing":
                procThreshold1 = threshold
        for taskType, threshold in submitThresholds["testSite2"]["thresholds"].items():
            if taskType == "Merge":
                mergeThreshold2 = threshold
            elif taskType == "Processing":
                procThreshold2 = threshold

        self.assertEqual(submitThresholds["testSite1"]["total_running_jobs"], 0,
                         "Error: Wrong number of running jobs for submit thresholds.")
        self.assertEqual(submitThresholds["testSite2"]["total_running_jobs"], 1,
                         "Error: Wrong number of running jobs for submit thresholds.")
        self.assertEqual(submitThresholds["testSite1"]["total_pending_jobs"], 1,
                         "Error: Wrong number of pending jobs for submit thresholds.")
        self.assertEqual(submitThresholds["testSite2"]["total_pending_jobs"], 0,
                         "Error: Wrong number of pending jobs for submit thresholds.")

        self.assertEqual(mergeThreshold1["task_running_jobs"], 0,
                         "Error: Wrong number of task running jobs for submit thresholds.")
        self.assertEqual(mergeThreshold1["task_pending_jobs"], 0,
                         "Error: Wrong number of task running jobs for submit thresholds.")
        self.assertEqual(procThreshold1["task_running_jobs"], 0,
                         "Error: Wrong number of task running jobs for submit thresholds.")
        self.assertEqual(procThreshold1["task_pending_jobs"], 1,
                         "Error: Wrong number of task running jobs for submit thresholds.")
        self.assertEqual(mergeThreshold2["task_running_jobs"], 0,
                         "Error: Wrong number of task running jobs for submit thresholds.")
        self.assertEqual(mergeThreshold2["task_pending_jobs"], 0,
                         "Error: Wrong number of task running jobs for submit thresholds.")
        self.assertEqual(procThreshold2["task_running_jobs"], 1,
                         "Error: Wrong number of task running jobs for submit thresholds.")
        self.assertEqual(procThreshold2["task_pending_jobs"], 0,
                         "Error: Wrong number of task running jobs for submit thresholds.")

        return

    def testListSiteInfo(self):
        """
        _testListSiteInfo_

        Verify that the listSiteInfo() methods works properly.
        """
        myResourceControl = ResourceControl()
        myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1")
        myResourceControl.insertSite("testSite2", 100, 200, "testSE2", "testCE2")

        siteInfo = myResourceControl.listSiteInfo("testSite1")

        self.assertEqual(siteInfo["site_name"], "testSite1",
                         "Error: Site name is wrong.")

        self.assertEqual(siteInfo["pnn"], ["testSE1"],
                         "Error: SE name is wrong.")

        self.assertEqual(siteInfo["ce_name"], "testCE1",
                         "Error: CE name is wrong.")

        self.assertEqual(siteInfo["pending_slots"], 10,
                         "Error: Pending slots is wrong.")

        self.assertEqual(siteInfo["running_slots"], 20,
                         "Error: Pending slots is wrong.")

        return

    def testUpdateJobSlots(self):
        """
        _testUpdateJobSlots_

        Verify that it is possible to update the number of job slots at a site.
        """
        myResourceControl = ResourceControl()
        myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1")

        siteInfo = myResourceControl.listSiteInfo("testSite1")

        self.assertEqual(siteInfo["pending_slots"], 10, "Error: Pending slots is wrong.")
        self.assertEqual(siteInfo["running_slots"], 20, "Error: Running slots is wrong.")

        myResourceControl.setJobSlotsForSite("testSite1", pendingJobSlots=20)

        siteInfo = myResourceControl.listSiteInfo("testSite1")

        self.assertEqual(siteInfo["pending_slots"], 20, "Error: Pending slots is wrong.")

        myResourceControl.setJobSlotsForSite("testSite1", runningJobSlots=40)

        siteInfo = myResourceControl.listSiteInfo("testSite1")

        self.assertEqual(siteInfo["running_slots"], 40, "Error: Running slots is wrong.")

        myResourceControl.setJobSlotsForSite("testSite1", 5, 10)

        siteInfo = myResourceControl.listSiteInfo("testSite1")

        self.assertEqual(siteInfo["pending_slots"], 5, "Error: Pending slots is wrong.")
        self.assertEqual(siteInfo["running_slots"], 10, "Error: Running slots is wrong.")

        return

    def testThresholdsForSite(self):
        """
        _testThresholdsForSite_

        Check that we can get the thresholds in intelligible form
        for each site
        """

        myResourceControl = ResourceControl()
        myResourceControl.insertSite("testSite1", 20, 40, "testSE1", "testCE1")
        myResourceControl.insertThreshold("testSite1", "Processing", 10, 8)
        myResourceControl.insertThreshold("testSite1", "Merge", 5, 3)

        result = myResourceControl.thresholdBySite(siteName="testSite1")
        procInfo = {}
        mergInfo = {}
        for res in result:
            if res['task_type'] == 'Processing':
                procInfo = res
            elif res['task_type'] == 'Merge':
                mergInfo = res
        self.assertEqual(procInfo.get('pending_slots', None), 20)
        self.assertEqual(procInfo.get('running_slots', None), 40)
        self.assertEqual(procInfo.get('max_slots', None), 10)
        self.assertEqual(procInfo.get('task_pending_slots', None), 8)
        self.assertEqual(mergInfo.get('pending_slots', None), 20)
        self.assertEqual(mergInfo.get('running_slots', None), 40)
        self.assertEqual(mergInfo.get('max_slots', None), 5)
        self.assertEqual(mergInfo.get('task_pending_slots', None), 3)

        return

    def testThresholdPriority(self):
        """
        _testThresholdPriority_

        Test that we get things back in priority order
        """

        myResourceControl = ResourceControl()
        myResourceControl.insertSite("testSite1", 20, 40, "testSE1", "testCE1")
        myResourceControl.insertThreshold("testSite1", "Processing", 10, 8)
        myResourceControl.insertThreshold("testSite1", "Merge", 5, 3)

        # test default task priorities
        result = myResourceControl.listThresholdsForSubmit()
        self.assertEqual(result['testSite1']['thresholds']['Merge']['priority'], 4)
        self.assertEqual(result['testSite1']['thresholds']['Processing']['priority'], 0)

        myResourceControl.changeTaskPriority("Merge", 3)
        myResourceControl.changeTaskPriority("Processing", 1)

        result = myResourceControl.listThresholdsForSubmit()
        self.assertEqual(result['testSite1']['thresholds']['Merge']['priority'], 3)
        self.assertEqual(result['testSite1']['thresholds']['Processing']['priority'], 1)

        myResourceControl.changeTaskPriority("Merge", 1)
        myResourceControl.changeTaskPriority("Processing", 3)

        result = myResourceControl.listThresholdsForSubmit()
        self.assertEqual(result['testSite1']['thresholds']['Merge']['priority'], 1)
        self.assertEqual(result['testSite1']['thresholds']['Processing']['priority'], 3)

        return

    def testChangeSiteState(self):
        """
        _testNewState_

        Check that we can change the state between different values and
        retrieve it through the threshold methods
        """
        self.tempDir = self.testInit.generateWorkDir()
        config = self.createConfig()
        myResourceControl = ResourceControl(config)
        myResourceControl.insertSite("testSite1", 20, 40, "testSE1", "testCE1")
        myResourceControl.insertThreshold("testSite1", "Processing", 10, 5)

        result = myResourceControl.listThresholdsForCreate()
        self.assertEqual(result['testSite1']['state'], 'Normal', 'Error: Wrong site state')

        myResourceControl.changeSiteState("testSite1", "Down")
        result = myResourceControl.listThresholdsForCreate()
        self.assertEqual(result['testSite1']['state'], 'Down', 'Error: Wrong site state')

        # If you set the value to 'Normal' instead of 'Down' this test should FAIL
        # self.assertEqual(result['testSite1']['state'], 'Normal', 'Error: Wrong site state')

    def testAbortedState(self):
        """
        _testAbortedState_

        Check that we can kill jobs when a site is set to aborted
        ### We no longer need this test as we are not killing jobs that are running
        """
        self.tempDir = self.testInit.generateWorkDir()
        config = self.createConfig()
        myResourceControl = ResourceControl(config)
        myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1", "T1_US_FNAL", "MockPlugin")
        myResourceControl.insertSite("testSite2", 20, 40, "testSE2", "testCE2", "T1_IT_CNAF", "MockPlugin")

        myResourceControl.insertThreshold("testSite1", "Processing", 20, 10)
        myResourceControl.insertThreshold("testSite1", "Merge", 200, 100)
        myResourceControl.insertThreshold("testSite2", "Processing", 50, 25)
        myResourceControl.insertThreshold("testSite2", "Merge", 135, 65)

        self.createJobs()

        myResourceControl.changeSiteState("testSite1", "Aborted")

        ## Now check the tempDir for a FWJR for the killed job
        reportPath = os.path.join(self.tempDir, "Report.0.pkl")
        report = Report()
        report.load(reportPath)
        self.assertEqual(report.getExitCode(), 71301)
        return

    def createConfig(self):
        """
        _createConfig_

        Create a config and save it to the temp dir.  Set the WMAGENT_CONFIG
        environment variable so the config gets picked up.
        """
        config = Configuration()
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", os.getcwd())
        config.section_("Agent")
        config.Agent.componentName = "resource_control_t"
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket = os.getenv("DBSOCK")
        config.section_("JobStateMachine")
        config.JobStateMachine.couchurl = os.getenv('COUCHURL')
        config.JobStateMachine.couchDBName = "bossair_t"
        config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t'
        config.JobStateMachine.summaryStatsDBName = 'stat_summary_t'
        config.section_("BossAir")
        config.BossAir.pluginDir = "WMCore.BossAir.Plugins"
        config.BossAir.pluginNames = ["MockPlugin"]
        config.BossAir.section_("MockPlugin")
        config.BossAir.MockPlugin.fakeReport = os.path.join(getTestBase(),
                                                            'WMComponent_t/JobAccountant_t/fwjrs',
                                                            "MergeSuccess.pkl")

        configHandle = open(os.path.join(self.tempDir, "config.py"), "w")
        configHandle.write(str(config))
        configHandle.close()

        os.environ["WMAGENT_CONFIG"] = os.path.join(self.tempDir, "config.py")
        return config

    def testInsertSite(self):
        """
        _testInsertSite_

        Test to see if we can insert a fake test site alone
        with a single option
        """
        self.createConfig()

        resControlPath = os.path.join(getTestBase(), "../../bin/wmagent-resource-control")
        env = os.environ
        env['PYTHONPATH'] = ":".join(sys.path)
        cmdline = [resControlPath, "--add-Test"]
        retval = subprocess.Popen(cmdline,
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.STDOUT,
                                  env=env)
        (_, _) = retval.communicate()
        myResourceControl = ResourceControl()
        result = myResourceControl.listThresholdsForSubmit()
        self.assertEqual(len(result), 1)
        self.assertTrue('CERN' in result)
        for x in result:
            self.assertEqual(len(result[x]['thresholds']), 7)
            self.assertEqual(result[x]['total_pending_slots'], 500)
            self.assertEqual(result[x]['total_running_slots'], 1)
            for taskType, thresh in result[x]['thresholds'].items():
                if taskType == 'Processing':
                    self.assertEqual(thresh['priority'], 0)
                    self.assertEqual(thresh['max_slots'], 1)

        # Verify that sites with more than one SE were added correctly.
        cernInfo = myResourceControl.listSiteInfo("CERN")
        self.assertTrue(len(cernInfo["pnn"]) == 2)
        return
コード例 #21
0
ファイル: scaleTest.py プロジェクト: alexanderrichards/WMCore
class scaleTestFiller:
    """
    _scaleTestFiller_

    Initializes the DB and the DBSUploader
    On __call__() it creates data and uploads it.
    """

    def __init__(self):
        """
        __init__

        Init the DB
        """

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection(destroyAllDatabase = True)
        self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer"],
                                useDefault = False)
        self.configFile = EmulatorSetup.setupWMAgentConfig()

        myThread = threading.currentThread()
        self.bufferFactory = DAOFactory(package = "WMComponent.DBS3Buffer",
                                        logger = myThread.logger,
                                        dbinterface = myThread.dbi)

        locationAction = self.bufferFactory(classname = "DBSBufferFiles.AddLocation")
        locationAction.execute(siteName = "se1.cern.ch")
        locationAction.execute(siteName = "se1.fnal.gov")
        locationAction.execute(siteName = "malpaquet")

        config = self.getConfig()
        self.dbsUploader = DBSUploadPoller(config = config)

        return


    def __call__(self):
        """
        __call__

        Generate some random data
        """

        # Generate somewhere between one and a thousand files
        name = "ThisIsATest_%s" % (makeUUID())
        nFiles = random.randint(10, 2000)
        name = name.replace('-', '_')
        name = '%s-v0' % name
        files = self.getFiles(name = name, nFiles = nFiles)

        print("Inserting %i files for dataset %s" % (nFiles * 2, name))

        try:
            self.dbsUploader.algorithm()
        except:
            self.dbsUploader.close()
            raise

        # Repeat just to make sure
        try:
            self.dbsUploader.algorithm()
        except:
            self.dbsUploader.close()
            raise


        return



    def getConfig(self):
        """
        _getConfig_

        This creates the actual config file used by the component

        """


        config = self.testInit.getConfiguration()
        self.testInit.generateWorkDir(config)

        #First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", os.getcwd())

        config.section_("Agent")
        config.Agent.componentName = 'DBSUpload'
        config.Agent.useHeartbeat  = False

        #Now the CoreDatabase information
        #This should be the dialect, dburl, etc
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket     = os.getenv("DBSOCK")


        config.component_("DBS3Upload")
        config.DBS3Upload.pollInterval     = 10
        config.DBS3Upload.logLevel         = 'DEBUG'
        config.DBS3Upload.DBSBlockMaxFiles = 500
        config.DBS3Upload.DBSBlockMaxTime  = 600
        config.DBS3Upload.DBSBlockMaxSize  = 999999999999
        config.DBS3Upload.dbsUrl           = 'http://cms-xen40.fnal.gov:8787/dbs/prod/global/DBSWriter'
        config.DBS3Upload.namespace        = 'WMComponent.DBS3Buffer.DBS3Upload'
        config.DBS3Upload.componentDir     = os.path.join(os.getcwd(), 'Components')
        config.DBS3Upload.nProcesses       = 1
        config.DBS3Upload.dbsWaitTime      = 1

        return config



    def getFiles(self, name, tier = 'RECO', nFiles = 12, site = "malpaquet", nLumis = 1):
        """
        Create some quick dummy test files

        """

        files = []

        for f in range(nFiles):
            testFile = DBSBufferFile(lfn = '/data/store/random/random/RANDOM/test/0/%s-%s-%i.root' % (name, site, f), size = 1024,
                                     events = 20, checksums = {'cksum': 1})
            testFile.setAlgorithm(appName = name, appVer = "CMSSW_3_1_1",
                                  appFam = "RECO", psetHash = "GIBBERISH",
                                  configContent = "MOREGIBBERISH")
            testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier))
            lumis = []
            for i in range(nLumis):
                lumis.append((f * 100000) + i)
            testFile.addRun(Run( 1, *lumis))
            testFile.setAcquisitionEra(name.split('-')[0])
            testFile.setProcessingVer("0")
            testFile.setGlobalTag("Weird")
            testFile.create()
            testFile.setLocation(site)
            files.append(testFile)

        count = 0
        for f in files:
            count += 1
            testFileChild = DBSBufferFile(lfn = '/data/store/random/random/RANDOM/test/0/%s-%s-%i-child.root' %(name, site, count), size = 1024,
                                          events = 10, checksums = {'cksum': 1})
            testFileChild.setAlgorithm(appName = name, appVer = "CMSSW_3_1_1",
                                       appFam = "RECO", psetHash = "GIBBERISH",
                                       configContent = "MOREGIBBERISH")
            testFileChild.setDatasetPath("/%s/%s_2/RECO" %(name, name))
            testFileChild.addRun(Run( 1, *[45]))
            testFileChild.create()
            testFileChild.setLocation(site)

            testFileChild.addParents([f['lfn']])


        return files
コード例 #22
0
class testJSONRequests(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        tmp = self.testInit.generateWorkDir()
        self.request = Requests.JSONRequests(idict={'req_cache_path' : tmp})

    def roundTrip(self, data):
        encoded = self.request.encode(data)
        #print encoded
        #print encoded.__class__.__name__
        decoded = self.request.decode(encoded)
        #print decoded.__class__.__name__
        self.assertEqual( data, decoded )

    def roundTripLax(self, data):
        encoded = self.request.encode(data)
        decoded = self.request.decode(encoded)
        datakeys = data.keys()

        for k in decoded.keys():
            assert k in datakeys
            datakeys.pop(datakeys.index(k))
        #print 'the following keys were dropped\n\t',datakeys

    def testSet1(self):
        self.roundTrip(set([]))

    def testSet2(self):
        self.roundTrip(set([1, 2, 3, 4, Run(1)]))

    def testSet3(self):
        self.roundTrip(set(['a', 'b', 'c', 'd']))

    def testSet4(self):
        self.roundTrip(set([1, 2, 3, 4, 'a', 'b']))

    def testRun1(self):
        self.roundTrip(Run(1))

    def testRun2(self):
        self.roundTrip(Run(1, 1))

    def testRun3(self):
        self.roundTrip(Run(1, 2, 3))

    def testMask1(self):
        self.roundTrip(Mask())

    def testMask2(self):
        mymask = Mask()
        mymask['FirstEvent'] = 9999
        mymask['LastEvent'] = 999
        self.roundTrip(mymask)

    def testMask3(self):
        mymask = Mask()
        mymask['FirstEvent'] = 9999
        mymask['LastEvent'] = 999
        myjob = DataStructsJob()
        myjob["mask"] = mymask
        self.roundTrip(myjob)

    def testMask4(self):
        self.roundTrip({'LastRun': None, 'FirstRun': None, 'LastEvent': None,
        'FirstEvent': None, 'LastLumi': None, 'FirstLumi': None})

    def testMask5(self):
        mymask = Mask()
        mymask['FirstEvent'] = 9999
        mymask['LastEvent'] = 999
        myjob = DataStructsJob()
        myjob["mask"] = mymask
        self.roundTripLax(myjob)

    def testMask6(self):
        mymask = Mask()
        myjob = DataStructsJob()
        myjob["mask"] = mymask
        self.roundTripLax(myjob)

    def testSpecialCharacterPasswords(self):
        url = 'http://*****:*****@ssw:rd@localhost:6666'
        req = JSONRequests(url)
        self.assertEqual(req['host'], 'http://localhost:6666')
        self.assertEqual(req.additionalHeaders['Authorization'], 'Basic dXNlcm5hbWU6cEBzc3c6cmQ=')
コード例 #23
0
ファイル: Harvest_t.py プロジェクト: vytjan/WMCore
class HarvestTest(unittest.TestCase):
    """
    _HarvestTest_

    Test for EndOfRun job splitter
    """

    def setUp(self):
        """
        _setUp_

        """
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()

        self.testInit.setSchema(customModules=["WMCore.WMBS"])

        self.splitterFactory = SplitterFactory(package="WMCore.JobSplitting")

        myThread = threading.currentThread()
        self.myThread = myThread
        daoFactory = DAOFactory(package="WMCore.WMBS",
                                logger=logging,
                                dbinterface=myThread.dbi)
        self.WMBSFactory = daoFactory

        config = self.getConfig()
        self.changer = ChangeState(config)

        myResourceControl = ResourceControl()
        myResourceControl.insertSite("T1_US_FNAL", 10, 20, "T1_US_FNAL_Disk", "T1_US_FNAL")
        myResourceControl.insertSite("T1_US_FNAL", 10, 20, "T3_US_FNALLPC", "T1_US_FNAL")
        myResourceControl.insertSite("T2_CH_CERN", 10, 20, "T2_CH_CERN", "T2_CH_CERN")

        self.fileset1 = Fileset(name="TestFileset1")
        for fileNum in range(11):
            newFile = File("/some/file/name%d" % fileNum, size=1000, events=100)
            newFile.addRun(Run(1, *[1]))
            newFile.setLocation('T1_US_FNAL_Disk')
            self.fileset1.addFile(newFile)

        self.fileset1.create()

        workflow1 = Workflow(spec="spec.xml", owner="hufnagel", name="TestWorkflow1", task="Test")
        workflow1.create()

        self.subscription1 = Subscription(fileset=self.fileset1,
                                          workflow=workflow1,
                                          split_algo="Harvest",
                                          type="Harvesting")

        self.subscription1.create()
        self.configFile = EmulatorSetup.setupWMAgentConfig()

        return

    def tearDown(self):
        """
        _tearDown_

        """
        self.testInit.clearDatabase()
        EmulatorSetup.deleteConfig(self.configFile)

        return

    def getConfig(self):
        """
        _getConfig_

        """
        config = self.testInit.getConfiguration()
        self.testInit.generateWorkDir(config)

        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket = os.getenv("DBSOCK")

        # JobStateMachine
        config.component_('JobStateMachine')
        config.JobStateMachine.couchurl = os.getenv('COUCHURL', None)
        config.JobStateMachine.couchDBName = 'wmagent_jobdump'

        return config

    def finishJobs(self, jobGroups, subscription=None):
        """
        _finishJobs_

        """
        if not subscription:
            subscription = self.subscription1
        for f in subscription.acquiredFiles():
            subscription.completeFiles(f)

        for jobGroup in jobGroups:
            self.changer.propagate(jobGroup.jobs, 'executing', 'created')
            self.changer.propagate(jobGroup.jobs, 'complete', 'executing')
            self.changer.propagate(jobGroup.jobs, 'success', 'complete')
            self.changer.propagate(jobGroup.jobs, 'cleanout', 'success')

        return

    def testHarvestEndOfRunTrigger(self):
        """
        _testDQMHarvestEndOfRunTrigger_

        Make sure that the basic splitting algo works, which is only, ExpressMerge is ALL done, fire a job against that fileset

        """
        self.assertEqual(self.fileset1.open, True, "Fileset is closed. Shouldn't")

        jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)

        jobGroups = jobFactory()

        self.assertEqual(len(jobGroups), 0,
                         "We got 1 or more jobGroups with an open fileset and no periodic configuration")

        self.fileset1.markOpen(False)
        self.assertEqual(self.fileset1.open, False, "Fileset is opened, why?")

        # We should also check if there are aqcuired files, if there are, there are jobs,
        # we don't want to fire another jobs while previous are running (output is integrating whatever  input)
        # TODO : The above one we can do when all is done. Not priority

        jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
        jobGroups = jobFactory()

        self.assertEqual(len(jobGroups), 1,
                         "Harvest jobsplitter didn't create a single jobGroup after the fileset was closed")

        return

    def testPeriodicTrigger(self):
        """
        _testPeriodicTrigger_

        """
        self.assertEqual(self.fileset1.open, True, "Fileset is not open, not testing periodic here")
        # Test timeout (5s for this first test)
        # there should be no acquired files, if there are, shouldn't be a job
        # self.subscription1.acquireFiles(self.subscription1.availableFiles().pop())

        jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
        jobGroups = jobFactory(periodic_harvest_interval=3)

        self.assertEqual(len(jobGroups), 1, "Didn't created the first periodic job when there were acquired files")

        # For the whole thing to work, faking the first job finishing, and putting the files as complete
        self.finishJobs(jobGroups)

        # Adding more of files, so we have new stuff to process
        for fileNum in range(12, 24):
            newFile = File("/some/file/name%d" % fileNum, size=1000, events=100)
            newFile.addRun(Run(1, *[1]))
            newFile.setLocation('T1_US_FNAL_Disk')
            self.fileset1.addFile(newFile)
        self.fileset1.commit()

        # Testing that it doesn't create a job unless the delay is past
        jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
        jobGroups = jobFactory(periodic_harvest_interval=2)

        self.assertEqual(len(jobGroups), 0,
                         "Created one or more job, when there were non-acquired file and the period is not passed by")

        time.sleep(2)

        jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
        jobGroups = jobFactory(periodic_harvest_interval=2)

        self.assertEqual(len(jobGroups), 1,
                         "Didn't created one or more job, and there weren't and the period is passed by")

        # Finishing out previous jobs
        self.finishJobs(jobGroups)

        # Adding more of files, so we have new stuff to process
        for fileNum in range(26, 36):
            newFile = File("/some/file/name%d" % fileNum, size=1000, events=100)
            newFile.addRun(Run(1, *[1]))
            newFile.setLocation('T1_US_FNAL_Disk')
            self.fileset1.addFile(newFile)
        self.fileset1.commit()

        # Trying to create another job just afterwards, it shouldn't, because it should respect the configured delay
        jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
        jobGroups = jobFactory(periodic_harvest_interval=2)

        self.assertEqual(len(jobGroups), 0, "Created one or more job, there are new files, but the delay is not past")

        time.sleep(2)

        jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
        jobGroups = jobFactory(periodic_harvest_interval=2)

        self.assertEqual(len(jobGroups), 1, "Didn't created one or more job, there are new files and the delay is past")

        # Last check is whether the job gets all the files or not

        numFilesJob = jobGroups[0].jobs[0].getFiles()
        numFilesFileset = self.fileset1.getFiles()
        self.assertEqual(numFilesJob, numFilesFileset, "Job didn't got all the files")

        # Finishing out previous jobs
        self.finishJobs(jobGroups)

        # Adding files for the first location
        for fileNum in range(38, 48):
            newFile = File("/some/file/name%d" % fileNum, size=1000, events=100)
            newFile.addRun(Run(1, *[1]))
            newFile.setLocation('T1_US_FNAL_Disk')
            self.fileset1.addFile(newFile)
        self.fileset1.commit()
        # Then another location
        for fileNum in range(50, 56):
            newFile = File("/some/file/name%d" % fileNum, size=1000, events=100)
            newFile.addRun(Run(1, *[1]))
            newFile.setLocation('T2_CH_CERN')
            self.fileset1.addFile(newFile)
        self.fileset1.commit()

        # We should have jobs in both locations
        time.sleep(2)

        jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
        jobGroups = jobFactory(periodic_harvest_interval=2)

        self.assertEqual(len(jobGroups[0].getJobs()), 2, "We didn't get 2 jobs for 2 locations")

        firstJobLocation = jobGroups[0].getJobs()[0].getFileLocations()[0]
        secondJobLocation = jobGroups[0].getJobs()[1].getFileLocations()[0]

        self.assertEqual(firstJobLocation, 'T2_CH_CERN')
        self.assertEqual(secondJobLocation, 'T1_US_FNAL')

        self.finishJobs(jobGroups)

        for fileNum in range(60, 65):
            newFile = File("/some/file/name%d" % fileNum, size=1000, events=100)
            newFile.addRun(Run(2, *[2]))
            newFile.setLocation('T2_CH_CERN')
            self.fileset1.addFile(newFile)
        self.fileset1.commit()

        for fileNum in range(70, 75):
            newFile = File("/some/file/name%d" % fileNum, size=1000, events=100)
            newFile.addRun(Run(3, *[3]))
            newFile.setLocation('T2_CH_CERN')
            self.fileset1.addFile(newFile)
        self.fileset1.commit()

        time.sleep(2)

        jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
        jobGroups = jobFactory(periodic_harvest_interval=2)

        # This is one of the most "complicated" tests so worth to comment, 4 jobs should be created
        # 1 - all previous files from SomeSE and run = 1 (a lot, like ~45)
        # 2 - Few files from SomeSE3, Run = 1
        # 3 - Few files from SomeSE3, Run = 2
        # 4 - Few files from SomeSE3, Run = 3
        self.assertEqual(len(jobGroups[0].getJobs()), 4, "We didn't get 4 jobs for adding 2 different runs to SomeSE3")

        return

    def testMultipleRunHarvesting(self):
        """
        _testMultipleRunHarvesting_

        Add some files with multiple runs in each, make sure the jobs
        are created by location and run. Verify each job mask afterwards.
        Note that in this test run are splitted between sites,
        in real life that MUST NOT happen we still don't support that.
        """
        multipleFilesFileset = Fileset(name="TestFileset")

        newFile = File("/some/file/test1", size=1000, events=100)
        newFile.addRun(Run(1, *[1, 3, 4, 5, 6, 7]))
        newFile.addRun(Run(2, *[1, 2, 4, 5, 6, 7]))
        newFile.setLocation('T1_US_FNAL_Disk')
        multipleFilesFileset.addFile(newFile)
        newFile = File("/some/file/test2", size=1000, events=100)
        newFile.addRun(Run(1, *[2, 8]))
        newFile.addRun(Run(2, *[3, 8]))
        newFile.setLocation('T2_CH_CERN')
        multipleFilesFileset.addFile(newFile)
        multipleFilesFileset.create()

        harvestingWorkflow = Workflow(spec="spec.xml",
                                      owner="hufnagel",
                                      name="TestWorkflow",
                                      task="Test")
        harvestingWorkflow.create()

        harvestSub = Subscription(fileset=multipleFilesFileset,
                                  workflow=harvestingWorkflow,
                                  split_algo="Harvest",
                                  type="Harvesting")
        harvestSub.create()

        jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub)
        jobGroups = jobFactory(periodic_harvest_interval=2)
        self.assertEqual(len(jobGroups), 1, "A single job group was not created")
        self.assertEqual(len(jobGroups[0].getJobs()), 4,
                         "Four jobs were not created")

        for job in jobGroups[0].getJobs():
            runs = job['mask'].getRunAndLumis()
            self.assertEqual(len(runs), 1, "Job has more than one run configured")
            ll = LumiList(compactList={1: [[1, 1], [3, 7], [2, 2], [8, 8]],
                                       2: [[1, 2], [4, 7], [3, 3], [8, 8]]})
            run = runs.keys()[0]
            for lumiPair in runs[run]:
                for lumi in range(lumiPair[0], lumiPair[1] + 1):
                    self.assertTrue((str(run), lumi) in ll, "All of %s not in %s" % (lumiPair, ll))

        self.finishJobs(jobGroups, harvestSub)

        newFile = File("/some/file/test3", size=1000, events=100)
        newFile.addRun(Run(1, *range(9, 15)))
        newFile.setLocation('T2_CH_CERN')
        multipleFilesFileset.addFile(newFile)
        multipleFilesFileset.commit()

        time.sleep(2)

        jobGroups = jobFactory(periodic_harvest_interval=2)
        self.assertEqual(len(jobGroups), 1, "A single job group was not created")
        self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created")

        for job in jobGroups[0].getJobs():
            runs = job['mask'].getRunAndLumis()
            self.assertEqual(len(runs), 1, "Job has more than one run configured")
            ll = LumiList(compactList={1: [[1, 1], [3, 7], [2, 2], [8, 8], [9, 14]],
                                       2: [[1, 2], [4, 7], [3, 3], [8, 8]]})
            run = runs.keys()[0]
            for lumiPair in runs[run]:
                for lumi in range(lumiPair[0], lumiPair[1] + 1):
                    self.assertTrue((run, lumi) in ll, "All of %s not in %s" % (lumiPair, ll))

        harvestingWorkflowSib = Workflow(spec="spec.xml",
                                         owner="hufnagel",
                                         name="TestWorkflowSib",
                                         task="TestSib")
        harvestingWorkflowSib.create()

        harvestSubSib = Subscription(fileset=multipleFilesFileset,
                                     workflow=harvestingWorkflowSib,
                                     split_algo="Harvest",
                                     type="Harvesting")
        harvestSubSib.create()

        jobFactorySib = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSubSib)

        multipleFilesFileset.markOpen(False)

        jobGroups = jobFactorySib(periodic_harvest_sibling=True)
        self.assertEqual(len(jobGroups), 0, "A single job group was created")

        self.finishJobs(jobGroups, harvestSub)

        jobGroups = jobFactorySib(periodic_harvest_sibling=True)
        self.assertEqual(len(jobGroups), 1, "A single job group was not created")
        self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created")

        for job in jobGroups[0].getJobs():
            runs = job['mask'].getRunAndLumis()
            self.assertEqual(len(runs), 1, "Job has more than one run configured")
            ll = LumiList(compactList={1: [[1, 1], [3, 7], [2, 2], [8, 8], [9, 14]],
                                       2: [[1, 2], [4, 7], [3, 3], [8, 8]]})
            run = runs.keys()[0]
            for lumiPair in runs[run]:
                for lumi in range(lumiPair[0], lumiPair[1] + 1):
                    self.assertTrue((run, lumi) in ll, "All of %s not in %s" % (lumiPair, ll))

    def testMultiRunHarvesting(self):
        """
        _testMultiRunHarvesting_

        Provided a fileset with a couple of files and different runs, create a
        single job for all the runs at a specific location, which also adds a
        baggage to the job (True) which is later on looked up by SetupCMSSWPSet.
        """
        multipleFilesFileset = createCommonFileset()
        self.assertEqual(multipleFilesFileset.open, True)

        harvestingWorkflow = Workflow(spec="spec.xml",
                                      owner="amaltaro",
                                      name="TestWorkflow",
                                      task="Test")
        harvestingWorkflow.create()

        harvestSub = Subscription(fileset=multipleFilesFileset,
                                  workflow=harvestingWorkflow,
                                  split_algo="Harvest",
                                  type="Harvesting")
        harvestSub.create()

        multipleFilesFileset.markOpen(False)
        self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed")

        jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub)
        jobGroups = jobFactory(dqmHarvestUnit="multiRun")
        self.assertEqual(len(jobGroups), 1)

        for jobGroup in jobGroups:
            self.assertEqual(len(jobGroup.jobs), 1)
            for job in jobGroup.jobs:
                baggage = job.getBaggage()
                self.assertTrue(getattr(baggage, "multiRun", False), "It's supposed to be a multiRun job")
                self.assertEqual(getattr(baggage, "runLimits", ""), "-1-6")

    def testByRunHarvesting(self):
        """
        _testByRunHarvesting_
        Provided a fileset with a couple of files and 4 different runs, create
        one single job per run and location.
        The multiRun baggage should be false in this case.
        """
        multipleFilesFileset = createCommonFileset()
        self.assertEqual(multipleFilesFileset.open, True, "Fileset should be open!")

        harvestingWorkflow = Workflow(spec="spec.xml",
                                      owner="amaltaro",
                                      name="TestWorkflow",
                                      task="Test")
        harvestingWorkflow.create()

        harvestSub = Subscription(fileset=multipleFilesFileset,
                                  workflow=harvestingWorkflow,
                                  split_algo="Harvest",
                                  type="Harvesting")
        harvestSub.create()

        multipleFilesFileset.markOpen(False)
        self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed")

        jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub)
        jobGroups = jobFactory()
        self.assertEqual(len(jobGroups), 1, "Should have created 1 job group")

        for jobGroup in jobGroups:
            self.assertEqual(len(jobGroup.jobs), 6, "Should have created 6 jobs")
            for job in jobGroup.jobs:
                baggage = job.getBaggage()
                self.assertFalse(getattr(baggage, "multiRun", False), "It's supposed to be a byRun job")

    def testByRunAndRunWhitelist(self):
        """
        _testByRunAndRunWhitelist_

        Create harvesting jobs by run for the runs provided in the RunWhitelist
        """
        multipleFilesFileset = createCommonFileset()
        self.assertEqual(multipleFilesFileset.open, True)

        harvestingWorkflow = Workflow(spec="spec.xml", owner="amaltaro",
                                      name="TestWorkflow", task="Test")
        harvestingWorkflow.create()

        harvestSub = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflow,
                                  split_algo="Harvest", type="Harvesting")
        harvestSub.create()

        multipleFilesFileset.markOpen(False)
        self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed")

        jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub)
        jobGroups = jobFactory(runWhitelist=[1, 3])
        self.assertEqual(len(jobGroups), 1, "One jobgroup per location")

        for jobGroup in jobGroups:
            self.assertEqual(len(jobGroup.jobs), 2)

    def testByRunAndRunBlacklist(self):
        """
        _testByRunAndRunWhitelist_

        Create harvesting jobs by run for the runs provided in the RunWhitelist
        """
        multipleFilesFileset = createCommonFileset()
        self.assertEqual(multipleFilesFileset.open, True)

        harvestingWorkflow = Workflow(spec="spec.xml", owner="amaltaro",
                                      name="TestWorkflow", task="Test")
        harvestingWorkflow.create()

        harvestSub = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflow,
                                  split_algo="Harvest", type="Harvesting")
        harvestSub.create()

        multipleFilesFileset.markOpen(False)
        self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed")

        jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub)
        jobGroups = jobFactory(runWhitelist=[1, 2, 3, 4, 5], runBlacklist=[1, 3])
        self.assertEqual(len(jobGroups), 1, "One jobgroup per location")

        for jobGroup in jobGroups:
            self.assertEqual(len(jobGroup.jobs), 3)
コード例 #24
0
class SetupCMSSWPsetTest(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testDir = self.testInit.generateWorkDir()
        sys.path.insert(
            0,
            os.path.join(WMCore.WMBase.getTestBase(),
                         "WMCore_t/WMRuntime_t/Scripts_t"))

    def tearDown(self):
        sys.path.remove(
            os.path.join(WMCore.WMBase.getTestBase(),
                         "WMCore_t/WMRuntime_t/Scripts_t"))
        if 'WMTaskSpace' in sys.modules:
            del sys.modules["WMTaskSpace"]
        self.testInit.delWorkDir()
        os.unsetenv("WMAGENT_SITE_CONFIG_OVERRIDE")

    def createTestStep(self):
        """
        _createTestStep_

        Create a test step that can be passed to the setup script.

        """
        newStep = WMStep("cmsRun1")
        stepTemplate = StepFactory.getStepTemplate("CMSSW")
        stepTemplate.install(newStep)
        newStepHelper = stepTemplate.helper(newStep)

        newStepHelper.setStepType("CMSSW")
        newStepHelper.setGlobalTag("SomeGlobalTag")
        newStepHelper.data.application.section_("setup")
        newStepHelper.cmsswSetup("CMSSW_11_0_2",
                                 scramArch=['slc7_amd64_gcc820'])

        return newStepHelper

    def createTestJob(self):
        """
        _createTestJob_

        Create a test job that has parents for each input file.

        """
        newJob = Job(name="TestJob")
        newJob.addFile(
            File(lfn="/some/file/one",
                 parents=set([File(lfn="/some/parent/one")])))
        newJob.addFile(
            File(lfn="/some/file/two",
                 parents=set([File(lfn="/some/parent/two")])))
        return newJob

    def loadProcessFromPSet(self, psetPath=None):
        """
        _loadProcessFromPSet_

        This requires changing the working directory,
        do so in a safe manner to encapsulate the change to this method only
        """
        from WMCore.WMRuntime.Scripts.SetupCMSSWPset import Unpickler
        currentPath = os.getcwd()
        loadedProcess = None

        if psetPath is None:
            psetPath = self.testDir
        with open(os.path.join(psetPath, "PSet.pkl")) as f:
            pset = Unpickler(f).load()

        os.chdir(currentPath)

        return pset

    def testPSetFixup(self):
        """
        _testPSetFixup_

        Verify that all necessary parameters are set in the PSet.

        """
        from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset

        setupScript = SetupCMSSWPset()
        setupScript.step = self.createTestStep()
        setupScript.stepSpace = ConfigSection(name="stepSpace")
        setupScript.stepSpace.location = self.testDir
        shutil.copyfile(
            os.path.join(os.path.dirname(__file__), "WMTaskSpace", "cmsRun1",
                         "PSet.py"),
            os.path.join(setupScript.stepSpace.location, "PSet.py"))
        setupScript.job = self.createTestJob()
        setupScript()

        fixedPSet = self.loadProcessFromPSet(setupScript.stepSpace.location)

        self.assertTrue(hasattr(fixedPSet.source, 'fileNames'))
        self.assertTrue(hasattr(fixedPSet.source, 'secondaryFileNames'))
        self.assertEqual(fixedPSet.maxEvents.input._value, -1,
                         "Error: Wrong maxEvents.")

    def testEventsPerLumi(self):
        """
        _testEventsPerLumi_
        Verify that you can put in events per lumi in the process.

        """
        from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset

        setupScript = SetupCMSSWPset()
        setupScript.step = self.createTestStep()
        setupScript.step.setEventsPerLumi(500)
        setupScript.stepSpace = ConfigSection(name="stepSpace")
        setupScript.stepSpace.location = self.testDir
        shutil.copyfile(
            os.path.join(os.path.dirname(__file__), "WMTaskSpace", "cmsRun1",
                         "PSet.py"),
            os.path.join(setupScript.stepSpace.location, "PSet.py"))
        setupScript.job = self.createTestJob()
        setupScript()

        fixedPSet = self.loadProcessFromPSet(setupScript.stepSpace.location)

        self.assertTrue(hasattr(fixedPSet.source, 'fileNames'))
        self.assertTrue(hasattr(fixedPSet.source, 'secondaryFileNames'))
        self.assertEqual(fixedPSet.source.numberEventsInLuminosityBlock._value,
                         500,
                         "Error: Wrong number of events per luminosity block")
        self.assertEqual(fixedPSet.maxEvents.input._value, -1,
                         "Error: Wrong maxEvents.")

    def testChainedProcesing(self):
        """
        test for chained CMSSW processing - check the overriden TFC, its values
        and that input files is / are set correctly.

        """
        from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset

        setupScript = SetupCMSSWPset()
        setupScript.step = self.createTestStep()
        setupScript.stepSpace = ConfigSection(name="stepSpace")
        setupScript.stepSpace.location = self.testDir
        shutil.copyfile(
            os.path.join(os.path.dirname(__file__), "WMTaskSpace", "cmsRun1",
                         "PSet.py"),
            os.path.join(setupScript.stepSpace.location, "PSet.py"))
        setupScript.job = self.createTestJob()
        setupScript.step.setupChainedProcessing("my_first_step",
                                                "my_input_module")
        setupScript()
        fixedPSet = self.loadProcessFromPSet(setupScript.stepSpace.location)

        # test if the overriden TFC is right
        print("DEBUG override: {0}".format(
            setupScript.step.data.application.overrideCatalog))
        self.assertTrue(
            hasattr(setupScript.step.data.application, "overrideCatalog"),
            "Error: overriden TFC was not set")
        tfc = loadTFC(setupScript.step.data.application.overrideCatalog)
        inputFile = "../my_first_step/my_input_module.root"
        self.assertEqual(tfc.matchPFN("direct", inputFile), inputFile)
        self.assertEqual(tfc.matchLFN("direct", inputFile), inputFile)
        self.assertTrue(hasattr(fixedPSet.source, 'fileNames'))

    def testPileupSetup(self):
        """
        Test the pileup setting.

        reference (setupScript.process instance):
        in test/python/WMCore_t/WMRuntime_t/Scripts_t/WMTaskSpace/cmsRun1/PSet.py

        """
        try:
            from dbs.apis.dbsClient import DbsApi
        except ImportError as ex:
            raise unittest.SkipTest

        # this is modified and shortened version of
        # WMCore/test/python/WMCore_t/Misc_t/site-local-config.xml
        # since the dataset name in question (below) is only present at
        # storm-fe-cms.cr.cnaf.infn.it, need to make the test think it's its local SE
        siteLocalConfigContent = \
        """
<site-local-config>
    <site name="-SOME-SITE-NAME-">
        <event-data>
            <catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/>
        </event-data>
        <local-stage-out>
            <!-- original cmssrm.fnal.gov -->
            <phedex-node value="T2_CH_CERN"/>
            <command value="test-copy"/>
            <catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/>
        </local-stage-out>
        <calib-data>
            <frontier-connect>
                <load balance="proxies"/>
                <proxy url="http://cmsfrontier1.fnal.gov:3128"/>
                <proxy url="http://cmsfrontier2.fnal.gov:3128"/>
            </frontier-connect>
        </calib-data>
    </site>
</site-local-config>
"""
        siteLocalConfig = os.path.join(self.testDir,
                                       "test-site-local-config.xml")
        f = open(siteLocalConfig, 'w')
        f.write(siteLocalConfigContent)
        f.close()

        from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
        setupScript = SetupCMSSWPset()
        setupScript.step = self.createTestStep()
        setupScript.stepSpace = ConfigSection(name="stepSpace")
        setupScript.stepSpace.location = os.path.join(self.testDir, "cmsRun1")
        setupScript.job = self.createTestJob()
        # define pileup configuration
        # despite of the implementation considering whichever type of pileup,
        # only "data" and "mc" types are eventually considered and lead to any
        # modifications of job input files
        pileupConfig = {
            "data": [
                "/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
            ],
            "mc": [
                "/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
            ]
        }
        dbsUrl = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader"
        setupScript.step.setupPileup(pileupConfig, dbsUrl)
        # SetupCMSSWPset pileup handling will be consulting SiteLocalConfig
        # to determine StorageElement (SE) name the job is running on
        # SiteLocalConfig loads the site-local-config.xml file from env.
        # variable defined location ; if the variable is not defined already, set it
        # obviously, if "WMAGENT_SITE_CONFIG_OVERRIDE" is already set here, the above
        # thick with SE name is not effective
        if not os.getenv("WMAGENT_SITE_CONFIG_OVERRIDE", None):
            os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = siteLocalConfig
        # find out local site name from the testing local site config,
        # will be needed later
        siteConfig = loadSiteLocalConfig()
        seLocalName = siteConfig.localStageOut["phedex-node"]
        print("Running on site '%s', local SE name: '%s'" %
              (siteConfig.siteName, seLocalName))

        # before calling the script, SetupCMSSWPset will try to load JSON
        # pileup configuration file, need to create it in self.testDir
        fetcher = PileupFetcher()
        fetcher.setWorkingDirectory(self.testDir)
        fetcher.createPileupConfigFile(setupScript.step)

        setupScript()

        # now test all modifications carried out in SetupCMSSWPset.__call__
        # which will also test that CMSSWStepHelper.setupPileup run correctly
        mixModules, dataMixModules = setupScript._getPileupMixingModules()

        # load in the pileup configuration in the form of dict which
        # PileupFetcher previously saved in a JSON file
        pileupDict = setupScript._getPileupConfigFromJson()

        # get the sub dict for particular pileup type
        # for pileupDict structure description - see PileupFetcher._queryDbsAndGetPileupConfig
        for pileupType, modules in zip(("data", "mc"),
                                       (dataMixModules, mixModules)):
            # getting KeyError here - above pileupConfig is not correct - need
            # to have these two types of pile type
            d = pileupDict[pileupType]
            self._mixingModulesInputFilesTest(modules, d, seLocalName)

    def _mixingModulesInputFilesTest(self, modules, pileupSubDict,
                                     seLocalName):
        """
        pileupSubDic - contains only dictionary for particular pile up type

        """
        # consider only locally available files
        filesInConfigDict = []
        for v in viewvalues(pileupSubDict):
            if seLocalName in v["phedexNodeNames"]:
                filesInConfigDict.extend(v["FileList"])

        for m in modules:
            inputTypeAttrib = getattr(m, "input", None) or getattr(
                m, "secsource", None)
            fileNames = inputTypeAttrib.fileNames.value
            if fileNames == None:
                fileNames = []
            m = (
                "Pileup configuration file list '%s' and mixing modules input "
                "filelist '%s' are not identical." %
                (filesInConfigDict, fileNames))
            self.assertEqual(sorted(filesInConfigDict), sorted(fileNames), m)
コード例 #25
0
ファイル: StoreResults_t.py プロジェクト: brij01/WMCore
class StoreResultsTest(unittest.TestCase):
    def setUp(self):
        """
        _setUp_

        Initialize the database.
        """
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules=["WMCore.WMBS"],
                                useDefault=False)
        self.testDir = self.testInit.generateWorkDir()
        return

    def tearDown(self):
        """
        _tearDown_

        Clear out the database.
        """
        self.testInit.clearDatabase()
        self.testInit.delWorkDir()
        return

    def testStoreResults(self):
        """
        _testStoreResults_

        Create a StoreResults workflow and verify it installs into WMBS
        correctly.
        """
        arguments = StoreResultsWorkloadFactory.getTestArguments()
        arguments.update({'CmsPath': "/uscmst1/prod/sw/cms"})

        factory = StoreResultsWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction(
            "TestWorkload", arguments)

        testWMBSHelper = WMBSHelper(testWorkload,
                                    "StoreResults",
                                    "SomeBlock",
                                    cachepath=self.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(
            testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        testWorkflow = Workflow(name="TestWorkload",
                                task="/TestWorkload/StoreResults")
        testWorkflow.load()

        self.assertEqual(len(testWorkflow.outputMap.keys()), 2,
                         "Error: Wrong number of WF outputs.")

        goldenOutputMods = ["Merged"]
        for goldenOutputMod in goldenOutputMods:
            mergedOutput = testWorkflow.outputMap[goldenOutputMod][0][
                "merged_output_fileset"]
            unmergedOutput = testWorkflow.outputMap[goldenOutputMod][0][
                "output_fileset"]

            mergedOutput.loadData()
            unmergedOutput.loadData()

            self.assertEqual(
                mergedOutput.name,
                "/TestWorkload/StoreResults/merged-%s" % goldenOutputMod,
                "Error: Merged output fileset is wrong: %s" %
                mergedOutput.name)
            self.assertEqual(
                unmergedOutput.name,
                "/TestWorkload/StoreResults/merged-%s" % goldenOutputMod,
                "Error: Unmerged output fileset is wrong: %s." %
                unmergedOutput.name)

        logArchOutput = testWorkflow.outputMap["logArchive"][0][
            "merged_output_fileset"]
        unmergedLogArchOutput = testWorkflow.outputMap["logArchive"][0][
            "output_fileset"]
        logArchOutput.loadData()
        unmergedLogArchOutput.loadData()

        self.assertEqual(logArchOutput.name,
                         "/TestWorkload/StoreResults/merged-logArchive",
                         "Error: LogArchive output fileset is wrong.")
        self.assertEqual(unmergedLogArchOutput.name,
                         "/TestWorkload/StoreResults/merged-logArchive",
                         "Error: LogArchive output fileset is wrong.")

        topLevelFileset = Fileset(name="TestWorkload-StoreResults-SomeBlock")
        topLevelFileset.loadData()

        procSubscription = Subscription(fileset=topLevelFileset,
                                        workflow=testWorkflow)
        procSubscription.loadData()

        self.assertEqual(procSubscription["type"], "Merge",
                         "Error: Wrong subscription type.")
        self.assertEqual(procSubscription["split_algo"],
                         "ParentlessMergeBySize", "Error: Wrong split algo.")

        return
コード例 #26
0
class ReportIntegrationTest(unittest.TestCase):
    """
    _ReportIntegrationTest_

    """
    def setUp(self):
        """
        _setUp_

        Setup the database and WMBS for the test.
        """
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer",
                                                 "WMCore.WMBS"],
                                useDefault = False)

        myThread = threading.currentThread()
        self.daofactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)
        self.dbsfactory = DAOFactory(package = "WMComponent.DBS3Buffer",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)
        locationAction = self.daofactory(classname = "Locations.New")
        locationAction.execute(siteName = "site1", seName = "cmssrm.fnal.gov")

        inputFile = File(lfn = "/path/to/some/lfn", size = 10, events = 10,
                         locations = "cmssrm.fnal.gov")
        inputFile.create()

        inputFileset = Fileset(name = "InputFileset")
        inputFileset.create()
        inputFileset.addFile(inputFile)
        inputFileset.commit()

        unmergedFileset = Fileset(name = "UnmergedFileset")
        unmergedFileset.create()

        mergedFileset = Fileset(name = "MergedFileset")
        mergedFileset.create()

        procWorkflow = Workflow(spec = "wf001.xml", owner = "Steve",
                                name = "TestWF", task = "/TestWF/None")
        procWorkflow.create()
        procWorkflow.addOutput("outputRECORECO", unmergedFileset)

        mergeWorkflow = Workflow(spec = "wf002.xml", owner = "Steve",
                                 name = "MergeWF", task = "/MergeWF/None")
        mergeWorkflow.create()
        mergeWorkflow.addOutput("Merged", mergedFileset)

        insertWorkflow = self.dbsfactory(classname = "InsertWorkflow")
        insertWorkflow.execute("TestWF", "/TestWF/None", 0, 0, 0, 0)
        insertWorkflow.execute("MergeWF", "/MergeWF/None", 0, 0, 0, 0)

        self.procSubscription = Subscription(fileset = inputFileset,
                                             workflow = procWorkflow,
                                             split_algo = "FileBased",
                                             type = "Processing")
        self.procSubscription.create()
        self.procSubscription.acquireFiles()

        self.mergeSubscription = Subscription(fileset = unmergedFileset,
                                             workflow = mergeWorkflow,
                                             split_algo = "WMBSMergeBySize",
                                             type = "Merge")
        self.mergeSubscription.create()

        self.procJobGroup = JobGroup(subscription = self.procSubscription)
        self.procJobGroup.create()
        self.mergeJobGroup = JobGroup(subscription = self.mergeSubscription)
        self.mergeJobGroup.create()

        self.testJob = Job(name = "testJob", files = [inputFile])
        self.testJob.create(group = self.procJobGroup)
        self.testJob["state"] = "complete"

        myThread = threading.currentThread()
        self.daofactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)
        self.stateChangeAction = self.daofactory(classname = "Jobs.ChangeState")
        self.setFWJRAction = self.daofactory(classname = "Jobs.SetFWJRPath")
        self.getJobTypeAction = self.daofactory(classname = "Jobs.GetType")
        locationAction = self.daofactory(classname = "Locations.New")
        locationAction.execute(siteName = "cmssrm.fnal.gov")

        self.stateChangeAction.execute(jobs = [self.testJob])

        self.tempDir = tempfile.mkdtemp()
        return

    def tearDown(self):
        """
        _tearDown_

        Clear out the database and the pickled report file.
        """
        self.testInit.clearDatabase()

        try:
            os.remove(os.path.join(self.tempDir, "ProcReport.pkl"))
            os.remove(os.path.join(self.tempDir, "MergeReport.pkl"))
        except Exception as ex:
            pass

        try:
            os.rmdir(self.tempDir)
        except Exception as ex:
            pass

        return

    def createConfig(self, workerThreads):
        """
        _createConfig_

        Create a config for the JobAccountant with the given number of worker
        threads.  This config needs to include information for connecting to the
        database as the component will create it's own database connections.
        These parameters are still pulled from the environment.
        """
        config = self.testInit.getConfiguration()
        self.testInit.generateWorkDir(config)

        config.section_("JobStateMachine")
        config.JobStateMachine.couchurl = os.getenv("COUCHURL")
        config.JobStateMachine.couchDBName = "report_integration_t"
        config.JobStateMachine.jobSummaryDBName = "report_integration_wmagent_summary_t"

        config.component_("JobAccountant")
        config.JobAccountant.pollInterval = 60
        config.JobAccountant.workerThreads = workerThreads
        config.JobAccountant.componentDir = os.getcwd()
        config.JobAccountant.logLevel = 'SQLDEBUG'

        config.component_("TaskArchiver")
        config.TaskArchiver.localWMStatsURL = "%s/%s" % (config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName)
        return config

    def verifyJobSuccess(self, jobID):
        """
        _verifyJobSuccess_

        Verify that the metadata for a successful job is correct.  This will
        check the outcome, retry count and state.
        """
        testJob = Job(id = jobID)
        testJob.load()

        assert testJob["state"] == "success", \
               "Error: test job in wrong state: %s" % testJob["state"]
        assert testJob["retry_count"] == 0, \
               "Error: test job has wrong retry count: %s" % testJob["retry_count"]
        assert testJob["outcome"] == "success", \
               "Error: test job has wrong outcome: %s" % testJob["outcome"]

        return

    def verifyFileMetaData(self, jobID, fwkJobReportFiles):
        """
        _verifyFileMetaData_

        Verify that all the files that were output by a job made it into WMBS
        correctly.  Compare the contents of WMBS to the files in the frameworks
        job report.

        Note that fwkJobReportFiles is a list of DataStructs File objects.
        """
        testJob = Job(id = jobID)
        testJob.loadData()

        inputLFNs = []
        for inputFile in testJob["input_files"]:
            inputLFNs.append(inputFile["lfn"])

        for fwkJobReportFile in fwkJobReportFiles:
            outputFile = File(lfn = fwkJobReportFile["lfn"])
            outputFile.loadData(parentage = 1)

            assert outputFile["events"] == int(fwkJobReportFile["events"]), \
                   "Error: Output file has wrong events: %s, %s" % \
                   (outputFile["events"], fwkJobReportFile["events"])
            assert outputFile["size"] == int(fwkJobReportFile["size"]), \
                   "Error: Output file has wrong size: %s, %s" % \
                   (outputFile["size"], fwkJobReportFile["size"])

            for ckType in fwkJobReportFile["checksums"].keys():
                assert ckType in outputFile["checksums"].keys(), \
                       "Error: Output file is missing checksums: %s" % ckType
                assert outputFile["checksums"][ckType] == fwkJobReportFile["checksums"][ckType], \
                       "Error: Checksums don't match."

            assert len(fwkJobReportFile["checksums"].keys()) == \
                   len(outputFile["checksums"].keys()), \
                   "Error: Wrong number of checksums."

            jobType = self.getJobTypeAction.execute(jobID = jobID)
            if jobType == "Merge":
                assert str(outputFile["merged"]) == "True", \
                       "Error: Merge jobs should output merged files."
            else:
                assert outputFile["merged"] == fwkJobReportFile["merged"], \
                       "Error: Output file merged output is wrong: %s, %s" % \
                       (outputFile["merged"], fwkJobReportFile["merged"])

            assert len(outputFile["locations"]) == 1, \
                   "Error: outputfile should have one location: %s" % outputFile["locations"]
            assert list(outputFile["locations"])[0] == list(fwkJobReportFile["locations"])[0], \
                   "Error: wrong location for file."

            assert len(outputFile["parents"]) == len(inputLFNs), \
                   "Error: Output file has wrong number of parents."
            for outputParent in outputFile["parents"]:
                assert outputParent["lfn"] in inputLFNs, \
                       "Error: Unknown parent file: %s" % outputParent["lfn"]

            fwjrRuns = {}
            for run in fwkJobReportFile["runs"]:
                fwjrRuns[run.run] = run.lumis

            for run in outputFile["runs"]:
                assert run.run in fwjrRuns, \
                       "Error: Extra run in output: %s" % run.run

                for lumi in run:
                    assert lumi in fwjrRuns[run.run], \
                           "Error: Extra lumi: %s" % lumi

                    fwjrRuns[run.run].remove(lumi)

                if len(fwjrRuns[run.run]) == 0:
                    del fwjrRuns[run.run]

            assert len(fwjrRuns.keys()) == 0, \
                   "Error: Missing runs, lumis: %s" % fwjrRuns

            testJobGroup = JobGroup(id = testJob["jobgroup"])
            testJobGroup.loadData()
            jobGroupFileset = testJobGroup.output
            jobGroupFileset.loadData()

            assert outputFile["id"] in jobGroupFileset.getFiles(type = "id"), \
                   "Error: output file not in jobgroup fileset."

            if testJob["mask"]["FirstEvent"] == None:
                assert outputFile["first_event"] == 0, \
                       "Error: first event not set correctly: 0, %s" % \
                       outputFile["first_event"]
            else:
                assert testJob["mask"]["FirstEvent"] == outputFile["first_event"], \
                       "Error: last event not set correctly: %s, %s" % \
                       (testJob["mask"]["FirstEvent"], outputFile["first_event"])

        return

    def testReportHandling(self):
        """
        _testReportHandling_

        Verify that we're able to parse a CMSSW report, convert it to a Report()
        style report, pickle it and then have the accountant process it.
        """
        self.procPath = os.path.join(WMCore.WMBase.getTestBase(),
                                    "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml")

        myReport = Report("cmsRun1")
        myReport.parse(self.procPath)

        # Fake some metadata that should be added by the stageout scripts.
        for fileRef in myReport.getAllFileRefsFromStep("cmsRun1"):
            fileRef.size = 1024
            fileRef.location = "cmssrm.fnal.gov"

        fwjrPath = os.path.join(self.tempDir, "ProcReport.pkl")
        cmsRunStep = myReport.retrieveStep("cmsRun1")
        cmsRunStep.status = 0
        myReport.setTaskName('/TestWF/None')
        myReport.persist(fwjrPath)

        self.setFWJRAction.execute(jobID = self.testJob["id"], fwjrPath = fwjrPath)

        pFile = DBSBufferFile(lfn = "/path/to/some/lfn", size = 600000, events = 60000)
        pFile.setAlgorithm(appName = "cmsRun", appVer = "UNKNOWN",
                           appFam = "RECO", psetHash = "GIBBERISH",
                           configContent = "MOREGIBBERISH")
        pFile.setDatasetPath("/bogus/dataset/path")
        #pFile.addRun(Run(1, *[45]))
        pFile.create()

        config = self.createConfig(workerThreads = 1)
        accountant = JobAccountantPoller(config)
        accountant.setup()
        accountant.algorithm()

        self.verifyJobSuccess(self.testJob["id"])
        self.verifyFileMetaData(self.testJob["id"], myReport.getAllFilesFromStep("cmsRun1"))

        inputFile = File(lfn = "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR09_R_34X_V5_All_v1/0000/outputRECORECO.root")
        inputFile.load()
        self.testMergeJob = Job(name = "testMergeJob", files = [inputFile])
        self.testMergeJob.create(group = self.mergeJobGroup)
        self.testMergeJob["state"] = "complete"
        self.stateChangeAction.execute(jobs = [self.testMergeJob])

        self.mergePath = os.path.join(WMCore.WMBase.getTestBase(),
                                         "WMCore_t/FwkJobReport_t/CMSSWMergeReport.xml")

        myReport = Report("mergeReco")
        myReport.parse(self.mergePath)

        # Fake some metadata that should be added by the stageout scripts.
        for fileRef in myReport.getAllFileRefsFromStep("mergeReco"):
            fileRef.size = 1024
            fileRef.location = "cmssrm.fnal.gov"
            fileRef.dataset = {"applicationName": "cmsRun", "applicationVersion": "CMSSW_3_4_2_patch1",
                               "primaryDataset": "MinimumBias", "processedDataset": "Rereco-v1",
                               "dataTier": "RECO"}

        fwjrPath = os.path.join(self.tempDir, "MergeReport.pkl")
        myReport.setTaskName('/MergeWF/None')
        cmsRunStep = myReport.retrieveStep("mergeReco")
        cmsRunStep.status = 0
        myReport.persist(fwjrPath)

        self.setFWJRAction.execute(jobID = self.testMergeJob["id"], fwjrPath = fwjrPath)
        accountant.algorithm()

        self.verifyJobSuccess(self.testMergeJob["id"])
        self.verifyFileMetaData(self.testMergeJob["id"], myReport.getAllFilesFromStep("mergeReco"))

        return
コード例 #27
0
class deleteFileTest(unittest.TestCase):


    def setUp(self):
        # stolen from CMSSWExecutor_t. thanks, dave
        self.testInit = TestInit(__file__)
        self.testDir = self.testInit.generateWorkDir()
        shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))

        self.workload = newWorkload("UnitTests")
        self.task = self.workload.newTask("DeleterTask")
        stepHelper = step = self.task.makeStep("DeleteTest")
        self.step = stepHelper.data
        self.actualStep = stepHelper
        template = DeleteTemplate()
        template(self.step)
        self.helper = template.helper(self.step)
        self.executor = StepFactory.getStepExecutor(self.actualStep.stepType())

        taskMaker = TaskMaker(self.workload, self.testDir)
        taskMaker.skipSubscription = True
        taskMaker.processWorkload()

        self.sandboxDir = "%s/UnitTests" % self.testDir

        self.task.build(self.testDir)
        sys.path.insert(0, self.testDir)
        sys.path.insert(0, self.sandboxDir)


        self.job = Job(name = "/UnitTest/DeleterTask/DeleteTest-test-job")

        binDir = inspect.getsourcefile(ModuleLocator)
        binDir = binDir.replace("__init__.py", "bin")

        if not binDir in os.environ['PATH']:
            os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)

    def tearDown(self):
        self.testInit.delWorkDir()
        sys.path.remove(self.testDir)
        sys.path.remove(self.sandboxDir)

    def setLocalOverride(self, step):
        step.section_('override')
        step.override.command    = 'test-copy'
        step.override.option     = ''
        step.override.__setattr__('lfn-prefix', '')
        step.override.__setattr__('phedex-node','DUMMYPNN')


    @attr('integration')
    def testManualDeleteOld(self):
        self.assertTrue(os.path.exists( os.path.join(self.testDir, 'testfile')))
        self.step.section_('filesToDelete')
        self.step.filesToDelete.file1 = os.path.join(self.testDir, 'testfile')
        self.setLocalOverride(self.step)
        self.executor.initialise(self.step, self.job)
        self.executor.execute()
        self.assertFalse(os.path.exists( os.path.join(self.testDir, 'testfile')))
        return

    @attr('integration')
    def testManualDeleteNew(self):
        self.assertTrue(os.path.exists( os.path.join(self.testDir, 'testfile')))
        self.step.section_('filesToDelete')
        self.step.filesToDelete.file1 = os.path.join(self.testDir, 'testfile')
        self.setLocalOverride(self.step)
        self.step.override.newStageOut = True
        self.executor.initialise(self.step, self.job)
        self.executor.execute()
        self.assertFalse(os.path.exists( os.path.join(self.testDir, 'testfile')))
        return

    @attr('integration')
    def testJobDeleteOld(self):
        self.assertTrue(os.path.exists( os.path.join(self.testDir, 'testfile')))
        self.setLocalOverride(self.step)
        self.job['input_files'] = [ {'lfn': os.path.join(self.testDir, 'testfile') } ]
        self.executor.initialise(self.step, self.job)
        self.executor.execute()
        self.assertFalse(os.path.exists( os.path.join(self.testDir, 'testfile')))
        return

    @attr('integration')
    def testJobDeleteNew(self):
        self.assertTrue(os.path.exists( os.path.join(self.testDir, 'testfile')))
        self.setLocalOverride(self.step)
        self.step.override.newStageOut = True
        self.job['input_files'] = [ {'lfn': os.path.join(self.testDir, 'testfile') } ]
        self.executor.initialise(self.step, self.job)
        self.executor.execute()
        self.assertFalse(os.path.exists( os.path.join(self.testDir, 'testfile')))
        return
コード例 #28
0
class ResourceControlTest(unittest.TestCase):
    def setUp(self):
        """
        _setUp_

        Install schema and create a DAO factory for WMBS.
        """
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules=[
            "WMCore.WMBS", "WMCore.ResourceControl", "WMCore.BossAir"
        ],
                                useDefault=False)

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package="WMCore.WMBS",
                                     logger=myThread.logger,
                                     dbinterface=myThread.dbi)

        self.baDaoFactory = DAOFactory(package="WMCore.BossAir",
                                       logger=myThread.logger,
                                       dbinterface=myThread.dbi)

        self.insertRunJob = self.baDaoFactory(classname="NewJobs")
        self.insertState = self.baDaoFactory(classname="NewState")
        states = ['PEND', 'RUN', 'Idle', 'Running']
        self.insertState.execute(states)

        self.tempDir = self.testInit.generateWorkDir()
        return

    def tearDown(self):
        """
        _tearDown_

        Clear the schema.
        """
        self.testInit.clearDatabase()
        return

    def testInsert(self):
        """
        _testInsert_

        Verify that inserting sites and thresholds works correctly, even if the
        site or threshold already exists.
        """
        myResourceControl = ResourceControl()
        myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1")
        myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1")
        myResourceControl.insertSite("testSite2", 100, 200, "testSE2",
                                     "testCE2")

        myResourceControl.insertThreshold("testSite1", "Processing", 20, 10)
        myResourceControl.insertThreshold("testSite1", "Merge", 200, 100)
        myResourceControl.insertThreshold("testSite1", "Merge", 250, 150)
        myResourceControl.insertThreshold("testSite2", "Processing", 50, 30)
        myResourceControl.insertThreshold("testSite2", "Merge", 135, 100)

        createThresholds = myResourceControl.listThresholdsForCreate()

        self.assertEqual(len(createThresholds.keys()), 2,
                         "Error: Wrong number of site in Resource Control DB")

        self.assertTrue("testSite1" in createThresholds.keys(),
                        "Error: Test Site 1 missing from thresholds.")

        self.assertTrue("testSite2" in createThresholds.keys(),
                        "Error: Test Site 2 missing from thresholds.")

        self.assertEqual(createThresholds["testSite1"]["total_slots"], 10,
                         "Error: Wrong number of total slots.")

        self.assertEqual(
            createThresholds["testSite1"]["pending_jobs"], 0,
            "Error: Wrong number of running jobs: %s" %
            createThresholds["testSite1"]["pending_jobs"])

        self.assertEqual(createThresholds["testSite2"]["total_slots"], 100,
                         "Error: Wrong number of total slots.")

        self.assertEqual(createThresholds["testSite2"]["pending_jobs"], 0,
                         "Error: Wrong number of running jobs.")

        thresholds = myResourceControl.listThresholdsForSubmit()

        self.assertEqual(
            len(thresholds.keys()), 2,
            "Error: Wrong number of sites in Resource Control DB")

        self.assertTrue("testSite1" in thresholds.keys(),
                        "Error: testSite1 missing from thresholds.")

        self.assertTrue("testSite2" in thresholds.keys(),
                        "Error: testSite2 missing from thresholds.")

        site1Info = thresholds["testSite1"]
        site2Info = thresholds["testSite2"]
        site1Thresholds = site1Info["thresholds"]
        site2Thresholds = site2Info["thresholds"]

        procThreshold1 = None
        procThreshold2 = None
        mergeThreshold1 = None
        mergeThreshold2 = None
        for threshold in site1Thresholds:
            if threshold["task_type"] == "Merge":
                mergeThreshold1 = threshold
            elif threshold["task_type"] == "Processing":
                procThreshold1 = threshold
        for threshold in site2Thresholds:
            if threshold["task_type"] == "Merge":
                mergeThreshold2 = threshold
            elif threshold["task_type"] == "Processing":
                procThreshold2 = threshold

        self.assertEqual(len(site1Thresholds), 2,
                         "Error: Wrong number of task types.")

        self.assertEqual(len(site2Thresholds), 2,
                         "Error: Wrong number of task types.")

        self.assertNotEqual(procThreshold1, None)
        self.assertNotEqual(procThreshold2, None)
        self.assertNotEqual(mergeThreshold1, None)
        self.assertNotEqual(mergeThreshold2, None)

        self.assertEqual(site1Info["total_pending_slots"], 10,
                         "Error: Site thresholds wrong")

        self.assertEqual(site1Info["total_running_slots"], 20,
                         "Error: Site thresholds wrong")

        self.assertEqual(site1Info["total_running_jobs"], 0,
                         "Error: Site thresholds wrong")

        self.assertEqual(site1Info["total_pending_jobs"], 0,
                         "Error: Site thresholds wrong")

        self.assertEqual(procThreshold1["task_running_jobs"], 0,
                         "Error: Site thresholds wrong")

        self.assertEqual(procThreshold1["task_pending_jobs"], 0,
                         "Error: Site thresholds wrong")

        self.assertEqual(procThreshold1["max_slots"], 20,
                         "Error: Site thresholds wrong")

        self.assertEqual(procThreshold1["pending_slots"], 10,
                         "Error: Site thresholds wrong")

        self.assertEqual(mergeThreshold1["task_running_jobs"], 0,
                         "Error: Site thresholds wrong")

        self.assertEqual(mergeThreshold1["task_pending_jobs"], 0,
                         "Error: Site thresholds wrong")

        self.assertEqual(mergeThreshold1["max_slots"], 250,
                         "Error: Site thresholds wrong")

        self.assertEqual(mergeThreshold1["pending_slots"], 150,
                         "Error: Site thresholds wrong")

        self.assertEqual(site2Info["total_pending_slots"], 100,
                         "Error: Site thresholds wrong")

        self.assertEqual(site2Info["total_running_slots"], 200,
                         "Error: Site thresholds wrong")

        self.assertEqual(site2Info["total_running_jobs"], 0,
                         "Error: Site thresholds wrong")

        self.assertEqual(site2Info["total_pending_jobs"], 0,
                         "Error: Site thresholds wrong")

        self.assertEqual(procThreshold2["task_running_jobs"], 0,
                         "Error: Site thresholds wrong")

        self.assertEqual(procThreshold2["task_pending_jobs"], 0,
                         "Error: Site thresholds wrong")

        self.assertEqual(procThreshold2["max_slots"], 50,
                         "Error: Site thresholds wrong")

        self.assertEqual(procThreshold2["pending_slots"], 30,
                         "Error: Site thresholds wrong")

        self.assertEqual(mergeThreshold2["task_running_jobs"], 0,
                         "Error: Site thresholds wrong")

        self.assertEqual(mergeThreshold2["task_pending_jobs"], 0,
                         "Error: Site thresholds wrong")

        self.assertEqual(mergeThreshold2["max_slots"], 135,
                         "Error: Site thresholds wrong")

        self.assertEqual(mergeThreshold2["pending_slots"], 100,
                         "Error: Site thresholds wrong")

    def testList(self):
        """
        _testList_

        Test the functions that list thresholds for creating jobs and submitting
        jobs.
        """
        myResourceControl = ResourceControl()
        myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1",
                                     "T1_US_FNAL", "LsfPlugin")
        myResourceControl.insertSite("testSite2", 20, 40, "testSE2", "testCE2")

        myResourceControl.insertThreshold("testSite1", "Processing", 20, 10)
        myResourceControl.insertThreshold("testSite1", "Merge", 200, 100)
        myResourceControl.insertThreshold("testSite2", "Processing", 50, 25)
        myResourceControl.insertThreshold("testSite2", "Merge", 135, 65)

        testWorkflow = Workflow(spec=makeUUID(),
                                owner="Steve",
                                name=makeUUID(),
                                task="Test")
        testWorkflow.create()

        testFilesetA = Fileset(name="TestFilesetA")
        testFilesetA.create()
        testFilesetB = Fileset(name="TestFilesetB")
        testFilesetB.create()
        testFilesetC = Fileset(name="TestFilesetC")
        testFilesetC.create()

        testFileA = File(lfn="testFileA",
                         locations=set(["testSE1", "testSE2"]))
        testFileA.create()
        testFilesetA.addFile(testFileA)
        testFilesetA.commit()
        testFilesetB.addFile(testFileA)
        testFilesetB.commit()
        testFilesetC.addFile(testFileA)
        testFilesetC.commit()

        testSubscriptionA = Subscription(fileset=testFilesetA,
                                         workflow=testWorkflow,
                                         type="Processing")
        testSubscriptionA.create()
        testSubscriptionA.addWhiteBlackList([{
            "site_name": "testSite1",
            "valid": True
        }])
        testSubscriptionB = Subscription(fileset=testFilesetB,
                                         workflow=testWorkflow,
                                         type="Processing")
        testSubscriptionB.create()
        testSubscriptionB.addWhiteBlackList([{
            "site_name": "testSite1",
            "valid": False
        }])
        testSubscriptionC = Subscription(fileset=testFilesetC,
                                         workflow=testWorkflow,
                                         type="Merge")
        testSubscriptionC.create()

        testJobGroupA = JobGroup(subscription=testSubscriptionA)
        testJobGroupA.create()
        testJobGroupB = JobGroup(subscription=testSubscriptionB)
        testJobGroupB.create()
        testJobGroupC = JobGroup(subscription=testSubscriptionC)
        testJobGroupC.create()

        # Site1, Has been assigned a location and is complete.
        testJobA = Job(name="testJobA", files=[testFileA])
        testJobA["couch_record"] = makeUUID()
        testJobA.create(group=testJobGroupA)
        testJobA["state"] = "success"

        # Site 1, Has been assigned a location and is incomplete.
        testJobB = Job(name="testJobB", files=[testFileA])
        testJobB["couch_record"] = makeUUID()
        testJobB.create(group=testJobGroupA)
        testJobB["state"] = "executing"
        runJobB = RunJob()
        runJobB.buildFromJob(testJobB)
        runJobB["status"] = "PEND"

        # Does not have a location, white listed to site 1
        testJobC = Job(name="testJobC", files=[testFileA])
        testJobC["couch_record"] = makeUUID()
        testJobC.create(group=testJobGroupA)
        testJobC["state"] = "new"

        # Site 2, Has been assigned a location and is complete.
        testJobD = Job(name="testJobD", files=[testFileA])
        testJobD["couch_record"] = makeUUID()
        testJobD.create(group=testJobGroupB)
        testJobD["state"] = "success"

        # Site 2, Has been assigned a location and is incomplete.
        testJobE = Job(name="testJobE", files=[testFileA])
        testJobE["couch_record"] = makeUUID()
        testJobE.create(group=testJobGroupB)
        testJobE["state"] = "executing"
        runJobE = RunJob()
        runJobE.buildFromJob(testJobE)
        runJobE["status"] = "RUN"

        # Does not have a location, site 1 is blacklisted.
        testJobF = Job(name="testJobF", files=[testFileA])
        testJobF["couch_record"] = makeUUID()
        testJobF.create(group=testJobGroupB)
        testJobF["state"] = "new"

        # Site 3, Has been assigned a location and is complete.
        testJobG = Job(name="testJobG", files=[testFileA])
        testJobG["couch_record"] = makeUUID()
        testJobG.create(group=testJobGroupC)
        testJobG["state"] = "cleanout"

        # Site 3, Has been assigned a location and is incomplete.
        testJobH = Job(name="testJobH", files=[testFileA])
        testJobH["couch_record"] = makeUUID()
        testJobH.create(group=testJobGroupC)
        testJobH["state"] = "new"

        # Site 3, Does not have a location.
        testJobI = Job(name="testJobI", files=[testFileA])
        testJobI["couch_record"] = makeUUID()
        testJobI.create(group=testJobGroupC)
        testJobI["state"] = "new"

        # Site 3, Does not have a location and is in cleanout.
        testJobJ = Job(name="testJobJ", files=[testFileA])
        testJobJ["couch_record"] = makeUUID()
        testJobJ.create(group=testJobGroupC)
        testJobJ["state"] = "cleanout"

        changeStateAction = self.daoFactory(classname="Jobs.ChangeState")
        changeStateAction.execute(jobs=[
            testJobA, testJobB, testJobC, testJobD, testJobE, testJobF,
            testJobG, testJobH, testJobI, testJobJ
        ])

        self.insertRunJob.execute([runJobB, runJobE])

        setLocationAction = self.daoFactory(classname="Jobs.SetLocation")
        setLocationAction.execute(testJobA["id"], "testSite1")
        setLocationAction.execute(testJobB["id"], "testSite1")
        setLocationAction.execute(testJobD["id"], "testSite1")
        setLocationAction.execute(testJobE["id"], "testSite1")
        setLocationAction.execute(testJobG["id"], "testSite1")
        setLocationAction.execute(testJobH["id"], "testSite1")

        createThresholds = myResourceControl.listThresholdsForCreate()
        submitThresholds = myResourceControl.listThresholdsForSubmit()

        self.assertEqual(len(createThresholds.keys()), 2,
                         "Error: Wrong number of sites in create thresholds")

        self.assertEqual(createThresholds["testSite1"]["total_slots"], 10,
                         "Error: Wrong number of slots for site 1")

        self.assertEqual(createThresholds["testSite2"]["total_slots"], 20,
                         "Error: Wrong number of slots for site 2")

        # We should have two running jobs with locations at site one,
        # two running jobs without locations at site two, and one running
        # job without a location at site one and two.
        self.assertEqual(createThresholds["testSite1"]["pending_jobs"], 4,
                         "Error: Wrong number of pending jobs for site 1")

        # We should have one running job with a location at site 2 and
        # another running job without a location.
        self.assertEqual(createThresholds["testSite2"]["pending_jobs"], 2,
                         "Error: Wrong number of pending jobs for site 2")

        # We should also have a phedex_name
        self.assertEqual(createThresholds["testSite1"]["cms_name"],
                         "T1_US_FNAL")
        self.assertEqual(createThresholds["testSite2"]["cms_name"], None)

        mergeThreshold1 = None
        mergeThreshold2 = None
        procThreshold1 = None
        procThreshold2 = None
        self.assertEqual(submitThresholds["testSite1"]['cms_name'],
                         'T1_US_FNAL')
        for threshold in submitThresholds["testSite1"]["thresholds"]:
            if threshold['task_type'] == "Merge":
                mergeThreshold1 = threshold
            elif threshold['task_type'] == "Processing":
                procThreshold1 = threshold
        self.assertEqual(submitThresholds["testSite2"]['cms_name'], None)
        for threshold in submitThresholds["testSite2"]["thresholds"]:
            if threshold['task_type'] == "Merge":
                mergeThreshold2 = threshold
            elif threshold['task_type'] == "Processing":
                procThreshold2 = threshold

        self.assertEqual(
            submitThresholds["testSite1"]["total_running_jobs"], 1,
            "Error: Wrong number of running jobs for submit thresholds.")
        self.assertEqual(
            submitThresholds["testSite2"]["total_running_jobs"], 0,
            "Error: Wrong number of running jobs for submit thresholds.")
        self.assertEqual(
            submitThresholds["testSite1"]["total_pending_jobs"], 1,
            "Error: Wrong number of pending jobs for submit thresholds.")
        self.assertEqual(
            submitThresholds["testSite2"]["total_pending_jobs"], 0,
            "Error: Wrong number of pending jobs for submit thresholds.")

        self.assertEqual(
            mergeThreshold1["task_running_jobs"], 0,
            "Error: Wrong number of task running jobs for submit thresholds.")
        self.assertEqual(
            mergeThreshold1["task_pending_jobs"], 0,
            "Error: Wrong number of task running jobs for submit thresholds.")
        self.assertEqual(
            procThreshold1["task_running_jobs"], 1,
            "Error: Wrong number of task running jobs for submit thresholds.")
        self.assertEqual(
            procThreshold1["task_pending_jobs"], 1,
            "Error: Wrong number of task running jobs for submit thresholds.")
        self.assertEqual(
            mergeThreshold2["task_running_jobs"], 0,
            "Error: Wrong number of task running jobs for submit thresholds.")
        self.assertEqual(
            mergeThreshold2["task_pending_jobs"], 0,
            "Error: Wrong number of task running jobs for submit thresholds.")
        self.assertEqual(
            procThreshold2["task_running_jobs"], 0,
            "Error: Wrong number of task running jobs for submit thresholds.")
        self.assertEqual(
            procThreshold2["task_pending_jobs"], 0,
            "Error: Wrong number of task running jobs for submit thresholds.")

        return

    def testListSiteInfo(self):
        """
        _testListSiteInfo_

        Verify that the listSiteInfo() methods works properly.
        """
        myResourceControl = ResourceControl()
        myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1")
        myResourceControl.insertSite("testSite2", 100, 200, "testSE2",
                                     "testCE2")

        siteInfo = myResourceControl.listSiteInfo("testSite1")

        self.assertEqual(siteInfo["site_name"], "testSite1",
                         "Error: Site name is wrong.")

        self.assertEqual(siteInfo["se_name"], ["testSE1"],
                         "Error: SE name is wrong.")

        self.assertEqual(siteInfo["ce_name"], "testCE1",
                         "Error: CE name is wrong.")

        self.assertEqual(siteInfo["pending_slots"], 10,
                         "Error: Pending slots is wrong.")

        self.assertEqual(siteInfo["running_slots"], 20,
                         "Error: Pending slots is wrong.")

        return

    def testUpdateJobSlots(self):
        """
        _testUpdateJobSlots_

        Verify that it is possible to update the number of job slots at a site.
        """
        myResourceControl = ResourceControl()
        myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1")

        siteInfo = myResourceControl.listSiteInfo("testSite1")

        self.assertEqual(siteInfo["pending_slots"], 10,
                         "Error: Pending slots is wrong.")
        self.assertEqual(siteInfo["running_slots"], 20,
                         "Error: Running slots is wrong.")

        myResourceControl.setJobSlotsForSite("testSite1", pendingJobSlots=20)

        siteInfo = myResourceControl.listSiteInfo("testSite1")

        self.assertEqual(siteInfo["pending_slots"], 20,
                         "Error: Pending slots is wrong.")

        myResourceControl.setJobSlotsForSite("testSite1", runningJobSlots=40)

        siteInfo = myResourceControl.listSiteInfo("testSite1")

        self.assertEqual(siteInfo["running_slots"], 40,
                         "Error: Running slots is wrong.")

        myResourceControl.setJobSlotsForSite("testSite1", 5, 10)

        siteInfo = myResourceControl.listSiteInfo("testSite1")

        self.assertEqual(siteInfo["pending_slots"], 5,
                         "Error: Pending slots is wrong.")
        self.assertEqual(siteInfo["running_slots"], 10,
                         "Error: Running slots is wrong.")

        return

    def testThresholdsForSite(self):
        """
        _testThresholdsForSite_

        Check that we can get the thresholds in intelligible form
        for each site
        """

        myResourceControl = ResourceControl()
        myResourceControl.insertSite("testSite1", 20, 40, "testSE1", "testCE1")
        myResourceControl.insertThreshold("testSite1", "Processing", 10, 8)
        myResourceControl.insertThreshold("testSite1", "Merge", 5, 3)

        result = myResourceControl.thresholdBySite(siteName="testSite1")
        procInfo = {}
        mergInfo = {}
        for res in result:
            if res['task_type'] == 'Processing':
                procInfo = res
            elif res['task_type'] == 'Merge':
                mergInfo = res
        self.assertEqual(procInfo.get('pending_slots', None), 20)
        self.assertEqual(procInfo.get('running_slots', None), 40)
        self.assertEqual(procInfo.get('max_slots', None), 10)
        self.assertEqual(procInfo.get('task_pending_slots', None), 8)
        self.assertEqual(mergInfo.get('pending_slots', None), 20)
        self.assertEqual(mergInfo.get('running_slots', None), 40)
        self.assertEqual(mergInfo.get('max_slots', None), 5)
        self.assertEqual(mergInfo.get('task_pending_slots', None), 3)

        return

    def testThresholdPriority(self):
        """
        _testThresholdPriority_

        Test that we get things back in priority order
        """

        myResourceControl = ResourceControl()
        myResourceControl.insertSite("testSite1", 20, 40, "testSE1", "testCE1")
        myResourceControl.insertThreshold("testSite1",
                                          "Processing",
                                          10,
                                          8,
                                          priority=1)
        myResourceControl.insertThreshold("testSite1",
                                          "Merge",
                                          5,
                                          3,
                                          priority=2)

        result = myResourceControl.listThresholdsForSubmit()

        self.assertEqual(result['testSite1']['thresholds'][0]['task_type'],
                         'Merge')
        self.assertEqual(result['testSite1']['thresholds'][1]['task_type'],
                         'Processing')

        myResourceControl.insertThreshold("testSite1",
                                          "Processing",
                                          10,
                                          8,
                                          priority=2)
        myResourceControl.insertThreshold("testSite1",
                                          "Merge",
                                          5,
                                          3,
                                          priority=1)

        # Should now be in reverse order
        result = myResourceControl.listThresholdsForSubmit()
        self.assertEqual(result['testSite1']['thresholds'][1]['task_type'],
                         'Merge')
        self.assertEqual(result['testSite1']['thresholds'][0]['task_type'],
                         'Processing')

        myResourceControl.insertSite("testSite2", 20, 40, "testSE2", "testCE2")
        myResourceControl.insertThreshold("testSite2",
                                          "Processing",
                                          10,
                                          8,
                                          priority=1)
        myResourceControl.insertThreshold("testSite2",
                                          "Merge",
                                          5,
                                          3,
                                          priority=2)

        # Should be in proper order for site 2
        result = myResourceControl.listThresholdsForSubmit()
        self.assertEqual(result['testSite2']['thresholds'][0]['task_type'],
                         'Merge')
        self.assertEqual(result['testSite2']['thresholds'][1]['task_type'],
                         'Processing')

        # Should now be in reverse order for site 1
        self.assertEqual(result['testSite1']['thresholds'][1]['task_type'],
                         'Merge')
        self.assertEqual(result['testSite1']['thresholds'][0]['task_type'],
                         'Processing')

        myResourceControl.insertThreshold("testSite2", "Merge", 20, 10)
        result = myResourceControl.listThresholdsForSubmit()
        self.assertEqual(result['testSite2']['thresholds'][0]['priority'], 2)

        return

    def testChangeState(self):
        """
        _testChangeState_

        Check that we can change the state between different values and
        retrieve it through the threshold methods
        """
        myResourceControl = ResourceControl()
        myResourceControl.insertSite("testSite1", 20, 40, "testSE1", "testCE1")
        myResourceControl.insertThreshold("testSite1",
                                          "Processing",
                                          10,
                                          5,
                                          priority=1)

        result = myResourceControl.listThresholdsForCreate()
        self.assertEqual(result['testSite1']['state'], 'Normal',
                         'Error: Wrong site state')

        myResourceControl.changeSiteState("testSite1", "Down")
        result = myResourceControl.listThresholdsForCreate()
        self.assertEqual(result['testSite1']['state'], 'Down',
                         'Error: Wrong site state')

    def createConfig(self):
        """
        _createConfig_

        Create a config and save it to the temp dir.  Set the WMAGENT_CONFIG
        environment variable so the config gets picked up.
        """
        config = Configuration()
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", os.getcwd())
        config.section_("Agent")
        config.Agent.componentName = "resource_control_t"
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket = os.getenv("DBSOCK")

        configHandle = open(os.path.join(self.tempDir, "config.py"), "w")
        configHandle.write(str(config))
        configHandle.close()

        os.environ["WMAGENT_CONFIG"] = os.path.join(self.tempDir, "config.py")
        return

    def testInsertAllSEs(self):
        """
        _testInsertAllSEs_

        Test to see if we can insert all SEs and Thresholds at once
        Depending on the WMCore.Services.SiteDB interface
        """
        self.createConfig()

        resControlPath = os.path.join(WMCore.WMBase.getTestBase(),
                                      "../../bin/wmagent-resource-control")
        env = os.environ
        env['PYTHONPATH'] = ":".join(sys.path)
        cmdline = [
            resControlPath, "--add-all-sites", "--plugin=CondorPlugin",
            "--pending-slots=100", "--running-slots=500"
        ]
        retval = subprocess.Popen(cmdline,
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.STDOUT,
                                  env=env)
        (output, _) = retval.communicate()

        myResourceControl = ResourceControl()
        result = myResourceControl.listThresholdsForSubmit()
        self.assertTrue('T1_US_FNAL' in result.keys())
        for x in result.keys():
            self.assertEqual(len(result[x]['thresholds']), 8)
            self.assertEqual(result[x]['total_pending_slots'], 100)
            self.assertEqual(result[x]['total_running_slots'], 500)
            for thresh in result[x]['thresholds']:
                if thresh['task_type'] == 'Processing':
                    self.assertEqual(thresh['priority'], 1)
                    self.assertEqual(thresh['max_slots'], 500)

        # Verify that sites with more than one SE were added correctly.
        nebInfo = myResourceControl.listSiteInfo("T2_US_Nebraska")
        self.assertTrue(len(nebInfo["se_name"]) == 3)
        return

    def testInsertAllSEs2(self):
        """
        _testInsertAllSEs2_

        Test to see if we can insert all SEs and Thresholds at once
        Depending on the WMCore.Services.SiteDB interface
        """
        myResourceControl = ResourceControl()
        taskList = [{
            'taskType': 'Processing',
            'maxSlots': 100,
            'pendingSlots': 80,
            'priority': 1
        }, {
            'taskType': 'Merge',
            'maxSlots': 50,
            'pendingSlots': 30,
            'priority': 2
        }]

        myResourceControl.insertAllSEs(siteName='test',
                                       pendingSlots=200,
                                       runningSlots=400,
                                       ceName='glidein-ce.fnal.gov',
                                       plugin='CondorPlugin',
                                       taskList=taskList)
        result = myResourceControl.listThresholdsForSubmit()
        self.assertTrue('test_cmssrm.fnal.gov' in result.keys())
        self.assertEqual(result['test_cmssrm.fnal.gov']['cms_name'],
                         'T1_US_FNAL')
        for x in result.keys():
            self.assertEqual(len(result[x]['thresholds']), 2)
            self.assertEqual(result[x]['total_pending_slots'], 200)
            self.assertEqual(result[x]['total_running_slots'], 400)
            for thresh in result[x]['thresholds']:
                if thresh['task_type'] == 'Processing':
                    self.assertEqual(thresh['priority'], 1)
                    self.assertEqual(thresh['max_slots'], 100)
                    self.assertEqual(thresh['pending_slots'], 80)

                else:
                    self.assertEqual(thresh['priority'], 2)
                    self.assertEqual(thresh['max_slots'], 50)
                    self.assertEqual(thresh['pending_slots'], 30)
        return

    def testInsertT0(self):
        """
        _testInsertT0_

        Test to see if we can insert the Tier-0 alone
        with a single option
        """
        self.createConfig()

        resControlPath = os.path.join(WMCore.WMBase.getTestBase(),
                                      "../../bin/wmagent-resource-control")
        env = os.environ
        env['PYTHONPATH'] = ":".join(sys.path)
        cmdline = [resControlPath, "--add-T0"]
        retval = subprocess.Popen(cmdline,
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.STDOUT,
                                  env=env)
        (_, _) = retval.communicate()

        myResourceControl = ResourceControl()
        result = myResourceControl.listThresholdsForSubmit()
        self.assertTrue(len(result), 1)
        self.assertTrue('CERN' in result)
        for x in result:
            self.assertEqual(len(result[x]['thresholds']), 10)
            self.assertEqual(result[x]['total_pending_slots'], 500)
            self.assertEqual(result[x]['total_running_slots'], -1)
            for thresh in result[x]['thresholds']:
                if thresh['task_type'] == 'Processing':
                    self.assertEqual(thresh['priority'], 1)
                    self.assertEqual(thresh['max_slots'], -1)

        # Verify that sites with more than one SE were added correctly.
        cernInfo = myResourceControl.listSiteInfo("CERN")
        self.assertTrue(len(cernInfo["se_name"]) == 2)
        return
コード例 #29
0
ファイル: LogArch_t.py プロジェクト: huohuo21/WMCore
class LogArchiveTest(unittest.TestCase):

    def setUp(self):
        self.testInit = TestInit(__file__)
        self.testDir = self.testInit.generateWorkDir()

        # shut up SiteLocalConfig
        os.environ['CMS_PATH'] = os.getcwd()
        workload = copy.deepcopy(testWorkloads.workload)
        task = workload.getTask("Production")
        step = task.getStep("stageOut1")
        # want to get the cmsstep so I can make the Report
        cmsstep = task.getStep('cmsRun1')
        self.cmsstepdir = os.path.join( self.testDir, 'cmsRun1')
        os.mkdir( self.cmsstepdir )
        open( os.path.join( self.cmsstepdir, '__init__.py'),'w').close()
        open( os.path.join( self.cmsstepdir, 'Report.pkl'),'w').close()

        cmsbuilder = CMSSWBuilder.CMSSW()
        cmsbuilder( cmsstep.data, 'Production', self.cmsstepdir )
        realstep = LogArchiveTemplate.LogArchiveStepHelper(step.data)
        realstep.disableRetries()
        self.realstep = realstep
        self.stepDir = os.path.join( self.testDir, 'stepdir')
        os.mkdir( self.stepDir )
        builder = LogArchiveBuilder.LogArchive()
        builder( step.data, 'Production', self.stepDir)
        
        # stolen from CMSSWExecutor_t. thanks, dave

        # first, delete all the sandboxen and taskspaces
        #    because of caching, this leaks from other tests in other files
        #    this sucks because the other tests are using sandboxen that
        #    are deleted after the test is over, which causes theses tests
        #    to break
        modsToDelete = []
        # not sure what happens if you delete from
        # an arrey you're iterating over. doing it in
        # two steps
        for modname in sys.modules.keys():
            # need to blow away things in sys.modules, otherwise
            # they are cached and we look at old taskspaces
            if modname.startswith('WMTaskSpace'):
                modsToDelete.append(modname)
            if modname.startswith('WMSandbox'):
                modsToDelete.append(modname)
        for modname in modsToDelete:
            try:
                reload(sys.modules[modname])
            except:
                pass
            del sys.modules[modname]

        self.oldpath = sys.path[:]
        self.testInit = TestInit(__file__)


        self.testDir = self.testInit.generateWorkDir()
        self.job = Job(name = "/UnitTests/DeleterTask/DeleteTest-test-job")
        shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))

        self.workload = newWorkload("UnitTests")
        self.task = self.workload.newTask("DeleterTask")

        cmsswHelper = self.task.makeStep("cmsRun1")
        cmsswHelper.setStepType('CMSSW')
        stepHelper = cmsswHelper.addStep("DeleteTest")
        stepHelper.setStepType('LogArchive')

        self.cmsswstep = cmsswHelper.data
        self.cmsswHelper = cmsswHelper


        self.stepdata = stepHelper.data
        self.stephelp = LogArchiveTemplate.LogArchiveStepHelper(stepHelper.data)
        self.task.applyTemplates()

        self.executor = StepFactory.getStepExecutor(self.stephelp.stepType())
        taskMaker = TaskMaker(self.workload, os.path.join(self.testDir))
        taskMaker.skipSubscription = True
        taskMaker.processWorkload()


        self.task.build(os.path.join(self.testDir, 'UnitTests'))

        sys.path.insert(0, self.testDir)
        sys.path.insert(0, os.path.join(self.testDir, 'UnitTests'))


#        binDir = inspect.getsourcefile(ModuleLocator)
#        binDir = binDir.replace("__init__.py", "bin")
#
#        if not binDir in os.environ['PATH']:
#            os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
        open( os.path.join( self.testDir, 'UnitTests', '__init__.py'),'w').close()
        shutil.copyfile( os.path.join( os.path.dirname( __file__ ), 'MergeSuccess.pkl'),
                         os.path.join( self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))

    def tearDown(self):
        sys.path = self.oldpath[:]
        self.testInit.delWorkDir()


        # making double sure WMTaskSpace and WMSandbox are gone
        modsToDelete = []
        # not sure what happens if you delete from
        # an arrey you're iterating over. doing it in
        # two steps
        for modname in sys.modules.keys():
            # need to blow away things in sys.modules, otherwise
            # they are cached and we look at old taskspaces
            if modname.startswith('WMTaskSpace'):
                modsToDelete.append(modname)
            if modname.startswith('WMSandbox'):
                modsToDelete.append(modname)
        for modname in modsToDelete:
            try:
                reload(sys.modules[modname])
            except:
                pass
            del sys.modules[modname]
        myThread = threading.currentThread()
        if hasattr(myThread, "factory"):
            myThread.factory = {}

    def makeReport(self, fileName):
        myReport = Report('oneitem')
        myReport.addStep('stageOut1')
        mod1 = myReport.addOutputModule('module1')
        mod2 = myReport.addOutputModule('module2')
        file1 = myReport.addOutputFile('module1', {'lfn': 'FILE1', 'size' : 1, 'events' : 1})
        file2 = myReport.addOutputFile('module2', {'lfn': 'FILE2', 'size' : 1, 'events' : 1})
        file3 = myReport.addOutputFile('module2', {'lfn': 'FILE3', 'size' : 1, 'events' : 1})
        myReport.persist( fileName )

    def testExecutorDoesntDetonate(self):
        myReport = Report()
        myReport.unpersist(os.path.join( self.testDir,'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
        myReport.data.cmsRun1.status = 1
        myReport.persist(os.path.join( self.testDir, 'UnitTests','WMTaskSpace', 'cmsRun1' , 'Report.pkl'))

        executor = LogArchiveExecutor.LogArchive()
        
        executor.initialise( self.stepdata, self.job)
        self.setLocalOverride(self.stepdata)
        executor.step = self.stepdata
        executor.execute( )
        self.assertFalse( os.path.exists( os.path.join( self.testDir, 'hosts' )))
        self.assertFalse( os.path.exists( os.path.join( self.testDir, 'test1', 'hosts')))
        return
    
    
    def testUnitTestBackend(self):
        myReport = Report()
        myReport.unpersist(os.path.join( self.testDir,'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
        myReport.data.cmsRun1.status = 1
        myReport.persist(os.path.join( self.testDir, 'UnitTests','WMTaskSpace', 'cmsRun1' , 'Report.pkl'))

        executor = LogArchiveExecutor.LogArchive()
        helper = LogArchiveTemplate.LogArchiveStepHelper(self.stepdata)
        helper.addOverride(override = 'command', overrideValue='test-win')
        helper.addOverride(override = 'option', overrideValue='')
        helper.addOverride(override = 'se-name', overrideValue='charlie.sheen.biz')
        helper.addOverride(override = 'lfn-prefix', overrideValue='test-win')
        
        executor.initialise( self.stepdata, self.job)
        self.setLocalOverride(self.stepdata)
        executor.step = self.stepdata
        executor.execute( )
        self.assertFalse( os.path.exists( os.path.join( self.testDir, 'hosts' )))
        self.assertFalse( os.path.exists( os.path.join( self.testDir, 'test1', 'hosts')))

    def testUnitTestBackendNew(self):
        myReport = Report()
        myReport.unpersist(os.path.join( self.testDir,'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
        myReport.data.cmsRun1.status = 1
        myReport.persist(os.path.join( self.testDir, 'UnitTests','WMTaskSpace', 'cmsRun1' , 'Report.pkl'))

        executor = LogArchiveExecutor.LogArchive()
        helper = LogArchiveTemplate.LogArchiveStepHelper(self.stepdata)
        helper.addOverride(override = 'command', overrideValue='test-win')
        helper.addOverride(override = 'option', overrideValue='')
        helper.addOverride(override = 'se-name', overrideValue='charlie.sheen.biz')
        helper.addOverride(override = 'lfn-prefix', overrideValue='test-win')
        helper.setNewStageoutOverride( True )
        
        executor.initialise( self.stepdata, self.job)
        self.setLocalOverride(self.stepdata)
        executor.step = self.stepdata
        executor.execute( )
        self.assertFalse( os.path.exists( os.path.join( self.testDir, 'hosts' )))
        self.assertFalse( os.path.exists( os.path.join( self.testDir, 'test1', 'hosts')))
        
    def setLocalOverride(self, step):
        step.section_('override')
        step.override.command    = 'cp'
        step.override.option     = ''
        step.override.__setattr__('lfn-prefix', self.testDir +"/")
        step.override.__setattr__('se-name','DUMMYSE')
        step.override.__setattr__('phedex-node','DUMMYPNN')
コード例 #30
0
ファイル: StageOut_t.py プロジェクト: vytjan/WMCore
class otherStageOutTexst(unittest.TestCase):
    def setUp(self):
        # stolen from CMSSWExecutor_t. thanks, dave

        # first, delete all the sandboxen and taskspaces
        #    because of caching, this leaks from other tests in other files
        #    this sucks because the other tests are using sandboxen that
        #    are deleted after the test is over, which causes theses tests
        #    to break
        modsToDelete = []
        # not sure what happens if you delete from
        # an arrey you're iterating over. doing it in
        # two steps
        for modname in sys.modules.keys():
            # need to blow away things in sys.modules, otherwise
            # they are cached and we look at old taskspaces
            if modname.startswith('WMTaskSpace'):
                modsToDelete.append(modname)
            if modname.startswith('WMSandbox'):
                modsToDelete.append(modname)
        for modname in modsToDelete:
            try:
                reload(sys.modules[modname])
            except Exception:
                pass
            del sys.modules[modname]

        self.oldpath = sys.path[:]
        self.testInit = TestInit(__file__)

        self.testDir = self.testInit.generateWorkDir()
        self.job = Job(name="/UnitTests/DeleterTask/DeleteTest-test-job")
        shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))

        self.workload = newWorkload("UnitTests")
        self.task = self.workload.newTask("DeleterTask")

        cmsswHelper = self.task.makeStep("cmsRun1")
        cmsswHelper.setStepType('CMSSW')
        stepHelper = cmsswHelper.addStep("DeleteTest")
        stepHelper.setStepType('StageOut')

        self.cmsswstep = cmsswHelper.data
        self.cmsswHelper = cmsswHelper

        self.stepdata = stepHelper.data
        self.stephelp = StageOutTemplate.StageOutStepHelper(stepHelper.data)
        self.task.applyTemplates()

        self.executor = StepFactory.getStepExecutor(self.stephelp.stepType())
        taskMaker = TaskMaker(self.workload, os.path.join(self.testDir))
        taskMaker.skipSubscription = True
        taskMaker.processWorkload()

        self.task.build(os.path.join(self.testDir, 'UnitTests'))

        sys.path.insert(0, self.testDir)
        sys.path.insert(0, os.path.join(self.testDir, 'UnitTests'))

        #        binDir = inspect.getsourcefile(ModuleLocator)
        #        binDir = binDir.replace("__init__.py", "bin")
        #
        #        if not binDir in os.environ['PATH']:
        #            os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
        open(os.path.join(self.testDir, 'UnitTests', '__init__.py'),
             'w').close()
        shutil.copyfile(
            os.path.join(os.path.dirname(__file__), 'MergeSuccess.pkl'),
            os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
                         'Report.pkl'))

    def tearDown(self):
        sys.path = self.oldpath[:]
        self.testInit.delWorkDir()

        # making double sure WMTaskSpace and WMSandbox are gone
        modsToDelete = []
        # not sure what happens if you delete from
        # an arrey you're iterating over. doing it in
        # two steps
        for modname in sys.modules.keys():
            # need to blow away things in sys.modules, otherwise
            # they are cached and we look at old taskspaces
            if modname.startswith('WMTaskSpace'):
                modsToDelete.append(modname)
            if modname.startswith('WMSandbox'):
                modsToDelete.append(modname)
        for modname in modsToDelete:
            try:
                reload(sys.modules[modname])
            except Exception:
                pass
            del sys.modules[modname]
        myThread = threading.currentThread()
        if hasattr(myThread, "factory"):
            myThread.factory = {}

    @attr('integration')
    def testCPBackendStageOutAgainstReportNew(self):
        myReport = Report()
        myReport.unpersist(
            os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
                         'Report.pkl'))
        myReport.data.cmsRun1.status = 0
        myReport.persist(
            os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
                         'Report.pkl'))
        executor = StageOutExecutor.StageOut()
        executor.initialise(self.stepdata, self.job)
        self.setLocalOverride(self.stepdata)
        self.stepdata.override.newStageOut = True
        executor.step = self.stepdata
        executor.execute()
        self.assertTrue(os.path.exists(os.path.join(self.testDir, 'hosts')))
        self.assertTrue(
            os.path.exists(os.path.join(self.testDir, 'test1', 'hosts')))

    @attr('integration')
    def testCPBackendStageOutAgainstReportFailedStepNew(self):
        myReport = Report()
        myReport.unpersist(
            os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
                         'Report.pkl'))
        myReport.data.cmsRun1.status = 1
        myReport.persist(
            os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
                         'Report.pkl'))
        executor = StageOutExecutor.StageOut()
        executor.initialise(self.stepdata, self.job)
        self.setLocalOverride(self.stepdata)
        self.stepdata.override.newStageOut = True
        executor.step = self.stepdata
        executor.execute()
        self.assertFalse(os.path.exists(os.path.join(self.testDir, 'hosts')))
        self.assertFalse(
            os.path.exists(os.path.join(self.testDir, 'test1', 'hosts')))
        return

    @attr('integration')
    def testCPBackendStageOutAgainstReportOld(self):

        myReport = Report()
        myReport.unpersist(
            os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
                         'Report.pkl'))
        myReport.data.cmsRun1.status = 0
        myReport.persist(
            os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
                         'Report.pkl'))
        executor = StageOutExecutor.StageOut()
        executor.initialise(self.stepdata, self.job)
        self.setLocalOverride(self.stepdata)
        executor.step = self.stepdata
        executor.execute()
        self.assertTrue(os.path.exists(os.path.join(self.testDir, 'hosts')))
        self.assertTrue(
            os.path.exists(os.path.join(self.testDir, 'test1', 'hosts')))
        return

    @attr('integration')
    def testCPBackendStageOutAgainstReportFailedStepOld(self):
        myReport = Report()
        myReport.unpersist(
            os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
                         'Report.pkl'))
        myReport.data.cmsRun1.status = 1
        myReport.persist(
            os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
                         'Report.pkl'))

        executor = StageOutExecutor.StageOut()
        executor.initialise(self.stepdata, self.job)
        self.setLocalOverride(self.stepdata)
        executor.step = self.stepdata
        executor.execute()
        self.assertFalse(os.path.exists(os.path.join(self.testDir, 'hosts')))
        self.assertFalse(
            os.path.exists(os.path.join(self.testDir, 'test1', 'hosts')))
        return

    @attr('workerNodeTest')
    def testOnWorkerNodes(self):
        raise RuntimeError
        # Stage a file out, stage it back in, check it, delete it
        myReport = Report()
        myReport.unpersist(
            os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
                         'Report.pkl'))
        myReport.data.cmsRun1.status = 1
        del myReport.data.cmsRun1.output
        myReport.data.cmsRun1.section_('output')
        myReport.data.cmsRun1.output.section_('stagingTestOutput')
        myReport.data.cmsRun1.output.stagingTestOutput.section_('files')
        myReport.data.cmsRun1.output.stagingTestOutput.fileCount = 0
        targetFiles = [
            '/store/temp/WMAgent/storetest-%s' % time.time(),
            '/store/unmerged/WMAgent/storetest-%s' % time.time()
        ]

        for file in targetFiles:
            print("Adding file for StageOut %s" % file)
            self.addStageOutFile(myReport, file)

        myReport.persist(
            os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
                         'Report.pkl'))
        executor = StageOutExecutor.StageOut()

        executor.initialise(self.stepdata, self.job)
        executor.step = self.stepdata
        print("beginning stageout")
        executor.execute()
        print("stageout done")

        # pull in the report with the stage out info
        myReport = Report()
        myReport.unpersist(
            os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
                         'Report.pkl'))
        print("Got the stage out data back")
        print(myReport.data)

        # now, transfer them back
        # TODO make a stagein step in the task - Melo
        import WMCore.Storage.FileManager as FileManagerModule
        fileManager = FileManagerModule.FileManager(numberOfRetries=10,
                                                    retryPauseTime=1)
        for file in targetFiles:
            print("Staging in %s" % file)

            fileManager.stageOut(fileToStage={
                'LFN': file,
                'PFN': '%s/%s' % (self.testDir, file)
            })
            self.assertTrue(os.path.exists('%s/%s' % (self.testDir, file)))
            # self.assertEqual(os.path.getsize('/etc/hosts', '%s/%s' % (self.testDir, file))) This makes no sense - EWV

        # now, should delete the files we made
        for file in targetFiles:
            print("deleting %s" % file)
            fileManager.deleteLFN(file)

        # try staging in again to make sure teh files are gone
        for file in targetFiles:
            print("Staging in (should fail) %s" % file)
            self.assertRaises(StageOutError,
                              FileManagerModule.FileManager.stageOut,
                              fileManager,
                              fileToStage={
                                  'LFN': file,
                                  'PFN': '%s/%s' % (self.testDir, file)
                              },
                              stageOut=False)

        # need to make sure files didn't show up
        self.assertFalse(os.path.exists(os.path.join(self.testDir, 'hosts')))
        self.assertFalse(
            os.path.exists(os.path.join(self.testDir, 'test1', 'hosts')))

    def addStageOutFile(self, myReport, lfn):
        myId = myReport.data.cmsRun1.output.stagingTestOutput.fileCount
        mySection = myReport.data.cmsRun1.output.stagingTestOutput.section_(
            'file%s' % myId)
        mySection.section_('runs')
        setattr(mySection.runs, '114475', [33])
        mySection.section_('branches')
        mySection.lfn = lfn
        mySection.dataset = {
            'applicationName': 'cmsRun',
            'primaryDataset': 'Calo',
            'processedDataset': 'Commissioning09-PromptReco-v8',
            'dataTier': 'ALCARECO',
            'applicationVersion': 'CMSSW_3_2_7'
        }
        mySection.module_label = 'ALCARECOStreamCombined'
        mySection.parents = []
        mySection.location = 'srm-cms.cern.ch'
        mySection.checksums = {'adler32': 'bbcf2215', 'cksum': '2297542074'}
        mySection.pfn = '/etc/hosts'
        mySection.events = 20000
        mySection.merged = False
        mySection.size = 37556367
        myReport.data.cmsRun1.output.stagingTestOutput.fileCount = myId + 1

    def setLocalOverride(self, step):
        step.section_('override')
        step.override.command = 'cp'
        step.override.option = ''
        step.override.__setattr__('lfn-prefix', self.testDir + "/")
        step.override.__setattr__('phedex-node', 'DUMMYPNN')
コード例 #31
0
ファイル: scaleTest.py プロジェクト: tsarangi/WMCore
class scaleTestFiller:
    """
    _scaleTestFiller_

    Initializes the DB and the DBSUploader
    On __call__() it creates data and uploads it.
    """
    def __init__(self):
        """
        __init__

        Init the DB
        """

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection(destroyAllDatabase=True)
        self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer"],
                                useDefault=False)
        self.configFile = EmulatorSetup.setupWMAgentConfig()

        myThread = threading.currentThread()
        self.bufferFactory = DAOFactory(
            package="WMComponent.DBSBuffer.Database",
            logger=myThread.logger,
            dbinterface=myThread.dbi)

        locationAction = self.bufferFactory(
            classname="DBSBufferFiles.AddLocation")
        locationAction.execute(siteName="se1.cern.ch")
        locationAction.execute(siteName="se1.fnal.gov")
        locationAction.execute(siteName="malpaquet")

        config = self.getConfig()
        self.dbsUploader = DBSUploadPoller(config=config)

        return

    def __call__(self):
        """
        __call__

        Generate some random data
        """

        # Generate somewhere between one and a thousand files
        name = "ThisIsATest_%s" % (makeUUID())
        nFiles = random.randint(10, 2000)
        name = name.replace('-', '_')
        name = '%s-v0' % name
        files = self.getFiles(name=name, nFiles=nFiles)

        print "Inserting %i files for dataset %s" % (nFiles * 2, name)

        try:
            self.dbsUploader.algorithm()
        except:
            self.dbsUploader.close()
            raise

        # Repeat just to make sure
        try:
            self.dbsUploader.algorithm()
        except:
            self.dbsUploader.close()
            raise

        return

    def getConfig(self):
        """
        _getConfig_

        This creates the actual config file used by the component

        """

        config = self.testInit.getConfiguration()
        self.testInit.generateWorkDir(config)

        #First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", os.getcwd())

        config.section_("Agent")
        config.Agent.componentName = 'DBSUpload'
        config.Agent.useHeartbeat = False

        #Now the CoreDatabase information
        #This should be the dialect, dburl, etc
        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket = os.getenv("DBSOCK")

        config.component_("DBS3Upload")
        config.DBS3Upload.pollInterval = 10
        config.DBS3Upload.logLevel = 'DEBUG'
        config.DBS3Upload.DBSBlockMaxFiles = 500
        config.DBS3Upload.DBSBlockMaxTime = 600
        config.DBS3Upload.DBSBlockMaxSize = 999999999999
        config.DBS3Upload.dbsUrl = 'http://cms-xen40.fnal.gov:8787/dbs/prod/global/DBSWriter'
        config.DBS3Upload.namespace = 'WMComponent.DBS3Buffer.DBSUpload'
        config.DBS3Upload.componentDir = os.path.join(os.getcwd(),
                                                      'Components')
        config.DBS3Upload.nProcesses = 1
        config.DBS3Upload.dbsWaitTime = 1

        return config

    def getFiles(self,
                 name,
                 tier='RECO',
                 nFiles=12,
                 site="malpaquet",
                 nLumis=1):
        """
        Create some quick dummy test files

        """

        files = []

        for f in range(nFiles):
            testFile = DBSBufferFile(
                lfn='/data/store/random/random/RANDOM/test/0/%s-%s-%i.root' %
                (name, site, f),
                size=1024,
                events=20,
                checksums={'cksum': 1})
            testFile.setAlgorithm(appName=name,
                                  appVer="CMSSW_3_1_1",
                                  appFam="RECO",
                                  psetHash="GIBBERISH",
                                  configContent="MOREGIBBERISH")
            testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier))
            lumis = []
            for i in range(nLumis):
                lumis.append((f * 100000) + i)
            testFile.addRun(Run(1, *lumis))
            testFile.setAcquisitionEra(name.split('-')[0])
            testFile.setProcessingVer("0")
            testFile.setGlobalTag("Weird")
            testFile.create()
            testFile.setLocation(site)
            files.append(testFile)

        count = 0
        for f in files:
            count += 1
            testFileChild = DBSBufferFile(
                lfn=
                '/data/store/random/random/RANDOM/test/0/%s-%s-%i-child.root' %
                (name, site, count),
                size=1024,
                events=10,
                checksums={'cksum': 1})
            testFileChild.setAlgorithm(appName=name,
                                       appVer="CMSSW_3_1_1",
                                       appFam="RECO",
                                       psetHash="GIBBERISH",
                                       configContent="MOREGIBBERISH")
            testFileChild.setDatasetPath("/%s/%s_2/RECO" % (name, name))
            testFileChild.addRun(Run(1, *[45]))
            testFileChild.create()
            testFileChild.setLocation(site)

            testFileChild.addParents([f['lfn']])

        return files
コード例 #32
0
ファイル: StageOut_t.py プロジェクト: vytjan/WMCore
class StageOutTest(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInit(__file__)
        self.testDir = self.testInit.generateWorkDir()

        # shut up SiteLocalConfig
        os.environ['CMS_PATH'] = os.getcwd()
        workload = copy.deepcopy(testWorkloads.workload)
        task = workload.getTask("Production")
        step = task.getStep("stageOut1")
        # want to get the cmsstep so I can make the Report
        cmsstep = task.getStep('cmsRun1')
        self.cmsstepdir = os.path.join(self.testDir, 'cmsRun1')
        os.mkdir(self.cmsstepdir)
        open(os.path.join(self.cmsstepdir, '__init__.py'), 'w').close()
        open(os.path.join(self.cmsstepdir, 'Report.pkl'), 'w').close()

        cmsbuilder = CMSSWBuilder.CMSSW()
        cmsbuilder(cmsstep.data, 'Production', self.cmsstepdir)
        realstep = StageOutTemplate.StageOutStepHelper(step.data)
        realstep.disableRetries()
        self.realstep = realstep
        self.stepDir = os.path.join(self.testDir, 'stepdir')
        os.mkdir(self.stepDir)
        builder = StageOutBuilder.StageOut()
        builder(step.data, 'Production', self.stepDir)

        # stolen from CMSSWExecutor_t. thanks, dave

        # first, delete all the sandboxen and taskspaces
        #    because of caching, this leaks from other tests in other files
        #    this sucks because the other tests are using sandboxen that
        #    are deleted after the test is over, which causes theses tests
        #    to break
        modsToDelete = []
        # not sure what happens if you delete from
        # an arrey you're iterating over. doing it in
        # two steps
        for modname in sys.modules.keys():
            # need to blow away things in sys.modules, otherwise
            # they are cached and we look at old taskspaces
            if modname.startswith('WMTaskSpace'):
                modsToDelete.append(modname)
            if modname.startswith('WMSandbox'):
                modsToDelete.append(modname)
        for modname in modsToDelete:
            try:
                reload(sys.modules[modname])
            except Exception:
                pass
            del sys.modules[modname]

        self.oldpath = sys.path[:]
        self.testInit = TestInit(__file__)

        self.testDir = self.testInit.generateWorkDir()
        self.job = Job(name="/UnitTests/DeleterTask/DeleteTest-test-job")
        shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))

        self.workload = newWorkload("UnitTests")
        self.task = self.workload.newTask("DeleterTask")

        cmsswHelper = self.task.makeStep("cmsRun1")
        cmsswHelper.setStepType('CMSSW')
        stepHelper = cmsswHelper.addStep("DeleteTest")
        stepHelper.setStepType('StageOut')

        self.cmsswstep = cmsswHelper.data
        self.cmsswHelper = cmsswHelper

        self.stepdata = stepHelper.data
        self.stephelp = StageOutTemplate.StageOutStepHelper(stepHelper.data)
        self.task.applyTemplates()

        self.executor = StepFactory.getStepExecutor(self.stephelp.stepType())
        taskMaker = TaskMaker(self.workload, os.path.join(self.testDir))
        taskMaker.skipSubscription = True
        taskMaker.processWorkload()

        self.task.build(os.path.join(self.testDir, 'UnitTests'))

        sys.path.insert(0, self.testDir)
        sys.path.insert(0, os.path.join(self.testDir, 'UnitTests'))

        #        binDir = inspect.getsourcefile(ModuleLocator)
        #        binDir = binDir.replace("__init__.py", "bin")
        #
        #        if not binDir in os.environ['PATH']:
        #            os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
        open(os.path.join(self.testDir, 'UnitTests', '__init__.py'),
             'w').close()
        shutil.copyfile(
            os.path.join(os.path.dirname(__file__), 'MergeSuccess.pkl'),
            os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
                         'Report.pkl'))

    def tearDown(self):
        sys.path = self.oldpath[:]
        self.testInit.delWorkDir()

        # making double sure WMTaskSpace and WMSandbox are gone
        modsToDelete = []
        # not sure what happens if you delete from
        # an arrey you're iterating over. doing it in
        # two steps
        for modname in sys.modules.keys():
            # need to blow away things in sys.modules, otherwise
            # they are cached and we look at old taskspaces
            if modname.startswith('WMTaskSpace'):
                modsToDelete.append(modname)
            if modname.startswith('WMSandbox'):
                modsToDelete.append(modname)
        for modname in modsToDelete:
            try:
                reload(sys.modules[modname])
            except Exception:
                pass
            del sys.modules[modname]
        myThread = threading.currentThread()
        if hasattr(myThread, "factory"):
            myThread.factory = {}

    def makeReport(self, fileName):
        myReport = Report('oneitem')
        myReport.addStep('stageOut1')
        myReport.addOutputModule('module1')
        myReport.addOutputModule('module2')
        myReport.addOutputFile('module1', {
            'lfn': 'FILE1',
            'size': 1,
            'events': 1
        })
        myReport.addOutputFile('module2', {
            'lfn': 'FILE2',
            'size': 1,
            'events': 1
        })
        myReport.addOutputFile('module2', {
            'lfn': 'FILE3',
            'size': 1,
            'events': 1
        })
        myReport.persist(fileName)

    def testExecutorDoesntDetonate(self):
        myReport = Report()
        myReport.unpersist(
            os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
                         'Report.pkl'))
        myReport.data.cmsRun1.status = 1
        myReport.persist(
            os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
                         'Report.pkl'))

        executor = StageOutExecutor.StageOut()

        executor.initialise(self.stepdata, self.job)
        self.setLocalOverride(self.stepdata)
        executor.step = self.stepdata
        executor.execute()
        self.assertFalse(os.path.exists(os.path.join(self.testDir, 'hosts')))
        self.assertFalse(
            os.path.exists(os.path.join(self.testDir, 'test1', 'hosts')))
        return

    def testUnitTestBackend(self):
        myReport = Report()
        myReport.unpersist(
            os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
                         'Report.pkl'))
        myReport.data.cmsRun1.status = 1
        myReport.persist(
            os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
                         'Report.pkl'))

        executor = StageOutExecutor.StageOut()
        helper = StageOutTemplate.StageOutStepHelper(self.stepdata)
        helper.addOverride(override='command', overrideValue='test-win')
        helper.addOverride(override='option', overrideValue='')
        helper.addOverride(override='phedex-node',
                           overrideValue='charlie.sheen.biz')
        helper.addOverride(override='lfn-prefix', overrideValue='test-win')

        executor.initialise(self.stepdata, self.job)
        self.setLocalOverride(self.stepdata)
        executor.step = self.stepdata
        executor.execute()
        self.assertFalse(os.path.exists(os.path.join(self.testDir, 'hosts')))
        self.assertFalse(
            os.path.exists(os.path.join(self.testDir, 'test1', 'hosts')))

    def testUnitTestBackendNew(self):
        myReport = Report()
        myReport.unpersist(
            os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
                         'Report.pkl'))
        myReport.data.cmsRun1.status = 1
        myReport.persist(
            os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
                         'Report.pkl'))

        executor = StageOutExecutor.StageOut()
        helper = StageOutTemplate.StageOutStepHelper(self.stepdata)
        helper.addOverride(override='command', overrideValue='test-win')
        helper.addOverride(override='option', overrideValue='')
        helper.addOverride(override='phedex-node',
                           overrideValue='charlie.sheen.biz')
        helper.addOverride(override='lfn-prefix', overrideValue='test-win')
        helper.setNewStageoutOverride(True)

        executor.initialise(self.stepdata, self.job)
        self.setLocalOverride(self.stepdata)
        executor.step = self.stepdata
        executor.execute()
        self.assertFalse(os.path.exists(os.path.join(self.testDir, 'hosts')))
        self.assertFalse(
            os.path.exists(os.path.join(self.testDir, 'test1', 'hosts')))

    def setLocalOverride(self, step):
        step.section_('override')
        step.override.command = 'cp'
        step.override.option = ''
        step.override.__setattr__('lfn-prefix', self.testDir + "/")
        step.override.__setattr__('phedex-node', 'DUMMYPNN')
コード例 #33
0
ファイル: API_t.py プロジェクト: AndrewLevin/WMCore
class APITest(unittest.TestCase):
    def setUp(self):
        """
        This much stuff for simple alerts client API testing is
        needed because it's also testing setting up alert fw as
        done in the BaseWorkerThread.

        """
        myThread = threading.currentThread()
        self.testInit = TestInit(__file__)
        self.testInit.setLogging(logLevel = logging.DEBUG)
        self.testDir = self.testInit.generateWorkDir()
        # needed for instantiating BaseWorkerThread
        self.testInit.setDatabaseConnection()
        self.alertsReceiver = None


    def tearDown(self):
        self.testInit.clearDatabase()
        self.testInit.delWorkDir()
        if self.alertsReceiver:
            self.alertsReceiver.shutdown()


    def testAlertsMessagingBasic(self):
        config = getConfig("/tmp")
        self.assertTrue(hasattr(config, "Alert"))
        # initialization
        # sender: instance of Alert messages Sender
        # preAlert: pre-defined values for Alert instances generated from this class
        self.config = config # needed in setUpAlertsMessaging
        preAlert, sender = alertAPI.setUpAlertsMessaging(self,
                                                         compName = "testBasic")
        sendAlert = alertAPI.getSendAlert(sender = sender,
                                          preAlert = preAlert)

        # set up a temporary alert message receiver
        handler, receiver = utils.setUpReceiver(config.Alert.address,
                                                config.Alert.controlAddr)
        # test sending alert
        msg = "this is my message Basic"
        sendAlert(100, msg = msg)

        # wait for the alert to arrive
        while len(handler.queue) == 0:
            time.sleep(0.3)
            print "%s waiting for alert to arrive ..." % inspect.stack()[0][3]

        self.assertEqual(len(handler.queue), 1)
        alert = handler.queue[0]
        self.assertEqual(alert["Component"], "testBasic")
        self.assertEqual(alert["Level"], 100)
        self.assertEqual(alert["Source"], self.__class__.__name__)
        self.assertEqual(alert["Details"]["msg"], msg)

        sender.unregister()
        receiver.shutdown()


    def testAlertsMessagingNotSetUpViaBaseWorkerThread(self):
        # alerts will not be set up if 'config.Alert' etc is not provided
        config = Configuration()
        self.assertFalse(hasattr(config, "Alert"))
        # test the same way alerts are set up in the client code (currently
        # all cases via BaseWorkerThread)
        # this call creates .sender, but here will be set to None
        thread = BaseWorkerThread()
        thread.config = config
        thread.initAlerts(compName = "test1")
        self.assertFalse(thread.sender)
        # shall do nothing and not fail
        thread.sendAlert("nonsense", msg = "nonsense")


    def testAlertsSetUpAndSendingViaBaseWorkerThread(self):
        # calls as they are made from child/client classes of BaseWorkerThread
        config = getConfig("/tmp")
        self.assertTrue(hasattr(config, "Alert"))
        # test the same way alerts are set up in the client code (currently
        # all cases via BaseWorkerThread)
        # this call creates .sender, but here will be set to None
        thread = BaseWorkerThread()
        thread.config = config
        thread.initAlerts(compName = "test2")
        self.assertTrue(thread.sender)

        # set up a temporary alert message receiver
        handler, receiver = utils.setUpReceiver(config.Alert.address,
                                                config.Alert.controlAddr)

        # send an alert message
        msg = "this is my message 1"
        thread.sendAlert(10, msg = msg)

        # wait for the alert to arrive
        while len(handler.queue) == 0:
            time.sleep(0.3)
            print "%s waiting for alert to arrive ..." % inspect.stack()[0][3]

        self.assertEqual(len(handler.queue), 1)
        alert = handler.queue[0]
        self.assertEqual(alert["Component"], "test2")
        self.assertEqual(alert["Level"], 10)
        self.assertEqual(alert["Source"], thread.__class__.__name__)
        self.assertEqual(alert["Details"]["msg"], msg)

        thread.sender.unregister()
        receiver.shutdown()


    def testAgentConfigurationRetrieving(self):
        """
        Test that getting some agent details (config values from config.Agent
        section) will be correctly propagated into Alert instances.
        Alert instance is obtained via API.getPredefinedAlert factory.

        """
        d = dict(Additional = "detail")
        # instantiate just plain Alert, no configuration to take
        # into account at this point
        a = Alert(**d)
        self.assertEqual(a["HostName"], None)
        self.assertEqual(a["Contact"], None)
        self.assertEqual(a["TeamName"], None)
        self.assertEqual(a["AgentName"], None)
        self.assertEqual(a["Additional"], "detail")
        # instantiate via factory which reads configuration instance
        config = Configuration()
        config.section_("Agent")
        config.Agent.hostName = "some1"
        config.Agent.contact = "some2"
        config.Agent.teamName = "some3"
        config.Agent.agentName = "some4"
        a = alertAPI.getPredefinedAlert(**d)
        self.assertEqual(a["HostName"], "some1")
        self.assertEqual(a["Contact"], "some2")
        self.assertEqual(a["TeamName"], "some3")
        self.assertEqual(a["AgentName"], "some4")
        self.assertEqual(a["Additional"], "detail")
コード例 #34
0
ファイル: DataProcessing_t.py プロジェクト: cinquo/WMCore
class DataProcessingTest(unittest.TestCase):
    def setUp(self):
        """
        _setUp_

        Initialize the database.
        """
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = ["WMCore.WMBS"],
                                useDefault = False)
        self.testDir = self.testInit.generateWorkDir()
        return

    def tearDown(self):
        """
        _tearDown_

        Clear out the database.
        """
        self.testInit.clearDatabase()
        self.testInit.delWorkDir()
        return

    def testDataProcessing(self):
        """
        _testDataProcessing_

        Create a data processing workflow and verify it installs into WMBS
        correctly. Check that we can drop an output module.
        """
        testArgs = getTestArguments()
        testArgs['TransientOutputModules'] = ['RECOoutput']
        testWorkload = dataProcessingWorkload("TestWorkload", testArgs)
        testWorkload.setSpecUrl("somespec")
        testWorkload.setOwnerDetails("*****@*****.**", "DMWM")

        testWMBSHelper = WMBSHelper(testWorkload, "DataProcessing", "SomeBlock", cachepath = self.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        procWorkflow = Workflow(name = "TestWorkload",
                                task = "/TestWorkload/DataProcessing")
        procWorkflow.load()

        self.assertEqual(len(procWorkflow.outputMap.keys()), 3,
                         "Error: Wrong number of WF outputs.")

        goldenOutputMods = ["RECOoutput", "ALCARECOoutput"]
        for goldenOutputMod in goldenOutputMods:
            mergedOutput = procWorkflow.outputMap[goldenOutputMod][0]["merged_output_fileset"]
            unmergedOutput = procWorkflow.outputMap[goldenOutputMod][0]["output_fileset"]

            mergedOutput.loadData()
            unmergedOutput.loadData()

            if goldenOutputMod in testArgs["TransientOutputModules"]:
                self.assertEqual(mergedOutput.name, "/TestWorkload/DataProcessing/unmerged-%s" % goldenOutputMod,
                                 "Error: Merged output fileset is wrong: %s" % mergedOutput.name)
            else:
                self.assertEqual(mergedOutput.name, "/TestWorkload/DataProcessing/DataProcessingMerge%s/merged-Merged" % goldenOutputMod,
                                 "Error: Merged output fileset is wrong: %s" % mergedOutput.name)
            self.assertEqual(unmergedOutput.name, "/TestWorkload/DataProcessing/unmerged-%s" % goldenOutputMod,
                             "Error: Unmerged output fileset is wrong.")

        logArchOutput = procWorkflow.outputMap["logArchive"][0]["merged_output_fileset"]
        unmergedLogArchOutput = procWorkflow.outputMap["logArchive"][0]["output_fileset"]
        logArchOutput.loadData()
        unmergedLogArchOutput.loadData()

        self.assertEqual(logArchOutput.name, "/TestWorkload/DataProcessing/unmerged-logArchive",
                         "Error: LogArchive output fileset is wrong.")
        self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/DataProcessing/unmerged-logArchive",
                         "Error: LogArchive output fileset is wrong.")

        for goldenOutputMod in goldenOutputMods:
            if goldenOutputMod in testArgs["TransientOutputModules"]:
                # No merge for this output module
                continue
            mergeWorkflow = Workflow(name = "TestWorkload",
                                     task = "/TestWorkload/DataProcessing/DataProcessingMerge%s" % goldenOutputMod)
            mergeWorkflow.load()

            self.assertEqual(len(mergeWorkflow.outputMap.keys()), 2,
                             "Error: Wrong number of WF outputs.")

            mergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["merged_output_fileset"]
            unmergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["output_fileset"]

            mergedMergeOutput.loadData()
            unmergedMergeOutput.loadData()

            self.assertEqual(mergedMergeOutput.name, "/TestWorkload/DataProcessing/DataProcessingMerge%s/merged-Merged" % goldenOutputMod,
                             "Error: Merged output fileset is wrong.")
            self.assertEqual(unmergedMergeOutput.name, "/TestWorkload/DataProcessing/DataProcessingMerge%s/merged-Merged" % goldenOutputMod,
                             "Error: Unmerged output fileset is wrong.")

            logArchOutput = mergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"]
            unmergedLogArchOutput = mergeWorkflow.outputMap["logArchive"][0]["output_fileset"]
            logArchOutput.loadData()
            unmergedLogArchOutput.loadData()

            self.assertEqual(logArchOutput.name, "/TestWorkload/DataProcessing/DataProcessingMerge%s/merged-logArchive" % goldenOutputMod,
                             "Error: LogArchive output fileset is wrong: %s" % logArchOutput.name)
            self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/DataProcessing/DataProcessingMerge%s/merged-logArchive" % goldenOutputMod,
                             "Error: LogArchive output fileset is wrong.")

        topLevelFileset = Fileset(name = "TestWorkload-DataProcessing-SomeBlock")
        topLevelFileset.loadData()

        procSubscription = Subscription(fileset = topLevelFileset, workflow = procWorkflow)
        procSubscription.loadData()

        self.assertEqual(procSubscription["type"], "Processing",
                         "Error: Wrong subscription type.")
        self.assertEqual(procSubscription["split_algo"], "LumiBased",
                         "Error: Wrong split algo.")

        unmergedReco = Fileset(name = "/TestWorkload/DataProcessing/unmerged-RECOoutput")
        unmergedReco.loadData()
        recoMergeWorkflow = Workflow(name = "TestWorkload",
                                     task = "/TestWorkload/DataProcessing/DataProcessingMergeRECOoutput")
        # No merge workflow should exist in WMBS
        self.assertRaises(IndexError, recoMergeWorkflow.load)

        unmergedAlca = Fileset(name = "/TestWorkload/DataProcessing/unmerged-ALCARECOoutput")
        unmergedAlca.loadData()
        alcaMergeWorkflow = Workflow(name = "TestWorkload",
                                     task = "/TestWorkload/DataProcessing/DataProcessingMergeALCARECOoutput")
        alcaMergeWorkflow.load()
        mergeSubscription = Subscription(fileset = unmergedAlca, workflow = alcaMergeWorkflow)
        mergeSubscription.loadData()

        self.assertEqual(mergeSubscription["type"], "Merge",
                         "Error: Wrong subscription type.")
        self.assertEqual(mergeSubscription["split_algo"], "ParentlessMergeBySize",
                         "Error: Wrong split algo.")

        for procOutput in ["RECOoutput", "ALCARECOoutput"]:
            unmerged = Fileset(name = "/TestWorkload/DataProcessing/unmerged-%s" % procOutput)
            unmerged.loadData()
            cleanupWorkflow = Workflow(name = "TestWorkload",
                                      task = "/TestWorkload/DataProcessing/DataProcessingCleanupUnmerged%s" % procOutput)
            cleanupWorkflow.load()
            cleanupSubscription = Subscription(fileset = unmerged, workflow = cleanupWorkflow)
            cleanupSubscription.loadData()

            self.assertEqual(cleanupSubscription["type"], "Cleanup",
                             "Error: Wrong subscription type.")
            self.assertEqual(cleanupSubscription["split_algo"], "SiblingProcessingBased",
                             "Error: Wrong split algo.")

        procLogCollect = Fileset(name = "/TestWorkload/DataProcessing/unmerged-logArchive")
        procLogCollect.loadData()
        procLogCollectWorkflow = Workflow(name = "TestWorkload",
                                          task = "/TestWorkload/DataProcessing/LogCollect")
        procLogCollectWorkflow.load()
        logCollectSub = Subscription(fileset = procLogCollect, workflow = procLogCollectWorkflow)
        logCollectSub.loadData()

        self.assertEqual(logCollectSub["type"], "LogCollect",
                         "Error: Wrong subscription type.")
        self.assertEqual(logCollectSub["split_algo"], "MinFileBased",
                         "Error: Wrong split algo.")

        procLogCollect = Fileset(name = "/TestWorkload/DataProcessing/DataProcessingMergeALCARECOoutput/merged-logArchive")
        procLogCollect.loadData()
        procLogCollectWorkflow = Workflow(name = "TestWorkload",
                                          task = "/TestWorkload/DataProcessing/DataProcessingMergeALCARECOoutput/DataProcessingALCARECOoutputMergeLogCollect")
        procLogCollectWorkflow.load()
        logCollectSub = Subscription(fileset = procLogCollect, workflow = procLogCollectWorkflow)
        logCollectSub.loadData()

        self.assertEqual(logCollectSub["type"], "LogCollect",
                         "Error: Wrong subscription type.")
        self.assertEqual(logCollectSub["split_algo"], "MinFileBased",
                         "Error: Wrong split algo.")

        return
コード例 #35
0
ファイル: JobPackage_t.py プロジェクト: vytjan/WMCore
class JobPackageTest(unittest.TestCase):
    def setUp(self):
        """
        _setUp_

        Create a temporary file to presist the JobPackage to.
        """
        self.testInit = TestInit(__file__)
        self.persistFile = os.path.join(self.testInit.generateWorkDir(),
                                        "JobPackage.pkl")
        return

    def tearDown(self):
        pass

    def testAddingJobs(self):
        """
        _testAddingJobs_

        Verify that adding jobs to the package works as expected.
        """
        package = JobPackage()

        for i in range(100):
            newJob = Job("Job%s" % i)
            newJob["id"] = i
            package[i] = newJob

        # There is an extra key for the directory the package is stored in.
        assert len(package.keys()) == 101, \
               "Error: Wrong number of jobs in package."

        for i in range(100):
            job = package[i]
            assert job["id"] == i, \
                   "Error: Jobs has wrong ID."
            assert job["name"] == "Job%d" % i, \
                   "Error: Job has wrong name."

        return

    def testPersist(self):
        """
        _testPersist_

        Verify that we're able to save and load the job package.
        """
        package = JobPackage()

        for i in range(100):
            newJob = Job("Job%s" % i)
            newJob["id"] = i
            package[i] = newJob

        package.save(self.persistFile)

        assert os.path.exists(self.persistFile), \
               "Error: Package file was never created."

        newPackage = JobPackage()
        newPackage.load(self.persistFile)

        # There is an extra key for the directory the package is stored in.
        assert len(newPackage.keys()) == 101, \
               "Error: Wrong number of jobs in package."

        for i in range(100):
            job = newPackage[i]
            assert job["id"] == i, \
                   "Error: Jobs has wrong ID."
            assert job["name"] == "Job%d" % i, \
                   "Error: Job has wrong name."

        return

    def testBaggage(self):
        """
        _testBaggage_

        Verify that job baggage is persisted with the package.
        """
        package = JobPackage()

        for i in range(100):
            newJob = Job("Job%s" % i)
            newJob["id"] = i
            baggage = newJob.getBaggage()
            setattr(baggage, "thisJob", newJob["name"])
            setattr(baggage, "seed1", 11111111)
            setattr(baggage, "seed2", 22222222)
            setattr(baggage, "seed3", 33333333)
            setattr(baggage, "seed4", 44444444)
            setattr(baggage, "seed5", 55555555)
            package[i] = newJob

        package.save(self.persistFile)

        assert os.path.exists(self.persistFile), \
               "Error: Package file was never created."

        newPackage = JobPackage()
        newPackage.load(self.persistFile)

        # There is an extra key for the directory the package is stored in.
        assert len(newPackage.keys()) == 101, \
               "Error: Wrong number of jobs in package."

        for i in range(100):
            job = newPackage[i]
            assert job["id"] == i, \
                   "Error: Jobs has wrong ID."
            assert job["name"] == "Job%d" % i, \
                   "Error: Job has wrong name."
            jobBaggage = job.getBaggage()

            assert jobBaggage.thisJob == "Job%d" % i, \
                   "Error: Job baggage has wrong name."
            assert jobBaggage.seed1 == 11111111, \
                   "Error: Job baggee has wrong value for seed1."
            assert jobBaggage.seed2 == 22222222, \
                   "Error: Job baggee has wrong value for seed2."
            assert jobBaggage.seed3 == 33333333, \
                   "Error: Job baggee has wrong value for seed3."
            assert jobBaggage.seed4 == 44444444, \
                   "Error: Job baggee has wrong value for seed4."
            assert jobBaggage.seed5 == 55555555, \
                   "Error: Job baggee has wrong value for seed5."

        return
コード例 #36
0
class CouchTest(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInit(__file__)
        self.testInit.setLogging(logLevel = logging.DEBUG)
        self.testDir = self.testInit.generateWorkDir()
        self.config = getConfig(self.testDir)
        # mock generator instance to communicate some configuration values
        self.generator = utils.AlertGeneratorMock(self.config)
        self.testName = self.id().split('.')[-1]


    def tearDown(self):
        self.testInit.delWorkDir()
        self.generator = None


    def testAlertGeneratorCouchDbSizePollerBasic(self):
        config = getConfig("/tmp")
        try:
            poller = CouchDbSizePoller(config.AlertGenerator.couchDbSizePoller, self.generator)
        except Exception as ex:
            self.fail("%s: exception: %s" % (self.testName, ex))
        poller.check() # -> on real system dir may result in permission denied
        poller._dbDirectory = "/dev"
        poller.check() # -> OK

        # test failing during set up
        poller = CouchDbSizePoller(config.AlertGenerator.couchDbSizePoller, self.generator)
        poller._query = "nonsense query"
        # this will fail on the above query
        self.assertRaises(Exception, poller._getDbDir)
        poller.check()


    def testAlertGeneratorCouchDbSizePollerSoftThreshold(self):
        self.config.AlertGenerator.couchDbSizePoller.couchURL = os.getenv("COUCHURL", None)
        self.config.AlertGenerator.couchDbSizePoller.soft = 5
        self.config.AlertGenerator.couchDbSizePoller.critical = 10
        self.config.AlertGenerator.couchDbSizePoller.pollInterval = 0.2
        ti = utils.TestInput() # see attributes comments at the class
        ti.pollerClass = CouchDbSizePoller
        ti.config = self.config.AlertGenerator.couchDbSizePoller
        ti.thresholdToTest = self.config.AlertGenerator.couchDbSizePoller.soft
        ti.level = self.config.AlertProcessor.soft.level
        ti.expected = 1
        ti.thresholdDiff = 4
        ti.testCase = self
        utils.doGenericValueBasedPolling(ti)


    def testAlertGeneratorCouchDbSizePollerCriticalThreshold(self):
        self.config.AlertGenerator.couchDbSizePoller.couchURL = os.getenv("COUCHURL", None)
        self.config.AlertGenerator.couchDbSizePoller.soft = 5
        self.config.AlertGenerator.couchDbSizePoller.critical = 10
        self.config.AlertGenerator.couchDbSizePoller.pollInterval = 0.2
        ti = utils.TestInput() # see attributes comments at the class
        ti.pollerClass = CouchDbSizePoller
        ti.config = self.config.AlertGenerator.couchDbSizePoller
        ti.thresholdToTest = self.config.AlertGenerator.couchDbSizePoller.critical
        ti.level = self.config.AlertProcessor.critical.level
        ti.expected = 1
        ti.thresholdDiff = 1
        ti.testCase = self
        utils.doGenericValueBasedPolling(ti)


    def testAlertGeneratorCouchDbSizePollerNoAlert(self):
        self.config.AlertGenerator.couchDbSizePoller.couchURL = os.getenv("COUCHURL", None)
        self.config.AlertGenerator.couchDbSizePoller.soft = 5
        self.config.AlertGenerator.couchDbSizePoller.critical = 10
        self.config.AlertGenerator.couchDbSizePoller.pollInterval = 0.2
        ti = utils.TestInput() # see attributes comments at the class
        ti.pollerClass = CouchDbSizePoller
        ti.config = self.config.AlertGenerator.couchDbSizePoller
        ti.thresholdToTest = self.config.AlertGenerator.couchDbSizePoller.soft - 3
        ti.expected = 0
        ti.thresholdDiff = 2
        ti.testCase = self
        utils.doGenericValueBasedPolling(ti)


    def testAlertGeneratorCouchPollerBasic(self):
        self.config.AlertGenerator.section_("bogusCouchPoller")
        self.config.AlertGenerator.bogusCouchPoller.couchURL = os.getenv("COUCHURL", None)
        self.config.AlertGenerator.bogusCouchPoller.soft = 1000
        self.config.AlertGenerator.bogusCouchPoller.critical = 2000
        self.config.AlertGenerator.bogusCouchPoller.pollInterval = 0.2
        self.config.AlertGenerator.bogusCouchPoller.period = 1
        try:
            poller = CouchPoller(self.config.AlertGenerator.bogusCouchPoller,
                                 self.generator)
        except Exception as ex:
            self.fail("%s: exception: %s" % (self.testName, ex))
        # this class would not have defined polling sample function, give it one
        poller.sample = lambda proc: float(12)
        poller.check()
        # assuming CouchDb server is running, check that 1 sensible measurement value was collected
        self.assertEqual(len(poller._measurements), 1)
        self.assertTrue(isinstance(poller._measurements[0], types.FloatType))
        # test handling of a non-existing process
        CouchPoller._getProcessPID = lambda inst: 1212121212
        self.assertRaises(Exception, CouchPoller,
                          self.config.AlertGenerator.bogusCouchPoller, self.generator)


    def testAlertGeneratorCouchCPUPollerSoftThreshold(self):
        self.config.AlertGenerator.couchCPUPoller.couchURL = os.getenv("COUCHURL", None)
        self.config.AlertGenerator.couchCPUPoller.soft = 70
        self.config.AlertGenerator.couchCPUPoller.critical = 80
        self.config.AlertGenerator.couchCPUPoller.pollInterval = 0.2
        self.config.AlertGenerator.couchCPUPoller.period = 1
        ti = utils.TestInput() # see attributes comments at the class
        ti.pollerClass = CouchCPUPoller
        ti.config = self.config.AlertGenerator.couchCPUPoller
        ti.thresholdToTest = self.config.AlertGenerator.couchCPUPoller.soft
        ti.level = self.config.AlertProcessor.soft.level
        ti.expected = 1
        ti.thresholdDiff = 10
        ti.testCase = self
        utils.doGenericPeriodAndProcessPolling(ti)


    def testAlertGeneratorCouchCPUPollerCriticalThreshold(self):
        self.config.AlertGenerator.couchCPUPoller.couchURL = os.getenv("COUCHURL", None)
        self.config.AlertGenerator.couchCPUPoller.soft = 70
        self.config.AlertGenerator.couchCPUPoller.critical = 80
        self.config.AlertGenerator.couchCPUPoller.pollInterval = 0.2
        self.config.AlertGenerator.couchCPUPoller.period = 1
        ti = utils.TestInput() # see attributes comments at the class
        ti.pollerClass = CouchCPUPoller
        ti.config = self.config.AlertGenerator.couchCPUPoller
        ti.thresholdToTest = self.config.AlertGenerator.couchCPUPoller.critical
        ti.level = self.config.AlertProcessor.critical.level
        ti.expected = 1
        ti.thresholdDiff = 10
        ti.testCase = self
        utils.doGenericPeriodAndProcessPolling(ti)


    def testAlertGeneratorCouchCPUPollerNoAlert(self):
        self.config.AlertGenerator.couchCPUPoller.couchURL = os.getenv("COUCHURL", None)
        self.config.AlertGenerator.couchCPUPoller.soft = 70
        self.config.AlertGenerator.couchCPUPoller.critical = 80
        self.config.AlertGenerator.couchCPUPoller.pollInterval = 0.2
        self.config.AlertGenerator.couchCPUPoller.period = 1
        ti = utils.TestInput() # see attributes comments at the class
        ti.pollerClass = CouchCPUPoller
        ti.config = self.config.AlertGenerator.couchCPUPoller
        # lower the threshold so that the alert is never generated
        ti.thresholdToTest = self.config.AlertGenerator.couchCPUPoller.soft - 20
        ti.level = 0
        ti.expected = 0
        ti.thresholdDiff = 10
        ti.testCase = self
        utils.doGenericPeriodAndProcessPolling(ti)


    def testAlertGeneratorCouchMemoryPollerSoftThreshold(self):
        self.config.AlertGenerator.couchMemPoller.couchURL = os.getenv("COUCHURL", None)
        self.config.AlertGenerator.couchMemPoller.soft = 70
        self.config.AlertGenerator.couchMemPoller.critical = 80
        self.config.AlertGenerator.couchMemPoller.pollInterval = 0.2
        self.config.AlertGenerator.couchMemPoller.period = 1
        ti = utils.TestInput() # see attributes comments at the class
        ti.pollerClass = CouchMemoryPoller
        ti.config = self.config.AlertGenerator.couchMemPoller
        ti.thresholdToTest = self.config.AlertGenerator.couchMemPoller.soft
        ti.level = self.config.AlertProcessor.soft.level
        ti.expected = 1
        ti.thresholdDiff = 10
        ti.testCase = self
        utils.doGenericPeriodAndProcessPolling(ti)


    def testAlertGeneratorCouchMemoryPollerCriticalThreshold(self):
        self.config.AlertGenerator.couchMemPoller.couchURL = os.getenv("COUCHURL", None)
        self.config.AlertGenerator.couchMemPoller.soft = 70
        self.config.AlertGenerator.couchMemPoller.critical = 80
        self.config.AlertGenerator.couchMemPoller.pollInterval = 0.2
        self.config.AlertGenerator.couchMemPoller.period = 1
        ti = utils.TestInput() # see attributes comments at the class
        ti.pollerClass = CouchMemoryPoller
        ti.config = self.config.AlertGenerator.couchMemPoller
        ti.thresholdToTest = self.config.AlertGenerator.couchMemPoller.critical
        ti.level = self.config.AlertProcessor.critical.level
        ti.expected = 1
        ti.thresholdDiff = 10
        ti.testCase = self
        utils.doGenericPeriodAndProcessPolling(ti)


    def testAlertGeneratorCouchMemoryPollerNoAlert(self):
        self.config.AlertGenerator.couchMemPoller.couchURL = os.getenv("COUCHURL", None)
        self.config.AlertGenerator.couchMemPoller.soft = 70
        self.config.AlertGenerator.couchMemPoller.critical = 80
        self.config.AlertGenerator.couchMemPoller.pollInterval = 0.2
        self.config.AlertGenerator.couchMemPoller.period = 1
        ti = utils.TestInput() # see attributes comments at the class
        ti.pollerClass = CouchMemoryPoller
        ti.config = self.config.AlertGenerator.couchMemPoller
        # lower the threshold so that the alert is never generated
        ti.thresholdToTest = self.config.AlertGenerator.couchMemPoller.soft - 20
        ti.level = 0
        ti.expected = 0
        ti.thresholdDiff = 10
        ti.testCase = self
        utils.doGenericPeriodAndProcessPolling(ti)


    def testAlertGeneratorCouchErrorsPollerBasic(self):
        try:
            poller = CouchErrorsPoller(self.config.AlertGenerator.couchErrorsPoller, self.generator)
        except Exception as ex:
            self.fail("Exception, reason: %s" % ex)

        # even a single values to observe shall be turned into particular iterables
        obs = self.config.AlertGenerator.couchErrorsPoller.observables
        self.config.AlertGenerator.couchErrorsPoller.observables = 400
        try:
            poller = CouchErrorsPoller(self.config.AlertGenerator.couchErrorsPoller, self.generator)
        except Exception as ex:
            self.fail("Exception, reason: %s" % ex)
        #self.assertTrue(isinstance(obs, (types.ListType, types.TupleType)))
        self.assertTrue(isinstance(self.config.AlertGenerator.couchErrorsPoller.observables,
                                   (types.ListType, types.TupleType)))

        # test return value on non-sense HTTP status code
        res = poller.sample("40000")
        self.assertFalse(res)
        # test definitely existing value
        res = poller.sample("200")
        # on a freshly started couch, this status code may not exist in the
        # stats table, so despite correct and meaningful HTTP status code, the
        # query may still return None, hence don't assume any particular response
        #self.assertTrue(isinstance(res, types.IntType))
        poller.check()


    def testAlertGeneratorCouchErrorsPollerSoftThreshold(self):
        self.config.AlertGenerator.couchErrorsPoller.couchURL = os.getenv("COUCHURL", None)
        self.config.AlertGenerator.couchErrorsPoller.soft = 100
        self.config.AlertGenerator.couchErrorsPoller.critical = 200
        # shall expect corresponding number of generated alerts for each observable value
        self.config.AlertGenerator.couchErrorsPoller.observables = (5, 6, 7)
        self.config.AlertGenerator.couchErrorsPoller.pollInterval = 0.2
        ti = utils.TestInput() # see attributes comments at the class
        ti.pollerClass = CouchErrorsPoller
        ti.config = self.config.AlertGenerator.couchErrorsPoller
        ti.thresholdToTest = self.config.AlertGenerator.couchErrorsPoller.soft
        ti.level = self.config.AlertProcessor.soft.level
        ti.expected = len(self.config.AlertGenerator.couchErrorsPoller.observables)
        ti.thresholdDiff = 10
        ti.testCase = self
        utils.doGenericValueBasedPolling(ti)


    def testAlertGeneratorCouchErrorsPollerCriticalThreshold(self):
        self.config.AlertGenerator.couchErrorsPoller.couchURL = os.getenv("COUCHURL", None)
        self.config.AlertGenerator.couchErrorsPoller.soft = 100
        self.config.AlertGenerator.couchErrorsPoller.critical = 200
        # shall expect corresponding number of generated alerts for each observable value
        self.config.AlertGenerator.couchErrorsPoller.observables = (5, 6, 7)
        self.config.AlertGenerator.couchErrorsPoller.pollInterval = 0.2
        ti = utils.TestInput() # see attributes comments at the class
        ti.pollerClass = CouchErrorsPoller
        ti.config = self.config.AlertGenerator.couchErrorsPoller
        ti.thresholdToTest = self.config.AlertGenerator.couchErrorsPoller.critical
        ti.level = self.config.AlertProcessor.critical.level
        ti.expected = len(self.config.AlertGenerator.couchErrorsPoller.observables)
        ti.thresholdDiff = 50
        ti.testCase = self
        utils.doGenericValueBasedPolling(ti)


    def testAlertGeneratorCouchErrorsPollerNoAlert(self):
        self.config.AlertGenerator.couchErrorsPoller.couchURL = os.getenv("COUCHURL", None)
        self.config.AlertGenerator.couchErrorsPoller.soft = 100
        self.config.AlertGenerator.couchErrorsPoller.critical = 200
        # shall expect corresponding number of generated alerts for each observable value
        self.config.AlertGenerator.couchErrorsPoller.observables = (5, 6, 7)
        self.config.AlertGenerator.couchErrorsPoller.pollInterval = 0.2
        ti = utils.TestInput() # see attributes comments at the class
        ti.pollerClass = CouchErrorsPoller
        ti.config = self.config.AlertGenerator.couchErrorsPoller
        ti.thresholdToTest = self.config.AlertGenerator.couchErrorsPoller.soft - 30
        ti.expected = 0
        ti.thresholdDiff = 20
        ti.testCase = self
        utils.doGenericValueBasedPolling(ti)
コード例 #37
0
ファイル: FeederManager_t.py プロジェクト: tsarangi/WMCore
class FeederManagerTest(unittest.TestCase):
    """
    TestCase for TestFeederManager module
    """

    _maxMessage = 10

    def setUp(self):
        """
        _setUp_

        Setup the database and logging connection.  Try to create all needed
        tables.
        """

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.generateWorkDir()
        self.testInit.setSchema(customModules = \
                         ['WMCore.Agent.Database',
                          'WMComponent.FeederManager.Database',
                          'WMCore.ThreadPool',
                          'WMCore.WMBS'],
                                useDefault = False)

        return

    def tearDown(self):
        """
        _tearDown_

        Database deletion
        """
        self.testInit.clearDatabase()

        return

    def getConfig(self):
        """
        _createConfig_

        Create a config for the JobAccountant.  This config needs to include
        information for connecting to the database as the component will create
        it's own database connections.  These parameters are still pulled from
        the environment.
        """
        config = self.testInit.getConfiguration()
        self.testInit.generateWorkDir(config)

        config.component_("FeederManager")
        config.FeederManager.logLevel = "INFO"
        config.FeederManager.componentName = "FeederManager"
        config.FeederManager.componentDir = \
            os.path.join(os.getenv("TESTDIR"), "FeederManager")
        config.FeederManager.addDatasetWatchHandler = \
            'WMComponent.FeederManager.Handler.DefaultAddDatasetWatch'

        # The maximum number of threads to process each message type
        config.FeederManager.maxThreads = 10

        # The poll interval at which to look for new fileset/feeder association
        config.FeederManager.pollInterval = 60

        return config

    def testA(self):
        """
        _testA_

        Handle AddDatasetWatch events
        """
        myThread = threading.currentThread()
        config = self.getConfig()
        testFeederManager = FeederManager(config)
        testFeederManager.prepareToStart()

        for i in xrange(0, FeederManagerTest._maxMessage):
            for j in xrange(0, 3):
                feederManagerdict = {'payload':{'FeederType':'NO Feeder',
                                     'dataset' : 'NO DATASET', 'FileType' : 'NO FILE TYPE',
                                     'StartRun' : 'NO START RUN' }}

                testFeederManager.handleMessage( type = 'AddDatasetWatch',
                                                 payload = feederManagerdict )

        time.sleep(30)

        myThread.workerThreadManager.terminateWorkers()

        while threading.activeCount() > 1:
            print('Currently: '+str(threading.activeCount())+\
                ' Threads. Wait until all our threads have finished')
            time.sleep(1)
コード例 #38
0
class JobPackageTest(unittest.TestCase):
    def setUp(self):
        """
        _setUp_

        Create a temporary file to presist the JobPackage to.
        """
        self.testInit = TestInit(__file__)
        self.persistFile = os.path.join(self.testInit.generateWorkDir(),
                                        "JobPackage.pkl")
        return

    def tearDown(self):
        pass

    def testAddingJobs(self):
        """
        _testAddingJobs_

        Verify that adding jobs to the package works as expected.
        """
        package = JobPackage()

        for i in range(100):
            newJob = Job("Job%s" % i)
            newJob["id"] = i
            package[i] = newJob

        # There is an extra key for the directory the package is stored in.
        assert len(package.keys()) == 101, \
               "Error: Wrong number of jobs in package."

        for i in range(100):
            job = package[i]
            assert job["id"] == i, \
                   "Error: Jobs has wrong ID."
            assert job["name"] == "Job%d" % i, \
                   "Error: Job has wrong name."

        return

    def testPersist(self):
        """
        _testPersist_

        Verify that we're able to save and load the job package.
        """
        package = JobPackage()

        for i in range(100):
            newJob = Job("Job%s" % i)
            newJob["id"] = i
            package[i] = newJob

        package.save(self.persistFile)

        assert os.path.exists(self.persistFile), \
               "Error: Package file was never created."

        newPackage = JobPackage()
        newPackage.load(self.persistFile)

        # There is an extra key for the directory the package is stored in.
        assert len(newPackage.keys()) == 101, \
               "Error: Wrong number of jobs in package."

        for i in range(100):
            job = newPackage[i]
            assert job["id"] == i, \
                   "Error: Jobs has wrong ID."
            assert job["name"] == "Job%d" % i, \
                   "Error: Job has wrong name."

        return

    def testBaggage(self):
        """
        _testBaggage_

        Verify that job baggage is persisted with the package.
        """
        package = JobPackage()

        for i in range(100):
            newJob = Job("Job%s" % i)
            newJob["id"] = i
            baggage = newJob.getBaggage()
            setattr(baggage, "thisJob", newJob["name"])
            setattr(baggage, "seed1", 11111111)
            setattr(baggage, "seed2", 22222222)
            setattr(baggage, "seed3", 33333333)
            setattr(baggage, "seed4", 44444444)
            setattr(baggage, "seed5", 55555555)
            package[i] = newJob

        package.save(self.persistFile)

        assert os.path.exists(self.persistFile), \
               "Error: Package file was never created."

        newPackage = JobPackage()
        newPackage.load(self.persistFile)

        # There is an extra key for the directory the package is stored in.
        assert len(newPackage.keys()) == 101, \
               "Error: Wrong number of jobs in package."

        for i in range(100):
            job = newPackage[i]
            assert job["id"] == i, \
                   "Error: Jobs has wrong ID."
            assert job["name"] == "Job%d" % i, \
                   "Error: Job has wrong name."
            jobBaggage = job.getBaggage()

            assert jobBaggage.thisJob == "Job%d" % i, \
                   "Error: Job baggage has wrong name."
            assert jobBaggage.seed1 == 11111111, \
                   "Error: Job baggee has wrong value for seed1."
            assert jobBaggage.seed2 == 22222222, \
                   "Error: Job baggee has wrong value for seed2."
            assert jobBaggage.seed3 == 33333333, \
                   "Error: Job baggee has wrong value for seed3."
            assert jobBaggage.seed4 == 44444444, \
                   "Error: Job baggee has wrong value for seed4."
            assert jobBaggage.seed5 == 55555555, \
                   "Error: Job baggee has wrong value for seed5."


        return
コード例 #39
0
class SetupCMSSWPsetTest(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testDir = self.testInit.generateWorkDir()
        sys.path.insert(0, os.path.join(WMCore.WMBase.getTestBase(),
                                        "WMCore_t/WMRuntime_t/Scripts_t"))

    def tearDown(self):
        sys.path.remove(os.path.join(WMCore.WMBase.getTestBase(), "WMCore_t/WMRuntime_t/Scripts_t"))
        if 'WMTaskSpace' in sys.modules:
            del sys.modules["WMTaskSpace"]
        self.testInit.delWorkDir()
        os.unsetenv("WMAGENT_SITE_CONFIG_OVERRIDE")

    def createTestStep(self):
        """
        _createTestStep_

        Create a test step that can be passed to the setup script.

        """
        newStep = WMStep("cmsRun1")
        newStepHelper = CMSSWStepHelper(newStep)
        newStepHelper.setStepType("CMSSW")
        newStepHelper.setGlobalTag("SomeGlobalTag")
        stepTemplate = StepFactory.getStepTemplate("CMSSW")
        stepTemplate(newStep)
        newStep.application.command.configuration = "PSet.py"
        return newStepHelper


    def createTestJob(self):
        """
        _createTestJob_

        Create a test job that has parents for each input file.

        """
        newJob = Job(name = "TestJob")
        newJob.addFile(File(lfn = "/some/file/one",
                            parents = set([File(lfn = "/some/parent/one")])))
        newJob.addFile(File(lfn = "/some/file/two",
                            parents = set([File(lfn = "/some/parent/two")])))
        return newJob

    def loadProcessFromPSet(self):
        """
        _loadProcessFromPSet_

        This requires changing the working directory,
        do so in a safe manner to encapsulate the change to this method only
        """

        currentPath = os.getcwd()
        loadedProcess = None
        try:
            if not os.path.isdir(self.testDir):
                raise
            os.chdir(self.testDir)
            testFile = "PSet.py"
            pset = imp.load_source('process', testFile)
            loadedProcess = pset.process
        except Exception as ex:
            self.fail("An exception was caught while trying to load the PSet, %s" % str(ex))
        finally:
            os.chdir(currentPath)

        return loadedProcess

    def testPSetFixup(self):
        """
        _testPSetFixup_

        Verify that all necessary parameters are set in the PSet.

        """
        from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset

        if os.environ.get('CMSSW_VERSION', None):
            del os.environ['CMSSW_VERSION']

        setupScript = SetupCMSSWPset()
        setupScript.step = self.createTestStep()
        setupScript.stepSpace = ConfigSection(name = "stepSpace")
        setupScript.stepSpace.location = self.testDir
        setupScript.job = self.createTestJob()
        setupScript()

        fixedPSet = self.loadProcessFromPSet()

        self.assertEqual(len(fixedPSet.source.fileNames.value), 2,
                         "Error: Wrong number of files.")
        self.assertEqual(len(fixedPSet.source.secondaryFileNames.value), 2,
                         "Error: Wrong number of secondary files.")
        self.assertEqual(fixedPSet.source.fileNames.value[0], "/some/file/one",
                         "Error: Wrong input file.")
        self.assertEqual(fixedPSet.source.fileNames.value[1], "/some/file/two",
                         "Error: Wrong input file.")
        self.assertEqual(fixedPSet.source.secondaryFileNames.value[0], "/some/parent/one",
                         "Error: Wrong input file.")
        self.assertEqual(fixedPSet.source.secondaryFileNames.value[1], "/some/parent/two",
                         "Error: Wrong input file.")
        self.assertEqual(fixedPSet.maxEvents.input.value, -1,
                         "Error: Wrong maxEvents.")

    def testEventsPerLumi(self):
        """
        _testEventsPerLumi_
        Verify that you can put in events per lumi in the process.

        """
        from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset

        os.environ['CMSSW_VERSION'] = "CMSSW_7_4_0"

        setupScript = SetupCMSSWPset()
        setupScript.step = self.createTestStep()
        setupScript.step.setEventsPerLumi(500)
        setupScript.stepSpace = ConfigSection(name = "stepSpace")
        setupScript.stepSpace.location = self.testDir
        setupScript.job = self.createTestJob()
        setupScript()

        fixedPSet = self.loadProcessFromPSet()

        self.assertEqual(len(fixedPSet.source.fileNames.value), 2,
                         "Error: Wrong number of files.")
        self.assertEqual(len(fixedPSet.source.secondaryFileNames.value), 2,
                         "Error: Wrong number of secondary files.")
        self.assertEqual(fixedPSet.source.fileNames.value[0], "/some/file/one",
                         "Error: Wrong input file.")
        self.assertEqual(fixedPSet.source.fileNames.value[1], "/some/file/two",
                         "Error: Wrong input file.")
        self.assertEqual(fixedPSet.source.secondaryFileNames.value[0], "/some/parent/one",
                         "Error: Wrong input file.")
        self.assertEqual(fixedPSet.source.secondaryFileNames.value[1], "/some/parent/two",
                         "Error: Wrong input file.")
        self.assertEqual(fixedPSet.source.numberEventsInLuminosityBlock.value,
                         500, "Error: Wrong number of events per luminosity block")
        self.assertEqual(fixedPSet.maxEvents.input.value, -1,
                         "Error: Wrong maxEvents.")

    def testChainedProcesing(self):
        """
        test for chained CMSSW processing - check the overriden TFC, its values
        and that input files is / are set correctly.

        """
        from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset

        os.environ['CMSSW_VERSION'] = "CMSSW_7_6_0"

        setupScript = SetupCMSSWPset()
        setupScript.step = self.createTestStep()
        setupScript.stepSpace = ConfigSection(name = "stepSpace")
        setupScript.stepSpace.location = self.testDir
        setupScript.job = self.createTestJob()
        setupScript.step.setupChainedProcessing("my_first_step", "my_input_module")
        setupScript()

        # test if the overriden TFC is right
        self.assertTrue(hasattr(setupScript.step.data.application, "overrideCatalog"),
                        "Error: overriden TFC was not set")
        tfc = loadTFC(setupScript.step.data.application.overrideCatalog)
        inputFile = "../my_first_step/my_input_module.root"
        self.assertEqual(tfc.matchPFN("direct", inputFile), inputFile)
        self.assertEqual(tfc.matchLFN("direct", inputFile), inputFile)

        self.assertEqual(setupScript.process.source.fileNames.value, [inputFile])


    def testPileupSetup(self):
        """
        Test the pileup setting.

        reference (setupScript.process instance):
        in test/python/WMCore_t/WMRuntime_t/Scripts_t/WMTaskSpace/cmsRun1/PSet.py

        """
        try:
            from dbs.apis.dbsClient import DbsApi
        except ImportError as ex:
            raise nose.SkipTest

        # this is modified and shortened version of
        # WMCore/test/python/WMCore_t/Misc_t/site-local-config.xml
        # since the dataset name in question (below) is only present at
        # storm-fe-cms.cr.cnaf.infn.it, need to make the test think it's its local SE
        siteLocalConfigContent = \
        """
<site-local-config>
    <site name="-SOME-SITE-NAME-">
        <event-data>
            <catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/>
        </event-data>
        <local-stage-out>
            <!-- original cmssrm.fnal.gov -->
            <phedex-node value="T2_CH_CERN"/>
            <command value="test-copy"/>
            <catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/>
        </local-stage-out>
        <calib-data>
            <frontier-connect>
                <load balance="proxies"/>
                <proxy url="http://cmsfrontier1.fnal.gov:3128"/>
                <proxy url="http://cmsfrontier2.fnal.gov:3128"/>
            </frontier-connect>
        </calib-data>
    </site>
</site-local-config>
"""
        siteLocalConfig = os.path.join(self.testDir, "test-site-local-config.xml")
        f = open(siteLocalConfig, 'w')
        f.write(siteLocalConfigContent)
        f.close()

        from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
        setupScript = SetupCMSSWPset()
        setupScript.step = self.createTestStep()
        setupScript.stepSpace = ConfigSection(name = "stepSpace")
        setupScript.stepSpace.location = os.path.join(self.testDir, "cmsRun1")
        setupScript.job = self.createTestJob()
        # define pileup configuration
        # despite of the implementation considering whichever type of pileup,
        # only "data" and "mc" types are eventually considered and lead to any
        # modifications of job input files
        pileupConfig = {"data": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"],
                        "mc": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"]}
        dbsUrl = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
        setupScript.step.setupPileup(pileupConfig, dbsUrl)
        # SetupCMSSWPset pileup handling will be consulting SiteLocalConfig
        # to determine StorageElement (SE) name the job is running on
        # SiteLocalConfig loads the site-local-config.xml file from env.
        # variable defined location ; if the variable is not defined already, set it
        # obviously, if "WMAGENT_SITE_CONFIG_OVERRIDE" is already set here, the above
        # thick with SE name is not effective
        if not os.getenv("WMAGENT_SITE_CONFIG_OVERRIDE", None):
            os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = siteLocalConfig
        # find out local site name from the testing local site config,
        # will be needed later
        siteConfig = loadSiteLocalConfig()
        seLocalName = siteConfig.localStageOut["phedex-node"]
        print("Running on site '%s', local SE name: '%s'" % (siteConfig.siteName, seLocalName))

        # before calling the script, SetupCMSSWPset will try to load JSON
        # pileup configuration file, need to create it in self.testDir
        fetcher = PileupFetcher()
        fetcher.setWorkingDirectory(self.testDir)
        fetcher._createPileupConfigFile(setupScript.step, fakeSites=['T1_US_FNAL'])

        setupScript()

        # now test all modifications carried out in SetupCMSSWPset.__call__
        # which will also test that CMSSWStepHelper.setupPileup run correctly
        mixModules, dataMixModules = setupScript._getPileupMixingModules()

        # load in the pileup configuration in the form of dict which
        # PileupFetcher previously saved in a JSON file
        pileupDict = setupScript._getPileupConfigFromJson()

        # get the sub dict for particular pileup type
        # for pileupDict structure description - see PileupFetcher._queryDbsAndGetPileupConfig
        for pileupType, modules in zip(("data", "mc"), (dataMixModules, mixModules)):
            # getting KeyError here - above pileupConfig is not correct - need
            # to have these two types of pile type
            d = pileupDict[pileupType]
            self._mixingModulesInputFilesTest(modules, d, seLocalName)


    def _mixingModulesInputFilesTest(self, modules, pileupSubDict, seLocalName):
        """
        pileupSubDic - contains only dictionary for particular pile up type

        """
        # consider only locally available files
        filesInConfigDict = []
        for v in pileupSubDict.values():
            if seLocalName in v["phedexNodeNames"]:
                filesInConfigDict.extend(v["FileList"])

        for m in modules:
            inputTypeAttrib = getattr(m, "input", None) or getattr(m, "secsource", None)
            fileNames = inputTypeAttrib.fileNames.value
            if fileNames == None:
                fileNames = []
            m = ("Pileup configuration file list '%s' and mixing modules input "
                 "filelist '%s' are not identical." % (filesInConfigDict, fileNames))
            self.assertEqual(sorted(filesInConfigDict), sorted(fileNames), m)
コード例 #40
0
ファイル: Scram_t.py プロジェクト: todor-ivanov/WMCore
class Scram_t(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testDir = self.testInit.generateWorkDir()
        self.oldCwd = os.getcwd()
        if PY3:
            self.assertItemsEqual = self.assertCountEqual

    def tearDown(self):
        self.testInit.delWorkDir()

    def testA(self):
        """
        instantiate a Scram instance in test mode.
        """
        try:
            Scram(initialise="/bin/date",
                  architecture="slc5_amd64_gcc454",
                  version="CMSSW_X_Y_Z",
                  test=True)
        except Exception as ex:
            msg = "Failed to instantiate Scram in test mode:\n %s " % str(ex)
            self.fail(msg)

    def testB(self):
        """
        instantiante a Scram instance in non-test mode
        limited what we can test here since we dont have scram etc in unittest env
        """
        try:
            Scram(initialise="/bin/date",
                  architecture="slc5_amd64_gcc454",
                  version="CMSSW_X_Y_Z")
        except Exception as ex:
            msg = "Failed to instantiate Scram:\n %s " % str(ex)
            self.fail(msg)

    def testC(self):
        """
        test all method calls in test mode

        """
        s = Scram(initialise="/bin/date",
                  architecture="slc5_amd64_gcc454",
                  version="CMSSW_X_Y_Z",
                  directory=self.testDir,
                  test=True)

        try:
            status = s.project()
        except Exception as ex:
            msg = "Error running Scram.project:\n %s" % str(ex)
            self.fail(msg)

        self.assertEqual(status, 0)
        self.assertTrue(os.path.exists(s.projectArea))
        self.assertTrue("project" in s.lastExecuted)
        self.assertTrue("CMSSW_X_Y_Z" in s.lastExecuted)

        try:
            status = s.runtime()
        except Exception as ex:
            msg = "Error running Scram.runtime:\n %s" % str(ex)
            self.fail(msg)

        self.assertEqual(status, 0)
        self.assertTrue("ru -sh" in s.lastExecuted)
        self.assertTrue("TEST_MODE" in s.runtimeEnv)

        comm = "echo \"Hello World\""
        try:
            status = s(comm)
        except Exception as ex:
            msg = "Failed to call Scram object:\n %s" % str(ex)

        self.assertEqual(status, 0)
        self.assertEqual(s.lastExecuted, comm)

    def testArchMap(self):
        self.assertItemsEqual(OS_TO_ARCH['rhel6'], ['slc5', 'slc6'])
        self.assertItemsEqual(OS_TO_ARCH['rhel7'], ['slc7'])
        self.assertItemsEqual(ARCH_TO_OS['slc6'], ['rhel6'])
        self.assertItemsEqual(ARCH_TO_OS['slc7'], ['rhel7'])

    def testScramArchParsing(self):
        """
        Test the various modes of parsing for the scram arch
        """
        try:
            os.chdir(self.testDir)
            with tempfile.NamedTemporaryFile() as tf:
                tf.write(b'GLIDEIN_REQUIRED_OS = "rhel6" \n')
                tf.write(b'Memory = 2048\n')
                tf.flush()
                with tmpEnv(_CONDOR_MACHINE_AD=tf.name):
                    self.assertEqual(getSingleScramArch('slc6_blah_blah'),
                                     'slc6_blah_blah')
                    self.assertEqual(getSingleScramArch('slc5_blah_blah'),
                                     'slc5_blah_blah')
                    self.assertEqual(
                        getSingleScramArch(
                            ['slc6_blah_blah', 'slc7_blah_blah']),
                        'slc6_blah_blah')
                    self.assertEqual(
                        getSingleScramArch(
                            ['slc6_blah_blah', 'slc5_blah_blah']),
                        'slc6_blah_blah')
                    self.assertEqual(
                        getSingleScramArch(
                            ['slc7_blah_blah', 'slc8_blah_blah']), None)
            with tempfile.NamedTemporaryFile() as tf:
                tf.write(b'GLIDEIN_REQUIRED_OS = "rhel7" \n')
                tf.write(b'Memory = 2048\n')
                tf.flush()
                with tmpEnv(_CONDOR_MACHINE_AD=tf.name):
                    self.assertEqual(getSingleScramArch('slc6_blah_blah'),
                                     'slc6_blah_blah')
                    self.assertEqual(getSingleScramArch('slc7_blah_blah'),
                                     'slc7_blah_blah')
                    self.assertEqual(
                        getSingleScramArch(
                            ['slc6_blah_blah', 'slc7_blah_blah']),
                        'slc7_blah_blah')
                    self.assertEqual(
                        getSingleScramArch(
                            ['slc6_blah_blah', 'slc5_blah_blah']), None)
                    self.assertEqual(
                        getSingleScramArch(
                            ['slc7_blah_blah', 'slc8_blah_blah']),
                        'slc7_blah_blah')
        except Exception:
            raise
        finally:
            os.chdir(self.oldCwd)
        return

    def testCMSSWSupported(self):
        """
        Test the functionality of isCMSSWSupported function
        """
        self.assertFalse(isCMSSWSupported('CMSSW_1_2_3', ''))
        self.assertFalse(isCMSSWSupported(None, 'a'))
        self.assertFalse(isCMSSWSupported('CMSSW_1_2_3', 'CMSSW_2_2_3'))
        self.assertFalse(isCMSSWSupported('CMSSW_1_2_3', 'CMSSW_1_3_3'))
        self.assertFalse(isCMSSWSupported('CMSSW_1_2_3', 'CMSSW_1_2_4'))
        self.assertFalse(isCMSSWSupported('CMSSW_1_2_3_pre1', 'CMSSW_1_2_3'))
        self.assertFalse(isCMSSWSupported('CMSSW_1_2_3', 'CMSSW_1_2_3_pre1'))
        self.assertFalse(
            isCMSSWSupported('CMSSW_1_2_3_pre1', 'CMSSW_1_2_3_pre2'))
        self.assertFalse(
            isCMSSWSupported('CMSSW_1_2_3_pre2', 'CMSSW_1_2_3_pre1'))
        self.assertFalse(isCMSSWSupported('CMSSW_7_1_25_patch2',
                                          'CMSSW_7_6_0'))
        self.assertFalse(isCMSSWSupported('CMSSW_7_3_2', 'CMSSW_10_4_0'))

        self.assertTrue(
            isCMSSWSupported('CMSSW_1_2_3_pre1', 'CMSSW_1_2_3_pre1'))
        self.assertTrue(isCMSSWSupported('CMSSW_1_2_3', 'CMSSW_1_2_3'))
        self.assertTrue(isCMSSWSupported('CMSSW_2_2_3', 'CMSSW_1_2_3'))
        self.assertTrue(isCMSSWSupported('CMSSW_1_3_3', 'CMSSW_1_2_3'))
        self.assertTrue(isCMSSWSupported('CMSSW_1_2_4', 'CMSSW_1_2_3'))

    def testisEnforceGUIDInFileNameSupported(self):
        """
        Test functionality of the `isEnforceGUIDInFileNameSupported` function
        """
        ### invalid input
        self.assertFalse(isEnforceGUIDInFileNameSupported(None))
        self.assertFalse(isEnforceGUIDInFileNameSupported(''))

        ### forever supported
        self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_11_0_0'))
        self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_11_0_2'))
        self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_11_1_0_pre1'))
        self.assertTrue(
            isEnforceGUIDInFileNameSupported('CMSSW_11_1_0_patch1'))
        self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_11_1_1'))

        ### specific releases
        self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_10_2_20_UL'))
        self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_9_4_16_UL'))
        self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_8_0_34_UL'))
        self.assertTrue(
            isEnforceGUIDInFileNameSupported('CMSSW_7_1_45_patch3'))

        ### minor supported releases
        self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_10_6_8'))
        self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_10_6_9'))
        self.assertTrue(
            isEnforceGUIDInFileNameSupported('CMSSW_10_6_8_patch1'))
        self.assertTrue(
            isEnforceGUIDInFileNameSupported('CMSSW_10_6_9_patch1'))
        self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_10_2_20'))
        self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_9_4_16'))
        self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_9_3_17'))
        self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_8_0_34'))

        ### releases not supported
        self.assertFalse(isEnforceGUIDInFileNameSupported('CMSSW_10_6_7'))
        self.assertFalse(isEnforceGUIDInFileNameSupported('CMSSW_10_7_0'))
        self.assertFalse(isEnforceGUIDInFileNameSupported('CMSSW_10_2_19'))
        self.assertFalse(isEnforceGUIDInFileNameSupported('CMSSW_10_3_10'))
        self.assertFalse(isEnforceGUIDInFileNameSupported('CMSSW_5_3_10'))
コード例 #41
0
ファイル: CMSSW_t.py プロジェクト: vkuznet/WMCore
class CMSSW_t(unittest.TestCase):
    def setUp(self):
        """
        _setUp_

        Build a testing environment similar to a WN
        """
        self.testInit = TestInit(__file__)
        self.testDir = self.testInit.generateWorkDir()

        # Build a workload/task/step with the basic required information
        self.workload = newWorkload("UnitTests")
        self.task = self.workload.newTask("CMSSWExecutor")
        stepHelper = self.task.makeStep("ExecutorTest")
        self.step = stepHelper.data
        template = CMSSWTemplate()
        template(self.step)
        self.helper = template.helper(self.step)
        self.step.application.setup.scramCommand = "scramulator.py"
        self.step.application.command.executable = "cmsRun.py"
        self.step.application.setup.scramProject = "CMSSW"
        self.step.application.setup.scramArch = "slc5_ia32_gcc434"
        self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
        self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";"
        self.step.output.jobReport = "FrameworkJobReport.xml"
        self.helper.addOutputModule("outputRECORECO", primaryDataset="Bogus",
                                    processedDataset="Test-Era-v1",
                                    dataTier="DATA")
        self.helper.addOutputModule("outputALCARECORECO", primaryDataset="Bogus",
                                    processedDataset="Test-Era-v1",
                                    dataTier="DATA")
        self.helper.setGlobalTag("Bogus")
        taskMaker = TaskMaker(self.workload, self.testDir)
        taskMaker.skipSubscription = True
        taskMaker.processWorkload()

        # Build the TaskSpace/StepSpace
        self.sandboxDir = os.path.join(self.testDir, "UnitTests")
        self.task.build(self.testDir)
        sys.path.append(self.testDir)
        sys.path.append(self.sandboxDir)

        # Copy the files that cmsRun would have generated in the step space
        open(os.path.join(self.step.builder.workingDir, "outputRECORECO.root"), "w").close()
        open(os.path.join(self.step.builder.workingDir, "outputALCARECORECO.root"), "w").close()
        shutil.copy(os.path.join(getTestBase(),
                                 "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml"),
                    os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))

        # Create a job
        self.job = Job(name="/UnitTest/CMSSWExecutor/ExecutorTest-test-job")
        self.job["id"] = 1

        # Set the PATH
        binDir = inspect.getsourcefile(ModuleLocator)
        binDir = binDir.replace("__init__.py", "bin")

        if not binDir in os.environ['PATH']:
            os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)

        self.oldCwd = os.getcwd()

    def tearDown(self):
        """
        _tearDown_

        Clean up
        """
        self.testInit.delWorkDir()
        sys.path.remove(self.testDir)
        sys.path.remove(self.sandboxDir)
        try:
            sys.modules.pop("WMTaskSpace")
            sys.modules.pop("WMTaskSpace.ExecutorTest")
            sys.modules.pop("WMTaskSpace.ExecutorTest.WMCore")
            sys.modules.pop("WMSandbox")
            sys.modules.pop("WMSandbox.CMSSWExecutor")
            sys.modules.pop("WMSandbox.CMSSWExecutor.ExecutorTest")
        except KeyError:
            return

        return

    def testA_Execute(self):
        """
        _Execute_

        Test the full path, including execute.
        """
        try:
            os.chdir(self.step.builder.workingDir)
            executor = CMSSWExecutor()
            executor.initialise(self.step, self.job)
            executor.pre()
            # Remove the scram pre-script as it requires an actual SCRAM environment
            executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
            executor.execute()
            executor.post()
            # Check that there were no errors
            self.assertEqual(0, executor.report.getExitCode())
            # Check that we processed the XML
            self.assertEqual(2, len(executor.report.getAllFiles()))
            # Check that the executable was really executed
            self.assertTrue(os.path.isfile(os.path.join(self.step.builder.workingDir, "BogusFile.txt")))
        except Exception as ex:
            self.fail("Failure encountered, %s" % str(ex))
        finally:
            os.chdir(self.oldCwd)
        return

    def testB_ExecuteNonZeroExit(self):
        """
        _ExecuteNonZeroExit_

        Test the execution of a script
        which exits with non-zero code.
        """
        self.step.application.command.executable = "brokenCmsRun.py"
        shutil.copy(os.path.join(getTestBase(),
                                 "WMCore_t/FwkJobReport_t/CMSSWFailReport.xml"),
                    os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))
        try:
            os.chdir(self.step.builder.workingDir)
            executor = StepFactory.getStepExecutor("CMSSW")
            executor.initialise(self.step, self.job)
            executor.pre()
            executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
            try:
                executor.execute()
                self.fail("An exception should have been raised")
            except WMExecutionFailure as ex:
                executor.diagnostic(ex.code, executor, ExceptionInstance=ex)
                self.assertEqual(8001, executor.report.getExitCode())
                report = Report()
                report.load("Report.pkl")
                self.assertEqual(8001, report.getExitCode())
        except Exception as ex:
            self.fail("Failure encountered, %s" % str(ex))
        finally:
            os.chdir(self.oldCwd)
        return

    def testC_ExecuteSegfault(self):
        """
        _ExecuteSegfault_

        Test the execution of a script
        which raises a ABRT signal which
        is the normal CMSSW response
        to a SEGFAULT.
        """
        self.step.application.command.executable = "test.sh"
        # CMSSW leaves an empty FWJR when a SEGFAULT is present
        open(os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"), "w").close()
        try:
            os.chdir(self.step.builder.workingDir)
            executor = StepFactory.getStepExecutor("CMSSW")
            executor.initialise(self.step, self.job)
            executor.pre()
            executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
            try:
                executor.execute()
                self.fail("An exception should have been raised")
            except WMExecutionFailure as ex:
                executor.diagnostic(ex.code, executor, ExceptionInstance=ex)
                self.assertEqual(50115, executor.report.getExitCode())
                report = Report()
                report.load("Report.pkl")
                self.assertEqual(50115, report.getExitCode())
        except Exception as ex:
            self.fail("Failure encountered, %s" % str(ex))
        finally:
            os.chdir(self.oldCwd)
        return

    def testD_ExecuteNoOutput(self):
        """
        _ExecuteNoOutput_

        Test what happens when no output is produced,
        the proper error should be included.
        """
        self.step.application.command.executable = "cmsRun.py"
        shutil.copy(os.path.join(getTestBase(),
                                 "WMCore_t/FwkJobReport_t/CMSSWSkippedAll.xml"),
                    os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))
        try:
            os.chdir(self.step.builder.workingDir)
            executor = StepFactory.getStepExecutor("CMSSW")
            executor.initialise(self.step, self.job)
            executor.pre()
            executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
            executor.execute()
            executor.post()
            self.assertEqual(60450, executor.report.getExitCode())
        except Exception as ex:
            self.fail("Failure encountered, %s" % str(ex))
        finally:
            os.chdir(self.oldCwd)
        return
コード例 #42
0
ファイル: DBSUpload_t.py プロジェクト: AndrewLevin/WMCore
class DBSUploadTest(unittest.TestCase):
    """
    TestCase for DBSUpload module

    """
    def setUp(self):
        """
        _setUp_

        setUp function for unittest

        """

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer"],
                                useDefault = False)
        self.testDir = self.testInit.generateWorkDir(deleteOnDestruction = False)
        self.configFile = EmulatorSetup.setupWMAgentConfig()

        myThread = threading.currentThread()
        self.bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database",
                                        logger = myThread.logger,
                                        dbinterface = myThread.dbi)

        self.buffer3Factory = DAOFactory(package = "WMComponent.DBS3Buffer",
                                         logger = myThread.logger,
                                         dbinterface = myThread.dbi)

        locationAction = self.bufferFactory(classname = "DBSBufferFiles.AddLocation")
        locationAction.execute(siteName = "se1.cern.ch")
        locationAction.execute(siteName = "se1.fnal.gov")
        locationAction.execute(siteName = "malpaquet")
        self.dbsUrl = "https://*****:*****@attr("integration")
    def testBasicUpload(self):
        """
        _testBasicUpload_

        Verify that we can successfully upload to DBS3.  Also verify that the
        uploader correctly handles files parentage when uploading.
        """
        self.dbsApi = DbsApi(url = self.dbsUrl)
        config = self.getConfig()
        dbsUploader = DBSUploadPoller(config = config)

        # First test verifies that uploader will poll and then not do anything
        # as the database is empty.
        dbsUploader.algorithm()

        acqEra = "Summer%s" % (int(time.time()))
        parentFiles = self.createParentFiles(acqEra)

        # The algorithm needs to be run twice.  On the first iteration it will
        # create all the blocks and upload one.  On the second iteration it will
        # timeout and upload the second block.
        dbsUploader.algorithm()
        time.sleep(5)
        dbsUploader.algorithm()
        time.sleep(5)

        # Verify the files made it into DBS3.
        self.verifyData(parentFiles[0]["datasetPath"], parentFiles)

        # Inject some more parent files and some child files into DBSBuffer.
        # Run the uploader twice, only the parent files should be added to DBS3.
        (moreParentFiles, childFiles) = \
                          self.createFilesWithChildren(parentFiles, acqEra)
        dbsUploader.algorithm()
        time.sleep(5)
        dbsUploader.algorithm()
        time.sleep(5)

        self.verifyData(parentFiles[0]["datasetPath"],
                        parentFiles + moreParentFiles)

        # Run the uploader another two times to upload the child files.  Verify
        # that the child files were uploaded.
        dbsUploader.algorithm()
        time.sleep(5)
        dbsUploader.algorithm()
        time.sleep(5)

        self.verifyData(childFiles[0]["datasetPath"], childFiles)
        return

    @attr("integration")
    def testDualUpload(self):
        """
        _testDualUpload_

        Verify that the dual upload mode works correctly.
        """
        self.dbsApi = DbsApi(url = self.dbsUrl)
        config = self.getConfig(dbs3UploadOnly = True)
        dbsUploader = DBSUploadPoller(config = config)
        dbsUtil = DBSBufferUtil()

        # First test verifies that uploader will poll and then not do anything
        # as the database is empty.
        dbsUploader.algorithm()

        acqEra = "Summer%s" % (int(time.time()))
        parentFiles = self.createParentFiles(acqEra)
        (moreParentFiles, childFiles) = \
                          self.createFilesWithChildren(parentFiles, acqEra)

        allFiles = parentFiles + moreParentFiles
        allBlocks = []
        for i in range(4):
            blockName = parentFiles[0]["datasetPath"] + "#" + makeUUID()
            dbsBlock = DBSBlock(blockName, "malpaquet", 1)
            dbsBlock.status = "Open"                
            dbsUtil.createBlocks([dbsBlock])
            for file in allFiles[i * 5 : (i * 5) + 5]:
                dbsBlock.addFile(file)
                dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]})
                if i < 2:
                    dbsBlock.status = "InDBS"
                dbsUtil.updateBlocks([dbsBlock])
            dbsUtil.updateFileStatus([dbsBlock], "InDBS")
            allBlocks.append(dbsBlock)            

        blockName = childFiles[0]["datasetPath"] + "#" + makeUUID()
        dbsBlock = DBSBlock(blockName, "malpaquet", 1)
        dbsBlock.status = "InDBS"
        dbsUtil.createBlocks([dbsBlock])
        for file in childFiles:
            dbsBlock.addFile(file)
            dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]})

        dbsUtil.updateFileStatus([dbsBlock], "InDBS")

        dbsUploader.algorithm()
        time.sleep(5)
        dbsUploader.algorithm()
        time.sleep(5)

        self.verifyData(parentFiles[0]["datasetPath"], parentFiles)

        # Change the status of the rest of the parent blocks so we can upload
        # them and the children.
        for dbsBlock in allBlocks:
            dbsBlock.status = "InDBS"            
            dbsUtil.updateBlocks([dbsBlock])

        dbsUploader.algorithm()
        time.sleep(5)

        self.verifyData(parentFiles[0]["datasetPath"], parentFiles + moreParentFiles)

        # Run the uploader one more time to upload the children.
        dbsUploader.algorithm()
        time.sleep(5)        
    
        self.verifyData(childFiles[0]["datasetPath"], childFiles)
        return

    def testCloseSettingsPerWorkflow(self):
        """
        _testCloseSettingsPerWorkflow_

        Test the block closing mechanics in the DBS3 uploader,
        this uses a fake dbs api to avoid reliance on external services.
        """
        # Signal trapExit that we are a friend
        os.environ["DONT_TRAP_EXIT"] = "True"
        try:
            # Monkey patch the imports of DbsApi
            from WMComponent.DBS3Buffer import DBSUploadPoller as MockDBSUploadPoller
            MockDBSUploadPoller.DbsApi = MockDbsApi
    
            # Set the poller and the dbsUtil for verification
            myThread = threading.currentThread()
            (_, dbsFilePath) = mkstemp(dir = self.testDir)
            self.dbsUrl = dbsFilePath
            config = self.getConfig()
            dbsUploader = MockDBSUploadPoller.DBSUploadPoller(config = config)
            dbsUtil = DBSBufferUtil()
    
            # First test is event based limits and timeout with no new files.
            # Set the files and workflow
            acqEra = "TropicalSeason%s" % (int(time.time()))
            workflowName = 'TestWorkload%s' % (int(time.time()))
            taskPath = '/%s/TestProcessing' % workflowName
            self.injectWorkflow(workflowName, taskPath,
                                MaxWaitTime = 2, MaxFiles = 100,
                                MaxEvents = 150)
            self.createParentFiles(acqEra, nFiles = 20,
                                   workflowName = workflowName,
                                   taskPath = taskPath)
    
            # The algorithm needs to be run twice.  On the first iteration it will
            # create all the blocks and upload one with less than 150 events.
            # On the second iteration the second block is uploaded.
            dbsUploader.algorithm()
            dbsUploader.checkBlocks()
            openBlocks = dbsUtil.findOpenBlocks()
            self.assertEqual(len(openBlocks), 1)
            globalFiles = myThread.dbi.processData("SELECT id FROM dbsbuffer_file WHERE status = 'InDBS'")[0].fetchall()
            notUploadedFiles = myThread.dbi.processData("SELECT id FROM dbsbuffer_file WHERE status = 'NOTUPLOADED'")[0].fetchall()
            self.assertEqual(len(globalFiles), 14)
            self.assertEqual(len(notUploadedFiles), 6)
            # Check the fake DBS for data
            fakeDBS = open(self.dbsUrl, 'r')
            fakeDBSInfo = json.load(fakeDBS)
            fakeDBS.close()
            self.assertEqual(len(fakeDBSInfo), 2)
            for block in fakeDBSInfo:
                self.assertTrue('block_events' not in block['block'])
                self.assertEqual(block['block']['file_count'], 7)
                self.assertEqual(block['block']['open_for_writing'], 0)
                self.assertTrue('close_settings' not in block)
            time.sleep(3)
            dbsUploader.algorithm()
            dbsUploader.checkBlocks()
            openBlocks = dbsUtil.findOpenBlocks()
            self.assertEqual(len(openBlocks), 0)
            fakeDBS = open(self.dbsUrl, 'r')
            fakeDBSInfo = json.load(fakeDBS)
            fakeDBS.close()
            self.assertEqual(len(fakeDBSInfo), 3)
            for block in fakeDBSInfo:
                if block['block']['file_count'] != 6:
                    self.assertEqual(block['block']['file_count'], 7)
                self.assertTrue('block_events' not in block['block'])
                self.assertEqual(block['block']['open_for_writing'], 0)
                self.assertTrue('close_settings' not in block)

            # Now check the limit by size and timeout with new files
            acqEra = "TropicalSeason%s" % (int(time.time()))
            workflowName = 'TestWorkload%s' % (int(time.time()))
            taskPath = '/%s/TestProcessing' % workflowName
            self.injectWorkflow(workflowName, taskPath,
                                MaxWaitTime = 2, MaxFiles = 5,
                                MaxEvents = 200000000)
            self.createParentFiles(acqEra, nFiles = 16,
                                   workflowName = workflowName,
                                   taskPath = taskPath)
            dbsUploader.algorithm()
            dbsUploader.checkBlocks()
            openBlocks = dbsUtil.findOpenBlocks()
            self.assertEqual(len(openBlocks), 1)
            fakeDBS = open(self.dbsUrl, 'r')
            fakeDBSInfo = json.load(fakeDBS)
            fakeDBS.close()
            self.assertEqual(len(fakeDBSInfo), 6)
            for block in fakeDBSInfo:
                if acqEra in block['block']['block_name']:
                    self.assertEqual(block['block']['file_count'], 5)
                self.assertTrue('block_events' not in block['block'])
                self.assertTrue('close_settings' not in block)
                self.assertEqual(block['block']['open_for_writing'], 0)
    
            # Put more files, they will go into the same block and then it will be closed
            # after timeout
            time.sleep(3)
            self.createParentFiles(acqEra, nFiles = 3,
                                   workflowName = workflowName,
                                   taskPath = taskPath)
            dbsUploader.algorithm()
            dbsUploader.checkBlocks()
            openBlocks = dbsUtil.findOpenBlocks()
            self.assertEqual(len(openBlocks), 0)
            fakeDBS = open(self.dbsUrl, 'r')
            fakeDBSInfo = json.load(fakeDBS)
            fakeDBS.close()
            self.assertEqual(len(fakeDBSInfo), 7)
            for block in fakeDBSInfo:
                if acqEra in block['block']['block_name']:
                    if block['block']['file_count'] < 5:
                        self.assertEqual(block['block']['file_count'], 4)
                    else:
                        self.assertEqual(block['block']['file_count'], 5)
                self.assertTrue('block_events' not in block['block'])
                self.assertEqual(block['block']['open_for_writing'], 0)
                self.assertTrue('close_settings' not in block)

            # Finally test size limits
            acqEra = "TropicalSeason%s" % (int(time.time()))
            workflowName = 'TestWorkload%s' % (int(time.time()))
            taskPath = '/%s/TestProcessing' % workflowName
            self.injectWorkflow(workflowName, taskPath,
                                MaxWaitTime = 1, MaxFiles = 500,
                                MaxEvents = 200000000, MaxSize = 2048)
            self.createParentFiles(acqEra, nFiles = 7,
                                   workflowName = workflowName,
                                   taskPath = taskPath)
            dbsUploader.algorithm()
            dbsUploader.checkBlocks()
            time.sleep(2)
            dbsUploader.algorithm()
            dbsUploader.checkBlocks()
    
            self.assertEqual(len(openBlocks), 0)
            fakeDBS = open(self.dbsUrl, 'r')
            fakeDBSInfo = json.load(fakeDBS)
            fakeDBS.close()
            self.assertEqual(len(fakeDBSInfo), 11)
            for block in fakeDBSInfo:
                if acqEra in block['block']['block_name']:
                    if block['block']['file_count'] != 1:
                        self.assertEqual(block['block']['block_size'], 2048)
                        self.assertEqual(block['block']['file_count'], 2)
                self.assertTrue('block_events' not in block['block'])
                self.assertEqual(block['block']['open_for_writing'], 0)
                self.assertTrue('close_settings' not in block)
        except:
            self.fail("We failed at some point in the test")
        finally:
            # We don't trust anyone else with _exit
            del os.environ["DONT_TRAP_EXIT"]
        return
コード例 #43
0
class ReportIntegrationTest(unittest.TestCase):
    """
    _ReportIntegrationTest_

    """
    def setUp(self):
        """
        _setUp_

        Setup the database and WMBS for the test.
        """
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer",
                                                 "WMCore.WMBS"],
                                useDefault = False)

        myThread = threading.currentThread()
        self.daofactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)
        self.dbsfactory = DAOFactory(package = "WMComponent.DBS3Buffer",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)
        locationAction = self.daofactory(classname = "Locations.New")
        locationAction.execute(siteName = "site1", pnn = "T1_US_FNAL_Disk")

        inputFile = File(lfn = "/path/to/some/lfn", size = 10, events = 10,
                         locations = "T1_US_FNAL_Disk")
        inputFile.create()

        inputFileset = Fileset(name = "InputFileset")
        inputFileset.create()
        inputFileset.addFile(inputFile)
        inputFileset.commit()

        unmergedFileset = Fileset(name = "UnmergedFileset")
        unmergedFileset.create()

        mergedFileset = Fileset(name = "MergedFileset")
        mergedFileset.create()

        procWorkflow = Workflow(spec = "wf001.xml", owner = "Steve",
                                name = "TestWF", task = "/TestWF/None")
        procWorkflow.create()
        procWorkflow.addOutput("outputRECORECO", unmergedFileset)

        mergeWorkflow = Workflow(spec = "wf002.xml", owner = "Steve",
                                 name = "MergeWF", task = "/MergeWF/None")
        mergeWorkflow.create()
        mergeWorkflow.addOutput("Merged", mergedFileset)

        insertWorkflow = self.dbsfactory(classname = "InsertWorkflow")
        insertWorkflow.execute("TestWF", "/TestWF/None", 0, 0, 0, 0)
        insertWorkflow.execute("MergeWF", "/MergeWF/None", 0, 0, 0, 0)

        self.procSubscription = Subscription(fileset = inputFileset,
                                             workflow = procWorkflow,
                                             split_algo = "FileBased",
                                             type = "Processing")
        self.procSubscription.create()
        self.procSubscription.acquireFiles()

        self.mergeSubscription = Subscription(fileset = unmergedFileset,
                                             workflow = mergeWorkflow,
                                             split_algo = "WMBSMergeBySize",
                                             type = "Merge")
        self.mergeSubscription.create()

        self.procJobGroup = JobGroup(subscription = self.procSubscription)
        self.procJobGroup.create()
        self.mergeJobGroup = JobGroup(subscription = self.mergeSubscription)
        self.mergeJobGroup.create()

        self.testJob = Job(name = "testJob", files = [inputFile])
        self.testJob.create(group = self.procJobGroup)
        self.testJob["state"] = "complete"

        myThread = threading.currentThread()
        self.daofactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)
        self.stateChangeAction = self.daofactory(classname = "Jobs.ChangeState")
        self.setFWJRAction = self.daofactory(classname = "Jobs.SetFWJRPath")
        self.getJobTypeAction = self.daofactory(classname = "Jobs.GetType")
        locationAction = self.daofactory(classname = "Locations.New")
        locationAction.execute(siteName = "cmssrm.fnal.gov")

        self.stateChangeAction.execute(jobs = [self.testJob])

        self.tempDir = tempfile.mkdtemp()
        return

    def tearDown(self):
        """
        _tearDown_

        Clear out the database and the pickled report file.
        """
        self.testInit.clearDatabase()

        try:
            os.remove(os.path.join(self.tempDir, "ProcReport.pkl"))
            os.remove(os.path.join(self.tempDir, "MergeReport.pkl"))
        except Exception as ex:
            pass

        try:
            os.rmdir(self.tempDir)
        except Exception as ex:
            pass

        return

    def createConfig(self, workerThreads):
        """
        _createConfig_

        Create a config for the JobAccountant with the given number of worker
        threads.  This config needs to include information for connecting to the
        database as the component will create it's own database connections.
        These parameters are still pulled from the environment.
        """
        config = self.testInit.getConfiguration()
        self.testInit.generateWorkDir(config)

        config.section_("JobStateMachine")
        config.JobStateMachine.couchurl = os.getenv("COUCHURL")
        config.JobStateMachine.couchDBName = "report_integration_t"
        config.JobStateMachine.jobSummaryDBName = "report_integration_wmagent_summary_t"

        config.component_("JobAccountant")
        config.JobAccountant.pollInterval = 60
        config.JobAccountant.workerThreads = workerThreads
        config.JobAccountant.componentDir = os.getcwd()
        config.JobAccountant.logLevel = 'SQLDEBUG'

        config.component_("TaskArchiver")
        config.TaskArchiver.localWMStatsURL = "%s/%s" % (config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName)
        return config

    def verifyJobSuccess(self, jobID):
        """
        _verifyJobSuccess_

        Verify that the metadata for a successful job is correct.  This will
        check the outcome, retry count and state.
        """
        testJob = Job(id = jobID)
        testJob.load()

        assert testJob["state"] == "success", \
               "Error: test job in wrong state: %s" % testJob["state"]
        assert testJob["retry_count"] == 0, \
               "Error: test job has wrong retry count: %s" % testJob["retry_count"]
        assert testJob["outcome"] == "success", \
               "Error: test job has wrong outcome: %s" % testJob["outcome"]

        return

    def verifyFileMetaData(self, jobID, fwkJobReportFiles):
        """
        _verifyFileMetaData_

        Verify that all the files that were output by a job made it into WMBS
        correctly.  Compare the contents of WMBS to the files in the frameworks
        job report.

        Note that fwkJobReportFiles is a list of DataStructs File objects.
        """
        testJob = Job(id = jobID)
        testJob.loadData()

        inputLFNs = []
        for inputFile in testJob["input_files"]:
            inputLFNs.append(inputFile["lfn"])

        for fwkJobReportFile in fwkJobReportFiles:
            outputFile = File(lfn = fwkJobReportFile["lfn"])
            outputFile.loadData(parentage = 1)

            assert outputFile["events"] == int(fwkJobReportFile["events"]), \
                   "Error: Output file has wrong events: %s, %s" % \
                   (outputFile["events"], fwkJobReportFile["events"])
            assert outputFile["size"] == int(fwkJobReportFile["size"]), \
                   "Error: Output file has wrong size: %s, %s" % \
                   (outputFile["size"], fwkJobReportFile["size"])

            for ckType in fwkJobReportFile["checksums"]:
                assert ckType in outputFile["checksums"], \
                       "Error: Output file is missing checksums: %s" % ckType
                assert outputFile["checksums"][ckType] == fwkJobReportFile["checksums"][ckType], \
                       "Error: Checksums don't match."

            assert len(fwkJobReportFile["checksums"]) == \
                   len(outputFile["checksums"]), \
                   "Error: Wrong number of checksums."

            jobType = self.getJobTypeAction.execute(jobID = jobID)
            if jobType == "Merge":
                assert str(outputFile["merged"]) == "True", \
                       "Error: Merge jobs should output merged files."
            else:
                assert outputFile["merged"] == fwkJobReportFile["merged"], \
                       "Error: Output file merged output is wrong: %s, %s" % \
                       (outputFile["merged"], fwkJobReportFile["merged"])

            assert len(outputFile["locations"]) == 1, \
                   "Error: outputfile should have one location: %s" % outputFile["locations"]
            assert list(outputFile["locations"])[0] == list(fwkJobReportFile["locations"])[0], \
                   "Error: wrong location for file."

            assert len(outputFile["parents"]) == len(inputLFNs), \
                   "Error: Output file has wrong number of parents."
            for outputParent in outputFile["parents"]:
                assert outputParent["lfn"] in inputLFNs, \
                       "Error: Unknown parent file: %s" % outputParent["lfn"]

            fwjrRuns = {}
            for run in fwkJobReportFile["runs"]:
                fwjrRuns[run.run] = run.lumis

            for run in outputFile["runs"]:
                assert run.run in fwjrRuns, \
                       "Error: Extra run in output: %s" % run.run

                for lumi in run:
                    assert lumi in fwjrRuns[run.run], \
                           "Error: Extra lumi: %s" % lumi

                    fwjrRuns[run.run].remove(lumi)

                if len(fwjrRuns[run.run]) == 0:
                    del fwjrRuns[run.run]

            assert len(fwjrRuns) == 0, \
                   "Error: Missing runs, lumis: %s" % fwjrRuns

            testJobGroup = JobGroup(id = testJob["jobgroup"])
            testJobGroup.loadData()
            jobGroupFileset = testJobGroup.output
            jobGroupFileset.loadData()

            assert outputFile["id"] in jobGroupFileset.getFiles(type = "id"), \
                   "Error: output file not in jobgroup fileset."

            if testJob["mask"]["FirstEvent"] == None:
                assert outputFile["first_event"] == 0, \
                       "Error: first event not set correctly: 0, %s" % \
                       outputFile["first_event"]
            else:
                assert testJob["mask"]["FirstEvent"] == outputFile["first_event"], \
                       "Error: last event not set correctly: %s, %s" % \
                       (testJob["mask"]["FirstEvent"], outputFile["first_event"])

        return

    def testReportHandling(self):
        """
        _testReportHandling_

        Verify that we're able to parse a CMSSW report, convert it to a Report()
        style report, pickle it and then have the accountant process it.
        """
        self.procPath = os.path.join(WMCore.WMBase.getTestBase(),
                                    "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml")

        myReport = Report("cmsRun1")
        myReport.parse(self.procPath)

        # Fake some metadata that should be added by the stageout scripts.
        for fileRef in myReport.getAllFileRefsFromStep("cmsRun1"):
            fileRef.size = 1024
            fileRef.location = "cmssrm.fnal.gov"

        fwjrPath = os.path.join(self.tempDir, "ProcReport.pkl")
        cmsRunStep = myReport.retrieveStep("cmsRun1")
        cmsRunStep.status = 0
        myReport.setTaskName('/TestWF/None')
        myReport.persist(fwjrPath)

        self.setFWJRAction.execute(jobID = self.testJob["id"], fwjrPath = fwjrPath)

        pFile = DBSBufferFile(lfn = "/path/to/some/lfn", size = 600000, events = 60000)
        pFile.setAlgorithm(appName = "cmsRun", appVer = "UNKNOWN",
                           appFam = "RECO", psetHash = "GIBBERISH",
                           configContent = "MOREGIBBERISH")
        pFile.setDatasetPath("/bogus/dataset/path")
        #pFile.addRun(Run(1, *[45]))
        pFile.create()

        config = self.createConfig(workerThreads = 1)
        accountant = JobAccountantPoller(config)
        accountant.setup()
        accountant.algorithm()

        self.verifyJobSuccess(self.testJob["id"])
        self.verifyFileMetaData(self.testJob["id"], myReport.getAllFilesFromStep("cmsRun1"))

        inputFile = File(lfn = "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR09_R_34X_V5_All_v1/0000/outputRECORECO.root")
        inputFile.load()
        self.testMergeJob = Job(name = "testMergeJob", files = [inputFile])
        self.testMergeJob.create(group = self.mergeJobGroup)
        self.testMergeJob["state"] = "complete"
        self.stateChangeAction.execute(jobs = [self.testMergeJob])

        self.mergePath = os.path.join(WMCore.WMBase.getTestBase(),
                                         "WMCore_t/FwkJobReport_t/CMSSWMergeReport.xml")

        myReport = Report("mergeReco")
        myReport.parse(self.mergePath)

        # Fake some metadata that should be added by the stageout scripts.
        for fileRef in myReport.getAllFileRefsFromStep("mergeReco"):
            fileRef.size = 1024
            fileRef.location = "cmssrm.fnal.gov"
            fileRef.dataset = {"applicationName": "cmsRun", "applicationVersion": "CMSSW_3_4_2_patch1",
                               "primaryDataset": "MinimumBias", "processedDataset": "Rereco-v1",
                               "dataTier": "RECO"}

        fwjrPath = os.path.join(self.tempDir, "MergeReport.pkl")
        myReport.setTaskName('/MergeWF/None')
        cmsRunStep = myReport.retrieveStep("mergeReco")
        cmsRunStep.status = 0
        myReport.persist(fwjrPath)

        self.setFWJRAction.execute(jobID = self.testMergeJob["id"], fwjrPath = fwjrPath)
        accountant.algorithm()

        self.verifyJobSuccess(self.testMergeJob["id"])
        self.verifyFileMetaData(self.testMergeJob["id"], myReport.getAllFilesFromStep("mergeReco"))

        return
コード例 #44
0
class Scram_t(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testDir = self.testInit.generateWorkDir()
        self.oldCwd = os.getcwd()

    def tearDown(self):
        self.testInit.delWorkDir()

    def testA(self):
        """
        instantiate a Scram instance in test mode.
        """
        try:
            Scram(
                initialise="/bin/date",
                architecture="slc5_amd64_gcc454",
                version="CMSSW_X_Y_Z",
                test=True
            )
        except Exception as ex:
            msg = "Failed to instantiate Scram in test mode:\n %s " % str(ex)
            self.fail(msg)

    def testB(self):
        """
        instantiante a Scram instance in non-test mode
        limited what we can test here since we dont have scram etc in unittest env
        """
        try:
            Scram(
                initialise="/bin/date",
                architecture="slc5_amd64_gcc454",
                version="CMSSW_X_Y_Z"
            )
        except Exception as ex:
            msg = "Failed to instantiate Scram:\n %s " % str(ex)
            self.fail(msg)

    def testC(self):
        """
        test all method calls in test mode

        """
        s = Scram(
            initialise="/bin/date",
            architecture="slc5_amd64_gcc454",
            version="CMSSW_X_Y_Z",
            directory=self.testDir,
            test=True

        )

        try:
            status = s.project()
        except Exception as ex:
            msg = "Error running Scram.project:\n %s" % str(ex)
            self.fail(msg)

        self.assertEqual(status, 0)
        self.assertTrue(os.path.exists(s.projectArea))
        self.assertTrue("project" in s.lastExecuted)
        self.assertTrue("CMSSW_X_Y_Z" in s.lastExecuted)

        try:
            status = s.runtime()
        except Exception as ex:
            msg = "Error running Scram.runtime:\n %s" % str(ex)
            self.fail(msg)

        self.assertEqual(status, 0)
        self.assertTrue("ru -sh" in s.lastExecuted)
        self.assertTrue("TEST_MODE" in s.runtimeEnv)

        comm = "echo \"Hello World\""
        try:
            status = s(comm)
        except Exception as ex:
            msg = "Failed to call Scram object:\n %s" % str(ex)

        self.assertEqual(status, 0)
        self.assertEqual(s.lastExecuted, comm)

    def testArchMap(self):
        self.assertItemsEqual(OS_TO_ARCH['rhel6'], ['slc5', 'slc6'])
        self.assertItemsEqual(OS_TO_ARCH['rhel7'], ['slc7'])
        self.assertItemsEqual(ARCH_TO_OS['slc6'], ['rhel6'])
        self.assertItemsEqual(ARCH_TO_OS['slc7'], ['rhel7'])

    def testScramArchParsing(self):
        """
        Test the various modes of parsing for the scram arch
        """
        try:
            os.chdir(self.testDir)
            with tempfile.NamedTemporaryFile() as tf:
                tf.write('GLIDEIN_REQUIRED_OS = "rhel6" \n')
                tf.write('Memory = 2048\n')
                tf.flush()
                with tmpEnv(_CONDOR_MACHINE_AD=tf.name):
                    self.assertEqual(getSingleScramArch('slc6_blah_blah'), 'slc6_blah_blah')
                    self.assertEqual(getSingleScramArch('slc5_blah_blah'), 'slc5_blah_blah')
                    self.assertEqual(getSingleScramArch(['slc6_blah_blah', 'slc7_blah_blah']),
                                      'slc6_blah_blah')
                    self.assertEqual(getSingleScramArch(['slc6_blah_blah', 'slc5_blah_blah']),
                                      'slc6_blah_blah')
                    self.assertEqual(getSingleScramArch(['slc7_blah_blah', 'slc8_blah_blah']), None)
            with tempfile.NamedTemporaryFile() as tf:
                tf.write('GLIDEIN_REQUIRED_OS = "rhel7" \n')
                tf.write('Memory = 2048\n')
                tf.flush()
                with tmpEnv(_CONDOR_MACHINE_AD=tf.name):
                    self.assertEqual(getSingleScramArch('slc6_blah_blah'), 'slc6_blah_blah')
                    self.assertEqual(getSingleScramArch('slc7_blah_blah'), 'slc7_blah_blah')
                    self.assertEqual(getSingleScramArch(['slc6_blah_blah', 'slc7_blah_blah']),
                                      'slc7_blah_blah')
                    self.assertEqual(getSingleScramArch(['slc6_blah_blah', 'slc5_blah_blah']), None)
                    self.assertEqual(getSingleScramArch(['slc7_blah_blah', 'slc8_blah_blah']),
                                      'slc7_blah_blah')
        except Exception:
            raise
        finally:
            os.chdir(self.oldCwd)
        return
コード例 #45
0
ファイル: Base_t.py プロジェクト: tsarangi/WMCore
class BaseTest(unittest.TestCase):
    """
    Some methods of this class are made static and are used from
    other test cases.

    """
    def setUp(self):
        self.testInit = TestInit(__file__)
        self.testInit.setLogging(logLevel=logging.DEBUG)
        self.testDir = self.testInit.generateWorkDir()
        self.config = getConfig(self.testDir)
        self.testComponentDaemonXml = "/tmp/TestComponent/Daemon.xml"

    def tearDown(self):
        self.testInit.delWorkDir()
        self.generator = None
        # if the directory and file "/tmp/TestComponent/Daemon.xml" after
        # ComponentsPoller test exist, then delete it
        d = os.path.dirname(self.testComponentDaemonXml)
        if os.path.exists(d):
            shutil.rmtree(d)

    def testSenderReceiverBasic(self):
        sender = Sender(self.config.Alert.address,
                        self.config.Alert.controlAddr, self.__class__.__name__)
        handler, receiver = utils.setUpReceiver(self.config.Alert.address,
                                                self.config.Alert.controlAddr)
        a = Alert(Component=inspect.stack()[0][3])
        sender(a)
        while len(handler.queue) == 0:
            time.sleep(0.5)
            print "%s waiting for alert to arrive" % inspect.stack()[0][3]
        receiver.shutdown()
        self.assertEqual(len(handler.queue), 1)
        self.assertEqual(handler.queue[0]["Component"], inspect.stack()[0][3])

    def testProcessDetailBasic(self):
        pid = os.getpid()
        name = inspect.stack()[0][3]  # test name
        pd = ProcessDetail(pid, name)
        self.assertEqual(pd.pid, pid)
        self.assertEqual(pd.name, name)
        self.assertEqual(pd.proc.pid, pid)
        numChildren = len(psutil.Process(pid).get_children())
        self.assertEqual(len(pd.children), numChildren)
        self.assertEqual(len(pd.allProcs), 1 + numChildren)
        d = pd.getDetails()
        self.assertEqual(d["pid"], pid)
        self.assertEqual(d["component"], name)
        self.assertEqual(d["numChildrenProcesses"], numChildren)
        pd.refresh()

    def testMeasurementsBasic(self):
        numMes = 10
        mes = Measurements(numMes)
        self.assertEqual(mes._numOfMeasurements, numMes)
        self.assertEqual(len(mes), 0)
        mes.append(20)
        self.assertEqual(len(mes), 1)
        self.assertEqual(mes[0], 20)
        mes.append(30)
        self.assertEqual(mes[1], 30)
        mes.clear()
        self.assertEqual(len(mes), 0)
        self.assertEqual(mes._numOfMeasurements, numMes)

    def testBasePollerBasic(self):
        config = getConfig("/tmp")
        # create some non-sence config section. just need a bunch of values defined
        config.AlertGenerator.section_("bogusPoller")
        config.AlertGenerator.bogusPoller.soft = 5  # [percent]
        config.AlertGenerator.bogusPoller.critical = 50  # [percent]
        config.AlertGenerator.bogusPoller.pollInterval = 2  # [second]
        config.AlertGenerator.bogusPoller.period = 10

        generator = utils.AlertGeneratorMock(config)
        poller = BasePoller(config.AlertGenerator.bogusPoller, generator)
        # define dummy check method
        poller.check = lambda: 1 + 1
        poller.start()
        # poller now runs
        time.sleep(config.AlertGenerator.bogusPoller.pollInterval * 2)
        poller.terminate()
        while poller.is_alive():
            time.sleep(0.2)
            print "%s waiting for test poller to terminate" % inspect.stack(
            )[0][3]

    def testBasePollerHandleFailedPolling(self):
        config = getConfig("/tmp")
        # create some non-sence config section. just need a bunch of values defined
        config.AlertGenerator.section_("bogusPoller")
        config.AlertGenerator.bogusPoller.soft = 5  # [percent]
        config.AlertGenerator.bogusPoller.critical = 50  # [percent]
        config.AlertGenerator.bogusPoller.pollInterval = 2  # [second]
        config.AlertGenerator.bogusPoller.period = 10

        generator = utils.AlertGeneratorMock(config)
        poller = BasePoller(config.AlertGenerator.bogusPoller, generator)
        ex = Exception("test exception")

        class Sender(object):
            def __call__(self, alert):
                self.alert = alert

        poller.sender = Sender()
        poller._handleFailedPolling(ex)
        self.assertEqual(poller.sender.alert["Source"], "BasePoller")

    def testPeriodPollerOnRealProcess(self):
        config = getConfig("/tmp")
        config.component_("AlertProcessor")
        config.AlertProcessor.section_("critical")
        config.AlertProcessor.section_("soft")
        config.AlertProcessor.critical.level = 5
        config.AlertProcessor.soft.level = 0
        config.component_("AlertGenerator")
        config.AlertGenerator.section_("bogusPoller")
        config.AlertGenerator.bogusPoller.soft = 5  # [percent]
        config.AlertGenerator.bogusPoller.critical = 50  # [percent]
        config.AlertGenerator.bogusPoller.pollInterval = 0.2  # [second]
        # period during which measurements are collected before evaluating for
        # possible alert triggering
        config.AlertGenerator.bogusPoller.period = 1

        generator = utils.AlertGeneratorMock(config)
        poller = PeriodPoller(config.AlertGenerator.bogusPoller, generator)
        poller.sender = utils.SenderMock()
        # get CPU usage percentage, it's like measuring CPU usage of a real
        # component, so use the appropriate poller's method for that
        # (PeriodPoller itself is higher-level class so it doesn't define
        # a method to provide sampling data)
        poller.sample = lambda processDetail: ComponentsCPUPoller.sample(
            processDetail)

        # get own pid
        pid = os.getpid()
        name = inspect.stack()[0][3]  # test name
        pd = ProcessDetail(pid, name)
        # need to repeat sampling required number of measurements
        numOfMeasurements = int(config.AlertGenerator.bogusPoller.period /
                                config.AlertGenerator.bogusPoller.pollInterval)
        mes = Measurements(numOfMeasurements)
        self.assertEqual(len(mes), 0)
        for i in range(mes._numOfMeasurements):
            poller.check(pd, mes)

        # since the whole measurement cycle was done, values should have been nulled
        self.assertEqual(len(mes), 0)

    def testPeriodPollerCalculationPredefinedInput(self):
        config = getConfig("/tmp")
        config.component_("AlertProcessor")
        config.AlertProcessor.section_("critical")
        config.AlertProcessor.section_("soft")
        config.AlertProcessor.critical.level = 5
        config.AlertProcessor.soft.level = 0
        config.component_("AlertGenerator")
        config.AlertGenerator.section_("bogusPoller")
        # put some threshold numbers, just need to check output calculation
        # from check() method
        config.AlertGenerator.bogusPoller.soft = 5  # [percent]
        config.AlertGenerator.bogusPoller.critical = 50  # [percent]
        config.AlertGenerator.bogusPoller.pollInterval = 0.2  # [second]
        config.AlertGenerator.bogusPoller.period = 1

        generator = utils.AlertGeneratorMock(config)
        poller = PeriodPoller(config.AlertGenerator.bogusPoller, generator)
        # since poller may trigger an alert, give it mock sender
        poller.sender = utils.SenderMock()
        # provide sample method with predefined input, float
        predefInput = 10.12
        poller.sample = lambda processDetail: predefInput

        processDetail = None
        numOfMeasurements = int(config.AlertGenerator.bogusPoller.period /
                                config.AlertGenerator.bogusPoller.pollInterval)
        mes = Measurements(numOfMeasurements)
        for i in range(mes._numOfMeasurements):
            poller.check(processDetail, mes)

        # the above loop should went 5 times, should reach evaluation of 5 x predefInput
        # values, the average should end up 10, which should trigger soft threshold
        self.assertEqual(len(poller.sender.queue), 1)
        a = poller.sender.queue[0]

        self.assertEqual(a["Component"], generator.__class__.__name__)
        self.assertEqual(a["Source"], poller.__class__.__name__)
        d = a["Details"]
        self.assertEqual(d["threshold"],
                         "%s%%" % config.AlertGenerator.bogusPoller.soft)
        self.assertEqual(d["numMeasurements"], mes._numOfMeasurements)
        self.assertEqual(d["period"], config.AlertGenerator.bogusPoller.period)
        self.assertEqual(d["average"], "%s%%" % predefInput)
        # since the whole measurement cycle was done, values should have been nulled
        self.assertEqual(len(mes), 0)
コード例 #46
0
class BaseTest(unittest.TestCase):
    """
    Some methods of this class are made static and are used from 
    other test cases.
    
    """
    def setUp(self):
        self.testInit = TestInit(__file__)
        self.testInit.setLogging(logLevel = logging.DEBUG)
        self.testDir = self.testInit.generateWorkDir()
        self.config = getConfig(self.testDir)
        self.testComponentDaemonXml = "/tmp/TestComponent/Daemon.xml" 
        
        
    def tearDown(self):       
        self.testInit.delWorkDir()
        self.generator = None
        # if the directory and file "/tmp/TestComponent/Daemon.xml" after
        # ComponentsPoller test exist, then delete it
        d = os.path.dirname(self.testComponentDaemonXml)
        if os.path.exists(d):
            shutil.rmtree(d)
                    

    def testSenderReceiverBasic(self):
        sender = Sender(self.config.Alert.address,
                        self.__class__.__name__,
                        self.config.Alert.controlAddr)
        handler, receiver = utils.setUpReceiver(self.config.Alert.address,
                                                self.config.Alert.controlAddr)
        a = Alert(Component = inspect.stack()[0][3])
        sender(a)
        while len(handler.queue) == 0:
            time.sleep(0.5)
            print "%s waiting for alert to arrive" % inspect.stack()[0][3]
        receiver.shutdown()
        self.assertEqual(len(handler.queue), 1)
        self.assertEqual(handler.queue[0]["Component"], inspect.stack()[0][3])
        
        
    def testProcessDetailBasic(self):
        pid = os.getpid()
        name = inspect.stack()[0][3] # test name
        pd = ProcessDetail(pid, name)
        self.assertEqual(pd.pid, pid)
        self.assertEqual(pd.name, name)
        self.assertEqual(pd.proc.pid, pid)
        numChildren = len(psutil.Process(pid).get_children())
        self.assertEqual(len(pd.children), numChildren)
        self.assertEqual(len(pd.allProcs), 1 + numChildren)
        d = pd.getDetails()
        self.assertEqual(d["pid"], pid)
        self.assertEqual(d["component"], name)
        self.assertEqual(d["numChildrenProcesses"], numChildren)
                
        
    def testMeasurementsBasic(self):
        numMes = 10
        mes = Measurements(numMes)
        self.assertEqual(mes._numOfMeasurements, numMes)
        self.assertEqual(len(mes), 0)
        mes.append(20)
        self.assertEqual(len(mes), 1)
        self.assertEqual(mes[0], 20)
        mes.append(30)
        self.assertEqual(mes[1], 30)
        mes.clear()
        self.assertEqual(len(mes), 0)
        self.assertEqual(mes._numOfMeasurements, numMes)
        

    def testBasePollerBasic(self):
        config = getConfig("/tmp")
        # create some non-sence config section. just need a bunch of values defined        
        config.AlertGenerator.section_("bogusPoller")
        config.AlertGenerator.bogusPoller.soft = 5 # [percent]
        config.AlertGenerator.bogusPoller.critical = 50 # [percent] 
        config.AlertGenerator.bogusPoller.pollInterval = 2  # [second]
        config.AlertGenerator.bogusPoller.period = 10
        
        generator = utils.AlertGeneratorMock(config)
        poller = BasePoller(config.AlertGenerator.bogusPoller, generator)
        # define dummy check method
        poller.check = lambda: 1+1
        poller.start()
        # poller now runs
        time.sleep(config.AlertGenerator.bogusPoller.pollInterval * 2)
        poller.terminate()
        while poller.is_alive():
            time.sleep(0.2)
            print "%s waiting for test poller to terminate" % inspect.stack()[0][3]
            
        
    def testPeriodPollerOnRealProcess(self):
        config = getConfig("/tmp")
        config.component_("AlertProcessor")
        config.AlertProcessor.section_("critical")
        config.AlertProcessor.section_("soft")
        config.AlertProcessor.critical.level = 5
        config.AlertProcessor.soft.level = 0        
        config.component_("AlertGenerator")
        config.AlertGenerator.section_("bogusPoller")
        config.AlertGenerator.bogusPoller.soft = 5 # [percent]
        config.AlertGenerator.bogusPoller.critical = 50 # [percent] 
        config.AlertGenerator.bogusPoller.pollInterval = 0.2  # [second]
        # period during which measurements are collected before evaluating for
        # possible alert triggering
        config.AlertGenerator.bogusPoller.period = 1
        
        generator = utils.AlertGeneratorMock(config)
        poller = PeriodPoller(config.AlertGenerator.bogusPoller, generator)
        poller.sender = utils.SenderMock()
        # get CPU usage percentage, it's like measuring CPU usage of a real
        # component, so use the appropriate poller's method for that
        # (PeriodPoller itself is higher-level class so it doesn't define
        # a method to provide sampling data)
        poller.sample = lambda processDetail: ComponentsCPUPoller.sample(processDetail)
        
        # get own pid
        pid = os.getpid()
        name = inspect.stack()[0][3] # test name
        pd = ProcessDetail(pid, name)
        # need to repeat sampling required number of measurements
        numOfMeasurements = int(config.AlertGenerator.bogusPoller.period / 
                                config.AlertGenerator.bogusPoller.pollInterval)
        mes = Measurements(numOfMeasurements)
        self.assertEqual(len(mes), 0)
        for i in range(mes._numOfMeasurements):
            poller.check(pd, mes)
                    
        # since the whole measurement cycle was done, values should have been nulled
        self.assertEqual(len(mes), 0)
        
        
    def testPeriodPollerCalculationPredefinedInput(self):
        config = getConfig("/tmp")
        config.component_("AlertProcessor")
        config.AlertProcessor.section_("critical")
        config.AlertProcessor.section_("soft")
        config.AlertProcessor.critical.level = 5
        config.AlertProcessor.soft.level = 0        
        config.component_("AlertGenerator")
        config.AlertGenerator.section_("bogusPoller")
        # put some threshold numbers, just need to check output calculation
        # from check() method
        config.AlertGenerator.bogusPoller.soft = 5 # [percent]
        config.AlertGenerator.bogusPoller.critical = 50 # [percent] 
        config.AlertGenerator.bogusPoller.pollInterval = 0.2  # [second]
        config.AlertGenerator.bogusPoller.period = 1
        
        generator = utils.AlertGeneratorMock(config)
        poller = PeriodPoller(config.AlertGenerator.bogusPoller, generator)
        # since poller may trigger an alert, give it mock sender
        poller.sender = utils.SenderMock()
        # provide sample method with predefined input, float
        predefInput = 10.12
        poller.sample = lambda processDetail: predefInput
        
        processDetail = None
        numOfMeasurements = int(config.AlertGenerator.bogusPoller.period / 
                                config.AlertGenerator.bogusPoller.pollInterval)
        mes = Measurements(numOfMeasurements)
        for i in range(mes._numOfMeasurements):
            poller.check(processDetail, mes)
            
        # the above loop should went 5 times, should reach evaluation of 5 x predefInput
        # values, the average should end up 10, which should trigger soft threshold
        self.assertEqual(len(poller.sender.queue), 1)
        a = poller.sender.queue[0]
        
        self.assertEqual(a["Component"], generator.__class__.__name__)
        self.assertEqual(a["Source"], poller.__class__.__name__)
        d = a["Details"]
        self.assertEqual(d["threshold"], "%s%%" % config.AlertGenerator.bogusPoller.soft)
        self.assertEqual(d["numMeasurements"], mes._numOfMeasurements)
        self.assertEqual(d["period"], config.AlertGenerator.bogusPoller.period)
        self.assertEqual(d["average"], "%s%%" % predefInput)
        # since the whole measurement cycle was done, values should have been nulled
        self.assertEqual(len(mes), 0)
コード例 #47
0
class DashboardInterfaceTest(unittest.TestCase):
    """
    Test for the dashboard interface and its monitoring interaction

    Well, once I've written them it will be
    """


    def setUp(self):
        """
        Basically, do nothing

        """

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()


        self.testDir = self.testInit.generateWorkDir()

        return


    def tearDown(self):
        """
        Clean up the test directory

        """

        self.testInit.delWorkDir()

        return


    def createWorkload(self):
        """
        Create a workload in order to test things

        """
        generator = WMSpecGenerator()
        workload = generator.createReRecoSpec("Tier1ReReco")
        return workload

    def createTestJob(self):
        """
        Create a test job to pass to the DashboardInterface

        """

        job = Job(name = "ThisIsASillyName")

        testFileA = File(lfn = "/this/is/a/lfnA", size = 1024, events = 10)
        testFileA.addRun(Run(1, *[45]))
        testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10)
        testFileB.addRun(Run(1, *[46]))

        job.addFile(testFileA)
        job.addFile(testFileB)

        job['id'] = 1

        return job


    def createReport(self, outcome = 0):
        """
        Create a test report

        """

        jobReport = Report()
        jobReport.addStep('cmsRun1')
        jobReport.setStepStartTime(stepName = 'cmsRun1')
        jobReport.setStepStopTime(stepName = 'cmsRun1')
        if outcome:
            jobReport.addError('cmsRun1', 200, 'FakeError', 'FakeError')

        return jobReport


    def setupJobEnvironment(self, name = 'test'):
        """
        _setupJobEnvironment_

        Make some sort of environment in which to run tests
        """

        os.environ['WMAGENT_SITE_CONFIG_OVERRIDE'] = os.path.join(getTestBase(),
                                            "WMCore_t/Storage_t",
                                            "T1_US_FNAL_SiteLocalConfig.xml")
        return

    def testASuccessfulJobMonitoring(self):
        """
        _testASuccessfulJobMonitoring_

        Check that the data packets make sense when a job completes successfully
        """

        # Get the necessary objects
        name     = 'testA'
        job      = self.createTestJob()
        workload = self.createWorkload()
        task     = workload.getTask(taskName = "DataProcessing")
        report   = self.createReport()

        # Fill the job environment
        self.setupJobEnvironment(name = name)

        # Instantiate DBInfo
        dbInfo   = DashboardInfo(job = job, task = task)
        dbInfo.addDestination('127.0.0.1', 8884)

        # Check jobStart information
        data = dbInfo.jobStart()
        self.assertEqual(data['MessageType'], 'JobStatus')
        self.assertEqual(data['StatusValue'], 'running')
        self.assertEqual(data['StatusDestination'], "T1_US_FNAL")
        self.assertEqual(data['taskId'], 'wmagent_Tier1ReReco')

        # Do the first step
        step = task.getStep(stepName = "cmsRun1")

        # Do the step start
        data = dbInfo.stepStart(step = step.data)
        self.assertNotEqual(data['jobStart'], None)
        self.assertEqual(data['jobStart']['ExeStart'], step.name())
        self.assertEqual(data['jobStart']['WNHostName'], socket.gethostname())
        self.assertEqual(data['1_ExeStart'], step.name())

        #Do the step end
        data = dbInfo.stepEnd(step = step.data, stepReport = report)
        self.assertEqual(data['1_ExeEnd'], step.name())
        self.assertEqual(data['1_ExeExitCode'], 0)
        self.assertTrue(data['1_ExeWCTime'] >= 0)
        self.assertEqual(report.retrieveStep("cmsRun1").counter, 1)

        #Do a second step
        step = task.getStep(stepName = "cmsRun1")

        #Do the step start (It's not the first step)
        data = dbInfo.stepStart(step = step.data)
        self.assertEqual(data['jobStart'], None)
        self.assertEqual(data['2_ExeStart'], step.name())

        #Do the step end
        data = dbInfo.stepEnd(step = step.data, stepReport = report)
        self.assertEqual(data['2_ExeEnd'], step.name())
        self.assertEqual(data['2_ExeExitCode'], 0)
        self.assertTrue(data['2_ExeWCTime'] >= 0)
        self.assertEqual(report.retrieveStep("cmsRun1").counter, 2)

        # End the job!
        data = dbInfo.jobEnd()
        self.assertEqual(data['ExeEnd'], "cmsRun1")
        self.assertEqual(data['JobExitCode'], 0)
        self.assertEqual(data['WrapperCPUTime'], 0)
        self.assertTrue(data['WrapperWCTime'] >= 0)
        self.assertNotEqual(data['JobExitReason'], "")

        return


    def testAFailedJobMonitoring(self):
        """
        _TestAFailedJobMonitoring_

        Simulate a job that completes but fails, check that the data sent is
        correct
        """

        # Get the necessary objects
        name     = 'testB'
        job      = self.createTestJob()
        workload = self.createWorkload()
        task     = workload.getTask(taskName = "DataProcessing")
        report   = self.createReport(outcome = 1)

        # Fill the job environment
        self.setupJobEnvironment(name = name)

        # Instantiate DBInfo
        dbInfo   = DashboardInfo(job = job, task = task)
        dbInfo.addDestination('127.0.0.1', 8884)

        # Check jobStart information
        data = dbInfo.jobStart()
        self.assertEqual(data['MessageType'], 'JobStatus')
        self.assertEqual(data['StatusValue'], 'running')
        self.assertEqual(data['StatusDestination'], "T1_US_FNAL")
        self.assertEqual(data['taskId'], 'wmagent_Tier1ReReco')

        # Do the first step
        step = task.getStep(stepName = "cmsRun1")

        # Do the step start
        data = dbInfo.stepStart(step = step.data)
        self.assertNotEqual(data['jobStart'], None)
        self.assertEqual(data['jobStart']['ExeStart'], step.name())
        self.assertEqual(data['jobStart']['WNHostName'], socket.gethostname())
        self.assertEqual(data['1_ExeStart'], step.name())

        #Do the step end
        data = dbInfo.stepEnd(step = step.data, stepReport = report)
        self.assertEqual(data['1_ExeEnd'], step.name())
        self.assertNotEqual(data['1_ExeExitCode'], 0)
        self.assertTrue(data['1_ExeWCTime'] >= 0)
        self.assertEqual(report.retrieveStep("cmsRun1").counter, 1)

        # End the job!
        data = dbInfo.jobEnd()
        self.assertEqual(data['ExeEnd'], "cmsRun1")
        self.assertNotEqual(data['JobExitCode'], 0)
        self.assertEqual(data['WrapperCPUTime'], 0)
        self.assertTrue(data['WrapperWCTime'] >= 0)
        self.assertNotEqual(data['JobExitReason'].find('cmsRun1'), -1)

        return

    def testAKilledJobMonitoring(self):
        """
        _TestAKilledJobMonitoring_

        Simulate a job that is killed check that the data sent is
        correct
        """

        # Get the necessary objects
        name     = 'testC'
        job      = self.createTestJob()
        workload = self.createWorkload()
        task     = workload.getTask(taskName = "DataProcessing")
        report   = self.createReport(outcome = 1)

        # Fill the job environment
        self.setupJobEnvironment(name = name)

        # Instantiate DBInfo
        dbInfo   = DashboardInfo(job = job, task = task)
        dbInfo.addDestination('127.0.0.1', 8884)

        # Check jobStart information
        data = dbInfo.jobStart()
        self.assertEqual(data['MessageType'], 'JobStatus')
        self.assertEqual(data['StatusValue'], 'running')
        self.assertEqual(data['StatusDestination'], "T1_US_FNAL")
        self.assertEqual(data['taskId'], 'wmagent_Tier1ReReco')

        # Do the first step
        step = task.getStep(stepName = "cmsRun1")

        # Do the step start
        data = dbInfo.stepStart(step = step.data)
        self.assertNotEqual(data['jobStart'], None)
        self.assertEqual(data['jobStart']['ExeStart'], step.name())
        self.assertEqual(data['jobStart']['WNHostName'], socket.gethostname())
        self.assertEqual(data['1_ExeStart'], step.name())

        #Do the step end
        data = dbInfo.stepEnd(step = step.data, stepReport = report)
        self.assertEqual(data['1_ExeEnd'], step.name())
        self.assertNotEqual(data['1_ExeExitCode'], 0)
        self.assertTrue(data['1_ExeWCTime'] >= 0)

        # Kill the job!
        data = dbInfo.jobKilled()
        self.assertEqual(data['ExeEnd'], "cmsRun1")
        self.assertNotEqual(data['JobExitCode'], 0)
        self.assertEqual(data['WrapperCPUTime'], 0)
        self.assertTrue(data['WrapperWCTime'] >= 0)
        self.assertNotEqual(data['JobExitReason'].find('killed'), -1)

        return

    @attr('integration')
    def testGetDN(self):
        """
        _testGetDN_

        Checks that we can get a DN
        """
        dn = getUserProxyDN()
        if 'X509_USER_PROXY' in os.environ:
            self.assertNotEqual(dn, None, 'Error: This should get a DN, if you have set one')
        else:
            self.assertEqual(dn, None, 'Error: There is no proxy in the environment, it should not get one')
コード例 #48
0
class DBSUploadTest(unittest.TestCase):
    """
    TestCase for DBSUpload module

    """
    def setUp(self):
        """
        _setUp_

        setUp function for unittest

        """

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer"],
                                useDefault = False)
        self.testDir = self.testInit.generateWorkDir(deleteOnDestruction = False)
        self.configFile = EmulatorSetup.setupWMAgentConfig()

        myThread = threading.currentThread()
        self.bufferFactory = DAOFactory(package = "WMComponent.DBS3Buffer",
                                         logger = myThread.logger,
                                         dbinterface = myThread.dbi)

        locationAction = self.bufferFactory(classname = "DBSBufferFiles.AddLocation")
        locationAction.execute(siteName = "se1.cern.ch")
        locationAction.execute(siteName = "se1.fnal.gov")
        locationAction.execute(siteName = "malpaquet")
        self.dbsUrl = "https://*****:*****@attr("integration")
    def testBasicUpload(self):
        """
        _testBasicUpload_

        Verify that we can successfully upload to DBS3.  Also verify that the
        uploader correctly handles files parentage when uploading.
        """
        self.dbsApi = DbsApi(url = self.dbsUrl)
        config = self.getConfig()
        dbsUploader = DBSUploadPoller(config = config)

        # First test verifies that uploader will poll and then not do anything
        # as the database is empty.
        dbsUploader.algorithm()

        acqEra = "Summer%s" % (int(time.time()))
        parentFiles = self.createParentFiles(acqEra)

        # The algorithm needs to be run twice.  On the first iteration it will
        # create all the blocks and upload one.  On the second iteration it will
        # timeout and upload the second block.
        dbsUploader.algorithm()
        time.sleep(5)
        dbsUploader.algorithm()
        time.sleep(5)

        # Verify the files made it into DBS3.
        self.verifyData(parentFiles[0]["datasetPath"], parentFiles)

        # Inject some more parent files and some child files into DBSBuffer.
        # Run the uploader twice, only the parent files should be added to DBS3.
        (moreParentFiles, childFiles) = \
                          self.createFilesWithChildren(parentFiles, acqEra)
        dbsUploader.algorithm()
        time.sleep(5)
        dbsUploader.algorithm()
        time.sleep(5)

        self.verifyData(parentFiles[0]["datasetPath"],
                        parentFiles + moreParentFiles)

        # Run the uploader another two times to upload the child files.  Verify
        # that the child files were uploaded.
        dbsUploader.algorithm()
        time.sleep(5)
        dbsUploader.algorithm()
        time.sleep(5)

        self.verifyData(childFiles[0]["datasetPath"], childFiles)
        return

    @attr("integration")
    def testDualUpload(self):
        """
        _testDualUpload_

        Verify that the dual upload mode works correctly.
        """
        self.dbsApi = DbsApi(url = self.dbsUrl)
        config = self.getConfig()
        dbsUploader = DBSUploadPoller(config = config)
        dbsUtil = DBSBufferUtil()

        # First test verifies that uploader will poll and then not do anything
        # as the database is empty.
        dbsUploader.algorithm()

        acqEra = "Summer%s" % (int(time.time()))
        parentFiles = self.createParentFiles(acqEra)
        (moreParentFiles, childFiles) = \
                          self.createFilesWithChildren(parentFiles, acqEra)

        allFiles = parentFiles + moreParentFiles
        allBlocks = []
        for i in range(4):
            DBSBufferDataset(parentFiles[0]["datasetPath"]).create()
            blockName = parentFiles[0]["datasetPath"] + "#" + makeUUID()
            dbsBlock = DBSBufferBlock(blockName,
                                      location = "malpaquet",
                                      datasetpath =  None)
            dbsBlock.status = "Open"
            dbsBlock.setDataset(parentFiles[0]["datasetPath"], 'data', 'VALID')
            dbsUtil.createBlocks([dbsBlock])
            for file in allFiles[i * 5 : (i * 5) + 5]:
                dbsBlock.addFile(file, 'data', 'VALID')
                dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]})
                if i < 2:
                    dbsBlock.status = "InDBS"
                dbsUtil.updateBlocks([dbsBlock])
            dbsUtil.updateFileStatus([dbsBlock], "InDBS")
            allBlocks.append(dbsBlock)

        DBSBufferDataset(childFiles[0]["datasetPath"]).create()
        blockName = childFiles[0]["datasetPath"] + "#" + makeUUID()
        dbsBlock = DBSBufferBlock(blockName,
                                  location = "malpaquet",
                                  datasetpath =  None)
        dbsBlock.status = "InDBS"
        dbsBlock.setDataset(childFiles[0]["datasetPath"], 'data', 'VALID')
        dbsUtil.createBlocks([dbsBlock])
        for file in childFiles:
            dbsBlock.addFile(file, 'data', 'VALID')
            dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]})

        dbsUtil.updateFileStatus([dbsBlock], "InDBS")

        dbsUploader.algorithm()
        time.sleep(5)
        dbsUploader.algorithm()
        time.sleep(5)

        self.verifyData(parentFiles[0]["datasetPath"], parentFiles)

        # Change the status of the rest of the parent blocks so we can upload
        # them and the children.
        for dbsBlock in allBlocks:
            dbsBlock.status = "InDBS"
            dbsUtil.updateBlocks([dbsBlock])

        dbsUploader.algorithm()
        time.sleep(5)

        self.verifyData(parentFiles[0]["datasetPath"], parentFiles + moreParentFiles)

        # Run the uploader one more time to upload the children.
        dbsUploader.algorithm()
        time.sleep(5)

        self.verifyData(childFiles[0]["datasetPath"], childFiles)
        return

    def testCloseSettingsPerWorkflow(self):
        """
        _testCloseSettingsPerWorkflow_

        Test the block closing mechanics in the DBS3 uploader,
        this uses a fake dbs api to avoid reliance on external services.
        """
        # Signal trapExit that we are a friend
        os.environ["DONT_TRAP_EXIT"] = "True"
        try:
            # Monkey patch the imports of DbsApi
            from WMComponent.DBS3Buffer import DBSUploadPoller as MockDBSUploadPoller
            MockDBSUploadPoller.DbsApi = MockDbsApi

            # Set the poller and the dbsUtil for verification
            myThread = threading.currentThread()
            (_, dbsFilePath) = mkstemp(dir = self.testDir)
            self.dbsUrl = dbsFilePath
            config = self.getConfig()
            dbsUploader = MockDBSUploadPoller.DBSUploadPoller(config = config)
            dbsUtil = DBSBufferUtil()

            # First test is event based limits and timeout with no new files.
            # Set the files and workflow
            acqEra = "TropicalSeason%s" % (int(time.time()))
            workflowName = 'TestWorkload%s' % (int(time.time()))
            taskPath = '/%s/TestProcessing' % workflowName
            self.injectWorkflow(workflowName, taskPath,
                                MaxWaitTime = 2, MaxFiles = 100,
                                MaxEvents = 150)
            self.createParentFiles(acqEra, nFiles = 20,
                                   workflowName = workflowName,
                                   taskPath = taskPath)

            # The algorithm needs to be run twice.  On the first iteration it will
            # create all the blocks and upload one with less than 150 events.
            # On the second iteration the second block is uploaded.
            dbsUploader.algorithm()
            dbsUploader.checkBlocks()
            openBlocks = dbsUtil.findOpenBlocks()
            self.assertEqual(len(openBlocks), 1)
            globalFiles = myThread.dbi.processData("SELECT id FROM dbsbuffer_file WHERE status = 'InDBS'")[0].fetchall()
            notUploadedFiles = myThread.dbi.processData("SELECT id FROM dbsbuffer_file WHERE status = 'NOTUPLOADED'")[0].fetchall()
            self.assertEqual(len(globalFiles), 14)
            self.assertEqual(len(notUploadedFiles), 6)
            # Check the fake DBS for data
            fakeDBS = open(self.dbsUrl, 'r')
            fakeDBSInfo = json.load(fakeDBS)
            fakeDBS.close()
            self.assertEqual(len(fakeDBSInfo), 2)
            for block in fakeDBSInfo:
                self.assertTrue('block_events' not in block['block'])
                self.assertEqual(block['block']['file_count'], 7)
                self.assertEqual(block['block']['open_for_writing'], 0)
                self.assertTrue('close_settings' not in block)
            time.sleep(3)
            dbsUploader.algorithm()
            dbsUploader.checkBlocks()
            openBlocks = dbsUtil.findOpenBlocks()
            self.assertEqual(len(openBlocks), 0)
            fakeDBS = open(self.dbsUrl, 'r')
            fakeDBSInfo = json.load(fakeDBS)
            fakeDBS.close()
            self.assertEqual(len(fakeDBSInfo), 3)
            for block in fakeDBSInfo:
                if block['block']['file_count'] != 6:
                    self.assertEqual(block['block']['file_count'], 7)
                self.assertTrue('block_events' not in block['block'])
                self.assertEqual(block['block']['open_for_writing'], 0)
                self.assertTrue('close_settings' not in block)

            # Now check the limit by size and timeout with new files
            acqEra = "TropicalSeason%s" % (int(time.time()))
            workflowName = 'TestWorkload%s' % (int(time.time()))
            taskPath = '/%s/TestProcessing' % workflowName
            self.injectWorkflow(workflowName, taskPath,
                                MaxWaitTime = 2, MaxFiles = 5,
                                MaxEvents = 200000000)
            self.createParentFiles(acqEra, nFiles = 16,
                                   workflowName = workflowName,
                                   taskPath = taskPath)
            dbsUploader.algorithm()
            dbsUploader.checkBlocks()
            openBlocks = dbsUtil.findOpenBlocks()
            self.assertEqual(len(openBlocks), 1)
            fakeDBS = open(self.dbsUrl, 'r')
            fakeDBSInfo = json.load(fakeDBS)
            fakeDBS.close()
            self.assertEqual(len(fakeDBSInfo), 6)
            for block in fakeDBSInfo:
                if acqEra in block['block']['block_name']:
                    self.assertEqual(block['block']['file_count'], 5)
                self.assertTrue('block_events' not in block['block'])
                self.assertTrue('close_settings' not in block)
                self.assertEqual(block['block']['open_for_writing'], 0)

            # Put more files, they will go into the same block and then it will be closed
            # after timeout
            time.sleep(3)
            self.createParentFiles(acqEra, nFiles = 3,
                                   workflowName = workflowName,
                                   taskPath = taskPath)
            dbsUploader.algorithm()
            dbsUploader.checkBlocks()
            openBlocks = dbsUtil.findOpenBlocks()
            self.assertEqual(len(openBlocks), 0)
            fakeDBS = open(self.dbsUrl, 'r')
            fakeDBSInfo = json.load(fakeDBS)
            fakeDBS.close()
            self.assertEqual(len(fakeDBSInfo), 7)
            for block in fakeDBSInfo:
                if acqEra in block['block']['block_name']:
                    if block['block']['file_count'] < 5:
                        self.assertEqual(block['block']['file_count'], 4)
                    else:
                        self.assertEqual(block['block']['file_count'], 5)
                self.assertTrue('block_events' not in block['block'])
                self.assertEqual(block['block']['open_for_writing'], 0)
                self.assertTrue('close_settings' not in block)

            # Finally test size limits
            acqEra = "TropicalSeason%s" % (int(time.time()))
            workflowName = 'TestWorkload%s' % (int(time.time()))
            taskPath = '/%s/TestProcessing' % workflowName
            self.injectWorkflow(workflowName, taskPath,
                                MaxWaitTime = 1, MaxFiles = 500,
                                MaxEvents = 200000000, MaxSize = 2048)
            self.createParentFiles(acqEra, nFiles = 7,
                                   workflowName = workflowName,
                                   taskPath = taskPath)
            dbsUploader.algorithm()
            dbsUploader.checkBlocks()
            time.sleep(2)
            dbsUploader.algorithm()
            dbsUploader.checkBlocks()

            self.assertEqual(len(openBlocks), 0)
            fakeDBS = open(self.dbsUrl, 'r')
            fakeDBSInfo = json.load(fakeDBS)
            fakeDBS.close()
            self.assertEqual(len(fakeDBSInfo), 11)
            for block in fakeDBSInfo:
                if acqEra in block['block']['block_name']:
                    if block['block']['file_count'] != 1:
                        self.assertEqual(block['block']['block_size'], 2048)
                        self.assertEqual(block['block']['file_count'], 2)
                self.assertTrue('block_events' not in block['block'])
                self.assertEqual(block['block']['open_for_writing'], 0)
                self.assertTrue('close_settings' not in block)
        except:
            self.fail("We failed at some point in the test")
        finally:
            # We don't trust anyone else with _exit
            del os.environ["DONT_TRAP_EXIT"]
        return
コード例 #49
0
ファイル: ProcessPool_t.py プロジェクト: todor-ivanov/WMCore
class ProcessPoolTest(unittest.TestCase):
    def setUp(self):
        """
        _setUp_

        """
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection(destroyAllDatabase = True)
        self.testInit.setSchema(customModules = ["WMCore.Agent.Database"],
                                useDefault = False)
        return

    def tearDown(self):
        """
        _tearDown_

        """
        self.testInit.clearDatabase()
        return

    def testA_ProcessPool(self):
        """
        _testProcessPool_

        """
        raise nose.SkipTest
        config = self.testInit.getConfiguration()
        config.Agent.useHeartbeat = False
        self.testInit.generateWorkDir(config)

        processPool = ProcessPool("ProcessPool_t.ProcessPoolTestWorker",
                                  totalSlaves = 1,
                                  componentDir = config.General.workDir,
                                  config = config,
                                  namespace = "WMCore_t")

        processPool.enqueue(["One", "Two", "Three"])
        result =  processPool.dequeue(3)

        self.assertEqual(len(result), 3, "Error: Expected three items back.")
        self.assertTrue( "One" in result)
        self.assertTrue( "Two" in result)
        self.assertTrue( "Three" in result)

        return

    def testB_ProcessPoolStress(self):
        """
        _testProcessPoolStress_

        """
        raise nose.SkipTest
        config = self.testInit.getConfiguration()
        config.Agent.useHeartbeat = False
        self.testInit.generateWorkDir(config)

        processPool = ProcessPool("ProcessPool_t.ProcessPoolTestWorker",
                                  totalSlaves = 1,
                                  componentDir = config.General.workDir,
                                  namespace = "WMCore_t",
                                  config = config)

        result = None
        input  = None
        for i in range(1000):
            input = []
            while i > 0:
                input.append("COMMAND%s" % i)
                i -= 1

            processPool.enqueue(input)
            result =  processPool.dequeue(len(input))

            self.assertEqual(len(result), len(input),
                             "Error: Wrong number of results returned.")

        for k in result:
            self.assertTrue(k in input)

        return


    def testC_MultiPool(self):
        """
        _testMultiPool_

        Run a test with multiple workers
        """
        raise nose.SkipTest
        config = self.testInit.getConfiguration()
        config.Agent.useHeartbeat = False
        self.testInit.generateWorkDir(config)

        processPool = ProcessPool("ProcessPool_t.ProcessPoolTestWorker",
                                  totalSlaves = 3,
                                  componentDir = config.General.workDir,
                                  namespace = "WMCore_t",
                                  config = config)

        for i in range(100):
            input = []
            while i > 0:
                input.append("COMMAND%s" % i)
                i -= 1

            processPool.enqueue(input)
            result =  processPool.dequeue(len(input))

            self.assertEqual(len(result), len(input),
                             "Error: Wrong number of results returned.")
コード例 #50
0
class StoreResultsTest(unittest.TestCase):
    def setUp(self):
        """
        _setUp_

        Initialize the database.
        """
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules=["WMCore.WMBS"],
                                useDefault=False)
        self.testDir = self.testInit.generateWorkDir()

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package="WMCore.WMBS",
                                     logger=myThread.logger,
                                     dbinterface=myThread.dbi)
        self.listTasksByWorkflow = self.daoFactory(
            classname="Workflow.LoadFromName")
        self.listFilesets = self.daoFactory(classname="Fileset.List")
        self.listSubsMapping = self.daoFactory(
            classname="Subscriptions.ListSubsAndFilesetsFromWorkflow")

        return

    def tearDown(self):
        """
        _tearDown_

        Clear out the database.
        """
        self.testInit.clearDatabase()
        self.testInit.delWorkDir()
        return

    def testStoreResults(self):
        """
        _testStoreResults_

        Create a StoreResults workflow and verify it installs into WMBS
        correctly.
        """
        arguments = StoreResultsWorkloadFactory.getTestArguments()

        factory = StoreResultsWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction(
            "TestWorkload", arguments)

        testWMBSHelper = WMBSHelper(testWorkload,
                                    "StoreResults",
                                    "SomeBlock",
                                    cachepath=self.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(
            testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        testWorkflow = Workflow(name="TestWorkload",
                                task="/TestWorkload/StoreResults")
        testWorkflow.load()

        self.assertEqual(len(testWorkflow.outputMap.keys()), 2,
                         "Error: Wrong number of WF outputs.")
        goldenOutputMods = {"Merged": "USER"}
        for goldenOutputMod, tier in goldenOutputMods.items():
            fset = goldenOutputMod + tier
            mergedOutput = testWorkflow.outputMap[fset][0][
                "merged_output_fileset"]
            unmergedOutput = testWorkflow.outputMap[fset][0]["output_fileset"]

            mergedOutput.loadData()
            unmergedOutput.loadData()

            self.assertEqual(
                mergedOutput.name,
                "/TestWorkload/StoreResults/merged-%s" % fset,
                "Error: Merged output fileset is wrong: %s" %
                mergedOutput.name)
            self.assertEqual(
                unmergedOutput.name,
                "/TestWorkload/StoreResults/merged-%s" % fset,
                "Error: Unmerged output fileset is wrong: %s." %
                unmergedOutput.name)

        logArchOutput = testWorkflow.outputMap["logArchive"][0][
            "merged_output_fileset"]
        unmergedLogArchOutput = testWorkflow.outputMap["logArchive"][0][
            "output_fileset"]
        logArchOutput.loadData()
        unmergedLogArchOutput.loadData()

        self.assertEqual(logArchOutput.name,
                         "/TestWorkload/StoreResults/merged-logArchive",
                         "Error: LogArchive output fileset is wrong.")
        self.assertEqual(unmergedLogArchOutput.name,
                         "/TestWorkload/StoreResults/merged-logArchive",
                         "Error: LogArchive output fileset is wrong.")

        topLevelFileset = Fileset(name="TestWorkload-StoreResults-SomeBlock")
        topLevelFileset.loadData()

        procSubscription = Subscription(fileset=topLevelFileset,
                                        workflow=testWorkflow)
        procSubscription.loadData()

        self.assertEqual(procSubscription["type"], "Merge",
                         "Error: Wrong subscription type.")
        self.assertEqual(procSubscription["split_algo"],
                         "ParentlessMergeBySize", "Error: Wrong split algo.")

        return

    def testFilesets(self):
        """
        Test workflow tasks, filesets and subscriptions creation
        """
        # expected tasks, filesets, subscriptions, etc
        expOutTasks = ['/TestWorkload/StoreResults']
        expWfTasks = [
            '/TestWorkload/StoreResults',
            '/TestWorkload/StoreResults/StoreResultsLogCollect'
        ]
        expFsets = [
            'TestWorkload-StoreResults-/MinimumBias/ComissioningHI-v1/RAW',
            '/TestWorkload/StoreResults/merged-MergedUSER',
            '/TestWorkload/StoreResults/merged-logArchive'
        ]
        subMaps = [
            (2, '/TestWorkload/StoreResults/merged-logArchive',
             '/TestWorkload/StoreResults/StoreResultsLogCollect',
             'MinFileBased', 'LogCollect'),
            (1, 'TestWorkload-StoreResults-/MinimumBias/ComissioningHI-v1/RAW',
             '/TestWorkload/StoreResults', 'ParentlessMergeBySize', 'Merge')
        ]

        testArguments = StoreResultsWorkloadFactory.getTestArguments()

        factory = StoreResultsWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction(
            "TestWorkload", testArguments)

        testWMBSHelper = WMBSHelper(testWorkload,
                                    "StoreResults",
                                    blockName=testArguments['InputDataset'],
                                    cachepath=self.testInit.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(
            testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        self.assertItemsEqual(testWorkload.listOutputProducingTasks(),
                              expOutTasks)

        workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload")
        self.assertItemsEqual([item['task'] for item in workflows], expWfTasks)

        # returns a tuple of id, name, open and last_update
        filesets = self.listFilesets.execute()
        self.assertItemsEqual([item[1] for item in filesets], expFsets)

        subscriptions = self.listSubsMapping.execute(workflow="TestWorkload",
                                                     returnTuple=True)
        self.assertItemsEqual(subscriptions, subMaps)
コード例 #51
0
ファイル: Agent_t.py プロジェクト: stuartw/WMCore
class AgentTest(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInit(__file__)
        self.testInit.setLogging(logLevel = logging.DEBUG)
        self.testDir = self.testInit.generateWorkDir()
        self.config = getConfig(self.testDir)
        # mock generator instance to communicate some configuration values
        self.generator = utils.AlertGeneratorMock(self.config)        
        self.testComponentDaemonXml = os.path.join(self.testDir, "Daemon.xml") 
        
        
    def tearDown(self):       
        self.testInit.delWorkDir()
        self.generator = None
            

    def testComponentsPollerBasic(self):
        """
        Test ComponentsPoller class.
        Beware of different process context in real running.
        
        """
        config = getConfig("/tmp")
        config.component_("AlertProcessor")
        config.AlertProcessor.section_("critical")
        config.AlertProcessor.section_("soft")
        config.AlertProcessor.critical.level = 5
        config.AlertProcessor.soft.level = 0        
        config.component_("AlertGenerator")
        config.AlertGenerator.section_("bogusPoller")
        config.AlertGenerator.bogusPoller.soft = 5 # [percent]
        config.AlertGenerator.bogusPoller.critical = 90 # [percent] 
        config.AlertGenerator.bogusPoller.pollInterval = 2  # [second]
        # period during which measurements are collected before evaluating for possible alert triggering
        config.AlertGenerator.bogusPoller.period = 10
        
        # need to create some temp directory, real process and it's
        # Daemon.xml so that is looks like agents component process 
        # and check back the information
        pid = os.getpid()
        config.component_("TestComponent")
        d = os.path.dirname(self.testComponentDaemonXml)
        config.TestComponent.componentDir = d
        if not os.path.exists(d):
            os.mkdir(d)
        f = open(self.testComponentDaemonXml, 'w')
        f.write(utils.daemonXmlContent % dict(PID_TO_PUT = pid))
        f.close()
                
        generator = utils.AlertGeneratorMock(config)
        poller = ComponentsPoller(config.AlertGenerator.bogusPoller, generator)
        
        # only 1 component should have valid workDir with proper Daemon.xml content
        # other components present in the configuration (AlertProcessor, AlertGenerator)
        # should have been ignored
        self.assertEqual(len(poller._components), 1)
        pd = poller._components[0]
        self.assertEqual(pd.pid, pid)
        self.assertEqual(pd.name, "TestComponent")
        self.assertEqual(len(pd.children), 0)
        self.assertEqual(len(poller._compMeasurements), 1)
        mes = poller._compMeasurements[0]
        numMeasurements = round(config.AlertGenerator.bogusPoller.period / config.AlertGenerator.bogusPoller.pollInterval, 0)
        self.assertEqual(mes._numOfMeasurements, numMeasurements)
        
        shutil.rmtree(d)
        

    def _doComponentsPoller(self, thresholdToTest, level, config,
                            pollerClass, expected = 0):
        """
        Components pollers have array of Measurements and ProcessDetails
        which make it more difficult to factory with test methods from the
        utils module.
        
        """
        handler, receiver = utils.setUpReceiver(self.generator.config.Alert.address,
                                                self.generator.config.Alert.controlAddr)
        
        # need some real process to poll, give itself
        pid = os.getpid()
        # the input configuration doesn't have component work directories set right, rectify:
        # the configuration will have, see with what _doComponentsPoller is called
        # two components: AlertGenerator and AlertProcessor defined
        configInstance = Configuration.getInstance()
        for comp in Configuration.getInstance().listComponents_():
            compDir = getattr(configInstance, comp).componentDir
            compDir = os.path.join(compDir, comp)
            setattr(getattr(configInstance, comp), "componentDir", compDir)
            os.makedirs(compDir)
            f = open(os.path.join(compDir, "Daemon.xml"), 'w')
            f.write(utils.daemonXmlContent % dict(PID_TO_PUT = pid))
            f.close()
   
        numMeasurements = config.period / config.pollInterval
        poller = pollerClass(config, self.generator)
        # inject own input sample data provider
        # there is in fact input argument in this case which needs be ignored
        poller.sample = lambda proc_: random.randint(thresholdToTest, thresholdToTest + 10)
        
        # the poller will run upon components (as defined in the configuration)
        # and poll them. the PID, etc will be run from the compomentsDir
        poller.start()
        self.assertTrue(poller.is_alive())

        if expected != 0:
            # watch so that the test can't take for ever, fail in 2mins
            timeLimitExceeded = False
            startTime = datetime.datetime.now()
            limitTime = 2 * 60 # seconds
            while len(handler.queue) == 0:
                time.sleep(config.pollInterval / 5)
                if (datetime.datetime.now() - startTime).seconds > limitTime:
                    timeLimitExceeded = True
                    break                
        else:
            time.sleep(config.period * 2)
            
        poller.terminate()
        receiver.shutdown()
        self.assertFalse(poller.is_alive())
        
        if expected != 0:
            if timeLimitExceeded:
                self.fail("No alert received in %s seconds." % limitTime)            
            # there should be just one alert received, poller should have the
            # change to send a second
            self.assertEqual(len(handler.queue), expected)
            a = handler.queue[0]
            # soft threshold - alert should have 'soft' level
            self.assertEqual(a["Level"], level)
            self.assertEqual(a["Component"], self.generator.__class__.__name__)
            self.assertEqual(a["Source"], poller.__class__.__name__)
            
            
    def testComponentsCPUPollerSoftThreshold(self):
        self.config.AlertGenerator.componentsCPUPoller.soft = 70
        self.config.AlertGenerator.componentsCPUPoller.critical = 80
        self.config.AlertGenerator.componentsCPUPoller.pollInterval = 0.2
        self.config.AlertGenerator.componentsCPUPoller.period = 1
        level = self.config.AlertProcessor.soft.level
        thresholdToTest = self.config.AlertGenerator.componentsCPUPoller.soft
        # expected 2: 2 components are defined by the configuration, an alert
        # will be sent for each of them
        self._doComponentsPoller(thresholdToTest, level,
                                 self.config.AlertGenerator.componentsCPUPoller,
                                 ComponentsCPUPoller, expected = 2)        
       

    def testComponentsCPUPollerCriticalThreshold(self):
        self.config.AlertGenerator.componentsCPUPoller.soft = 70
        self.config.AlertGenerator.componentsCPUPoller.critical = 80
        self.config.AlertGenerator.componentsCPUPoller.pollInterval = 0.2
        self.config.AlertGenerator.componentsCPUPoller.period = 1
        level = self.config.AlertProcessor.critical.level
        thresholdToTest = self.config.AlertGenerator.componentsCPUPoller.critical
        # expected 2: 2 components are defined by the configuration, an alert
        # will be sent for each of them
        self._doComponentsPoller(thresholdToTest, level,
                                 self.config.AlertGenerator.componentsCPUPoller,
                                 ComponentsCPUPoller, expected = 2)


    def testComponentsCPUPollerNoAlert(self):
        self.config.AlertGenerator.componentsCPUPoller.soft = 70
        self.config.AlertGenerator.componentsCPUPoller.critical = 80
        self.config.AlertGenerator.componentsCPUPoller.pollInterval = 0.2
        self.config.AlertGenerator.componentsCPUPoller.period = 1
        level = 0
        # lower the threshold so that the alert never happens
        thresholdToTest = self.config.AlertGenerator.componentsCPUPoller.soft - 10
        self._doComponentsPoller(thresholdToTest, level,
                                 self.config.AlertGenerator.componentsCPUPoller,
                                 ComponentsCPUPoller, expected = 0)
        

    def testComponentsMemoryPollerSoftThreshold(self):
        self.config.AlertGenerator.componentsMemPoller.soft = 70
        self.config.AlertGenerator.componentsMemPoller.critical = 80
        self.config.AlertGenerator.componentsMemPoller.pollInterval = 0.2
        self.config.AlertGenerator.componentsMemPoller.period = 1
        level = self.config.AlertProcessor.soft.level
        thresholdToTest = self.config.AlertGenerator.componentsMemPoller.soft
        # expected 2: 2 components are defined by the configuration, an alert
        # will be sent for each of them
        self._doComponentsPoller(thresholdToTest, level,
                                 self.config.AlertGenerator.componentsMemPoller,
                                 ComponentsMemoryPoller, expected = 2)


    def testComponentsMemoryPollerCriticalThreshold(self):
        self.config.AlertGenerator.componentsMemPoller.soft = 70
        self.config.AlertGenerator.componentsMemPoller.critical = 80
        self.config.AlertGenerator.componentsMemPoller.pollInterval = 0.2
        self.config.AlertGenerator.componentsMemPoller.period = 1
        level = self.config.AlertProcessor.critical.level
        thresholdToTest = self.config.AlertGenerator.componentsMemPoller.critical
        # expected 2: 2 components are defined by the configuration, an alert
        # will be sent for each of them
        self._doComponentsPoller(thresholdToTest, level,
                                 self.config.AlertGenerator.componentsMemPoller,
                                 ComponentsMemoryPoller, expected = 2)


    def testComponentsMemoryPollerNoAlert(self):
        self.config.AlertGenerator.componentsMemPoller.soft = 70
        self.config.AlertGenerator.componentsMemPoller.critical = 80
        self.config.AlertGenerator.componentsMemPoller.pollInterval = 0.2
        self.config.AlertGenerator.componentsMemPoller.period = 1
        level = 0
        # lower the threshold so that the alert never happens
        thresholdToTest = self.config.AlertGenerator.componentsCPUPoller.soft - 10
        self._doComponentsPoller(thresholdToTest, level,
                                 self.config.AlertGenerator.componentsMemPoller,
                                 ComponentsMemoryPoller, expected = 0)
                

    def testComponentsCPUPollerPossiblyOnLiveAgent(self):
        """
        If there is currently running agent upon WMAGENT_CONFIG
        configuration, then the test will pick up live processes
        and poll them.
        
        """
        # check if the live agent configuration was loaded (above this class)    
        if globals().has_key("config"):
            self.config = config
            # AlertProcessor values - values for Level soft, resp. critical
            # are also needed by this AlertGenerator test
            self.config.component_("AlertProcessor")
            self.config.AlertProcessor.componentDir = "/tmp"
            self.config.AlertProcessor.section_("critical")
            self.config.AlertProcessor.section_("soft")
            self.config.AlertProcessor.critical.level = 5
            self.config.AlertProcessor.soft.level = 0
            
            self.config.component_("AlertGenerator")
            self.config.AlertGenerator.componentDir = "/tmp"
            self.config.section_("Alert")
            self.config.Alert.address = "tcp://127.0.0.1:6557"
            self.config.Alert.controlAddr = "tcp://127.0.0.1:6559"
            
            self.config.AlertGenerator.section_("componentsCPUPoller")
        else:
            self.config = getConfig("/tmp")
        
        self.config.AlertGenerator.componentsCPUPoller.soft = 70
        self.config.AlertGenerator.componentsCPUPoller.critical = 80
        self.config.AlertGenerator.componentsCPUPoller.pollInterval = 0.2
        self.config.AlertGenerator.componentsCPUPoller.period = 0.3
                
        # generator has already been instantiated, but need another one
        # with just defined configuration
        # mock generator instance to communicate some configuration values
        self.generator = utils.AlertGeneratorMock(self.config)
        
        handler, receiver = utils.setUpReceiver(self.generator.config.Alert.address,
                                                self.generator.config.Alert.controlAddr)

        numMeasurements = self.config.AlertGenerator.componentsCPUPoller.period / self.config.AlertGenerator.componentsCPUPoller.pollInterval
        poller = ComponentsCPUPoller(self.config.AlertGenerator.componentsCPUPoller, self.generator)
        # inject own input sample data provider
        thresholdToTest = self.config.AlertGenerator.componentsCPUPoller.soft
        # there is in fact input argument in this case which needs be ignored
        poller.sample = lambda proc_: random.randint(thresholdToTest - 10, thresholdToTest)
        
        poller.start()
        self.assertTrue(poller.is_alive())

        # no alert shall arrive
        time.sleep(5 * self.config.AlertGenerator.componentsCPUPoller.period)
        
        poller.terminate()
        receiver.shutdown()
        self.assertFalse(poller.is_alive())
コード例 #52
0
ファイル: WMAgent_t.py プロジェクト: ticoann/WMCore
class WMAgentTest(unittest.TestCase):
    """
    _WMAgentTest_

    Global unittest for all WMAgent components
    """

    # This is an integration test
    __integration__ = "Any old bollocks"

    sites = ['T2_US_Florida', 'T2_US_UCSD', 'T2_TW_Taiwan', 'T1_CH_CERN']
    components = ['JobCreator', 'JobSubmitter', 'JobTracker',
                  'JobAccountant', 'JobArchiver', 'TaskArchiver',
                  'RetryManager', 'ErrorHandler']

    def setUp(self):
        """
        _setUp_

        Set up vital components
        """

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = ["WMCore.WMBS",'WMCore.MsgService',
                                                 'WMCore.ResourceControl', 'WMCore.ThreadPool',
                                                 'WMCore.Agent.Database'],
                                useDefault = False)

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)



        locationAction = self.daoFactory(classname = "Locations.New")
        pendingSlots  = self.daoFactory(classname = "Locations.SetPendingSlots")


        for site in self.sites:
            locationAction.execute(siteName = site, seName = 'se.%s' % (site), ceName = site)
            pendingSlots.execute(siteName = site, pendingSlots = 1000)


        #Create sites in resourceControl
        resourceControl = ResourceControl()
        for site in self.sites:
            resourceControl.insertSite(siteName = site, seName = 'se.%s' % (site), ceName = site)
            resourceControl.insertThreshold(siteName = site, taskType = 'Processing', \
                                            maxSlots = 10000, pendingSlots = 10000)


        self.testDir = self.testInit.generateWorkDir()


        # Set heartbeat
        for component in self.components:
            heartbeatAPI = HeartbeatAPI(component)
            heartbeatAPI.registerComponent()




        return


    def tearDown(self):
        """
        _tearDown_

        Tear down everything and go home.
        """

        self.testInit.clearDatabase()

        self.testInit.delWorkDir()

        return




    def createTestWorkload(self, workloadName = 'Test', emulator = True):
        """
        _createTestWorkload_

        Creates a test workload for us to run on, hold the basic necessities.
        """


        workload = testWorkload("TestWorkload")
        rereco = workload.getTask("ReReco")


        taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest'))
        taskMaker.skipSubscription = True
        taskMaker.processWorkload()

        workload.save(workloadName)

        return workload




    def getConfig(self):
        """
        _getConfig_

        This is the global test configuration object
        """



        config = Configuration()

        config.component_("Agent")
        config.Agent.WMSpecDirectory = self.testDir
        config.Agent.agentName       = 'testAgent'
        config.Agent.componentName   = 'test'


        # First the general stuff
        config.section_("General")
        config.General.workDir = os.getenv("TESTDIR", self.testDir)

        # Now the CoreDatabase information
        # This should be the dialect, dburl, etc

        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket     = os.getenv("DBSOCK")



        # JobCreator
        config.component_("JobCreator")
        config.JobCreator.namespace = 'WMComponent.JobCreator.JobCreator'
        config.JobCreator.logLevel  = 'DEBUG'
        config.JobCreator.maxThreads                = 1
        config.JobCreator.UpdateFromResourceControl = True
        config.JobCreator.pollInterval              = 10
        config.JobCreator.jobCacheDir               = self.testDir
        config.JobCreator.defaultJobType            = 'processing' #Type of jobs that we run, used for resource control
        config.JobCreator.workerThreads             = 2
        config.JobCreator.componentDir              = os.path.join(os.getcwd(), 'Components')



        # JobSubmitter
        config.component_("JobSubmitter")
        config.JobSubmitter.namespace     = 'WMComponent.JobSubmitter.JobSubmitter'
        config.JobSubmitter.logLevel      = 'INFO'
        config.JobSubmitter.maxThreads    = 1
        config.JobSubmitter.pollInterval  = 10
        config.JobSubmitter.pluginName    = 'CondorGlobusPlugin'
        config.JobSubmitter.pluginDir     = 'JobSubmitter.Plugins'
        config.JobSubmitter.submitDir     = os.path.join(self.testDir, 'submit')
        config.JobSubmitter.submitNode    = os.getenv("HOSTNAME", 'badtest.fnal.gov')
        config.JobSubmitter.submitScript  = os.path.join(getWMBASE(),
                                                         'test/python/WMComponent_t/JobSubmitter_t',
                                                         'submit.sh')
        config.JobSubmitter.componentDir  = os.path.join(os.getcwd(), 'Components')
        config.JobSubmitter.workerThreads = 2
        config.JobSubmitter.jobsPerWorker = 200




        # JobTracker
        config.component_("JobTracker")
        config.JobTracker.logLevel      = 'DEBUG'
        config.JobTracker.pollInterval  = 10
        config.JobTracker.trackerName   = 'CondorTracker'
        config.JobTracker.pluginDir     = 'WMComponent.JobTracker.Plugins'
        config.JobTracker.componentDir  = os.path.join(os.getcwd(), 'Components')
        config.JobTracker.runTimeLimit  = 7776000 #Jobs expire after 90 days
        config.JobTracker.idleTimeLimit = 7776000
        config.JobTracker.heldTimeLimit = 7776000
        config.JobTracker.unknTimeLimit = 7776000



        # JobAccountant
        config.component_("JobAccountant")
        config.JobAccountant.pollInterval = 60
        config.JobAccountant.componentDir = os.path.join(os.getcwd(), 'Components')
        config.JobAccountant.logLevel     = 'INFO'



        # JobArchiver
        config.component_("JobArchiver")
        config.JobArchiver.pollInterval          = 60
        config.JobArchiver.logLevel              = 'INFO'
        config.JobArchiver.logDir                = os.path.join(self.testDir, 'logs')
        config.JobArchiver.componentDir          = os.path.join(os.getcwd(), 'Components')
        config.JobArchiver.numberOfJobsToCluster = 1000



        # Task Archiver
        config.component_("TaskArchiver")
        config.TaskArchiver.componentDir    = self.testInit.generateWorkDir()
        config.TaskArchiver.WorkQueueParams = {}
        config.TaskArchiver.pollInterval    = 60
        config.TaskArchiver.logLevel        = 'INFO'
        config.TaskArchiver.timeOut         = 0



        # JobStateMachine
        config.component_('JobStateMachine')
        config.JobStateMachine.couchurl        = os.getenv('COUCHURL',
                                                           'mnorman:[email protected]:5984')
        config.JobStateMachine.couchDBName     = "mnorman_test"


        # Needed, because this is a test
        os.makedirs(config.JobSubmitter.submitDir)


        return config



    def createFileCollection(self, name, nSubs, nFiles, workflowURL = 'test', site = None):
        """
        _createFileCollection_

        Create a collection of files for splitting into jobs
        """

        myThread = threading.currentThread()

        testWorkflow = Workflow(spec = workflowURL, owner = "mnorman",
                                name = name, task="/TestWorkload/ReReco")
        testWorkflow.create()

        for sub in range(nSubs):

            nameStr = '%s-%i' % (name, sub)

            testFileset = Fileset(name = nameStr)
            testFileset.create()

            for f in range(nFiles):
                # pick a random site
                if not site:
                    tmpSite = 'se.%s' % (random.choice(self.sites))
                else:
                    tmpSite = 'se.%s' % (site)
                testFile = File(lfn = "/lfn/%s/%i" % (nameStr, f), size = 1024, events = 10)
                testFile.setLocation(tmpSite)
                testFile.create()
                testFileset.addFile(testFile)

            testFileset.commit()
            testFileset.markOpen(isOpen = 0)
            testSubscription = Subscription(fileset = testFileset,
                                            workflow = testWorkflow,
                                            type = "Processing",
                                            split_algo = "FileBased")
            testSubscription.create()


        return



    def createReports(self, jobs, retryCount = 0):
        """
        _createReports_

        Create some dummy job reports for each job
        """


        report = Report()
        report.addStep('testStep', 0)

        for job in jobs:
            #reportPath = os.path.join(job['cache_dir'], 'Report.%i.pkl' % (retryCount))
            reportPath = job['fwjr_path']
            if os.path.exists(reportPath):
                os.remove(reportPath)
            report.save(reportPath)


        return



    def testA_StraightThrough(self):
        """
        _StraightThrough_

        Just run everything straight through without any variations
        """
        # Do pre-submit job check
        nRunning = getCondorRunningJobs()
        self.assertEqual(nRunning, 0, "User currently has %i running jobs.  Test will not continue" % (nRunning))

        myThread = threading.currentThread()
        workload = self.createTestWorkload()
        config   = self.getConfig()


        name         = 'WMAgent_Test1'
        site         = self.sites[0]
        nSubs        = 5
        nFiles       = 10
        workloadPath = os.path.join(self.testDir, 'workloadTest',
                                    'TestWorkload', 'WMSandbox',
                                    'WMWorkload.pkl')

        # Create a collection of files
        self.createFileCollection(name = name, nSubs = nSubs,
                                  nFiles = nFiles,
                                  workflowURL = workloadPath,
                                  site = site)



        ############################################################
        # Test the JobCreator


        config.Agent.componentName = 'JobCreator'
        testJobCreator = JobCreatorPoller(config = config)

        testJobCreator.algorithm()
        time.sleep(5)


        # Did all jobs get created?
        getJobsAction = self.daoFactory(classname = "Jobs.GetAllJobs")
        result = getJobsAction.execute(state = 'Created', jobType = "Processing")
        self.assertEqual(len(result), nSubs*nFiles)


        # Count database objects
        result = myThread.dbi.processData('SELECT * FROM wmbs_sub_files_acquired')[0].fetchall()
        self.assertEqual(len(result), nSubs * nFiles)


        # Find the test directory
        testDirectory = os.path.join(self.testDir, 'TestWorkload', 'ReReco')
        self.assertTrue('JobCollection_1_0' in os.listdir(testDirectory))
        self.assertTrue(len(os.listdir(testDirectory)) <= 20)

        groupDirectory = os.path.join(testDirectory, 'JobCollection_1_0')

        # First job should be in here
        self.assertTrue('job_1' in os.listdir(groupDirectory))
        jobFile = os.path.join(groupDirectory, 'job_1', 'job.pkl')
        self.assertTrue(os.path.isfile(jobFile))
        f = open(jobFile, 'r')
        job = cPickle.load(f)
        f.close()


        self.assertEqual(job['workflow'], name)
        self.assertEqual(len(job['input_files']), 1)
        self.assertEqual(os.path.basename(job['sandbox']), 'TestWorkload-Sandbox.tar.bz2')










        ###############################################################
        # Now test the JobSubmitter

        config.Agent.componentName = 'JobSubmitter'
        testJobSubmitter = JobSubmitterPoller(config = config)


        testJobSubmitter.algorithm()


        # Check that jobs are in the right state
        result = getJobsAction.execute(state = 'Created', jobType = "Processing")
        self.assertEqual(len(result), 0)
        result = getJobsAction.execute(state = 'Executing', jobType = "Processing")
        self.assertEqual(len(result), nSubs * nFiles)



        # Check assigned locations
        getLocationAction = self.daoFactory(classname = "Jobs.GetLocation")
        for id in result:
            loc = getLocationAction.execute(jobid = id)
            self.assertEqual(loc, [[site]])


        # Check to make sure we have running jobs
        nRunning = getCondorRunningJobs()
        self.assertEqual(nRunning, nFiles * nSubs)


        #################################################################
        # Now the JobTracker


        config.Agent.componentName = 'JobTracker'
        testJobTracker = JobTrackerPoller(config = config)
        testJobTracker.setup()

        testJobTracker.algorithm()

        # Running the algo without removing the jobs should do nothing
        result = getJobsAction.execute(state = 'Executing', jobType = "Processing")
        self.assertEqual(len(result), nSubs * nFiles)


        condorRM()
        time.sleep(1)

        # All jobs gone?
        nRunning = getCondorRunningJobs()
        self.assertEqual(nRunning, 0)


        testJobTracker.algorithm()
        time.sleep(5)

        # Running the algo without removing the jobs should do nothing
        result = getJobsAction.execute(state = 'Executing', jobType = "Processing")
        self.assertEqual(len(result), 0)
        result = getJobsAction.execute(state = 'Complete', jobType = "Processing")
        self.assertEqual(len(result), nSubs * nFiles)




        #################################################################
        # Now the JobAccountant

        # First you need to load all jobs


        self.getFWJRAction = self.daoFactory(classname = "Jobs.GetFWJRByState")
        completeJobs       = self.getFWJRAction.execute(state = "complete")


        # Create reports for all jobs
        self.createReports(jobs = completeJobs, retryCount = 0)






        config.Agent.componentName = 'JobAccountant'
        testJobAccountant = JobAccountantPoller(config = config)
        testJobAccountant.setup()


        # It should do something with the jobs
        testJobAccountant.algorithm()


        # All the jobs should be done now
        result = getJobsAction.execute(state = 'Complete', jobType = "Processing")
        self.assertEqual(len(result), 0)
        result = getJobsAction.execute(state = 'Success', jobType = "Processing")
        self.assertEqual(len(result), nSubs * nFiles)



        #######################################################################
        # Now the JobArchiver


        config.Agent.componentName = 'JobArchiver'
        testJobArchiver = JobArchiverPoller(config = config)


        testJobArchiver.algorithm()

        # All the jobs should be cleaned up
        result = getJobsAction.execute(state = 'Success', jobType = "Processing")
        self.assertEqual(len(result), 0)
        result = getJobsAction.execute(state = 'Cleanout', jobType = "Processing")
        self.assertEqual(len(result), nSubs * nFiles)


        logDir = os.path.join(self.testDir, 'logs')

        for job in completeJobs:
            self.assertFalse(os.path.exists(job['fwjr_path']))
            jobFolder = 'JobCluster_%i' \
                    % (int(job['id']/config.JobArchiver.numberOfJobsToCluster))
            jobPath = os.path.join(logDir, jobFolder, 'Job_%i.tar' %(job['id']))
            self.assertTrue(os.path.isfile(jobPath))
            self.assertTrue(os.path.getsize(jobPath) > 0)




        ###########################################################################
        # Now the TaskAchiver


        config.Agent.componentName = 'TaskArchiver'
        testTaskArchiver = TaskArchiverPoller(config = config)


        testTaskArchiver.algorithm()


        result = getJobsAction.execute(state = 'Cleanout', jobType = "Processing")
        self.assertEqual(len(result), 0)


        for jdict in completeJobs:
            job = Job(id = jdict['id'])
            self.assertFalse(job.exists())





        if os.path.isdir('testDir'):
            shutil.rmtree('testDir')
        shutil.copytree('%s' %self.testDir, os.path.join(os.getcwd(), 'testDir'))




        return
コード例 #53
0
ファイル: Harvest_t.py プロジェクト: AndrewLevin/WMCore
class HarvestTest(unittest.TestCase):
    """
    _HarvestTest_

    Test for EndOfRun job splitter
    """

    def setUp(self):
        """
        _setUp_

        """
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()

        self.testInit.setSchema(customModules = ["WMCore.WMBS"])

        self.splitterFactory = SplitterFactory(package = "WMCore.JobSplitting")

        myThread = threading.currentThread()
        self.myThread = myThread
        daoFactory = DAOFactory(package = "WMCore.WMBS",
                                logger = logging,
                                dbinterface = myThread.dbi)
        self.WMBSFactory = daoFactory

        config = self.getConfig()
        self.changer = ChangeState(config)

        myResourceControl = ResourceControl()
        myResourceControl.insertSite("SomeSite", 10, 20, "SomeSE", "SomeCE")
        myResourceControl.insertSite("SomeSite", 10, 20, "SomeSE2", "SomeCE")
        myResourceControl.insertSite("SomeSite2", 10, 20, "SomeSE3", "SomeCE2")

        self.fileset1 = Fileset(name = "TestFileset1")
        for file in range(11):
            newFile = File("/some/file/name%d" % file, size = 1000, events = 100)
            newFile.addRun(Run(1,*[1]))
            newFile.setLocation('SomeSE')
            self.fileset1.addFile(newFile)

        self.fileset1.create()

        workflow1 = Workflow(spec = "spec.xml", owner = "hufnagel", name = "TestWorkflow1", task="Test")
        workflow1.create()

        self.subscription1  = Subscription(fileset = self.fileset1,
                                           workflow = workflow1,
                                           split_algo = "Harvest",
                                           type = "Harvesting")

        self.subscription1.create()
        self.configFile = EmulatorSetup.setupWMAgentConfig()

        return

    def tearDown(self):
        """
        _tearDown_

        """
        self.testInit.clearDatabase()
        EmulatorSetup.deleteConfig(self.configFile)

        return

    def getConfig(self):
        """
        _getConfig_

        """
        config = self.testInit.getConfiguration()
        self.testInit.generateWorkDir(config)

        config.section_("CoreDatabase")
        config.CoreDatabase.connectUrl = os.getenv("DATABASE")
        config.CoreDatabase.socket     = os.getenv("DBSOCK")

        # JobStateMachine
        config.component_('JobStateMachine')
        config.JobStateMachine.couchurl        = os.getenv('COUCHURL', None)
        config.JobStateMachine.couchDBName     = 'wmagent_jobdump'

        return config

    def finishJobs(self, jobGroups, subscription = None):
        """
        _finishJobs_

        """
        if not subscription:
            subscription = self.subscription1
        for f in subscription.acquiredFiles():
            subscription.completeFiles(f)

        for jobGroup in jobGroups:
            self.changer.propagate(jobGroup.jobs, 'executing', 'created')
            self.changer.propagate(jobGroup.jobs, 'complete', 'executing')
            self.changer.propagate(jobGroup.jobs, 'success', 'complete')
            self.changer.propagate(jobGroup.jobs, 'cleanout', 'success')

        return

    def testHarvestEndOfRunTrigger(self):
        """
        _testDQMHarvestEndOfRunTrigger_

        Make sure that the basic splitting algo works, which is only, ExpressMerge is ALL done, fire a job against that fileset

        """
        self.assertEqual(self.fileset1.open, True, "Fileset is closed. Shouldn't")

        jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)

        jobGroups = jobFactory()

        self.assertEqual(len(jobGroups), 0 , "We got 1 or more jobGroups with an open fileset and no periodic configuration")

        self.fileset1.markOpen(False)
        self.assertEqual(self.fileset1.open, False, "Fileset is opened, why?")

        # We should also check if there are aqcuired files, if there are, there are jobs,
        # we don't want to fire another jobs while previous are running (output is integrating whatever  input)
        # TODO : The above one we can do when all is done. Not priority

        jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
        jobGroups = jobFactory()

        self.assertEqual(len(jobGroups), 1 , "Harvest jobsplitter didn't create a single jobGroup after the fileset was closed")

        return

    def testPeriodicTrigger(self):
        """
        _testPeriodicTrigger_

        """
        self.assertEqual(self.fileset1.open, True, "Fileset is not open, not testing periodic here")
        # Test timeout (5s for this first test)
        # there should be no acquired files, if there are, shouldn't be a job
        #self.subscription1.acquireFiles(self.subscription1.availableFiles().pop())

        jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
        jobGroups = jobFactory(periodic_harvest_interval = 3)

        self.assertEqual(len(jobGroups), 1 , "Didn't created the first periodic job when there were acquired files")

        # For the whole thing to work, faking the first job finishing, and putting the files as complete
        self.finishJobs(jobGroups)

        # Adding more of files, so we have new stuff to process
        for file in range(12,24):
            newFile = File("/some/file/name%d" % file, size = 1000, events = 100)
            newFile.addRun(Run(1,*[1]))
            newFile.setLocation('SomeSE')
            self.fileset1.addFile(newFile)
        self.fileset1.commit()

        # Testing that it doesn't create a job unless the delay is past
        jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
        jobGroups = jobFactory(periodic_harvest_interval = 2)

        self.assertEqual(len(jobGroups), 0 , "Created one or more job, when there were non-acquired file and the period is not passed by")

        time.sleep(2)

        jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
        jobGroups = jobFactory(periodic_harvest_interval = 2)

        self.assertEqual(len(jobGroups), 1 , "Didn't created one or more job, and there weren't and the period is passed by")

        # Finishing out previous jobs
        self.finishJobs(jobGroups)

        # Adding more of files, so we have new stuff to process
        for file in range(26,36):
            newFile = File("/some/file/name%d" % file, size = 1000, events = 100)
            newFile.addRun(Run(1,*[1]))
            newFile.setLocation('SomeSE')
            self.fileset1.addFile(newFile)
        self.fileset1.commit()

        # Trying to create another job just afterwards, it shouldn't, because it should respect the configured delay
        jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
        jobGroups = jobFactory(periodic_harvest_interval = 2)

        self.assertEqual(len(jobGroups), 0 , "Created one or more job, there are new files, but the delay is not past")

        time.sleep(2)

        jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
        jobGroups = jobFactory(periodic_harvest_interval = 2)

        self.assertEqual(len(jobGroups), 1 , "Didn't created one or more job, there are new files and the delay is past")

        # Last check is whether the job gets all the files or not

        numFilesJob = jobGroups[0].jobs[0].getFiles()
        numFilesFileset = self.fileset1.getFiles()
        self.assertEqual(numFilesJob, numFilesFileset, "Job didn't got all the files")

        # Finishing out previous jobs
        self.finishJobs(jobGroups)

        # Adding files for the first location
        for file in range(38,48):
            newFile = File("/some/file/name%d" % file, size = 1000, events = 100)
            newFile.addRun(Run(1,*[1]))
            newFile.setLocation('SomeSE')
            self.fileset1.addFile(newFile)
        self.fileset1.commit()
        # Then another location
        for file in range(50,56):
            newFile = File("/some/file/name%d" % file, size = 1000, events = 100)
            newFile.addRun(Run(1,*[1]))
            newFile.setLocation('SomeSE3')
            self.fileset1.addFile(newFile)
        self.fileset1.commit()

        # We should have jobs in both locations
        time.sleep(2)

        jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
        jobGroups = jobFactory(periodic_harvest_interval = 2)

        self.assertEqual(len(jobGroups[0].getJobs()), 2 , "We didn't get 2 jobs for 2 locations")

        firstJobLocation = jobGroups[0].getJobs()[0].getFileLocations()[0]
        secondJobLocation = jobGroups[0].getJobs()[1].getFileLocations()[0]

        self.assertEqual(firstJobLocation, 'SomeSite', "First job location is not SomeSite")
        self.assertEqual(secondJobLocation, 'SomeSite2', "Second job location is not SomeSite2")

        self.finishJobs(jobGroups)

        for file in range(60,65):
            newFile = File("/some/file/name%d" % file, size = 1000, events = 100)
            newFile.addRun(Run(2,*[2]))
            newFile.setLocation('SomeSE3')
            self.fileset1.addFile(newFile)
        self.fileset1.commit()

        for file in range(70,75):
            newFile = File("/some/file/name%d" % file, size = 1000, events = 100)
            newFile.addRun(Run(3,*[3]))
            newFile.setLocation('SomeSE3')
            self.fileset1.addFile(newFile)
        self.fileset1.commit()

        time.sleep(2)

        jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
        jobGroups = jobFactory(periodic_harvest_interval = 2)

        # This is one of the most "complicated" tests so worth to comment, 4 jobs should be created
        # 1 - all previous files from SomeSE and run = 1 (a lot, like ~45)
        # 2 - Few files from SomeSE3, Run = 1
        # 3 - Few files from SomeSE3, Run = 2
        # 4 - Few files from SomeSE3, Run = 3
        self.assertEqual(len(jobGroups[0].getJobs()), 4 , "We didn't get 4 jobs for adding 2 different runs to SomeSE3")

        return

    def testMultipleRunHarvesting(self):
        """
        _testMultipleRunHarvesting_

        Add some files with multiple runs in each, make sure the jobs
        are created by location and run. Verify each job mask afterwards.
        Note that in this test run are splitted between sites,
        in real life that MUST NOT happen we still don't support that.
        """
        multipleFilesFileset = Fileset(name = "TestFileset")

        newFile = File("/some/file/test1", size = 1000, events = 100)
        newFile.addRun(Run(1,*[1,3,4,5,6,7]))
        newFile.addRun(Run(2,*[1,2,4,5,6,7]))
        newFile.setLocation('SomeSE')
        multipleFilesFileset.addFile(newFile)
        newFile = File("/some/file/test2", size = 1000, events = 100)
        newFile.addRun(Run(1,*[2,8]))
        newFile.addRun(Run(2,*[3,8]))
        newFile.setLocation('SomeSE3')
        multipleFilesFileset.addFile(newFile)
        multipleFilesFileset.create()

        harvestingWorkflow = Workflow(spec = "spec.xml",
                                      owner = "hufnagel",
                                      name = "TestWorkflow",
                                      task="Test")
        harvestingWorkflow.create()

        harvestSub  = Subscription(fileset = multipleFilesFileset,
                                   workflow = harvestingWorkflow,
                                   split_algo = "Harvest",
                                   type = "Harvesting")
        harvestSub.create()

        jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = harvestSub)
        jobGroups = jobFactory(periodic_harvest_interval = 2)
        self.assertEqual(len(jobGroups), 1, "A single job group was not created")
        self.assertEqual(len(jobGroups[0].getJobs()), 4,
                             "Four jobs were not created")

        for job in jobGroups[0].getJobs():
            runs = job['mask'].getRunAndLumis()
            self.assertEqual(len(runs), 1, "Job has more than one run configured")
            possibleLumiPairs = {1 : [[1,1],[3,7],[2,2],[8,8]],
                                 2 : [[1,2],[4,7],[3,3],[8,8]]}
            run = runs.keys()[0]
            for lumiPair in runs[run]:
                self.assertTrue(lumiPair in possibleLumiPairs[run], "Strange lumi pair in the job mask")

        self.finishJobs(jobGroups, harvestSub)

        newFile = File("/some/file/test3", size = 1000, events = 100)
        newFile.addRun(Run(1,*range(9,15)))
        newFile.setLocation('SomeSE3')
        multipleFilesFileset.addFile(newFile)
        multipleFilesFileset.commit()

        time.sleep(2)

        jobGroups = jobFactory(periodic_harvest_interval = 2)
        self.assertEqual(len(jobGroups), 1, "A single job group was not created")
        self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created")

        for job in jobGroups[0].getJobs():
            runs = job['mask'].getRunAndLumis()
            self.assertEqual(len(runs), 1, "Job has more than one run configured")
            possibleLumiPairs = {1 : [[1,1],[3,7],[2,2],[8,8],[9,14]],
                                 2 : [[1,2],[4,7],[3,3],[8,8]]}
            run = runs.keys()[0]
            for lumiPair in runs[run]:
                self.assertTrue(lumiPair in possibleLumiPairs[run], "Strange lumi pair in the job mask")

        harvestingWorkflowSib = Workflow(spec = "spec.xml",
                                         owner = "hufnagel",
                                         name = "TestWorkflowSib",
                                         task="TestSib")
        harvestingWorkflowSib.create()

        harvestSubSib  = Subscription(fileset = multipleFilesFileset,
                                      workflow = harvestingWorkflowSib,
                                      split_algo = "Harvest",
                                      type = "Harvesting")
        harvestSubSib.create()

        jobFactorySib = self.splitterFactory(package = "WMCore.WMBS", subscription = harvestSubSib)

        multipleFilesFileset.markOpen(False)

        jobGroups = jobFactorySib(periodic_harvest_sibling = True)
        self.assertEqual(len(jobGroups), 0, "A single job group was created")
                
        self.finishJobs(jobGroups, harvestSub)

        jobGroups = jobFactorySib(periodic_harvest_sibling = True)
        self.assertEqual(len(jobGroups), 1, "A single job group was not created")
        self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created")

        for job in jobGroups[0].getJobs():
            runs = job['mask'].getRunAndLumis()
            self.assertEqual(len(runs), 1, "Job has more than one run configured")
            possibleLumiPairs = {1 : [[1,1],[3,7],[2,2],[8,8],[9,14]],
                                 2 : [[1,2],[4,7],[3,3],[8,8]]}
            run = runs.keys()[0]
            for lumiPair in runs[run]:
                self.assertTrue(lumiPair in possibleLumiPairs[run], "Strange lumi pair in the job mask")