class ConfigurationTest(unittest.TestCase):
    """
    test case for Configuration object

    """
    def setUp(self):
        """set up"""
        self.testInit = TestInit(__file__)
        self.testDir  = self.testInit.generateWorkDir()
        self.normalSave = "%s/WMCore_Agent_Configuration_t_normal.py" % self.testDir
        self.docSave = "%s/WMCore_Agent_Configuration_t_documented.py" % self.testDir
        self.commentSave = "%s/WMCore_Agent_Configuration_t_commented.py" % self.testDir


    def tearDown(self):
        """clean up"""
        self.testInit.delWorkDir()


    def testA(self):
        """ctor"""
        try:
            config = Configuration()
        except Exception, ex:
            msg = "Failed to instantiate Configuration\n"
            msg += str(ex)
            self.fail(msg)
class AnalysisCollectionService_t(unittest.TestCase):
    """
    Unit tests for AnalysisCollectionService
    """

    def setUp(self):
        """Set up test couch instance"""
        self.dbsURL  = 'http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet'
        self.dataset = '/RelValProdTTbar/JobRobot-MC_3XY_V24_JobRobot-v1/GEN-SIM-DIGI-RECO'

        """Set up couch test environment"""
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setupCouch("wmcore-acdc-acservice", "GroupUser", "ACDC")

        self.acService = AnalysisCollectionService(url=self.testInit.couchUrl, database = self.testInit.couchDbName)
        self.collection = self.acService.createCollection('ewv-testcoll', 'ewv2', 'Analysis')


    def tearDown(self):
        """Clean up couch instance"""
        self.testInit.tearDownCouch()


    def testCreateFilesetFromDBS(self):
        """Test creating an analysis fileset from DBS"""

        rangesMask = Mask()
        rangesMask.addRunWithLumiRanges(run=1, lumiList=[[1, 9], [12, 12], [31, 31], [38, 39], [49, 49], [51, 52], [64, 65], [82, 82], [92, 98]])

        fs, fl = self.acService.createFilesetFromDBS(self.collection, filesetName='test_fs',
                                                     dbsURL=self.dbsURL, dataset=self.dataset, mask=rangesMask)

        self.assertTrue(fl['_id'])
        self.assertEqual(len(fl['files']), 21)
class ReproducibleSeedingTests(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInitCouchApp("ReproducibleSeedingTest")
        self.testInit.setupCouch("seeding_config_cache", "GroupUser", "ConfigCache")
        self.database = Database(self.testInit.couchDbName, self.testInit.couchUrl)

        self.documentId = None

    def tearDown(self):
        self.testInit.tearDownCouch()
        return


    def testA(self):
        """instantiate"""

        document = Document()
        document[u'pset_tweak_details'] = {}
        document[u'pset_tweak_details'][u'process'] = {}
        document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'] = {}
        document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed1'] = {}
        document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed2'] = {}
        document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed3'] = {}

        document = self.database.commitOne(document)[0]
        seeder = ReproducibleSeeding(CouchUrl = self.testInit.couchUrl,
                                     CouchDBName = self.testInit.couchDbName,
                                     ConfigCacheDoc = document[u'id'])

        job = Job("testjob")
        seeder(job)
        baggage = job.getBaggage()
        seed1 = getattr(baggage.process.RandomNumberGeneratorService, "seed1", None)
        self.failUnless(seed1 != None)
Exemple #4
0
class RequestDBTest(unittest.TestCase):
    """
    """

    def setUp(self):
        """
        _setUp_
        """
        self.schema = []
        self.couchApps = ["ReqMgr"]
        self.testInit = TestInitCouchApp("RequestDBServiceTest")
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules=self.schema, useDefault=False)
        dbName = "requsetdb_t"
        self.testInit.setupCouch(dbName, *self.couchApps)
        self.requestWriter = RequestDBWriter(self.testInit.couchUrl, dbName)
        self.requestReader = RequestDBReader(self.testInit.couchUrl, dbName)
        self.requestWriter.defaultStale = {}
        self.requestReader.defaultStale = {}
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()

    def testRequestDBWriter(self):
        # test getWork
        schema = generate_reqmgr_schema()
        result = self.requestWriter.insertGenericRequest(schema[0])

        self.assertEquals(len(result), 1, "insert fail")

        self.assertEquals(
            self.requestWriter.updateRequestStatus(schema[0]["RequestName"], "failed"), "OK", "update fail"
        )
        self.assertEquals(
            self.requestWriter.updateRequestStatus("not_exist_schema", "assigned"), "Error: document not found"
        )
        result = self.requestWriter.updateRequestProperty(schema[0]["RequestName"], {"Teams": ["teamA"]})
        self.assertEquals(
            self.requestWriter.updateRequestProperty(schema[0]["RequestName"], {"Teams": ["teamA"]}),
            "OK",
            "update fail",
        )
        self.assertEquals(
            self.requestWriter.updateRequestProperty("not_exist_schema", {"Teams": "teamA"}),
            "Error: document not found",
        )

        result = self.requestWriter.getRequestByNames([schema[0]["RequestName"]])
        self.assertEquals(len(result), 1, "should be 1")
        result = self.requestWriter.getRequestByStatus(["failed"], False, 1)
        self.assertEquals(len(result), 1, "should be 1")
Exemple #5
0
class T0RequestDBTest(unittest.TestCase):
    """
    """
    def setUp(self):
        """
        _setUp_
        """
        self.schema = []
        self.couchApps = ["T0Request"]
        self.testInit = TestInitCouchApp('RequestDBServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        dbName = 't0_requsetdb_t'
        self.testInit.setupCouch(dbName, *self.couchApps)
        reqDBURL = "%s/%s" % (self.testInit.couchUrl, dbName)
        self.requestWriter = RequestDBWriter(reqDBURL, self.couchApps[0])
        self.requestReader = RequestDBReader(reqDBURL, self.couchApps[0])
        self.requestWriter.defaultStale = {}
        self.requestReader.defaultStale = {}
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()

    def testRequestDBWriter(self):
        # test getWork
        schema = generate_reqmgr_schema()
        result =  self.requestWriter.insertGenericRequest(schema[0])

        self.assertEqual(len(result), 1, 'insert fail');
        
        result = self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "assigned")

        self.assertEqual(result, 'not allowed state assigned', 'update fail')
        self.assertEqual(self.requestWriter.updateRequestStatus("not_exist_schema", "new"),
                          'Error: document not found')
        
        allowedStates = ["Closed", "Merge", "AlcaSkim", "Harvesting",  
                         "Processing Done", "completed"]
        for state in allowedStates:
            self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], state),
                          'OK')
        
        self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "Processing Done"),
                          'not allowed transition completed to Processing Done')  
        
        self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "normal-archived"),
                          'OK')  
        result = self.requestWriter.getRequestByStatus(["normal-archived"], False, 1)
        self.assertEqual(len(result), 1, "should be 1 but %s" % result)
Exemple #6
0
class Interface_t(unittest.TestCase):

    def setUp(self):
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setupCouch("wmcore-groupuser-interface", "GroupUser")

        #create a test owner for the collections in this test
        self.owner1 = makeUser("DMWM", "evansde77", self.testInit.couchUrl, self.testInit.couchDbName)
        self.owner1.connect()
        self.owner1.create()

        self.owner2 = makeUser("DMWM", "drsm79", self.testInit.couchUrl, self.testInit.couchDbName)
        self.owner2.connect()
        self.owner2.create()

        #self.url = os.getenv("COUCHURL", "http://127.0.0.1:5984")
        #self.database = "awesome_acdc"

    def tearDown(self):
        self.testInit.tearDownCouch()

    def testA(self):
        """ make some documents and own them"""
        guInt = Interface(self.testInit.couchUrl, self.testInit.couchDbName)


        #create a couple of docs
        couch = Database(self.testInit.couchDbName, self.testInit.couchUrl)
        for x in range(10):
            doc = Document("document%s" % x, {"Test Data": [1,2,3,4] })
            couch.queue(doc)
        couch.commit()

        self.assertEqual(len(guInt.documentsOwned(self.owner1.group.name, self.owner1.name)), 0)
        self.assertEqual(len(guInt.documentsOwned(self.owner2.group.name, self.owner2.name)), 0)

        guInt.callUpdate("ownthis","document1", group = self.owner1.group.name, user = self.owner1.name)

        self.failUnless("document1" in guInt.documentsOwned(self.owner1.group.name, self.owner1.name))
        self.assertEqual(len(guInt.documentsOwned(self.owner1.group.name, self.owner1.name)), 1)
        self.assertEqual(len(guInt.documentsOwned(self.owner2.group.name, self.owner2.name)), 0)

        guInt.callUpdate("ownthis","document2", group = self.owner2.group.name, user = self.owner2.name)

        self.failUnless("document2" in guInt.documentsOwned(self.owner2.group.name, self.owner2.name))
        self.assertEqual(len(guInt.documentsOwned(self.owner1.group.name, self.owner1.name)), 1)
        self.assertEqual(len(guInt.documentsOwned(self.owner2.group.name, self.owner2.name)), 1)


        guInt.callUpdate("newgroup", "group-DataOps", group = "DataOps")

        self.failUnless(couch.documentExists("group-DataOps") )

        guInt.callUpdate("newuser", "user-damason", group = "DataOps", user = "******")

        self.failUnless(couch.documentExists("user-damason") )
 def couchapp(self, couchappName):
     """
     Drop database in CouchDB.
     Reload (push) the couchapp couchappName (likely to be "OpsClipboard").
     
     """
     print "Pushing couchapp '%s' ..." % couchappName
     # here probably need to specify the couchUrl, otherwise default env COUCHURL
     # will be taken (as described above)
     testInit = TestInitCouchApp(__file__, dropExistingDb=True)
     # should now drop the CouchDB db, create a new one and push the couchapp
     testInit.setupCouch(self.dbName, couchappName)
Exemple #8
0
class WMStatsTest(unittest.TestCase):
    """
    """
    def setUp(self):
        """
        _setUp_
        """
        self.schema = []
        self.couchApps = ["WMStats"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        self.testInit.setupCouch('wmstats_t', *self.couchApps)
        self.wmstatsWriter = WMStatsWriter(self.testInit.couchUrl, 'wmstats_t');
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()

    def testWMStatsWriter(self):
        # test getWork
        schema = generate_reqmgr_schema()
        self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK', 'insert fail');
        self.assertEquals(self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail')
        self.assertEquals(self.wmstatsWriter.updateRequestStatus("not_exist_schema", "assigned"),
                          'ERROR: request not found - not_exist_schema')
        self.assertEquals(self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'), 'OK', 'update fail')
        self.assertEquals(self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'),
                          'ERROR: request not found - not_exist_schema')
        totalStats = {'total_jobs': 100, 'input_events': 1000, 'input_lumis': 1234, 'input_num_files': 5}
        self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'INSERTED', 'update fail')
        self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'UPDATED', 'update fail')
        self.assertEquals(self.wmstatsWriter.insertTotalStats("not_exist_schema", totalStats),
                          'ERROR: request not found - not_exist_schema')
        spec1 = newWorkload(schema[0]['RequestName'])
        production = spec1.newTask("Production")
        production.setTaskType("Merge")
        production.setSiteWhitelist(['TEST_SITE'])
        self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK', 'update fail')
        spec2 = newWorkload("not_exist_schema")
        production = spec2.newTask("Production")
        production.setTaskType("Merge")
        self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2),
                          'ERROR: request not found - not_exist_schema')
Exemple #9
0
class FWJRDBAPITest(unittest.TestCase):
    """
    """

    def setUp(self):
        """
        _setUp_
        """
        self.schema = []
        self.couchApps = ["FWJRDump"]
        self.testInit = TestInitCouchApp("FWJRAPITest")
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules=self.schema, useDefault=False)
        dbName = "fwjrdump_t"
        self.testInit.setupCouch(dbName, *self.couchApps)
        fwjrDBURL = "%s/%s" % (self.testInit.couchUrl, dbName)
        self.fwjrAPI = FWJRDBAPI(fwjrDBURL)
        self.fwjrAPI.defaultStale = {}
        return

    def tearDown(self):
        """
        _tearDown_

        Drop all the WMBS tables.
        """
        self.testInit.tearDownCouch()

    def testFWJRDBAPI(self):

        jobID = 1
        retryCount = 0
        fwjrDocument = {
            "_id": "%s-%s" % (jobID, retryCount),
            "jobid": jobID,
            "retrycount": retryCount,
            "archivestatus": "ready",
            "fwjr": SAMPLE_FWJR,
            "jobtype": "Merge",
            "type": "fwjr",
        }
        workflow = SAMPLE_FWJR["task"].split("/")[1]

        self.assertEqual(self.fwjrAPI.couchDB.commitOne(fwjrDocument)[0]["id"], fwjrDocument["_id"])
        self.assertEqual(self.fwjrAPI.getFWJRByArchiveStatus("ready")["rows"][0]["id"], fwjrDocument["_id"])
        self.assertEqual(self.fwjrAPI.isAllFWJRArchived(workflow), False)
        self.fwjrAPI.updateArchiveUploadedStatus(fwjrDocument["_id"])
        self.assertEqual(self.fwjrAPI.getFWJRByArchiveStatus("uploaded")["rows"][0]["id"], fwjrDocument["_id"])
        self.assertEqual(self.fwjrAPI.isAllFWJRArchived(workflow), True)
        self.assertEqual(self.fwjrAPI.getFWJRWithSkippedFiles()["rows"][0]["value"]["skippedFiles"], 2)
Exemple #10
0
class RESTBaseUnitTest(unittest.TestCase):
    
    def setUp(self):
        # default set
        self.schemaModules = []
        self.initialize()
        if self.schemaModules:
            self.testInit = TestInitCouchApp(__file__)
            self.testInit.setLogging() # logLevel = logging.SQLDEBUG
            self.testInit.setDatabaseConnection(self.config.getDBUrl())
            self.testInit.setSchema(customModules = self.schemaModules,
                                    useDefault = False)
        
        self.rt = Root(self.config)
        self.rt.start(blocking=False)
        
    def tearDown(self):
        self.rt.stop()
        if self.schemaModules:
            self.testInit.clearDatabase()
        self.config = None
        
    
    def initialize(self):
        """
        i.e.
        
        self.config = DefaultConfig('WMCore.WebTools.RESTModel')
        self.config.setDBUrl("sqlite://")
        self.schemaModules = ["WMCore.ThreadPool", WMCore.WMBS"]
        """
        
        message = "initialize method has to be implemented, self.restModel, self.schemaModules needs to be set"
        raise NotImplementedError, message
Exemple #11
0
    def setUp(self):
        """
        _setUp_

        """
        self.transitions = Transitions()
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setupCouch("changestate_t/jobs", "JobDump")
        self.testInit.setupCouch("changestate_t/fwjrs", "FWJRDump")        

        self.testInit.setSchema(customModules = ["WMCore.WMBS"],
                                useDefault = False)

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)

        self.couchServer = CouchServer(dburl = os.getenv("COUCHURL"))
        self.config = Configuration()
        self.config.component_("JobStateMachine")
        self.config.JobStateMachine.couchurl = os.getenv("COUCHURL")
        return
Exemple #12
0
 def setUp(self):
     """set up"""
     self.testInit = TestInit(__file__)
     self.testDir  = self.testInit.generateWorkDir()
     self.normalSave = "%s/WMCore_Agent_Configuration_t_normal.py" % self.testDir
     self.docSave = "%s/WMCore_Agent_Configuration_t_documented.py" % self.testDir
     self.commentSave = "%s/WMCore_Agent_Configuration_t_commented.py" % self.testDir
    def setUp(self):
        """
        _setUp_

        Initialize the database and couch.
        """
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setupCouch("promptreco_t", "ConfigCache")
        self.testInit.setSchema(customModules=["WMCore.WMBS"],
                                useDefault=False)
        couchServer = CouchServer(os.environ["COUCHURL"])
        self.configDatabase = couchServer.connectDatabase("promptreco_t")
        self.testDir = self.testInit.generateWorkDir()

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package="WMCore.WMBS",
                                     logger=myThread.logger,
                                     dbinterface=myThread.dbi)
        self.listTasksByWorkflow = self.daoFactory(classname="Workflow.LoadFromName")
        self.listFilesets = self.daoFactory(classname="Fileset.List")
        self.listSubsMapping = self.daoFactory(classname="Subscriptions.ListSubsAndFilesetsFromWorkflow")

        return
Exemple #14
0
    def testProcessorWithReceiverAndCouchSink(self):
        # set up couch first
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        dbName = "couch_sink"
        self.testInit.setupCouch(dbName)
        
        # add corresponding part of the configuration for CouchSink(s)
        config = self.config.AlertProcessor
        config.critical.sinks.section_("couch")
        config.critical.sinks.couch.url = self.testInit.couchUrl
        config.critical.sinks.couch.database = self.testInit.couchDbName

        # just send the Alert into couch

        processor = Processor(config)
        rec = Receiver(self.addr, processor, self.ctrl)
        rec.startReceiver() # non blocking call
        
        # run worker(), this time directly without Process as above,
        # worker will send 10 Alerts to Receiver
        worker(self.addr, self.ctrl, 10)
        
        # wait until the Receiver is shut by worker
        while rec.isReady():
            time.sleep(0.3)
            print "%s waiting for Receiver to shut ..." % inspect.stack()[0][3]
Exemple #15
0
    def setUp(self):
        """
        _setUp_

        """
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setupCouch("wmbshelper_t/jobs", "JobDump")
        self.testInit.setupCouch("wmbshelper_t/fwjrs", "FWJRDump")        
        os.environ["COUCHDB"] = "wmbshelper_t"
        self.testInit.setSchema(customModules = ["WMCore.WMBS",
                                                 "WMComponent.DBSBuffer.Database",
                                                 "WMCore.BossAir",
                                                 "WMCore.ResourceControl"],
                                useDefault = False)
        
        self.workDir = self.testInit.generateWorkDir()
        
        self.wmspec = self.createWMSpec()
        self.topLevelTask = getFirstTask(self.wmspec)
        self.inputDataset = self.topLevelTask.inputDataset()
        self.dataset = self.topLevelTask.getInputDatasetPath()
        self.dbs = MockDBSReader(self.inputDataset.dbsurl)
        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = threading.currentThread().logger,
                                     dbinterface = threading.currentThread().dbi)
        return
Exemple #16
0
    def setUp(self):
        """
        setup for test.
        """
        super(JobArchiverTest, self).setUp()

        myThread = threading.currentThread()
        super(JobArchiverTest, self).setUp()
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        # self.tearDown()
        self.testInit.setSchema(customModules=["WMCore.WMBS"],
                                useDefault=False)
        self.testInit.setupCouch("jobarchiver_t_0/jobs", "JobDump")
        self.testInit.setupCouch("jobarchiver_t_0/fwjrs", "FWJRDump")

        self.daofactory = DAOFactory(package="WMCore.WMBS",
                                     logger=myThread.logger,
                                     dbinterface=myThread.dbi)
        self.getJobs = self.daofactory(classname="Jobs.GetAllJobs")

        self.testDir = self.testInit.generateWorkDir(deleteOnDestruction=False)

        self.nJobs = 10

        self.configFile = EmulatorSetup.setupWMAgentConfig()

        return
Exemple #17
0
    def setUp(self):
        myThread = threading.currentThread()

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = ["WMCore.WMBS"],
                                useDefault = False)
        self.databaseName = "couchapp_t_0"
        self.testInit.setupCouch(self.databaseName, "WorkloadSummary")
        self.testInit.setupCouch("%s/jobs" % self.databaseName, "JobDump")
        self.testInit.setupCouch("%s/fwjrs" % self.databaseName, "FWJRDump")

        # Setup config for couch connections
        config = self.testInit.getConfiguration()
        config.section_("JobStateMachine")
        config.JobStateMachine.couchDBName  = self.databaseName

        # Create couch server and connect to databases
        self.couchdb      = CouchServer(config.JobStateMachine.couchurl)
        self.jobsdatabase = self.couchdb.connectDatabase("%s/jobs" % config.JobStateMachine.couchDBName)
        self.fwjrdatabase = self.couchdb.connectDatabase("%s/fwjrs" % config.JobStateMachine.couchDBName)

        # Create changeState
        self.changeState = ChangeState(config)
        self.config      = config

        # Create testDir
        self.testDir = self.testInit.generateWorkDir()

        return
Exemple #18
0
    def setUp(self):
        """
        _setUp_

        Standard setup: Now with 100% more couch
        """
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = ["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl", "WMCore.Agent.Database"])
        self.testInit.setupCouch("jobsubmitter_t/jobs", "JobDump")
        self.testInit.setupCouch("jobsubmitter_t/fwjrs", "FWJRDump")
        self.testInit.setupCouch("wmagent_summary_t", "WMStats")

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)
        self.baDaoFactory = DAOFactory(package = "WMCore.BossAir",
                                       logger = myThread.logger,
                                       dbinterface = myThread.dbi)

        self.testDir = self.testInit.generateWorkDir()

        # Set heartbeat
        self.componentName = 'JobSubmitter'
        self.heartbeatAPI = HeartbeatAPI(self.componentName)
        self.heartbeatAPI.registerComponent()

        return
Exemple #19
0
    def testProcessorWithReceiverAndCouchSink(self):
        # set up couch first
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        dbName = "couch_sink"
        self.testInit.setupCouch(dbName)
        
        # add corresponding part of the configuration for CouchSink(s)
        config = self.config.AlertProcessor
        config.critical.sinks.section_("couch")
        config.critical.sinks.couch.url = self.testInit.couchUrl
        config.critical.sinks.couch.database = self.testInit.couchDbName

        # just send the Alert into couch
        processor = Processor(config)
        # Receiver is waited for shutdown / shutdown explicitly in tearDown()
        self.receiver = Receiver(self.addr, processor, self.ctrl)
        self.receiver.startReceiver() # non blocking call
        
        
        
        # run worker(), this time directly without Process as above,
        # worker will send 10 Alerts to Receiver
        worker(self.addr, self.ctrl, 10)

        # wait until Receiver is shut down (by a message from worker()
        # also need to wait, otherwise tearDown kicks off and scrapes the
        # couch so half of the alerts will be undelivered
        
        while self.receiver.isReady():
            time.sleep(ReceiverLogic.TIMEOUT_AFTER_SHUTDOWN * 1.5)
            logging.info("%s: Waiting for Receiver shutdown ..." % inspect.stack()[0][3])
Exemple #20
0
    def setUp(self):
        """
        setup for test.
        """

        myThread = threading.currentThread()
        
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        #self.tearDown()
        self.testInit.setSchema(customModules = ["WMCore.WMBS"],
                                useDefault = False)
        self.testInit.setupCouch("jobarchiver_t_0/jobs", "JobDump")
        self.testInit.setupCouch("jobarchiver_t_0/fwjrs", "FWJRDump")

        self.daofactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)
        self.getJobs = self.daofactory(classname = "Jobs.GetAllJobs")

        self.testDir = self.testInit.generateWorkDir(deleteOnDestruction = False)

        self.nJobs = 10
        
        self.alertsReceiver = None
        
        EmulatorHelper.setEmulators(phedex = True, dbs = True,
                                    siteDB = True, requestMgr = False)

        return
Exemple #21
0
    def setUp(self):
        """
        _setUp_

        Setup the database and logging connection.  Try to create all of the
        WMBS tables.  Also, create some dummy locations.
        """
        
        myThread = threading.currentThread()

        self.sites = ['T2_US_Florida', 'T2_US_UCSD', 'T2_TW_Taiwan', 'T1_CH_CERN']
        
        self.testInit = TestInit(__file__)
        self.testInit.setLogging(logLevel = logging.DEBUG)
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = ['WMCore.WMBS', 
                                                 'WMCore.ResourceControl',
                                                 'WMCore.Agent.Database'], useDefault = False)
        self.testInit.setupCouch("dashboardreporter_t/jobs", "JobDump")
        self.testInit.setupCouch("dashboardreporter_t/fwjrs", "FWJRDump")


        resourceControl = ResourceControl()
        for site in self.sites:
            resourceControl.insertSite(siteName = site, seName = site, ceName = site)
            resourceControl.insertThreshold(siteName = site, taskType = 'Processing', \
                                            maxSlots = 10000)

        self.testDir = self.testInit.generateWorkDir()
        self.alertsReceiver = None
        return
Exemple #22
0
    def setUp(self):
        """
        _setUp_

        Setup the database and logging connection.
        """
        logging.basicConfig()
        self.logger = logging.getLogger('LogDBTest')
#        self.logger.setLevel(logging.DEBUG)

        self.schema = []
        self.couchApps = ["LogDB"]
        self.testInit = TestInitCouchApp('LogDBTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        dbName = 'logdb_t'
        self.testInit.setupCouch(dbName, *self.couchApps)
        url = "%s/%s" % (self.testInit.couchUrl, dbName)

        identifier = 'agentname'
        self.agent_inst = LogDB(url, identifier, logger=self.logger, thread_name="MainThread")
        self.agent_inst2 = LogDB(url, identifier, logger=self.logger, thread_name="Test")
        identifier = '/DC=org/DC=doegrids/OU=People/CN=First Last 123'
        self.user_inst = LogDB(url, identifier, logger=self.logger, thread_name="MainThread")
        self.report = LogDBReport(self.agent_inst)
    def setUp(self):
        """
        _setUp_

        Set everything up.
        """
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules = ["WMCore.WMBS", "WMCore.BossAir",
                                                 "WMCore.ResourceControl"],
                                useDefault = False)
        self.testInit.setupCouch("jobsubmittercaching_t/jobs", "JobDump")
        self.testInit.setupCouch("jobsubmittercaching_t/fwjrs", "FWJRDump")

        resourceControl = ResourceControl()
        for siteName in ["T1_US_FNAL", "T1_UK_RAL"]:
            resourceControl.insertSite(siteName = siteName, pnn = "se.%s" % (siteName),
                                       ceName = siteName, plugin = "CondorPlugin", cmsName = siteName)
            resourceControl.insertThreshold(siteName = siteName, taskType = "Processing",
                                            maxSlots = 10000, pendingSlots = 10000)

        self.testDir = self.testInit.generateWorkDir()
        self.configFile = EmulatorSetup.setupWMAgentConfig()
        return
Exemple #24
0
    def setUp(self):
        """
        Setup for unit tests
        """
        self.testInit = TestInit(__file__)
        self.testDir = self.testInit.generateWorkDir()
        testname = self.id().split('.')[-1]

        logging.basicConfig(level=logging.DEBUG,
                    format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
                    datefmt='%m-%d %H:%M',
                    filename='service_unittests.log',
                    filemode='w')

        logger_name = 'Service%s' % testname.replace('test', '', 1)

        self.logger = logging.getLogger(logger_name)

        #self.cache_path = tempfile.mkdtemp()
        test_dict = {'logger': self.logger,
                #'cachepath' : self.cache_path,
                #'req_cache_path': '%s/requests' % self.cache_path,
                'endpoint':'http://cmssw.cvs.cern.ch/cgi-bin/cmssw.cgi'}

        self.myService = Service(test_dict)

        test_dict['endpoint'] = 'http://cmssw-test.cvs.cern.ch/cgi-bin/cmssw.cgi'
        self.myService2 = Service(test_dict)
        self.testUrl = 'http://cern.ch'

        self.port = 8888
        cherrypy.config.update({'server.socket_port': self.port})
Exemple #25
0
    def setUp(self):
        """
        setup for test.
        """
        myThread = threading.currentThread()
        
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection(destroyAllDatabase = True)
        self.testInit.setSchema(customModules = ["WMCore.WMBS"],
                                useDefault = False)
        self.testInit.setupCouch("errorhandler_t", "GroupUser", "ACDC")
        self.testInit.setupCouch("errorhandler_t_jd/jobs", "JobDump")
        self.testInit.setupCouch("errorhandler_t_jd/fwjrs", "FWJRDump")

        self.daofactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)
        self.getJobs = self.daofactory(classname = "Jobs.GetAllJobs")
        self.setJobTime = self.daofactory(classname = "Jobs.SetStateTime")
        locationAction = self.daofactory(classname = "Locations.New")
        locationAction.execute(siteName = "malpaquet", seName = "malpaquet")
        self.testDir = self.testInit.generateWorkDir()
        self.nJobs = 10

        self.dataCS = DataCollectionService(url = self.testInit.couchUrl,
                                            database = "errorhandler_t")
        
        return
Exemple #26
0
    def setUp(self):
        """
        _setUp_

        """
        self.transitions = Transitions()
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setupCouch("changestate_t/jobs", "JobDump")
        self.testInit.setupCouch("changestate_t/fwjrs", "FWJRDump")
        self.testInit.setupCouch("job_summary", "WMStats")

        self.testInit.setSchema(customModules = ["WMCore.WMBS"],
                                useDefault = False)

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package = "WMCore.WMBS",
                                     logger = myThread.logger,
                                     dbinterface = myThread.dbi)
        
        couchurl = os.getenv("COUCHURL")
        self.couchServer = CouchServer(dburl = couchurl)
        self.config = self.testInit.getConfiguration()
        self.taskName = "/TestWorkflow/Task"
        return
Exemple #27
0
    def setUp(self):
        """
        _setUp_

        Setup couchdb and the test environment
        """
        super(ResubmitBlockTest, self).setUp()

        self.group = 'unknown'
        self.user = '******'

        # Set external test helpers
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setupCouch("resubmitblock_t", "ACDC", "GroupUser")

        # Define test environment
        self.couchUrl = os.environ["COUCHURL"]
        self.acdcDBName = 'resubmitblock_t'
        self.validLocations = ['T2_US_Nebraska', 'T1_US_FNAL_Disk', 'T1_UK_RAL_Disk']
        self.siteWhitelist = ['T2_XX_SiteA']
        siteDB = SiteDB()
        #Convert phedex node name to a valid processing site name
        self.PSNs = siteDB.PNNstoPSNs(self.validLocations)
        self.workflowName = 'dballest_ReReco_workflow'
        couchServer = CouchServer(dburl=self.couchUrl)
        self.acdcDB = couchServer.connectDatabase(self.acdcDBName, create=False)
        user = makeUser(self.group, '*****@*****.**', self.couchUrl, self.acdcDBName)
        user.create()

        return
Exemple #28
0
    def setUp(self):
        """
        _setUp_

        Figure out the location of the XML report produced by CMSSW.
        """
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection(destroyAllDatabase=True)
        self.testInit.setupCouch("report_t/fwjrs", "FWJRDump")

        self.xmlPath = os.path.join(getTestBase(),
                                    "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml")
        self.badxmlPath = os.path.join(getTestBase(),
                                       "WMCore_t/FwkJobReport_t/CMSSWFailReport2.xml")
        self.skippedFilesxmlPath = os.path.join(getTestBase(),
                                                "WMCore_t/FwkJobReport_t/CMSSWSkippedNonExistentFile.xml")
        self.skippedAllFilesxmlPath = os.path.join(getTestBase(),
                                                   "WMCore_t/FwkJobReport_t/CMSSWSkippedAll.xml")
        self.fallbackXmlPath = os.path.join(getTestBase(),
                                            "WMCore_t/FwkJobReport_t/CMSSWInputFallback.xml")
        self.twoFileFallbackXmlPath = os.path.join(getTestBase(),
                                                   "WMCore_t/FwkJobReport_t/CMSSWTwoFileRemote.xml")
        self.pileupXmlPath = os.path.join(getTestBase(),
                                          "WMCore_t/FwkJobReport_t/CMSSWPileup.xml")

        self.testDir = self.testInit.generateWorkDir()
        return
    def setUp(self):
        """
        _setUp_

        Create two subscriptions: One that contains a single file and one that
        contains multiple files.
        """

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        # self.testInit.clearDatabase(modules = ['WMCore.WMBS'])
        self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False)
        self.testInit.setupCouch("lumi_t", "GroupUser", "ACDC")

        myThread = threading.currentThread()
        daofactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi)

        locationAction = daofactory(classname = "Locations.New")
        locationAction.execute(siteName='s1', pnn="T1_US_FNAL_Disk")
        locationAction.execute(siteName='s2', pnn="T2_CH_CERN")

        self.testWorkflow = Workflow(spec="spec.xml", owner="mnorman", name="wf001", task="Test")
        self.testWorkflow.create()

        self.performanceParams = {'timePerEvent': 12,
                                  'memoryRequirement': 2300,
                                  'sizePerEvent': 400}

        return
    def setUp(self):
        """
        setup for test.
        """

        myThread = threading.currentThread()

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection(destroyAllDatabase=True)
        self.testInit.setSchema(customModules=["WMCore.WMBS", "WMComponent.DBS3Buffer"], useDefault=False)
        self.databaseName = "taskarchiver_t_0"
        self.testInit.setupCouch("%s/workloadsummary" % self.databaseName, "WorkloadSummary")
        self.testInit.setupCouch("%s/jobs" % self.databaseName, "JobDump")
        self.testInit.setupCouch("%s/fwjrs" % self.databaseName, "FWJRDump")
        self.testInit.setupCouch("wmagent_summary_t", "WMStats")
        self.testInit.setupCouch("wmagent_summary_central_t", "WMStats")

        self.daofactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi)

        self.getJobs = self.daofactory(classname="Jobs.GetAllJobs")
        self.inject = self.daofactory(classname="Workflow.MarkInjectedWorkflows")

        self.testDir = self.testInit.generateWorkDir()
        os.makedirs(os.path.join(self.testDir, "specDir"))

        self.nJobs = 10
        self.campaignName = "aCampaign"
        self.alertsReceiver = None

        self.uploadPublishInfo = False
        self.uploadPublishDir = None

        return
    def setUp(self):
        """
        Initialize the database and couch.

        """
        super(PileupFetcherTest, self).setUp()

        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setupCouch("pileupfetcher_t", "ConfigCache")
        self.testInit.setSchema(customModules=["WMCore.WMBS"],
                                useDefault=False)
        couchServer = CouchServer(os.environ["COUCHURL"])
        self.configDatabase = couchServer.connectDatabase("pileupfetcher_t")
        self.testDir = self.testInit.generateWorkDir()
        self.rucioAcct = "wmcore_transferor"
Exemple #32
0
 def setUp(self):
     """
     _setUp_
     """
     self.schema = []
     self.couchApps = ["FWJRDump"]
     self.testInit = TestInitCouchApp('FWJRAPITest')
     self.testInit.setLogging()
     self.testInit.setDatabaseConnection()
     self.testInit.setSchema(customModules = self.schema,
                             useDefault = False)
     dbName = 'fwjrdump_t'
     self.testInit.setupCouch(dbName, *self.couchApps)
     fwjrDBURL = "%s/%s" % (self.testInit.couchUrl, dbName)
     self.fwjrAPI= FWJRDBAPI(fwjrDBURL)
     self.fwjrAPI.defaultStale = {}
     return
Exemple #33
0
    def setUp(self):
        """
        _setUp_

        Initialize the database.
        """
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setupCouch("analysis_t", "ConfigCache")
        self.testInit.setSchema(customModules=["WMCore.WMBS"],
                                useDefault=False)

        couchServer = CouchServer(os.environ["COUCHURL"])
        self.configDatabase = couchServer.connectDatabase("analysis_t")
        self.testDir = self.testInit.generateWorkDir()
        return
Exemple #34
0
    def setUp(self):
        """
        _setUp_

        Setup the database and logging connection.  Try to create all of the
        WMBS tables.  Also, create some dummy locations.
        """
        super(JobCreatorTest, self).setUp()

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()

        self.testInit.setSchema(customModules=[
            'WMCore.WMBS', 'WMCore.ResourceControl', 'WMCore.Agent.Database'
        ],
                                useDefault=False)
        self.couchdbname = "jobcreator_t"
        self.testInit.setupCouch("%s/jobs" % self.couchdbname, "JobDump")
        self.testInit.setupCouch("%s/fwjrs" % self.couchdbname, "FWJRDump")
        self.configFile = EmulatorSetup.setupWMAgentConfig()

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package="WMCore.WMBS",
                                     logger=myThread.logger,
                                     dbinterface=myThread.dbi)

        locationAction = self.daoFactory(classname="Locations.New")
        for site in self.sites:
            locationAction.execute(siteName=site, pnn=site)

        # Create sites in resourceControl

        resourceControl = ResourceControl()
        for site in self.sites:
            resourceControl.insertSite(siteName=site, pnn=site, ceName=site)
            resourceControl.insertThreshold(siteName=site,
                                            taskType='Processing',
                                            maxSlots=10000,
                                            pendingSlots=10000)

        self.resourceControl = resourceControl

        self._setup = True
        self._teardown = False

        self.testDir = self.testInit.generateWorkDir()
        self.cwd = os.getcwd()

        # Set heartbeat
        self.componentName = 'JobCreator'
        self.heartbeatAPI = HeartbeatAPI(self.componentName)
        self.heartbeatAPI.registerComponent()

        if PY3:
            self.assertItemsEqual = self.assertCountEqual

        return
Exemple #35
0
    def setUp(self):
        """
        _setUp_

        Initialize the database and couch.
        """
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setupCouch(TEST_DB_NAME, "ConfigCache")
        self.testInit.setSchema(customModules=["WMCore.WMBS"],
                                useDefault=False)
        self.testInit.generateWorkDir()

        couchServer = CouchServer(os.environ["COUCHURL"])
        self.configDatabase = couchServer.connectDatabase(TEST_DB_NAME)
        EmulatorHelper.setEmulators(dbs=True)
        return
Exemple #36
0
 def setUp(self):
     """
     _setUp_
     """
     self.schema = []
     self.couchApps = ["ReqMgr"]
     self.testInit = TestInitCouchApp('RequestDBServiceTest')
     self.testInit.setLogging()
     self.testInit.setDatabaseConnection()
     self.testInit.setSchema(customModules=self.schema, useDefault=False)
     dbName = 'requsetdb_t'
     self.testInit.setupCouch(dbName, *self.couchApps)
     reqDBURL = "%s/%s" % (self.testInit.couchUrl, dbName)
     self.requestWriter = RequestDBWriter(reqDBURL)
     self.requestReader = RequestDBReader(reqDBURL)
     self.requestWriter.defaultStale = {}
     self.requestReader.defaultStale = {}
     return
Exemple #37
0
class CouchUtils_t(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setupCouch("wmcore-acdc-couchutils", "GroupUser", "ACDC")

    def tearDown(self):
        self.testInit.tearDownCouch()


    def testA(self):
        """object driven connection via initialiseCouch method"""

        class Thingy(object):
            """misc object with couch access attrs"""
            def __init__(self):
                self.couchdb = None
                self.database = None
                self.url = None

            @CouchUtils.connectToCouch
            def __call__(self):
                return True

        couchThingy = Thingy()

        # test throws with everything None
        self.assertRaises(CouchUtils.CouchConnectionError, CouchUtils.initialiseCouch, couchThingy)
        couchThingy.url = self.testInit.couchUrl
        self.assertRaises(CouchUtils.CouchConnectionError, CouchUtils.initialiseCouch, couchThingy)
        couchThingy.database = self.testInit.couchDbName

        try:
            CouchUtils.initialiseCouch(couchThingy)
        except Exception, ex:
            msg = "Error initialising couch client for test object:\n %s " % str(ex)
            self.fail(msg)

        self.failUnless(couchThingy.couchdb != None)
        # test decorator on already connected object
        try:
            couchThingy()
        except Exception, ex:
            msg = "Error invoking connectToCouch decorator:\n %s" % str(msg)
            self.fail(msg)
    def setUp(self):
        """
        _setUp_

        Setup the database and logging connection.  Try to create all of the
        WMBS tables.  Also add some dummy locations.
        """
        super(WorkQueueTestCase, self).setUp()
        self.queueDB = 'workqueue_t'
        self.queueInboxDB = 'workqueue_t_inbox'
        self.globalQDB = 'workqueue_t_global'
        self.globalQInboxDB = 'workqueue_t_global_inbox'
        self.localQDB = 'workqueue_t_local'
        self.localQInboxDB = 'workqueue_t_local_inbox'
        self.localQDB2 = 'workqueue_t_local2'
        self.localQInboxDB2 = 'workqueue_t_local2_inbox'
        self.configCacheDB = 'workqueue_t_config_cache'
        self.logDBName = 'logdb_t'
        self.requestDBName = 'workqueue_t_reqmgr_workload_cache'
        
        self.setSchema()
        self.testInit = TestInit('WorkQueueTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection(destroyAllDatabase=True)
        self.addCleanup(self.testInit.clearDatabase)
        self.addCleanup(logging.debug, 'Cleanup called clearDatabase()')
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        self.testInit.setupCouch(self.queueDB, *self.couchApps)
        self.testInit.setupCouch(self.queueInboxDB, *self.couchApps)
        self.testInit.setupCouch(self.globalQDB, *self.couchApps)
        self.testInit.setupCouch(self.globalQInboxDB , *self.couchApps)
        self.testInit.setupCouch(self.localQDB, *self.couchApps)
        self.testInit.setupCouch(self.localQInboxDB, *self.couchApps)
        self.testInit.setupCouch(self.localQDB2, *self.couchApps)
        self.testInit.setupCouch(self.localQInboxDB2, *self.couchApps)
        self.testInit.setupCouch(self.configCacheDB, 'ConfigCache')
        self.testInit.setupCouch(self.logDBName, 'LogDB')
        self.testInit.setupCouch(self.requestDBName, 'ReqMgr')
        
        self.couchURL = os.environ.get("COUCHURL")
        couchServer = CouchServer(self.couchURL)
        self.configCacheDBInstance = couchServer.connectDatabase(self.configCacheDB)

        self.localCouchMonitor = CouchMonitor(self.couchURL)
        self.localCouchMonitor.deleteReplicatorDocs()
        self.addCleanup(self.localCouchMonitor.deleteReplicatorDocs)
        self.addCleanup(logging.debug, 'Cleanup called deleteReplicatorDocs()')
        self.addCleanup(self.testInit.tearDownCouch)
        self.addCleanup(logging.debug, 'Cleanup called tearDownCouch()')

        self.workDir = self.testInit.generateWorkDir()
        self.addCleanup(self.testInit.delWorkDir)
        self.addCleanup(logging.debug, 'Cleanup called delWorkDir()')

        return
Exemple #39
0
    def setUp(self):
        """
        _setUp_

        Figure out the location of the XML report produced by CMSSW.
        """
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection(destroyAllDatabase=True)
        self.testInit.setupCouch("report_t/fwjrs", "FWJRDump")

        self.xmlPath = os.path.join(
            WMCore.WMBase.getTestBase(),
            "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml")
        self.badxmlPath = os.path.join(
            WMCore.WMBase.getTestBase(),
            "WMCore_t/FwkJobReport_t/CMSSWFailReport2.xml")
        self.testDir = self.testInit.generateWorkDir()
        return
Exemple #40
0
    def setUp(self):
        """
        _setUp_

        setUp function for unittest
        """
        # Set constants
        self.couchDB = "config_test"
        self.configURL = "RANDOM;;URL;;NAME"
        self.configString = "This is a random string"

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(
            customModules=["WMComponent.DBS3Buffer", 'WMCore.Agent.Database'],
            useDefault=False)
        self.testInit.setupCouch(self.couchDB, "GroupUser", "ConfigCache")

        myThread = threading.currentThread()
        self.bufferFactory = DAOFactory(
            package="WMComponent.DBSBuffer.Database",
            logger=myThread.logger,
            dbinterface=myThread.dbi)
        self.buffer3Factory = DAOFactory(package="WMComponent.DBS3Buffer",
                                         logger=myThread.logger,
                                         dbinterface=myThread.dbi)

        locationAction = self.bufferFactory(
            classname="DBSBufferFiles.AddLocation")
        locationAction.execute(siteName="se1.cern.ch")
        locationAction.execute(siteName="se1.fnal.gov")
        locationAction.execute(siteName="malpaquet")

        # Set heartbeat
        self.componentName = 'JobSubmitter'
        self.heartbeatAPI = HeartbeatAPI(self.componentName)
        self.heartbeatAPI.registerComponent()

        # Set up a config cache
        configCache = ConfigCache(os.environ["COUCHURL"],
                                  couchDBName=self.couchDB)
        configCache.createUserGroup(groupname="testGroup", username='******')
        self.testDir = self.testInit.generateWorkDir()

        psetPath = os.path.join(self.testDir, "PSet.txt")
        f = open(psetPath, 'w')
        f.write(self.configString)
        f.close()

        configCache.addConfig(newConfig=psetPath, psetHash=None)
        configCache.save()
        self.configURL = "%s;;%s;;%s" % (os.environ["COUCHURL"], self.couchDB,
                                         configCache.getCouchID())
        return
Exemple #41
0
    def setUp(self):
        """
        _setUp_

        """
        super(WMBSHelperTest, self).setUp()

        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection(destroyAllDatabase=True)
        self.testInit.setupCouch("wmbshelper_t/jobs", "JobDump")
        self.testInit.setupCouch("wmbshelper_t/fwjrs", "FWJRDump")
        self.testInit.setupCouch("config_test", "GroupUser", "ConfigCache")
        os.environ["COUCHDB"] = "wmbshelper_t"
        self.testInit.setSchema(customModules=[
            "WMCore.WMBS", "WMComponent.DBS3Buffer", "WMCore.BossAir",
            "WMCore.ResourceControl"
        ],
                                useDefault=False)

        self.workDir = self.testInit.generateWorkDir()

        self.wmspec = self.createWMSpec()
        self.topLevelTask = getFirstTask(self.wmspec)
        self.inputDataset = self.topLevelTask.inputDataset()
        self.dataset = self.topLevelTask.getInputDatasetPath()
        self.dbs = DBSReader(self.inputDataset.dbsurl)
        self.rucioAcct = "wmcore_transferor"
        self.rucio = Rucio(self.rucioAcct)
        self.daoFactory = DAOFactory(package="WMCore.WMBS",
                                     logger=threading.currentThread().logger,
                                     dbinterface=threading.currentThread().dbi)

        self.configFile = EmulatorSetup.setupWMAgentConfig()
        self.config = loadConfigurationFile(self.configFile)

        self.config.component_("JobSubmitter")
        self.config.JobSubmitter.submitDir = self.workDir
        self.config.JobSubmitter.submitScript = os.path.join(
            getTestBase(), 'WMComponent_t/JobSubmitter_t', 'submit.sh')

        return
Exemple #42
0
    def setUp(self):
        self.testInit = TestInit('CouchWorkQueueTest')
        self.testInit.setLogging()
        self.testInit.setupCouch('wq_backend_test_inbox', 'WorkQueue')
        self.testInit.setupCouch('wq_backend_test', 'WorkQueue')
        self.couch_db = self.testInit.couch.couchServer.connectDatabase('wq_backend_test')
        self.backend = WorkQueueBackend(db_url = self.testInit.couchUrl,
                                        db_name = 'wq_backend_test',
                                        inbox_name = 'wq_backend_test_inbox')

        self.processingSpec = rerecoWorkload('testProcessing', rerecoArgs)
Exemple #43
0
    def setUp(self):
        """
        _setUp_
        """
        EmulatorHelper.setEmulators(phedex=True,
                                    dbs=True,
                                    siteDB=True,
                                    requestMgr=True)

        self.specGenerator = WMSpecGenerator("WMSpecs")
        #self.configFile = EmulatorSetup.setupWMAgentConfig()
        self.schema = []
        self.couchApps = ["WorkQueue"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules=self.schema, useDefault=False)
        self.testInit.setupCouch('workqueue_t', *self.couchApps)
        self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps)
        return
Exemple #44
0
    def setUp(self):
        """
        _setUp_
        """
        super(WorkQueueTest, self).setUp()

        self.specGenerator = WMSpecGenerator("WMSpecs")
        # self.configFile = EmulatorSetup.setupWMAgentConfig()
        self.schema = []
        self.couchApps = ["WorkQueue"]
        self.testInit = TestInitCouchApp('WorkQueueServiceTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules=self.schema, useDefault=False)
        self.testInit.setupCouch('workqueue_t', *self.couchApps)
        self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps)
        self.testInit.setupCouch('local_workqueue_t', *self.couchApps)
        self.testInit.setupCouch('local_workqueue_t_inbox', *self.couchApps)
        self.testInit.generateWorkDir()
        return
Exemple #45
0
    def setUp(self):
        """
        _setUp_

        Initialize the database and couch.
        """
        super(StepChainTests, self).setUp()
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection(destroyAllDatabase=True)
        self.testInit.setupCouch("stepchain_t", "ConfigCache")
        self.testInit.setSchema(customModules=["WMCore.WMBS"],
                                useDefault=False)

        couchServer = CouchServer(os.environ["COUCHURL"])
        self.configDatabase = couchServer.connectDatabase("stepchain_t")
        self.testInit.generateWorkDir()
        self.workload = None

        return
class AnalysisCollectionService_t(unittest.TestCase):
    """
    Unit tests for AnalysisCollectionService
    """
    def setUp(self):
        """Set up test couch instance"""
        self.dbsURL = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader'
        self.dataset = '/RelValProdTTbar/JobRobot-MC_3XY_V24_JobRobot-v1/GEN-SIM-DIGI-RECO'
        """Set up couch test environment"""
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setupCouch("wmcore-acdc-acservice", "GroupUser", "ACDC")

        self.acService = AnalysisCollectionService(
            url=self.testInit.couchUrl, database=self.testInit.couchDbName)
        self.collection = self.acService.createCollection(
            'ewv-testcoll', 'ewv2', 'Analysis')

    def tearDown(self):
        """Clean up couch instance"""
        self.testInit.tearDownCouch()

    def testCreateFilesetFromDBS(self):
        """Test creating an analysis fileset from DBS"""

        rangesMask = Mask()
        rangesMask.addRunWithLumiRanges(run=1,
                                        lumiList=[[1, 9], [12, 12], [31, 31],
                                                  [38, 39], [49, 49], [51, 52],
                                                  [64, 65], [82, 82], [92,
                                                                       98]])

        fs, fl = self.acService.createFilesetFromDBS(self.collection,
                                                     filesetName='test_fs',
                                                     dbsURL=self.dbsURL,
                                                     dataset=self.dataset,
                                                     mask=rangesMask)

        self.assertTrue(fl['_id'])
        self.assertEqual(len(fl['files']), 21)
        for file in fl['files']:
            self.assertTrue(fl['files'][file]['merged'])
Exemple #47
0
    def setUp(self):
        """
        _setUp_

        Setup a couch database for testing
        of produced JSON
        """

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setupCouch("histogram_dump_t")
        self.histogramDB = Database(dbname="histogram_dump_t")
Exemple #48
0
    def setUp(self):
        """
        setUP global values
        """
        testname = self.id().split('.')[-1]

        logging.basicConfig(
            level=logging.DEBUG,
            format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
            datefmt='%m-%d %H:%M',
            filename='service_unittests.log',
            filemode='w')

        logger_name = 'Service%s' % testname.replace('test', '', 1)

        self.logger = logging.getLogger(logger_name)

        self.cache_path = tempfile.mkdtemp()
        self.testInit = TestInitCouchApp("RegistrationTest")
        self.couch_db = "regsvc"
        self.testInit.setupCouch(self.couch_db)
Exemple #49
0
    def setUp(self):
        """
        Basic setUp

        """
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()

        self.testDir = self.testInit.generateWorkDir()

        return
Exemple #50
0
 def setUp(self):
     """
     _setUp_
     """
     self.schema = []
     self.couchApps = ["WMStats"]
     self.testInit = TestInitCouchApp('WorkQueueServiceTest')
     self.testInit.setLogging()
     self.testInit.setDatabaseConnection()
     self.testInit.setSchema(customModules=self.schema, useDefault=False)
     dbName = 'wmstats_t'
     self.testInit.setupCouch(dbName, "WMStats")
     reqDBName = "reqmgrdb_t"
     self.testInit.setupCouch(reqDBName, "ReqMgr")
     wmstatsURL = "%s/%s" % (self.testInit.couchUrl, dbName)
     reqDBURL = "%s/%s" % (self.testInit.couchUrl, reqDBName)
     self.reqDBWriter = RequestDBWriter(reqDBURL)
     self.wmstatsReader = WMStatsReader(wmstatsURL, reqdbURL=reqDBURL)
     self.wmstatsReader.defaultStale = {}
     self.wmstatsReader.reqDB.defaultStale = {}
     return
Exemple #51
0
    def setUp(self):
        """
        _setUp_

        Initialize the database and couch.
        """
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setupCouch("stepchain_t", "ConfigCache")
        self.testInit.setSchema(customModules=["WMCore.WMBS"],
                                useDefault=False)

        couchServer = CouchServer(os.environ["COUCHURL"])
        self.configDatabase = couchServer.connectDatabase("stepchain_t")
        self.testInit.generateWorkDir()
        self.workload = None
        self.jsonTemplate = getTestFile(
            'data/ReqMgr/requests/DMWM/StepChain_MC.json')

        return
class ReproducibleSeedingTests(unittest.TestCase):
    def setUp(self):
        self.testInit = TestInitCouchApp("ReproducibleSeedingTest")
        self.testInit.setupCouch("seeding_config_cache", "GroupUser",
                                 "ConfigCache")
        self.database = Database(self.testInit.couchDbName,
                                 self.testInit.couchUrl)

        self.documentId = None

    def tearDown(self):
        self.testInit.tearDownCouch()
        return

    def testA(self):
        """instantiate"""

        document = Document()
        document[u'pset_tweak_details'] = {}
        document[u'pset_tweak_details'][u'process'] = {}
        document[u'pset_tweak_details'][u'process'][
            u'RandomNumberGeneratorService'] = {}
        document[u'pset_tweak_details'][u'process'][
            u'RandomNumberGeneratorService'][u'seed1'] = {}
        document[u'pset_tweak_details'][u'process'][
            u'RandomNumberGeneratorService'][u'seed2'] = {}
        document[u'pset_tweak_details'][u'process'][
            u'RandomNumberGeneratorService'][u'seed3'] = {}

        document = self.database.commitOne(document)[0]
        seeder = ReproducibleSeeding(CouchUrl=self.testInit.couchUrl,
                                     CouchDBName=self.testInit.couchDbName,
                                     ConfigCacheDoc=document[u'id'])

        job = Job("testjob")
        seeder(job)
        baggage = job.getBaggage()
        seed1 = getattr(baggage.process.RandomNumberGeneratorService, "seed1",
                        None)
        self.assertTrue(seed1 != None)
Exemple #53
0
 def setUp(self):
     """
     setup for test.
     """
     self.testInit = TestInit(__file__)
     self.testInit.setLogging()
     self.testInit.setDatabaseConnection()
     self.reqmonDBName = "wmstat_t"
     self.localDBName = "wmstat_t_local"
     self.testInit.setupCouch(self.reqmonDBName, "WMStatsAgent")
     self.testInit.setupCouch(self.localDBName, "WMStatsAgent")
     self.testDir = self.testInit.generateWorkDir()
     return
Exemple #54
0
    def setUp(self):
        """
        Basic setUp

        """
        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setupCouch("config_test", "GroupUser", "ConfigCache")

        self.testDir = self.testInit.generateWorkDir()

        return
Exemple #55
0
    def setUp(self, initRoot=True):
        # default set
        self.schemaModules = []

        self.initialize()
        if self.schemaModules:
            import warnings
            warnings.warn("use RESTAndCouchUnitTest instead",
                          DeprecationWarning)
            from WMQuality.TestInitCouchApp import TestInitCouchApp
            self.testInit = TestInitCouchApp(__file__)
            self.testInit.setLogging()  # logLevel = logging.SQLDEBUG
            self.testInit.setDatabaseConnection(destroyAllDatabase=True)
            self.testInit.setSchema(customModules=self.schemaModules,
                                    useDefault=False)
            # Now pull the dbURL from the factory
            # I prefer this method because the factory has better error handling
            # Also because then you know everything is the same
            myThread = threading.currentThread()
            self.config.setDBUrl(myThread.dbFactory.dburl)

        logging.info("This is our config: %s" % self.config)

        self.initRoot = initRoot
        if initRoot:
            self.rt = Root(self.config, testName=self._testMethodName)
            try:
                self.rt.start(blocking=False)
            except RuntimeError as e:
                # there appears to be worker threads from a previous test
                # hanging out. Try to slay them so that we can keep going
                print("Failed to load cherrypy with exception: %s\n" % e)
                print("The threads are: \n%s\n" % threading.enumerate())
                print("The previous test was %s\n" % self.rt.getLastTest())
                print(traceback.format_exc())
                self.rt.stop()
                raise e

        return
Exemple #56
0
    def setUp(self):
        """
        _setUp_

        Initialize the database and couch.

        """
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setupCouch("taskchain_t", "ConfigCache")
        self.testInit.setSchema(customModules=["WMCore.WMBS"],
                                useDefault=False)

        couchServer = CouchServer(os.environ["COUCHURL"])
        self.configDatabase = couchServer.connectDatabase("taskchain_t")
        self.testInit.generateWorkDir()
        self.workload = None

        self.differentNCores = getTestFile(
            'data/ReqMgr/requests/Integration/TaskChain_RelVal_Multicore.json')
        return
Exemple #57
0
    def setUp(self):
        """
        setup for test.
        """

        myThread = threading.currentThread()

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection(destroyAllDatabase=True)
        self.testInit.setSchema(
            customModules=["WMCore.WMBS", "WMComponent.DBS3Buffer"],
            useDefault=False)
        self.databaseName = "taskarchiver_t_0"
        self.testInit.setupCouch("%s/workloadsummary" % self.databaseName,
                                 "WorkloadSummary")
        self.testInit.setupCouch("%s/jobs" % self.databaseName, "JobDump")
        self.testInit.setupCouch("%s/fwjrs" % self.databaseName, "FWJRDump")
        self.testInit.setupCouch("wmagent_summary_t", "WMStats")
        self.testInit.setupCouch("wmagent_summary_central_t", "WMStats")
        self.testInit.setupCouch("stat_summary_t", "SummaryStats")
        reqmgrdb = "reqmgrdb_t"
        self.testInit.setupCouch(reqmgrdb, "ReqMgr")

        reqDBURL = "%s/%s" % (self.testInit.couchUrl, reqmgrdb)
        self.requestWriter = RequestDBWriter(reqDBURL)
        self.requestWriter.defaultStale = {}

        self.daofactory = DAOFactory(package="WMCore.WMBS",
                                     logger=myThread.logger,
                                     dbinterface=myThread.dbi)

        self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer",
                                        logger=myThread.logger,
                                        dbinterface=myThread.dbi)

        self.getJobs = self.daofactory(classname="Jobs.GetAllJobs")
        self.inject = self.daofactory(
            classname="Workflow.MarkInjectedWorkflows")

        self.testDir = self.testInit.generateWorkDir()
        os.makedirs(os.path.join(self.testDir, 'specDir'))

        self.nJobs = 10
        self.campaignName = 'aCampaign'

        self.uploadPublishInfo = False
        self.uploadPublishDir = None

        return
Exemple #58
0
 def setUp(self):
     """
     setup for test.
     """
     self.testInit = TestInit(__file__)
     self.testInit.setLogging()
     self.reqmonDBName = "wmstat_t"
     self.localDBName = "wmstat_t_local"
     self.testInit.setupCouch(self.reqmonDBName, "WMStats")
     self.testInit.setupCouch(self.localDBName, "WMStats")
     self.testDir = self.testInit.generateWorkDir()
     EmulatorHelper.setEmulators(localCouch=True,
                                 reqMon=False,
                                 wmagentDB=True)
     return
Exemple #59
0
    def setUp(self):
        """
        _setUp_

        Initialize the database and couch.
        """
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema(customModules=["WMCore.WMBS"],
                                useDefault=False)
        self.testDir = self.testInit.generateWorkDir()

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package="WMCore.WMBS",
                                     logger=myThread.logger,
                                     dbinterface=myThread.dbi)
        self.listTasksByWorkflow = self.daoFactory(
            classname="Workflow.LoadFromName")
        self.listFilesets = self.daoFactory(classname="Fileset.List")
        self.listSubsMapping = self.daoFactory(
            classname="Subscriptions.ListSubsAndFilesetsFromWorkflow")

        return
Exemple #60
0
    def setUp(self):
        """
        _setUp_

        Figure out the location of the XML report produced by CMSSW.
        """
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection(destroyAllDatabase=True)
        self.testInit.setupCouch("report_t/fwjrs", "FWJRDump")

        testData = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t")
        self.xmlPath = os.path.join(testData, "CMSSWProcessingReport.xml")
        self.badxmlPath = os.path.join(testData, "CMSSWFailReport2.xml")
        self.skippedFilesxmlPath = os.path.join(testData, "CMSSWSkippedNonExistentFile.xml")
        self.skippedAllFilesxmlPath = os.path.join(testData, "CMSSWSkippedAll.xml")
        self.fallbackXmlPath = os.path.join(testData, "CMSSWInputFallback.xml")
        self.twoFileFallbackXmlPath = os.path.join(testData, "CMSSWTwoFileRemote.xml")
        self.pileupXmlPath = os.path.join(testData, "CMSSWPileup.xml")
        self.withEventsXmlPath = os.path.join(testData, "CMSSWWithEventCounts.xml")
        self.noLocationReport = os.path.join(testData, "Report.0.pkl")

        self.testDir = self.testInit.generateWorkDir()
        return