class ConfigurationTest(unittest.TestCase): """ test case for Configuration object """ def setUp(self): """set up""" self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() self.normalSave = "%s/WMCore_Agent_Configuration_t_normal.py" % self.testDir self.docSave = "%s/WMCore_Agent_Configuration_t_documented.py" % self.testDir self.commentSave = "%s/WMCore_Agent_Configuration_t_commented.py" % self.testDir def tearDown(self): """clean up""" self.testInit.delWorkDir() def testA(self): """ctor""" try: config = Configuration() except Exception, ex: msg = "Failed to instantiate Configuration\n" msg += str(ex) self.fail(msg)
class AnalysisCollectionService_t(unittest.TestCase): """ Unit tests for AnalysisCollectionService """ def setUp(self): """Set up test couch instance""" self.dbsURL = 'http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet' self.dataset = '/RelValProdTTbar/JobRobot-MC_3XY_V24_JobRobot-v1/GEN-SIM-DIGI-RECO' """Set up couch test environment""" self.testInit = TestInitCouchApp(__file__) self.testInit.setupCouch("wmcore-acdc-acservice", "GroupUser", "ACDC") self.acService = AnalysisCollectionService(url=self.testInit.couchUrl, database = self.testInit.couchDbName) self.collection = self.acService.createCollection('ewv-testcoll', 'ewv2', 'Analysis') def tearDown(self): """Clean up couch instance""" self.testInit.tearDownCouch() def testCreateFilesetFromDBS(self): """Test creating an analysis fileset from DBS""" rangesMask = Mask() rangesMask.addRunWithLumiRanges(run=1, lumiList=[[1, 9], [12, 12], [31, 31], [38, 39], [49, 49], [51, 52], [64, 65], [82, 82], [92, 98]]) fs, fl = self.acService.createFilesetFromDBS(self.collection, filesetName='test_fs', dbsURL=self.dbsURL, dataset=self.dataset, mask=rangesMask) self.assertTrue(fl['_id']) self.assertEqual(len(fl['files']), 21)
class ReproducibleSeedingTests(unittest.TestCase): def setUp(self): self.testInit = TestInitCouchApp("ReproducibleSeedingTest") self.testInit.setupCouch("seeding_config_cache", "GroupUser", "ConfigCache") self.database = Database(self.testInit.couchDbName, self.testInit.couchUrl) self.documentId = None def tearDown(self): self.testInit.tearDownCouch() return def testA(self): """instantiate""" document = Document() document[u'pset_tweak_details'] = {} document[u'pset_tweak_details'][u'process'] = {} document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'] = {} document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed1'] = {} document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed2'] = {} document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed3'] = {} document = self.database.commitOne(document)[0] seeder = ReproducibleSeeding(CouchUrl = self.testInit.couchUrl, CouchDBName = self.testInit.couchDbName, ConfigCacheDoc = document[u'id']) job = Job("testjob") seeder(job) baggage = job.getBaggage() seed1 = getattr(baggage.process.RandomNumberGeneratorService, "seed1", None) self.failUnless(seed1 != None)
class RequestDBTest(unittest.TestCase): """ """ def setUp(self): """ _setUp_ """ self.schema = [] self.couchApps = ["ReqMgr"] self.testInit = TestInitCouchApp("RequestDBServiceTest") self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=self.schema, useDefault=False) dbName = "requsetdb_t" self.testInit.setupCouch(dbName, *self.couchApps) self.requestWriter = RequestDBWriter(self.testInit.couchUrl, dbName) self.requestReader = RequestDBReader(self.testInit.couchUrl, dbName) self.requestWriter.defaultStale = {} self.requestReader.defaultStale = {} return def tearDown(self): """ _tearDown_ Drop all the WMBS tables. """ self.testInit.tearDownCouch() def testRequestDBWriter(self): # test getWork schema = generate_reqmgr_schema() result = self.requestWriter.insertGenericRequest(schema[0]) self.assertEquals(len(result), 1, "insert fail") self.assertEquals( self.requestWriter.updateRequestStatus(schema[0]["RequestName"], "failed"), "OK", "update fail" ) self.assertEquals( self.requestWriter.updateRequestStatus("not_exist_schema", "assigned"), "Error: document not found" ) result = self.requestWriter.updateRequestProperty(schema[0]["RequestName"], {"Teams": ["teamA"]}) self.assertEquals( self.requestWriter.updateRequestProperty(schema[0]["RequestName"], {"Teams": ["teamA"]}), "OK", "update fail", ) self.assertEquals( self.requestWriter.updateRequestProperty("not_exist_schema", {"Teams": "teamA"}), "Error: document not found", ) result = self.requestWriter.getRequestByNames([schema[0]["RequestName"]]) self.assertEquals(len(result), 1, "should be 1") result = self.requestWriter.getRequestByStatus(["failed"], False, 1) self.assertEquals(len(result), 1, "should be 1")
class T0RequestDBTest(unittest.TestCase): """ """ def setUp(self): """ _setUp_ """ self.schema = [] self.couchApps = ["T0Request"] self.testInit = TestInitCouchApp('RequestDBServiceTest') self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = self.schema, useDefault = False) dbName = 't0_requsetdb_t' self.testInit.setupCouch(dbName, *self.couchApps) reqDBURL = "%s/%s" % (self.testInit.couchUrl, dbName) self.requestWriter = RequestDBWriter(reqDBURL, self.couchApps[0]) self.requestReader = RequestDBReader(reqDBURL, self.couchApps[0]) self.requestWriter.defaultStale = {} self.requestReader.defaultStale = {} return def tearDown(self): """ _tearDown_ Drop all the WMBS tables. """ self.testInit.tearDownCouch() def testRequestDBWriter(self): # test getWork schema = generate_reqmgr_schema() result = self.requestWriter.insertGenericRequest(schema[0]) self.assertEqual(len(result), 1, 'insert fail'); result = self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "assigned") self.assertEqual(result, 'not allowed state assigned', 'update fail') self.assertEqual(self.requestWriter.updateRequestStatus("not_exist_schema", "new"), 'Error: document not found') allowedStates = ["Closed", "Merge", "AlcaSkim", "Harvesting", "Processing Done", "completed"] for state in allowedStates: self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], state), 'OK') self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "Processing Done"), 'not allowed transition completed to Processing Done') self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "normal-archived"), 'OK') result = self.requestWriter.getRequestByStatus(["normal-archived"], False, 1) self.assertEqual(len(result), 1, "should be 1 but %s" % result)
class Interface_t(unittest.TestCase): def setUp(self): self.testInit = TestInitCouchApp(__file__) self.testInit.setupCouch("wmcore-groupuser-interface", "GroupUser") #create a test owner for the collections in this test self.owner1 = makeUser("DMWM", "evansde77", self.testInit.couchUrl, self.testInit.couchDbName) self.owner1.connect() self.owner1.create() self.owner2 = makeUser("DMWM", "drsm79", self.testInit.couchUrl, self.testInit.couchDbName) self.owner2.connect() self.owner2.create() #self.url = os.getenv("COUCHURL", "http://127.0.0.1:5984") #self.database = "awesome_acdc" def tearDown(self): self.testInit.tearDownCouch() def testA(self): """ make some documents and own them""" guInt = Interface(self.testInit.couchUrl, self.testInit.couchDbName) #create a couple of docs couch = Database(self.testInit.couchDbName, self.testInit.couchUrl) for x in range(10): doc = Document("document%s" % x, {"Test Data": [1,2,3,4] }) couch.queue(doc) couch.commit() self.assertEqual(len(guInt.documentsOwned(self.owner1.group.name, self.owner1.name)), 0) self.assertEqual(len(guInt.documentsOwned(self.owner2.group.name, self.owner2.name)), 0) guInt.callUpdate("ownthis","document1", group = self.owner1.group.name, user = self.owner1.name) self.failUnless("document1" in guInt.documentsOwned(self.owner1.group.name, self.owner1.name)) self.assertEqual(len(guInt.documentsOwned(self.owner1.group.name, self.owner1.name)), 1) self.assertEqual(len(guInt.documentsOwned(self.owner2.group.name, self.owner2.name)), 0) guInt.callUpdate("ownthis","document2", group = self.owner2.group.name, user = self.owner2.name) self.failUnless("document2" in guInt.documentsOwned(self.owner2.group.name, self.owner2.name)) self.assertEqual(len(guInt.documentsOwned(self.owner1.group.name, self.owner1.name)), 1) self.assertEqual(len(guInt.documentsOwned(self.owner2.group.name, self.owner2.name)), 1) guInt.callUpdate("newgroup", "group-DataOps", group = "DataOps") self.failUnless(couch.documentExists("group-DataOps") ) guInt.callUpdate("newuser", "user-damason", group = "DataOps", user = "******") self.failUnless(couch.documentExists("user-damason") )
def couchapp(self, couchappName): """ Drop database in CouchDB. Reload (push) the couchapp couchappName (likely to be "OpsClipboard"). """ print "Pushing couchapp '%s' ..." % couchappName # here probably need to specify the couchUrl, otherwise default env COUCHURL # will be taken (as described above) testInit = TestInitCouchApp(__file__, dropExistingDb=True) # should now drop the CouchDB db, create a new one and push the couchapp testInit.setupCouch(self.dbName, couchappName)
class WMStatsTest(unittest.TestCase): """ """ def setUp(self): """ _setUp_ """ self.schema = [] self.couchApps = ["WMStats"] self.testInit = TestInitCouchApp('WorkQueueServiceTest') self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = self.schema, useDefault = False) self.testInit.setupCouch('wmstats_t', *self.couchApps) self.wmstatsWriter = WMStatsWriter(self.testInit.couchUrl, 'wmstats_t'); return def tearDown(self): """ _tearDown_ Drop all the WMBS tables. """ self.testInit.tearDownCouch() def testWMStatsWriter(self): # test getWork schema = generate_reqmgr_schema() self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK', 'insert fail'); self.assertEquals(self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail') self.assertEquals(self.wmstatsWriter.updateRequestStatus("not_exist_schema", "assigned"), 'ERROR: request not found - not_exist_schema') self.assertEquals(self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'), 'OK', 'update fail') self.assertEquals(self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'), 'ERROR: request not found - not_exist_schema') totalStats = {'total_jobs': 100, 'input_events': 1000, 'input_lumis': 1234, 'input_num_files': 5} self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'INSERTED', 'update fail') self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'UPDATED', 'update fail') self.assertEquals(self.wmstatsWriter.insertTotalStats("not_exist_schema", totalStats), 'ERROR: request not found - not_exist_schema') spec1 = newWorkload(schema[0]['RequestName']) production = spec1.newTask("Production") production.setTaskType("Merge") production.setSiteWhitelist(['TEST_SITE']) self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK', 'update fail') spec2 = newWorkload("not_exist_schema") production = spec2.newTask("Production") production.setTaskType("Merge") self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2), 'ERROR: request not found - not_exist_schema')
class FWJRDBAPITest(unittest.TestCase): """ """ def setUp(self): """ _setUp_ """ self.schema = [] self.couchApps = ["FWJRDump"] self.testInit = TestInitCouchApp("FWJRAPITest") self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=self.schema, useDefault=False) dbName = "fwjrdump_t" self.testInit.setupCouch(dbName, *self.couchApps) fwjrDBURL = "%s/%s" % (self.testInit.couchUrl, dbName) self.fwjrAPI = FWJRDBAPI(fwjrDBURL) self.fwjrAPI.defaultStale = {} return def tearDown(self): """ _tearDown_ Drop all the WMBS tables. """ self.testInit.tearDownCouch() def testFWJRDBAPI(self): jobID = 1 retryCount = 0 fwjrDocument = { "_id": "%s-%s" % (jobID, retryCount), "jobid": jobID, "retrycount": retryCount, "archivestatus": "ready", "fwjr": SAMPLE_FWJR, "jobtype": "Merge", "type": "fwjr", } workflow = SAMPLE_FWJR["task"].split("/")[1] self.assertEqual(self.fwjrAPI.couchDB.commitOne(fwjrDocument)[0]["id"], fwjrDocument["_id"]) self.assertEqual(self.fwjrAPI.getFWJRByArchiveStatus("ready")["rows"][0]["id"], fwjrDocument["_id"]) self.assertEqual(self.fwjrAPI.isAllFWJRArchived(workflow), False) self.fwjrAPI.updateArchiveUploadedStatus(fwjrDocument["_id"]) self.assertEqual(self.fwjrAPI.getFWJRByArchiveStatus("uploaded")["rows"][0]["id"], fwjrDocument["_id"]) self.assertEqual(self.fwjrAPI.isAllFWJRArchived(workflow), True) self.assertEqual(self.fwjrAPI.getFWJRWithSkippedFiles()["rows"][0]["value"]["skippedFiles"], 2)
class RESTBaseUnitTest(unittest.TestCase): def setUp(self): # default set self.schemaModules = [] self.initialize() if self.schemaModules: self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() # logLevel = logging.SQLDEBUG self.testInit.setDatabaseConnection(self.config.getDBUrl()) self.testInit.setSchema(customModules = self.schemaModules, useDefault = False) self.rt = Root(self.config) self.rt.start(blocking=False) def tearDown(self): self.rt.stop() if self.schemaModules: self.testInit.clearDatabase() self.config = None def initialize(self): """ i.e. self.config = DefaultConfig('WMCore.WebTools.RESTModel') self.config.setDBUrl("sqlite://") self.schemaModules = ["WMCore.ThreadPool", WMCore.WMBS"] """ message = "initialize method has to be implemented, self.restModel, self.schemaModules needs to be set" raise NotImplementedError, message
def setUp(self): """ _setUp_ """ self.transitions = Transitions() self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("changestate_t/jobs", "JobDump") self.testInit.setupCouch("changestate_t/fwjrs", "FWJRDump") self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) myThread = threading.currentThread() self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) self.couchServer = CouchServer(dburl = os.getenv("COUCHURL")) self.config = Configuration() self.config.component_("JobStateMachine") self.config.JobStateMachine.couchurl = os.getenv("COUCHURL") return
def setUp(self): """set up""" self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() self.normalSave = "%s/WMCore_Agent_Configuration_t_normal.py" % self.testDir self.docSave = "%s/WMCore_Agent_Configuration_t_documented.py" % self.testDir self.commentSave = "%s/WMCore_Agent_Configuration_t_commented.py" % self.testDir
def setUp(self): """ _setUp_ Initialize the database and couch. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("promptreco_t", "ConfigCache") self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) couchServer = CouchServer(os.environ["COUCHURL"]) self.configDatabase = couchServer.connectDatabase("promptreco_t") self.testDir = self.testInit.generateWorkDir() myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.listTasksByWorkflow = self.daoFactory(classname="Workflow.LoadFromName") self.listFilesets = self.daoFactory(classname="Fileset.List") self.listSubsMapping = self.daoFactory(classname="Subscriptions.ListSubsAndFilesetsFromWorkflow") return
def testProcessorWithReceiverAndCouchSink(self): # set up couch first self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() dbName = "couch_sink" self.testInit.setupCouch(dbName) # add corresponding part of the configuration for CouchSink(s) config = self.config.AlertProcessor config.critical.sinks.section_("couch") config.critical.sinks.couch.url = self.testInit.couchUrl config.critical.sinks.couch.database = self.testInit.couchDbName # just send the Alert into couch processor = Processor(config) rec = Receiver(self.addr, processor, self.ctrl) rec.startReceiver() # non blocking call # run worker(), this time directly without Process as above, # worker will send 10 Alerts to Receiver worker(self.addr, self.ctrl, 10) # wait until the Receiver is shut by worker while rec.isReady(): time.sleep(0.3) print "%s waiting for Receiver to shut ..." % inspect.stack()[0][3]
def setUp(self): """ _setUp_ """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("wmbshelper_t/jobs", "JobDump") self.testInit.setupCouch("wmbshelper_t/fwjrs", "FWJRDump") os.environ["COUCHDB"] = "wmbshelper_t" self.testInit.setSchema(customModules = ["WMCore.WMBS", "WMComponent.DBSBuffer.Database", "WMCore.BossAir", "WMCore.ResourceControl"], useDefault = False) self.workDir = self.testInit.generateWorkDir() self.wmspec = self.createWMSpec() self.topLevelTask = getFirstTask(self.wmspec) self.inputDataset = self.topLevelTask.inputDataset() self.dataset = self.topLevelTask.getInputDatasetPath() self.dbs = MockDBSReader(self.inputDataset.dbsurl) self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = threading.currentThread().logger, dbinterface = threading.currentThread().dbi) return
def setUp(self): """ setup for test. """ super(JobArchiverTest, self).setUp() myThread = threading.currentThread() super(JobArchiverTest, self).setUp() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() # self.tearDown() self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) self.testInit.setupCouch("jobarchiver_t_0/jobs", "JobDump") self.testInit.setupCouch("jobarchiver_t_0/fwjrs", "FWJRDump") self.daofactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.getJobs = self.daofactory(classname="Jobs.GetAllJobs") self.testDir = self.testInit.generateWorkDir(deleteOnDestruction=False) self.nJobs = 10 self.configFile = EmulatorSetup.setupWMAgentConfig() return
def setUp(self): myThread = threading.currentThread() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) self.databaseName = "couchapp_t_0" self.testInit.setupCouch(self.databaseName, "WorkloadSummary") self.testInit.setupCouch("%s/jobs" % self.databaseName, "JobDump") self.testInit.setupCouch("%s/fwjrs" % self.databaseName, "FWJRDump") # Setup config for couch connections config = self.testInit.getConfiguration() config.section_("JobStateMachine") config.JobStateMachine.couchDBName = self.databaseName # Create couch server and connect to databases self.couchdb = CouchServer(config.JobStateMachine.couchurl) self.jobsdatabase = self.couchdb.connectDatabase("%s/jobs" % config.JobStateMachine.couchDBName) self.fwjrdatabase = self.couchdb.connectDatabase("%s/fwjrs" % config.JobStateMachine.couchDBName) # Create changeState self.changeState = ChangeState(config) self.config = config # Create testDir self.testDir = self.testInit.generateWorkDir() return
def setUp(self): """ _setUp_ Standard setup: Now with 100% more couch """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl", "WMCore.Agent.Database"]) self.testInit.setupCouch("jobsubmitter_t/jobs", "JobDump") self.testInit.setupCouch("jobsubmitter_t/fwjrs", "FWJRDump") self.testInit.setupCouch("wmagent_summary_t", "WMStats") myThread = threading.currentThread() self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) self.baDaoFactory = DAOFactory(package = "WMCore.BossAir", logger = myThread.logger, dbinterface = myThread.dbi) self.testDir = self.testInit.generateWorkDir() # Set heartbeat self.componentName = 'JobSubmitter' self.heartbeatAPI = HeartbeatAPI(self.componentName) self.heartbeatAPI.registerComponent() return
def testProcessorWithReceiverAndCouchSink(self): # set up couch first self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() dbName = "couch_sink" self.testInit.setupCouch(dbName) # add corresponding part of the configuration for CouchSink(s) config = self.config.AlertProcessor config.critical.sinks.section_("couch") config.critical.sinks.couch.url = self.testInit.couchUrl config.critical.sinks.couch.database = self.testInit.couchDbName # just send the Alert into couch processor = Processor(config) # Receiver is waited for shutdown / shutdown explicitly in tearDown() self.receiver = Receiver(self.addr, processor, self.ctrl) self.receiver.startReceiver() # non blocking call # run worker(), this time directly without Process as above, # worker will send 10 Alerts to Receiver worker(self.addr, self.ctrl, 10) # wait until Receiver is shut down (by a message from worker() # also need to wait, otherwise tearDown kicks off and scrapes the # couch so half of the alerts will be undelivered while self.receiver.isReady(): time.sleep(ReceiverLogic.TIMEOUT_AFTER_SHUTDOWN * 1.5) logging.info("%s: Waiting for Receiver shutdown ..." % inspect.stack()[0][3])
def setUp(self): """ setup for test. """ myThread = threading.currentThread() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() #self.tearDown() self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) self.testInit.setupCouch("jobarchiver_t_0/jobs", "JobDump") self.testInit.setupCouch("jobarchiver_t_0/fwjrs", "FWJRDump") self.daofactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) self.getJobs = self.daofactory(classname = "Jobs.GetAllJobs") self.testDir = self.testInit.generateWorkDir(deleteOnDestruction = False) self.nJobs = 10 self.alertsReceiver = None EmulatorHelper.setEmulators(phedex = True, dbs = True, siteDB = True, requestMgr = False) return
def setUp(self): """ _setUp_ Setup the database and logging connection. Try to create all of the WMBS tables. Also, create some dummy locations. """ myThread = threading.currentThread() self.sites = ['T2_US_Florida', 'T2_US_UCSD', 'T2_TW_Taiwan', 'T1_CH_CERN'] self.testInit = TestInit(__file__) self.testInit.setLogging(logLevel = logging.DEBUG) self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ['WMCore.WMBS', 'WMCore.ResourceControl', 'WMCore.Agent.Database'], useDefault = False) self.testInit.setupCouch("dashboardreporter_t/jobs", "JobDump") self.testInit.setupCouch("dashboardreporter_t/fwjrs", "FWJRDump") resourceControl = ResourceControl() for site in self.sites: resourceControl.insertSite(siteName = site, seName = site, ceName = site) resourceControl.insertThreshold(siteName = site, taskType = 'Processing', \ maxSlots = 10000) self.testDir = self.testInit.generateWorkDir() self.alertsReceiver = None return
def setUp(self): """ _setUp_ Setup the database and logging connection. """ logging.basicConfig() self.logger = logging.getLogger('LogDBTest') # self.logger.setLevel(logging.DEBUG) self.schema = [] self.couchApps = ["LogDB"] self.testInit = TestInitCouchApp('LogDBTest') self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = self.schema, useDefault = False) dbName = 'logdb_t' self.testInit.setupCouch(dbName, *self.couchApps) url = "%s/%s" % (self.testInit.couchUrl, dbName) identifier = 'agentname' self.agent_inst = LogDB(url, identifier, logger=self.logger, thread_name="MainThread") self.agent_inst2 = LogDB(url, identifier, logger=self.logger, thread_name="Test") identifier = '/DC=org/DC=doegrids/OU=People/CN=First Last 123' self.user_inst = LogDB(url, identifier, logger=self.logger, thread_name="MainThread") self.report = LogDBReport(self.agent_inst)
def setUp(self): """ _setUp_ Set everything up. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl"], useDefault = False) self.testInit.setupCouch("jobsubmittercaching_t/jobs", "JobDump") self.testInit.setupCouch("jobsubmittercaching_t/fwjrs", "FWJRDump") resourceControl = ResourceControl() for siteName in ["T1_US_FNAL", "T1_UK_RAL"]: resourceControl.insertSite(siteName = siteName, pnn = "se.%s" % (siteName), ceName = siteName, plugin = "CondorPlugin", cmsName = siteName) resourceControl.insertThreshold(siteName = siteName, taskType = "Processing", maxSlots = 10000, pendingSlots = 10000) self.testDir = self.testInit.generateWorkDir() self.configFile = EmulatorSetup.setupWMAgentConfig() return
def setUp(self): """ Setup for unit tests """ self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() testname = self.id().split('.')[-1] logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', datefmt='%m-%d %H:%M', filename='service_unittests.log', filemode='w') logger_name = 'Service%s' % testname.replace('test', '', 1) self.logger = logging.getLogger(logger_name) #self.cache_path = tempfile.mkdtemp() test_dict = {'logger': self.logger, #'cachepath' : self.cache_path, #'req_cache_path': '%s/requests' % self.cache_path, 'endpoint':'http://cmssw.cvs.cern.ch/cgi-bin/cmssw.cgi'} self.myService = Service(test_dict) test_dict['endpoint'] = 'http://cmssw-test.cvs.cern.ch/cgi-bin/cmssw.cgi' self.myService2 = Service(test_dict) self.testUrl = 'http://cern.ch' self.port = 8888 cherrypy.config.update({'server.socket_port': self.port})
def setUp(self): """ setup for test. """ myThread = threading.currentThread() self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase = True) self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) self.testInit.setupCouch("errorhandler_t", "GroupUser", "ACDC") self.testInit.setupCouch("errorhandler_t_jd/jobs", "JobDump") self.testInit.setupCouch("errorhandler_t_jd/fwjrs", "FWJRDump") self.daofactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) self.getJobs = self.daofactory(classname = "Jobs.GetAllJobs") self.setJobTime = self.daofactory(classname = "Jobs.SetStateTime") locationAction = self.daofactory(classname = "Locations.New") locationAction.execute(siteName = "malpaquet", seName = "malpaquet") self.testDir = self.testInit.generateWorkDir() self.nJobs = 10 self.dataCS = DataCollectionService(url = self.testInit.couchUrl, database = "errorhandler_t") return
def setUp(self): """ _setUp_ """ self.transitions = Transitions() self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("changestate_t/jobs", "JobDump") self.testInit.setupCouch("changestate_t/fwjrs", "FWJRDump") self.testInit.setupCouch("job_summary", "WMStats") self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) myThread = threading.currentThread() self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) couchurl = os.getenv("COUCHURL") self.couchServer = CouchServer(dburl = couchurl) self.config = self.testInit.getConfiguration() self.taskName = "/TestWorkflow/Task" return
def setUp(self): """ _setUp_ Setup couchdb and the test environment """ super(ResubmitBlockTest, self).setUp() self.group = 'unknown' self.user = '******' # Set external test helpers self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setupCouch("resubmitblock_t", "ACDC", "GroupUser") # Define test environment self.couchUrl = os.environ["COUCHURL"] self.acdcDBName = 'resubmitblock_t' self.validLocations = ['T2_US_Nebraska', 'T1_US_FNAL_Disk', 'T1_UK_RAL_Disk'] self.siteWhitelist = ['T2_XX_SiteA'] siteDB = SiteDB() #Convert phedex node name to a valid processing site name self.PSNs = siteDB.PNNstoPSNs(self.validLocations) self.workflowName = 'dballest_ReReco_workflow' couchServer = CouchServer(dburl=self.couchUrl) self.acdcDB = couchServer.connectDatabase(self.acdcDBName, create=False) user = makeUser(self.group, '*****@*****.**', self.couchUrl, self.acdcDBName) user.create() return
def setUp(self): """ _setUp_ Figure out the location of the XML report produced by CMSSW. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.testInit.setupCouch("report_t/fwjrs", "FWJRDump") self.xmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml") self.badxmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWFailReport2.xml") self.skippedFilesxmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWSkippedNonExistentFile.xml") self.skippedAllFilesxmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWSkippedAll.xml") self.fallbackXmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWInputFallback.xml") self.twoFileFallbackXmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWTwoFileRemote.xml") self.pileupXmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWPileup.xml") self.testDir = self.testInit.generateWorkDir() return
def setUp(self): """ _setUp_ Create two subscriptions: One that contains a single file and one that contains multiple files. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() # self.testInit.clearDatabase(modules = ['WMCore.WMBS']) self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) self.testInit.setupCouch("lumi_t", "GroupUser", "ACDC") myThread = threading.currentThread() daofactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) locationAction = daofactory(classname = "Locations.New") locationAction.execute(siteName='s1', pnn="T1_US_FNAL_Disk") locationAction.execute(siteName='s2', pnn="T2_CH_CERN") self.testWorkflow = Workflow(spec="spec.xml", owner="mnorman", name="wf001", task="Test") self.testWorkflow.create() self.performanceParams = {'timePerEvent': 12, 'memoryRequirement': 2300, 'sizePerEvent': 400} return
def setUp(self): """ setup for test. """ myThread = threading.currentThread() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.testInit.setSchema(customModules=["WMCore.WMBS", "WMComponent.DBS3Buffer"], useDefault=False) self.databaseName = "taskarchiver_t_0" self.testInit.setupCouch("%s/workloadsummary" % self.databaseName, "WorkloadSummary") self.testInit.setupCouch("%s/jobs" % self.databaseName, "JobDump") self.testInit.setupCouch("%s/fwjrs" % self.databaseName, "FWJRDump") self.testInit.setupCouch("wmagent_summary_t", "WMStats") self.testInit.setupCouch("wmagent_summary_central_t", "WMStats") self.daofactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.getJobs = self.daofactory(classname="Jobs.GetAllJobs") self.inject = self.daofactory(classname="Workflow.MarkInjectedWorkflows") self.testDir = self.testInit.generateWorkDir() os.makedirs(os.path.join(self.testDir, "specDir")) self.nJobs = 10 self.campaignName = "aCampaign" self.alertsReceiver = None self.uploadPublishInfo = False self.uploadPublishDir = None return
def setUp(self): """ Initialize the database and couch. """ super(PileupFetcherTest, self).setUp() self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("pileupfetcher_t", "ConfigCache") self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) couchServer = CouchServer(os.environ["COUCHURL"]) self.configDatabase = couchServer.connectDatabase("pileupfetcher_t") self.testDir = self.testInit.generateWorkDir() self.rucioAcct = "wmcore_transferor"
def setUp(self): """ _setUp_ """ self.schema = [] self.couchApps = ["FWJRDump"] self.testInit = TestInitCouchApp('FWJRAPITest') self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = self.schema, useDefault = False) dbName = 'fwjrdump_t' self.testInit.setupCouch(dbName, *self.couchApps) fwjrDBURL = "%s/%s" % (self.testInit.couchUrl, dbName) self.fwjrAPI= FWJRDBAPI(fwjrDBURL) self.fwjrAPI.defaultStale = {} return
def setUp(self): """ _setUp_ Initialize the database. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("analysis_t", "ConfigCache") self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) couchServer = CouchServer(os.environ["COUCHURL"]) self.configDatabase = couchServer.connectDatabase("analysis_t") self.testDir = self.testInit.generateWorkDir() return
def setUp(self): """ _setUp_ Setup the database and logging connection. Try to create all of the WMBS tables. Also, create some dummy locations. """ super(JobCreatorTest, self).setUp() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=[ 'WMCore.WMBS', 'WMCore.ResourceControl', 'WMCore.Agent.Database' ], useDefault=False) self.couchdbname = "jobcreator_t" self.testInit.setupCouch("%s/jobs" % self.couchdbname, "JobDump") self.testInit.setupCouch("%s/fwjrs" % self.couchdbname, "FWJRDump") self.configFile = EmulatorSetup.setupWMAgentConfig() myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) locationAction = self.daoFactory(classname="Locations.New") for site in self.sites: locationAction.execute(siteName=site, pnn=site) # Create sites in resourceControl resourceControl = ResourceControl() for site in self.sites: resourceControl.insertSite(siteName=site, pnn=site, ceName=site) resourceControl.insertThreshold(siteName=site, taskType='Processing', maxSlots=10000, pendingSlots=10000) self.resourceControl = resourceControl self._setup = True self._teardown = False self.testDir = self.testInit.generateWorkDir() self.cwd = os.getcwd() # Set heartbeat self.componentName = 'JobCreator' self.heartbeatAPI = HeartbeatAPI(self.componentName) self.heartbeatAPI.registerComponent() if PY3: self.assertItemsEqual = self.assertCountEqual return
def setUp(self): """ _setUp_ Initialize the database and couch. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch(TEST_DB_NAME, "ConfigCache") self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) self.testInit.generateWorkDir() couchServer = CouchServer(os.environ["COUCHURL"]) self.configDatabase = couchServer.connectDatabase(TEST_DB_NAME) EmulatorHelper.setEmulators(dbs=True) return
def setUp(self): """ _setUp_ """ self.schema = [] self.couchApps = ["ReqMgr"] self.testInit = TestInitCouchApp('RequestDBServiceTest') self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=self.schema, useDefault=False) dbName = 'requsetdb_t' self.testInit.setupCouch(dbName, *self.couchApps) reqDBURL = "%s/%s" % (self.testInit.couchUrl, dbName) self.requestWriter = RequestDBWriter(reqDBURL) self.requestReader = RequestDBReader(reqDBURL) self.requestWriter.defaultStale = {} self.requestReader.defaultStale = {} return
class CouchUtils_t(unittest.TestCase): def setUp(self): self.testInit = TestInitCouchApp(__file__) self.testInit.setupCouch("wmcore-acdc-couchutils", "GroupUser", "ACDC") def tearDown(self): self.testInit.tearDownCouch() def testA(self): """object driven connection via initialiseCouch method""" class Thingy(object): """misc object with couch access attrs""" def __init__(self): self.couchdb = None self.database = None self.url = None @CouchUtils.connectToCouch def __call__(self): return True couchThingy = Thingy() # test throws with everything None self.assertRaises(CouchUtils.CouchConnectionError, CouchUtils.initialiseCouch, couchThingy) couchThingy.url = self.testInit.couchUrl self.assertRaises(CouchUtils.CouchConnectionError, CouchUtils.initialiseCouch, couchThingy) couchThingy.database = self.testInit.couchDbName try: CouchUtils.initialiseCouch(couchThingy) except Exception, ex: msg = "Error initialising couch client for test object:\n %s " % str(ex) self.fail(msg) self.failUnless(couchThingy.couchdb != None) # test decorator on already connected object try: couchThingy() except Exception, ex: msg = "Error invoking connectToCouch decorator:\n %s" % str(msg) self.fail(msg)
def setUp(self): """ _setUp_ Setup the database and logging connection. Try to create all of the WMBS tables. Also add some dummy locations. """ super(WorkQueueTestCase, self).setUp() self.queueDB = 'workqueue_t' self.queueInboxDB = 'workqueue_t_inbox' self.globalQDB = 'workqueue_t_global' self.globalQInboxDB = 'workqueue_t_global_inbox' self.localQDB = 'workqueue_t_local' self.localQInboxDB = 'workqueue_t_local_inbox' self.localQDB2 = 'workqueue_t_local2' self.localQInboxDB2 = 'workqueue_t_local2_inbox' self.configCacheDB = 'workqueue_t_config_cache' self.logDBName = 'logdb_t' self.requestDBName = 'workqueue_t_reqmgr_workload_cache' self.setSchema() self.testInit = TestInit('WorkQueueTest') self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.addCleanup(self.testInit.clearDatabase) self.addCleanup(logging.debug, 'Cleanup called clearDatabase()') self.testInit.setSchema(customModules = self.schema, useDefault = False) self.testInit.setupCouch(self.queueDB, *self.couchApps) self.testInit.setupCouch(self.queueInboxDB, *self.couchApps) self.testInit.setupCouch(self.globalQDB, *self.couchApps) self.testInit.setupCouch(self.globalQInboxDB , *self.couchApps) self.testInit.setupCouch(self.localQDB, *self.couchApps) self.testInit.setupCouch(self.localQInboxDB, *self.couchApps) self.testInit.setupCouch(self.localQDB2, *self.couchApps) self.testInit.setupCouch(self.localQInboxDB2, *self.couchApps) self.testInit.setupCouch(self.configCacheDB, 'ConfigCache') self.testInit.setupCouch(self.logDBName, 'LogDB') self.testInit.setupCouch(self.requestDBName, 'ReqMgr') self.couchURL = os.environ.get("COUCHURL") couchServer = CouchServer(self.couchURL) self.configCacheDBInstance = couchServer.connectDatabase(self.configCacheDB) self.localCouchMonitor = CouchMonitor(self.couchURL) self.localCouchMonitor.deleteReplicatorDocs() self.addCleanup(self.localCouchMonitor.deleteReplicatorDocs) self.addCleanup(logging.debug, 'Cleanup called deleteReplicatorDocs()') self.addCleanup(self.testInit.tearDownCouch) self.addCleanup(logging.debug, 'Cleanup called tearDownCouch()') self.workDir = self.testInit.generateWorkDir() self.addCleanup(self.testInit.delWorkDir) self.addCleanup(logging.debug, 'Cleanup called delWorkDir()') return
def setUp(self): """ _setUp_ Figure out the location of the XML report produced by CMSSW. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.testInit.setupCouch("report_t/fwjrs", "FWJRDump") self.xmlPath = os.path.join( WMCore.WMBase.getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml") self.badxmlPath = os.path.join( WMCore.WMBase.getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWFailReport2.xml") self.testDir = self.testInit.generateWorkDir() return
def setUp(self): """ _setUp_ setUp function for unittest """ # Set constants self.couchDB = "config_test" self.configURL = "RANDOM;;URL;;NAME" self.configString = "This is a random string" self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema( customModules=["WMComponent.DBS3Buffer", 'WMCore.Agent.Database'], useDefault=False) self.testInit.setupCouch(self.couchDB, "GroupUser", "ConfigCache") myThread = threading.currentThread() self.bufferFactory = DAOFactory( package="WMComponent.DBSBuffer.Database", logger=myThread.logger, dbinterface=myThread.dbi) self.buffer3Factory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) locationAction = self.bufferFactory( classname="DBSBufferFiles.AddLocation") locationAction.execute(siteName="se1.cern.ch") locationAction.execute(siteName="se1.fnal.gov") locationAction.execute(siteName="malpaquet") # Set heartbeat self.componentName = 'JobSubmitter' self.heartbeatAPI = HeartbeatAPI(self.componentName) self.heartbeatAPI.registerComponent() # Set up a config cache configCache = ConfigCache(os.environ["COUCHURL"], couchDBName=self.couchDB) configCache.createUserGroup(groupname="testGroup", username='******') self.testDir = self.testInit.generateWorkDir() psetPath = os.path.join(self.testDir, "PSet.txt") f = open(psetPath, 'w') f.write(self.configString) f.close() configCache.addConfig(newConfig=psetPath, psetHash=None) configCache.save() self.configURL = "%s;;%s;;%s" % (os.environ["COUCHURL"], self.couchDB, configCache.getCouchID()) return
def setUp(self): """ _setUp_ """ super(WMBSHelperTest, self).setUp() self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.testInit.setupCouch("wmbshelper_t/jobs", "JobDump") self.testInit.setupCouch("wmbshelper_t/fwjrs", "FWJRDump") self.testInit.setupCouch("config_test", "GroupUser", "ConfigCache") os.environ["COUCHDB"] = "wmbshelper_t" self.testInit.setSchema(customModules=[ "WMCore.WMBS", "WMComponent.DBS3Buffer", "WMCore.BossAir", "WMCore.ResourceControl" ], useDefault=False) self.workDir = self.testInit.generateWorkDir() self.wmspec = self.createWMSpec() self.topLevelTask = getFirstTask(self.wmspec) self.inputDataset = self.topLevelTask.inputDataset() self.dataset = self.topLevelTask.getInputDatasetPath() self.dbs = DBSReader(self.inputDataset.dbsurl) self.rucioAcct = "wmcore_transferor" self.rucio = Rucio(self.rucioAcct) self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=threading.currentThread().logger, dbinterface=threading.currentThread().dbi) self.configFile = EmulatorSetup.setupWMAgentConfig() self.config = loadConfigurationFile(self.configFile) self.config.component_("JobSubmitter") self.config.JobSubmitter.submitDir = self.workDir self.config.JobSubmitter.submitScript = os.path.join( getTestBase(), 'WMComponent_t/JobSubmitter_t', 'submit.sh') return
def setUp(self): self.testInit = TestInit('CouchWorkQueueTest') self.testInit.setLogging() self.testInit.setupCouch('wq_backend_test_inbox', 'WorkQueue') self.testInit.setupCouch('wq_backend_test', 'WorkQueue') self.couch_db = self.testInit.couch.couchServer.connectDatabase('wq_backend_test') self.backend = WorkQueueBackend(db_url = self.testInit.couchUrl, db_name = 'wq_backend_test', inbox_name = 'wq_backend_test_inbox') self.processingSpec = rerecoWorkload('testProcessing', rerecoArgs)
def setUp(self): """ _setUp_ """ EmulatorHelper.setEmulators(phedex=True, dbs=True, siteDB=True, requestMgr=True) self.specGenerator = WMSpecGenerator("WMSpecs") #self.configFile = EmulatorSetup.setupWMAgentConfig() self.schema = [] self.couchApps = ["WorkQueue"] self.testInit = TestInitCouchApp('WorkQueueServiceTest') self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=self.schema, useDefault=False) self.testInit.setupCouch('workqueue_t', *self.couchApps) self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps) return
def setUp(self): """ _setUp_ """ super(WorkQueueTest, self).setUp() self.specGenerator = WMSpecGenerator("WMSpecs") # self.configFile = EmulatorSetup.setupWMAgentConfig() self.schema = [] self.couchApps = ["WorkQueue"] self.testInit = TestInitCouchApp('WorkQueueServiceTest') self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=self.schema, useDefault=False) self.testInit.setupCouch('workqueue_t', *self.couchApps) self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps) self.testInit.setupCouch('local_workqueue_t', *self.couchApps) self.testInit.setupCouch('local_workqueue_t_inbox', *self.couchApps) self.testInit.generateWorkDir() return
def setUp(self): """ _setUp_ Initialize the database and couch. """ super(StepChainTests, self).setUp() self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.testInit.setupCouch("stepchain_t", "ConfigCache") self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) couchServer = CouchServer(os.environ["COUCHURL"]) self.configDatabase = couchServer.connectDatabase("stepchain_t") self.testInit.generateWorkDir() self.workload = None return
class AnalysisCollectionService_t(unittest.TestCase): """ Unit tests for AnalysisCollectionService """ def setUp(self): """Set up test couch instance""" self.dbsURL = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader' self.dataset = '/RelValProdTTbar/JobRobot-MC_3XY_V24_JobRobot-v1/GEN-SIM-DIGI-RECO' """Set up couch test environment""" self.testInit = TestInitCouchApp(__file__) self.testInit.setupCouch("wmcore-acdc-acservice", "GroupUser", "ACDC") self.acService = AnalysisCollectionService( url=self.testInit.couchUrl, database=self.testInit.couchDbName) self.collection = self.acService.createCollection( 'ewv-testcoll', 'ewv2', 'Analysis') def tearDown(self): """Clean up couch instance""" self.testInit.tearDownCouch() def testCreateFilesetFromDBS(self): """Test creating an analysis fileset from DBS""" rangesMask = Mask() rangesMask.addRunWithLumiRanges(run=1, lumiList=[[1, 9], [12, 12], [31, 31], [38, 39], [49, 49], [51, 52], [64, 65], [82, 82], [92, 98]]) fs, fl = self.acService.createFilesetFromDBS(self.collection, filesetName='test_fs', dbsURL=self.dbsURL, dataset=self.dataset, mask=rangesMask) self.assertTrue(fl['_id']) self.assertEqual(len(fl['files']), 21) for file in fl['files']: self.assertTrue(fl['files'][file]['merged'])
def setUp(self): """ _setUp_ Setup a couch database for testing of produced JSON """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setupCouch("histogram_dump_t") self.histogramDB = Database(dbname="histogram_dump_t")
def setUp(self): """ setUP global values """ testname = self.id().split('.')[-1] logging.basicConfig( level=logging.DEBUG, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', datefmt='%m-%d %H:%M', filename='service_unittests.log', filemode='w') logger_name = 'Service%s' % testname.replace('test', '', 1) self.logger = logging.getLogger(logger_name) self.cache_path = tempfile.mkdtemp() self.testInit = TestInitCouchApp("RegistrationTest") self.couch_db = "regsvc" self.testInit.setupCouch(self.couch_db)
def setUp(self): """ Basic setUp """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testDir = self.testInit.generateWorkDir() return
def setUp(self): """ _setUp_ """ self.schema = [] self.couchApps = ["WMStats"] self.testInit = TestInitCouchApp('WorkQueueServiceTest') self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=self.schema, useDefault=False) dbName = 'wmstats_t' self.testInit.setupCouch(dbName, "WMStats") reqDBName = "reqmgrdb_t" self.testInit.setupCouch(reqDBName, "ReqMgr") wmstatsURL = "%s/%s" % (self.testInit.couchUrl, dbName) reqDBURL = "%s/%s" % (self.testInit.couchUrl, reqDBName) self.reqDBWriter = RequestDBWriter(reqDBURL) self.wmstatsReader = WMStatsReader(wmstatsURL, reqdbURL=reqDBURL) self.wmstatsReader.defaultStale = {} self.wmstatsReader.reqDB.defaultStale = {} return
def setUp(self): """ _setUp_ Initialize the database and couch. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("stepchain_t", "ConfigCache") self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) couchServer = CouchServer(os.environ["COUCHURL"]) self.configDatabase = couchServer.connectDatabase("stepchain_t") self.testInit.generateWorkDir() self.workload = None self.jsonTemplate = getTestFile( 'data/ReqMgr/requests/DMWM/StepChain_MC.json') return
class ReproducibleSeedingTests(unittest.TestCase): def setUp(self): self.testInit = TestInitCouchApp("ReproducibleSeedingTest") self.testInit.setupCouch("seeding_config_cache", "GroupUser", "ConfigCache") self.database = Database(self.testInit.couchDbName, self.testInit.couchUrl) self.documentId = None def tearDown(self): self.testInit.tearDownCouch() return def testA(self): """instantiate""" document = Document() document[u'pset_tweak_details'] = {} document[u'pset_tweak_details'][u'process'] = {} document[u'pset_tweak_details'][u'process'][ u'RandomNumberGeneratorService'] = {} document[u'pset_tweak_details'][u'process'][ u'RandomNumberGeneratorService'][u'seed1'] = {} document[u'pset_tweak_details'][u'process'][ u'RandomNumberGeneratorService'][u'seed2'] = {} document[u'pset_tweak_details'][u'process'][ u'RandomNumberGeneratorService'][u'seed3'] = {} document = self.database.commitOne(document)[0] seeder = ReproducibleSeeding(CouchUrl=self.testInit.couchUrl, CouchDBName=self.testInit.couchDbName, ConfigCacheDoc=document[u'id']) job = Job("testjob") seeder(job) baggage = job.getBaggage() seed1 = getattr(baggage.process.RandomNumberGeneratorService, "seed1", None) self.assertTrue(seed1 != None)
def setUp(self): """ setup for test. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.reqmonDBName = "wmstat_t" self.localDBName = "wmstat_t_local" self.testInit.setupCouch(self.reqmonDBName, "WMStatsAgent") self.testInit.setupCouch(self.localDBName, "WMStatsAgent") self.testDir = self.testInit.generateWorkDir() return
def setUp(self): """ Basic setUp """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("config_test", "GroupUser", "ConfigCache") self.testDir = self.testInit.generateWorkDir() return
def setUp(self, initRoot=True): # default set self.schemaModules = [] self.initialize() if self.schemaModules: import warnings warnings.warn("use RESTAndCouchUnitTest instead", DeprecationWarning) from WMQuality.TestInitCouchApp import TestInitCouchApp self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() # logLevel = logging.SQLDEBUG self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.testInit.setSchema(customModules=self.schemaModules, useDefault=False) # Now pull the dbURL from the factory # I prefer this method because the factory has better error handling # Also because then you know everything is the same myThread = threading.currentThread() self.config.setDBUrl(myThread.dbFactory.dburl) logging.info("This is our config: %s" % self.config) self.initRoot = initRoot if initRoot: self.rt = Root(self.config, testName=self._testMethodName) try: self.rt.start(blocking=False) except RuntimeError as e: # there appears to be worker threads from a previous test # hanging out. Try to slay them so that we can keep going print("Failed to load cherrypy with exception: %s\n" % e) print("The threads are: \n%s\n" % threading.enumerate()) print("The previous test was %s\n" % self.rt.getLastTest()) print(traceback.format_exc()) self.rt.stop() raise e return
def setUp(self): """ _setUp_ Initialize the database and couch. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("taskchain_t", "ConfigCache") self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) couchServer = CouchServer(os.environ["COUCHURL"]) self.configDatabase = couchServer.connectDatabase("taskchain_t") self.testInit.generateWorkDir() self.workload = None self.differentNCores = getTestFile( 'data/ReqMgr/requests/Integration/TaskChain_RelVal_Multicore.json') return
def setUp(self): """ setup for test. """ myThread = threading.currentThread() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.testInit.setSchema( customModules=["WMCore.WMBS", "WMComponent.DBS3Buffer"], useDefault=False) self.databaseName = "taskarchiver_t_0" self.testInit.setupCouch("%s/workloadsummary" % self.databaseName, "WorkloadSummary") self.testInit.setupCouch("%s/jobs" % self.databaseName, "JobDump") self.testInit.setupCouch("%s/fwjrs" % self.databaseName, "FWJRDump") self.testInit.setupCouch("wmagent_summary_t", "WMStats") self.testInit.setupCouch("wmagent_summary_central_t", "WMStats") self.testInit.setupCouch("stat_summary_t", "SummaryStats") reqmgrdb = "reqmgrdb_t" self.testInit.setupCouch(reqmgrdb, "ReqMgr") reqDBURL = "%s/%s" % (self.testInit.couchUrl, reqmgrdb) self.requestWriter = RequestDBWriter(reqDBURL) self.requestWriter.defaultStale = {} self.daofactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) self.getJobs = self.daofactory(classname="Jobs.GetAllJobs") self.inject = self.daofactory( classname="Workflow.MarkInjectedWorkflows") self.testDir = self.testInit.generateWorkDir() os.makedirs(os.path.join(self.testDir, 'specDir')) self.nJobs = 10 self.campaignName = 'aCampaign' self.uploadPublishInfo = False self.uploadPublishDir = None return
def setUp(self): """ setup for test. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.reqmonDBName = "wmstat_t" self.localDBName = "wmstat_t_local" self.testInit.setupCouch(self.reqmonDBName, "WMStats") self.testInit.setupCouch(self.localDBName, "WMStats") self.testDir = self.testInit.generateWorkDir() EmulatorHelper.setEmulators(localCouch=True, reqMon=False, wmagentDB=True) return
def setUp(self): """ _setUp_ Initialize the database and couch. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) self.testDir = self.testInit.generateWorkDir() myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.listTasksByWorkflow = self.daoFactory( classname="Workflow.LoadFromName") self.listFilesets = self.daoFactory(classname="Fileset.List") self.listSubsMapping = self.daoFactory( classname="Subscriptions.ListSubsAndFilesetsFromWorkflow") return
def setUp(self): """ _setUp_ Figure out the location of the XML report produced by CMSSW. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.testInit.setupCouch("report_t/fwjrs", "FWJRDump") testData = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t") self.xmlPath = os.path.join(testData, "CMSSWProcessingReport.xml") self.badxmlPath = os.path.join(testData, "CMSSWFailReport2.xml") self.skippedFilesxmlPath = os.path.join(testData, "CMSSWSkippedNonExistentFile.xml") self.skippedAllFilesxmlPath = os.path.join(testData, "CMSSWSkippedAll.xml") self.fallbackXmlPath = os.path.join(testData, "CMSSWInputFallback.xml") self.twoFileFallbackXmlPath = os.path.join(testData, "CMSSWTwoFileRemote.xml") self.pileupXmlPath = os.path.join(testData, "CMSSWPileup.xml") self.withEventsXmlPath = os.path.join(testData, "CMSSWWithEventCounts.xml") self.noLocationReport = os.path.join(testData, "Report.0.pkl") self.testDir = self.testInit.generateWorkDir() return