class ConfigurationTest(unittest.TestCase):
"""
test case for Configuration object
"""
def setUp(self):
"""set up"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
self.normalSave = "%s/WMCore_Agent_Configuration_t_normal.py" % self.testDir
self.docSave = "%s/WMCore_Agent_Configuration_t_documented.py" % self.testDir
self.commentSave = "%s/WMCore_Agent_Configuration_t_commented.py" % self.testDir
def tearDown(self):
"""clean up"""
self.testInit.delWorkDir()
def testA(self):
"""ctor"""
try:
config = Configuration()
except Exception, ex:
msg = "Failed to instantiate Configuration\n"
msg += str(ex)
self.fail(msg)
class AnalysisCollectionService_t(unittest.TestCase):
"""
Unit tests for AnalysisCollectionService
"""
def setUp(self):
"""Set up test couch instance"""
self.dbsURL = 'http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet'
self.dataset = '/RelValProdTTbar/JobRobot-MC_3XY_V24_JobRobot-v1/GEN-SIM-DIGI-RECO'
"""Set up couch test environment"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setupCouch("wmcore-acdc-acservice", "GroupUser", "ACDC")
self.acService = AnalysisCollectionService(url=self.testInit.couchUrl, database = self.testInit.couchDbName)
self.collection = self.acService.createCollection('ewv-testcoll', 'ewv2', 'Analysis')
def tearDown(self):
"""Clean up couch instance"""
self.testInit.tearDownCouch()
def testCreateFilesetFromDBS(self):
"""Test creating an analysis fileset from DBS"""
rangesMask = Mask()
rangesMask.addRunWithLumiRanges(run=1, lumiList=[[1, 9], [12, 12], [31, 31], [38, 39], [49, 49], [51, 52], [64, 65], [82, 82], [92, 98]])
fs, fl = self.acService.createFilesetFromDBS(self.collection, filesetName='test_fs',
dbsURL=self.dbsURL, dataset=self.dataset, mask=rangesMask)
self.assertTrue(fl['_id'])
self.assertEqual(len(fl['files']), 21)
class ReproducibleSeedingTests(unittest.TestCase):
def setUp(self):
self.testInit = TestInitCouchApp("ReproducibleSeedingTest")
self.testInit.setupCouch("seeding_config_cache", "GroupUser", "ConfigCache")
self.database = Database(self.testInit.couchDbName, self.testInit.couchUrl)
self.documentId = None
def tearDown(self):
self.testInit.tearDownCouch()
return
def testA(self):
"""instantiate"""
document = Document()
document[u'pset_tweak_details'] = {}
document[u'pset_tweak_details'][u'process'] = {}
document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'] = {}
document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed1'] = {}
document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed2'] = {}
document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed3'] = {}
document = self.database.commitOne(document)[0]
seeder = ReproducibleSeeding(CouchUrl = self.testInit.couchUrl,
CouchDBName = self.testInit.couchDbName,
ConfigCacheDoc = document[u'id'])
job = Job("testjob")
seeder(job)
baggage = job.getBaggage()
seed1 = getattr(baggage.process.RandomNumberGeneratorService, "seed1", None)
self.failUnless(seed1 != None)
class RequestDBTest(unittest.TestCase):
"""
"""
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["ReqMgr"]
self.testInit = TestInitCouchApp("RequestDBServiceTest")
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=self.schema, useDefault=False)
dbName = "requsetdb_t"
self.testInit.setupCouch(dbName, *self.couchApps)
self.requestWriter = RequestDBWriter(self.testInit.couchUrl, dbName)
self.requestReader = RequestDBReader(self.testInit.couchUrl, dbName)
self.requestWriter.defaultStale = {}
self.requestReader.defaultStale = {}
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
self.testInit.tearDownCouch()
def testRequestDBWriter(self):
# test getWork
schema = generate_reqmgr_schema()
result = self.requestWriter.insertGenericRequest(schema[0])
self.assertEquals(len(result), 1, "insert fail")
self.assertEquals(
self.requestWriter.updateRequestStatus(schema[0]["RequestName"], "failed"), "OK", "update fail"
)
self.assertEquals(
self.requestWriter.updateRequestStatus("not_exist_schema", "assigned"), "Error: document not found"
)
result = self.requestWriter.updateRequestProperty(schema[0]["RequestName"], {"Teams": ["teamA"]})
self.assertEquals(
self.requestWriter.updateRequestProperty(schema[0]["RequestName"], {"Teams": ["teamA"]}),
"OK",
"update fail",
)
self.assertEquals(
self.requestWriter.updateRequestProperty("not_exist_schema", {"Teams": "teamA"}),
"Error: document not found",
)
result = self.requestWriter.getRequestByNames([schema[0]["RequestName"]])
self.assertEquals(len(result), 1, "should be 1")
result = self.requestWriter.getRequestByStatus(["failed"], False, 1)
self.assertEquals(len(result), 1, "should be 1")
class T0RequestDBTest(unittest.TestCase):
"""
"""
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["T0Request"]
self.testInit = TestInitCouchApp('RequestDBServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = self.schema,
useDefault = False)
dbName = 't0_requsetdb_t'
self.testInit.setupCouch(dbName, *self.couchApps)
reqDBURL = "%s/%s" % (self.testInit.couchUrl, dbName)
self.requestWriter = RequestDBWriter(reqDBURL, self.couchApps[0])
self.requestReader = RequestDBReader(reqDBURL, self.couchApps[0])
self.requestWriter.defaultStale = {}
self.requestReader.defaultStale = {}
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
self.testInit.tearDownCouch()
def testRequestDBWriter(self):
# test getWork
schema = generate_reqmgr_schema()
result = self.requestWriter.insertGenericRequest(schema[0])
self.assertEqual(len(result), 1, 'insert fail');
result = self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "assigned")
self.assertEqual(result, 'not allowed state assigned', 'update fail')
self.assertEqual(self.requestWriter.updateRequestStatus("not_exist_schema", "new"),
'Error: document not found')
allowedStates = ["Closed", "Merge", "AlcaSkim", "Harvesting",
"Processing Done", "completed"]
for state in allowedStates:
self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], state),
'OK')
self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "Processing Done"),
'not allowed transition completed to Processing Done')
self.assertEqual(self.requestWriter.updateRequestStatus(schema[0]['RequestName'], "normal-archived"),
'OK')
result = self.requestWriter.getRequestByStatus(["normal-archived"], False, 1)
self.assertEqual(len(result), 1, "should be 1 but %s" % result)
class Interface_t(unittest.TestCase):
def setUp(self):
self.testInit = TestInitCouchApp(__file__)
self.testInit.setupCouch("wmcore-groupuser-interface", "GroupUser")
#create a test owner for the collections in this test
self.owner1 = makeUser("DMWM", "evansde77", self.testInit.couchUrl, self.testInit.couchDbName)
self.owner1.connect()
self.owner1.create()
self.owner2 = makeUser("DMWM", "drsm79", self.testInit.couchUrl, self.testInit.couchDbName)
self.owner2.connect()
self.owner2.create()
#self.url = os.getenv("COUCHURL", "http://127.0.0.1:5984")
#self.database = "awesome_acdc"
def tearDown(self):
self.testInit.tearDownCouch()
def testA(self):
""" make some documents and own them"""
guInt = Interface(self.testInit.couchUrl, self.testInit.couchDbName)
#create a couple of docs
couch = Database(self.testInit.couchDbName, self.testInit.couchUrl)
for x in range(10):
doc = Document("document%s" % x, {"Test Data": [1,2,3,4] })
couch.queue(doc)
couch.commit()
self.assertEqual(len(guInt.documentsOwned(self.owner1.group.name, self.owner1.name)), 0)
self.assertEqual(len(guInt.documentsOwned(self.owner2.group.name, self.owner2.name)), 0)
guInt.callUpdate("ownthis","document1", group = self.owner1.group.name, user = self.owner1.name)
self.failUnless("document1" in guInt.documentsOwned(self.owner1.group.name, self.owner1.name))
self.assertEqual(len(guInt.documentsOwned(self.owner1.group.name, self.owner1.name)), 1)
self.assertEqual(len(guInt.documentsOwned(self.owner2.group.name, self.owner2.name)), 0)
guInt.callUpdate("ownthis","document2", group = self.owner2.group.name, user = self.owner2.name)
self.failUnless("document2" in guInt.documentsOwned(self.owner2.group.name, self.owner2.name))
self.assertEqual(len(guInt.documentsOwned(self.owner1.group.name, self.owner1.name)), 1)
self.assertEqual(len(guInt.documentsOwned(self.owner2.group.name, self.owner2.name)), 1)
guInt.callUpdate("newgroup", "group-DataOps", group = "DataOps")
self.failUnless(couch.documentExists("group-DataOps") )
guInt.callUpdate("newuser", "user-damason", group = "DataOps", user = "******")
self.failUnless(couch.documentExists("user-damason") )
def couchapp(self, couchappName):
"""
Drop database in CouchDB.
Reload (push) the couchapp couchappName (likely to be "OpsClipboard").
"""
print "Pushing couchapp '%s' ..." % couchappName
# here probably need to specify the couchUrl, otherwise default env COUCHURL
# will be taken (as described above)
testInit = TestInitCouchApp(__file__, dropExistingDb=True)
# should now drop the CouchDB db, create a new one and push the couchapp
testInit.setupCouch(self.dbName, couchappName)
class WMStatsTest(unittest.TestCase):
"""
"""
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["WMStats"]
self.testInit = TestInitCouchApp('WorkQueueServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = self.schema,
useDefault = False)
self.testInit.setupCouch('wmstats_t', *self.couchApps)
self.wmstatsWriter = WMStatsWriter(self.testInit.couchUrl, 'wmstats_t');
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
self.testInit.tearDownCouch()
def testWMStatsWriter(self):
# test getWork
schema = generate_reqmgr_schema()
self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK', 'insert fail');
self.assertEquals(self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail')
self.assertEquals(self.wmstatsWriter.updateRequestStatus("not_exist_schema", "assigned"),
'ERROR: request not found - not_exist_schema')
self.assertEquals(self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'), 'OK', 'update fail')
self.assertEquals(self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'),
'ERROR: request not found - not_exist_schema')
totalStats = {'total_jobs': 100, 'input_events': 1000, 'input_lumis': 1234, 'input_num_files': 5}
self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'INSERTED', 'update fail')
self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'UPDATED', 'update fail')
self.assertEquals(self.wmstatsWriter.insertTotalStats("not_exist_schema", totalStats),
'ERROR: request not found - not_exist_schema')
spec1 = newWorkload(schema[0]['RequestName'])
production = spec1.newTask("Production")
production.setTaskType("Merge")
production.setSiteWhitelist(['TEST_SITE'])
self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK', 'update fail')
spec2 = newWorkload("not_exist_schema")
production = spec2.newTask("Production")
production.setTaskType("Merge")
self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2),
'ERROR: request not found - not_exist_schema')
class FWJRDBAPITest(unittest.TestCase):
"""
"""
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["FWJRDump"]
self.testInit = TestInitCouchApp("FWJRAPITest")
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=self.schema, useDefault=False)
dbName = "fwjrdump_t"
self.testInit.setupCouch(dbName, *self.couchApps)
fwjrDBURL = "%s/%s" % (self.testInit.couchUrl, dbName)
self.fwjrAPI = FWJRDBAPI(fwjrDBURL)
self.fwjrAPI.defaultStale = {}
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
self.testInit.tearDownCouch()
def testFWJRDBAPI(self):
jobID = 1
retryCount = 0
fwjrDocument = {
"_id": "%s-%s" % (jobID, retryCount),
"jobid": jobID,
"retrycount": retryCount,
"archivestatus": "ready",
"fwjr": SAMPLE_FWJR,
"jobtype": "Merge",
"type": "fwjr",
}
workflow = SAMPLE_FWJR["task"].split("/")[1]
self.assertEqual(self.fwjrAPI.couchDB.commitOne(fwjrDocument)[0]["id"], fwjrDocument["_id"])
self.assertEqual(self.fwjrAPI.getFWJRByArchiveStatus("ready")["rows"][0]["id"], fwjrDocument["_id"])
self.assertEqual(self.fwjrAPI.isAllFWJRArchived(workflow), False)
self.fwjrAPI.updateArchiveUploadedStatus(fwjrDocument["_id"])
self.assertEqual(self.fwjrAPI.getFWJRByArchiveStatus("uploaded")["rows"][0]["id"], fwjrDocument["_id"])
self.assertEqual(self.fwjrAPI.isAllFWJRArchived(workflow), True)
self.assertEqual(self.fwjrAPI.getFWJRWithSkippedFiles()["rows"][0]["value"]["skippedFiles"], 2)
class RESTBaseUnitTest(unittest.TestCase):
def setUp(self):
# default set
self.schemaModules = []
self.initialize()
if self.schemaModules:
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging() # logLevel = logging.SQLDEBUG
self.testInit.setDatabaseConnection(self.config.getDBUrl())
self.testInit.setSchema(customModules = self.schemaModules,
useDefault = False)
self.rt = Root(self.config)
self.rt.start(blocking=False)
def tearDown(self):
self.rt.stop()
if self.schemaModules:
self.testInit.clearDatabase()
self.config = None
def initialize(self):
"""
i.e.
self.config = DefaultConfig('WMCore.WebTools.RESTModel')
self.config.setDBUrl("sqlite://")
self.schemaModules = ["WMCore.ThreadPool", WMCore.WMBS"]
"""
message = "initialize method has to be implemented, self.restModel, self.schemaModules needs to be set"
raise NotImplementedError, message
def setUp(self):
"""
_setUp_
"""
self.transitions = Transitions()
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setupCouch("changestate_t/jobs", "JobDump")
self.testInit.setupCouch("changestate_t/fwjrs", "FWJRDump")
self.testInit.setSchema(customModules = ["WMCore.WMBS"],
useDefault = False)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.couchServer = CouchServer(dburl = os.getenv("COUCHURL"))
self.config = Configuration()
self.config.component_("JobStateMachine")
self.config.JobStateMachine.couchurl = os.getenv("COUCHURL")
return
def setUp(self):
"""set up"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
self.normalSave = "%s/WMCore_Agent_Configuration_t_normal.py" % self.testDir
self.docSave = "%s/WMCore_Agent_Configuration_t_documented.py" % self.testDir
self.commentSave = "%s/WMCore_Agent_Configuration_t_commented.py" % self.testDir
def setUp(self):
"""
_setUp_
Initialize the database and couch.
"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setupCouch("promptreco_t", "ConfigCache")
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
couchServer = CouchServer(os.environ["COUCHURL"])
self.configDatabase = couchServer.connectDatabase("promptreco_t")
self.testDir = self.testInit.generateWorkDir()
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.listTasksByWorkflow = self.daoFactory(classname="Workflow.LoadFromName")
self.listFilesets = self.daoFactory(classname="Fileset.List")
self.listSubsMapping = self.daoFactory(classname="Subscriptions.ListSubsAndFilesetsFromWorkflow")
return
def testProcessorWithReceiverAndCouchSink(self):
# set up couch first
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
dbName = "couch_sink"
self.testInit.setupCouch(dbName)
# add corresponding part of the configuration for CouchSink(s)
config = self.config.AlertProcessor
config.critical.sinks.section_("couch")
config.critical.sinks.couch.url = self.testInit.couchUrl
config.critical.sinks.couch.database = self.testInit.couchDbName
# just send the Alert into couch
processor = Processor(config)
rec = Receiver(self.addr, processor, self.ctrl)
rec.startReceiver() # non blocking call
# run worker(), this time directly without Process as above,
# worker will send 10 Alerts to Receiver
worker(self.addr, self.ctrl, 10)
# wait until the Receiver is shut by worker
while rec.isReady():
time.sleep(0.3)
print "%s waiting for Receiver to shut ..." % inspect.stack()[0][3]
def setUp(self):
"""
_setUp_
"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setupCouch("wmbshelper_t/jobs", "JobDump")
self.testInit.setupCouch("wmbshelper_t/fwjrs", "FWJRDump")
os.environ["COUCHDB"] = "wmbshelper_t"
self.testInit.setSchema(customModules = ["WMCore.WMBS",
"WMComponent.DBSBuffer.Database",
"WMCore.BossAir",
"WMCore.ResourceControl"],
useDefault = False)
self.workDir = self.testInit.generateWorkDir()
self.wmspec = self.createWMSpec()
self.topLevelTask = getFirstTask(self.wmspec)
self.inputDataset = self.topLevelTask.inputDataset()
self.dataset = self.topLevelTask.getInputDatasetPath()
self.dbs = MockDBSReader(self.inputDataset.dbsurl)
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = threading.currentThread().logger,
dbinterface = threading.currentThread().dbi)
return
def setUp(self):
"""
setup for test.
"""
super(JobArchiverTest, self).setUp()
myThread = threading.currentThread()
super(JobArchiverTest, self).setUp()
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
# self.tearDown()
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
self.testInit.setupCouch("jobarchiver_t_0/jobs", "JobDump")
self.testInit.setupCouch("jobarchiver_t_0/fwjrs", "FWJRDump")
self.daofactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.getJobs = self.daofactory(classname="Jobs.GetAllJobs")
self.testDir = self.testInit.generateWorkDir(deleteOnDestruction=False)
self.nJobs = 10
self.configFile = EmulatorSetup.setupWMAgentConfig()
return
def setUp(self):
myThread = threading.currentThread()
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMCore.WMBS"],
useDefault = False)
self.databaseName = "couchapp_t_0"
self.testInit.setupCouch(self.databaseName, "WorkloadSummary")
self.testInit.setupCouch("%s/jobs" % self.databaseName, "JobDump")
self.testInit.setupCouch("%s/fwjrs" % self.databaseName, "FWJRDump")
# Setup config for couch connections
config = self.testInit.getConfiguration()
config.section_("JobStateMachine")
config.JobStateMachine.couchDBName = self.databaseName
# Create couch server and connect to databases
self.couchdb = CouchServer(config.JobStateMachine.couchurl)
self.jobsdatabase = self.couchdb.connectDatabase("%s/jobs" % config.JobStateMachine.couchDBName)
self.fwjrdatabase = self.couchdb.connectDatabase("%s/fwjrs" % config.JobStateMachine.couchDBName)
# Create changeState
self.changeState = ChangeState(config)
self.config = config
# Create testDir
self.testDir = self.testInit.generateWorkDir()
return
def setUp(self):
"""
_setUp_
Standard setup: Now with 100% more couch
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl", "WMCore.Agent.Database"])
self.testInit.setupCouch("jobsubmitter_t/jobs", "JobDump")
self.testInit.setupCouch("jobsubmitter_t/fwjrs", "FWJRDump")
self.testInit.setupCouch("wmagent_summary_t", "WMStats")
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.baDaoFactory = DAOFactory(package = "WMCore.BossAir",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.testDir = self.testInit.generateWorkDir()
# Set heartbeat
self.componentName = 'JobSubmitter'
self.heartbeatAPI = HeartbeatAPI(self.componentName)
self.heartbeatAPI.registerComponent()
return
def testProcessorWithReceiverAndCouchSink(self):
# set up couch first
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
dbName = "couch_sink"
self.testInit.setupCouch(dbName)
# add corresponding part of the configuration for CouchSink(s)
config = self.config.AlertProcessor
config.critical.sinks.section_("couch")
config.critical.sinks.couch.url = self.testInit.couchUrl
config.critical.sinks.couch.database = self.testInit.couchDbName
# just send the Alert into couch
processor = Processor(config)
# Receiver is waited for shutdown / shutdown explicitly in tearDown()
self.receiver = Receiver(self.addr, processor, self.ctrl)
self.receiver.startReceiver() # non blocking call
# run worker(), this time directly without Process as above,
# worker will send 10 Alerts to Receiver
worker(self.addr, self.ctrl, 10)
# wait until Receiver is shut down (by a message from worker()
# also need to wait, otherwise tearDown kicks off and scrapes the
# couch so half of the alerts will be undelivered
while self.receiver.isReady():
time.sleep(ReceiverLogic.TIMEOUT_AFTER_SHUTDOWN * 1.5)
logging.info("%s: Waiting for Receiver shutdown ..." % inspect.stack()[0][3])
def setUp(self):
"""
setup for test.
"""
myThread = threading.currentThread()
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
#self.tearDown()
self.testInit.setSchema(customModules = ["WMCore.WMBS"],
useDefault = False)
self.testInit.setupCouch("jobarchiver_t_0/jobs", "JobDump")
self.testInit.setupCouch("jobarchiver_t_0/fwjrs", "FWJRDump")
self.daofactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.getJobs = self.daofactory(classname = "Jobs.GetAllJobs")
self.testDir = self.testInit.generateWorkDir(deleteOnDestruction = False)
self.nJobs = 10
self.alertsReceiver = None
EmulatorHelper.setEmulators(phedex = True, dbs = True,
siteDB = True, requestMgr = False)
return
def setUp(self):
"""
_setUp_
Setup the database and logging connection. Try to create all of the
WMBS tables. Also, create some dummy locations.
"""
myThread = threading.currentThread()
self.sites = ['T2_US_Florida', 'T2_US_UCSD', 'T2_TW_Taiwan', 'T1_CH_CERN']
self.testInit = TestInit(__file__)
self.testInit.setLogging(logLevel = logging.DEBUG)
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ['WMCore.WMBS',
'WMCore.ResourceControl',
'WMCore.Agent.Database'], useDefault = False)
self.testInit.setupCouch("dashboardreporter_t/jobs", "JobDump")
self.testInit.setupCouch("dashboardreporter_t/fwjrs", "FWJRDump")
resourceControl = ResourceControl()
for site in self.sites:
resourceControl.insertSite(siteName = site, seName = site, ceName = site)
resourceControl.insertThreshold(siteName = site, taskType = 'Processing', \
maxSlots = 10000)
self.testDir = self.testInit.generateWorkDir()
self.alertsReceiver = None
return
def setUp(self):
"""
_setUp_
Setup the database and logging connection.
"""
logging.basicConfig()
self.logger = logging.getLogger('LogDBTest')
# self.logger.setLevel(logging.DEBUG)
self.schema = []
self.couchApps = ["LogDB"]
self.testInit = TestInitCouchApp('LogDBTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = self.schema,
useDefault = False)
dbName = 'logdb_t'
self.testInit.setupCouch(dbName, *self.couchApps)
url = "%s/%s" % (self.testInit.couchUrl, dbName)
identifier = 'agentname'
self.agent_inst = LogDB(url, identifier, logger=self.logger, thread_name="MainThread")
self.agent_inst2 = LogDB(url, identifier, logger=self.logger, thread_name="Test")
identifier = '/DC=org/DC=doegrids/OU=People/CN=First Last 123'
self.user_inst = LogDB(url, identifier, logger=self.logger, thread_name="MainThread")
self.report = LogDBReport(self.agent_inst)
def setUp(self):
"""
_setUp_
Set everything up.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMCore.WMBS", "WMCore.BossAir",
"WMCore.ResourceControl"],
useDefault = False)
self.testInit.setupCouch("jobsubmittercaching_t/jobs", "JobDump")
self.testInit.setupCouch("jobsubmittercaching_t/fwjrs", "FWJRDump")
resourceControl = ResourceControl()
for siteName in ["T1_US_FNAL", "T1_UK_RAL"]:
resourceControl.insertSite(siteName = siteName, pnn = "se.%s" % (siteName),
ceName = siteName, plugin = "CondorPlugin", cmsName = siteName)
resourceControl.insertThreshold(siteName = siteName, taskType = "Processing",
maxSlots = 10000, pendingSlots = 10000)
self.testDir = self.testInit.generateWorkDir()
self.configFile = EmulatorSetup.setupWMAgentConfig()
return
def setUp(self):
"""
Setup for unit tests
"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
testname = self.id().split('.')[-1]
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename='service_unittests.log',
filemode='w')
logger_name = 'Service%s' % testname.replace('test', '', 1)
self.logger = logging.getLogger(logger_name)
#self.cache_path = tempfile.mkdtemp()
test_dict = {'logger': self.logger,
#'cachepath' : self.cache_path,
#'req_cache_path': '%s/requests' % self.cache_path,
'endpoint':'http://cmssw.cvs.cern.ch/cgi-bin/cmssw.cgi'}
self.myService = Service(test_dict)
test_dict['endpoint'] = 'http://cmssw-test.cvs.cern.ch/cgi-bin/cmssw.cgi'
self.myService2 = Service(test_dict)
self.testUrl = 'http://cern.ch'
self.port = 8888
cherrypy.config.update({'server.socket_port': self.port})
def setUp(self):
"""
setup for test.
"""
myThread = threading.currentThread()
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase = True)
self.testInit.setSchema(customModules = ["WMCore.WMBS"],
useDefault = False)
self.testInit.setupCouch("errorhandler_t", "GroupUser", "ACDC")
self.testInit.setupCouch("errorhandler_t_jd/jobs", "JobDump")
self.testInit.setupCouch("errorhandler_t_jd/fwjrs", "FWJRDump")
self.daofactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.getJobs = self.daofactory(classname = "Jobs.GetAllJobs")
self.setJobTime = self.daofactory(classname = "Jobs.SetStateTime")
locationAction = self.daofactory(classname = "Locations.New")
locationAction.execute(siteName = "malpaquet", seName = "malpaquet")
self.testDir = self.testInit.generateWorkDir()
self.nJobs = 10
self.dataCS = DataCollectionService(url = self.testInit.couchUrl,
database = "errorhandler_t")
return
def setUp(self):
"""
_setUp_
"""
self.transitions = Transitions()
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setupCouch("changestate_t/jobs", "JobDump")
self.testInit.setupCouch("changestate_t/fwjrs", "FWJRDump")
self.testInit.setupCouch("job_summary", "WMStats")
self.testInit.setSchema(customModules = ["WMCore.WMBS"],
useDefault = False)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
couchurl = os.getenv("COUCHURL")
self.couchServer = CouchServer(dburl = couchurl)
self.config = self.testInit.getConfiguration()
self.taskName = "/TestWorkflow/Task"
return
def setUp(self):
"""
_setUp_
Setup couchdb and the test environment
"""
super(ResubmitBlockTest, self).setUp()
self.group = 'unknown'
self.user = '******'
# Set external test helpers
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setupCouch("resubmitblock_t", "ACDC", "GroupUser")
# Define test environment
self.couchUrl = os.environ["COUCHURL"]
self.acdcDBName = 'resubmitblock_t'
self.validLocations = ['T2_US_Nebraska', 'T1_US_FNAL_Disk', 'T1_UK_RAL_Disk']
self.siteWhitelist = ['T2_XX_SiteA']
siteDB = SiteDB()
#Convert phedex node name to a valid processing site name
self.PSNs = siteDB.PNNstoPSNs(self.validLocations)
self.workflowName = 'dballest_ReReco_workflow'
couchServer = CouchServer(dburl=self.couchUrl)
self.acdcDB = couchServer.connectDatabase(self.acdcDBName, create=False)
user = makeUser(self.group, '*****@*****.**', self.couchUrl, self.acdcDBName)
user.create()
return
def setUp(self):
"""
_setUp_
Figure out the location of the XML report produced by CMSSW.
"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase=True)
self.testInit.setupCouch("report_t/fwjrs", "FWJRDump")
self.xmlPath = os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml")
self.badxmlPath = os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWFailReport2.xml")
self.skippedFilesxmlPath = os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWSkippedNonExistentFile.xml")
self.skippedAllFilesxmlPath = os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWSkippedAll.xml")
self.fallbackXmlPath = os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWInputFallback.xml")
self.twoFileFallbackXmlPath = os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWTwoFileRemote.xml")
self.pileupXmlPath = os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWPileup.xml")
self.testDir = self.testInit.generateWorkDir()
return
def setUp(self):
"""
_setUp_
Create two subscriptions: One that contains a single file and one that
contains multiple files.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
# self.testInit.clearDatabase(modules = ['WMCore.WMBS'])
self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False)
self.testInit.setupCouch("lumi_t", "GroupUser", "ACDC")
myThread = threading.currentThread()
daofactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi)
locationAction = daofactory(classname = "Locations.New")
locationAction.execute(siteName='s1', pnn="T1_US_FNAL_Disk")
locationAction.execute(siteName='s2', pnn="T2_CH_CERN")
self.testWorkflow = Workflow(spec="spec.xml", owner="mnorman", name="wf001", task="Test")
self.testWorkflow.create()
self.performanceParams = {'timePerEvent': 12,
'memoryRequirement': 2300,
'sizePerEvent': 400}
return
def setUp(self):
"""
setup for test.
"""
myThread = threading.currentThread()
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase=True)
self.testInit.setSchema(customModules=["WMCore.WMBS", "WMComponent.DBS3Buffer"], useDefault=False)
self.databaseName = "taskarchiver_t_0"
self.testInit.setupCouch("%s/workloadsummary" % self.databaseName, "WorkloadSummary")
self.testInit.setupCouch("%s/jobs" % self.databaseName, "JobDump")
self.testInit.setupCouch("%s/fwjrs" % self.databaseName, "FWJRDump")
self.testInit.setupCouch("wmagent_summary_t", "WMStats")
self.testInit.setupCouch("wmagent_summary_central_t", "WMStats")
self.daofactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi)
self.getJobs = self.daofactory(classname="Jobs.GetAllJobs")
self.inject = self.daofactory(classname="Workflow.MarkInjectedWorkflows")
self.testDir = self.testInit.generateWorkDir()
os.makedirs(os.path.join(self.testDir, "specDir"))
self.nJobs = 10
self.campaignName = "aCampaign"
self.alertsReceiver = None
self.uploadPublishInfo = False
self.uploadPublishDir = None
return
def setUp(self):
"""
Initialize the database and couch.
"""
super(PileupFetcherTest, self).setUp()
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setupCouch("pileupfetcher_t", "ConfigCache")
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
couchServer = CouchServer(os.environ["COUCHURL"])
self.configDatabase = couchServer.connectDatabase("pileupfetcher_t")
self.testDir = self.testInit.generateWorkDir()
self.rucioAcct = "wmcore_transferor"
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["FWJRDump"]
self.testInit = TestInitCouchApp('FWJRAPITest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = self.schema,
useDefault = False)
dbName = 'fwjrdump_t'
self.testInit.setupCouch(dbName, *self.couchApps)
fwjrDBURL = "%s/%s" % (self.testInit.couchUrl, dbName)
self.fwjrAPI= FWJRDBAPI(fwjrDBURL)
self.fwjrAPI.defaultStale = {}
return
def setUp(self):
"""
_setUp_
Initialize the database.
"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setupCouch("analysis_t", "ConfigCache")
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
couchServer = CouchServer(os.environ["COUCHURL"])
self.configDatabase = couchServer.connectDatabase("analysis_t")
self.testDir = self.testInit.generateWorkDir()
return
def setUp(self):
"""
_setUp_
Setup the database and logging connection. Try to create all of the
WMBS tables. Also, create some dummy locations.
"""
super(JobCreatorTest, self).setUp()
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=[
'WMCore.WMBS', 'WMCore.ResourceControl', 'WMCore.Agent.Database'
],
useDefault=False)
self.couchdbname = "jobcreator_t"
self.testInit.setupCouch("%s/jobs" % self.couchdbname, "JobDump")
self.testInit.setupCouch("%s/fwjrs" % self.couchdbname, "FWJRDump")
self.configFile = EmulatorSetup.setupWMAgentConfig()
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
locationAction = self.daoFactory(classname="Locations.New")
for site in self.sites:
locationAction.execute(siteName=site, pnn=site)
# Create sites in resourceControl
resourceControl = ResourceControl()
for site in self.sites:
resourceControl.insertSite(siteName=site, pnn=site, ceName=site)
resourceControl.insertThreshold(siteName=site,
taskType='Processing',
maxSlots=10000,
pendingSlots=10000)
self.resourceControl = resourceControl
self._setup = True
self._teardown = False
self.testDir = self.testInit.generateWorkDir()
self.cwd = os.getcwd()
# Set heartbeat
self.componentName = 'JobCreator'
self.heartbeatAPI = HeartbeatAPI(self.componentName)
self.heartbeatAPI.registerComponent()
if PY3:
self.assertItemsEqual = self.assertCountEqual
return
def setUp(self):
"""
_setUp_
Initialize the database and couch.
"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setupCouch(TEST_DB_NAME, "ConfigCache")
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
self.testInit.generateWorkDir()
couchServer = CouchServer(os.environ["COUCHURL"])
self.configDatabase = couchServer.connectDatabase(TEST_DB_NAME)
EmulatorHelper.setEmulators(dbs=True)
return
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["ReqMgr"]
self.testInit = TestInitCouchApp('RequestDBServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=self.schema, useDefault=False)
dbName = 'requsetdb_t'
self.testInit.setupCouch(dbName, *self.couchApps)
reqDBURL = "%s/%s" % (self.testInit.couchUrl, dbName)
self.requestWriter = RequestDBWriter(reqDBURL)
self.requestReader = RequestDBReader(reqDBURL)
self.requestWriter.defaultStale = {}
self.requestReader.defaultStale = {}
return
class CouchUtils_t(unittest.TestCase):
def setUp(self):
self.testInit = TestInitCouchApp(__file__)
self.testInit.setupCouch("wmcore-acdc-couchutils", "GroupUser", "ACDC")
def tearDown(self):
self.testInit.tearDownCouch()
def testA(self):
"""object driven connection via initialiseCouch method"""
class Thingy(object):
"""misc object with couch access attrs"""
def __init__(self):
self.couchdb = None
self.database = None
self.url = None
@CouchUtils.connectToCouch
def __call__(self):
return True
couchThingy = Thingy()
# test throws with everything None
self.assertRaises(CouchUtils.CouchConnectionError, CouchUtils.initialiseCouch, couchThingy)
couchThingy.url = self.testInit.couchUrl
self.assertRaises(CouchUtils.CouchConnectionError, CouchUtils.initialiseCouch, couchThingy)
couchThingy.database = self.testInit.couchDbName
try:
CouchUtils.initialiseCouch(couchThingy)
except Exception, ex:
msg = "Error initialising couch client for test object:\n %s " % str(ex)
self.fail(msg)
self.failUnless(couchThingy.couchdb != None)
# test decorator on already connected object
try:
couchThingy()
except Exception, ex:
msg = "Error invoking connectToCouch decorator:\n %s" % str(msg)
self.fail(msg)
def setUp(self):
"""
_setUp_
Setup the database and logging connection. Try to create all of the
WMBS tables. Also add some dummy locations.
"""
super(WorkQueueTestCase, self).setUp()
self.queueDB = 'workqueue_t'
self.queueInboxDB = 'workqueue_t_inbox'
self.globalQDB = 'workqueue_t_global'
self.globalQInboxDB = 'workqueue_t_global_inbox'
self.localQDB = 'workqueue_t_local'
self.localQInboxDB = 'workqueue_t_local_inbox'
self.localQDB2 = 'workqueue_t_local2'
self.localQInboxDB2 = 'workqueue_t_local2_inbox'
self.configCacheDB = 'workqueue_t_config_cache'
self.logDBName = 'logdb_t'
self.requestDBName = 'workqueue_t_reqmgr_workload_cache'
self.setSchema()
self.testInit = TestInit('WorkQueueTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase=True)
self.addCleanup(self.testInit.clearDatabase)
self.addCleanup(logging.debug, 'Cleanup called clearDatabase()')
self.testInit.setSchema(customModules = self.schema,
useDefault = False)
self.testInit.setupCouch(self.queueDB, *self.couchApps)
self.testInit.setupCouch(self.queueInboxDB, *self.couchApps)
self.testInit.setupCouch(self.globalQDB, *self.couchApps)
self.testInit.setupCouch(self.globalQInboxDB , *self.couchApps)
self.testInit.setupCouch(self.localQDB, *self.couchApps)
self.testInit.setupCouch(self.localQInboxDB, *self.couchApps)
self.testInit.setupCouch(self.localQDB2, *self.couchApps)
self.testInit.setupCouch(self.localQInboxDB2, *self.couchApps)
self.testInit.setupCouch(self.configCacheDB, 'ConfigCache')
self.testInit.setupCouch(self.logDBName, 'LogDB')
self.testInit.setupCouch(self.requestDBName, 'ReqMgr')
self.couchURL = os.environ.get("COUCHURL")
couchServer = CouchServer(self.couchURL)
self.configCacheDBInstance = couchServer.connectDatabase(self.configCacheDB)
self.localCouchMonitor = CouchMonitor(self.couchURL)
self.localCouchMonitor.deleteReplicatorDocs()
self.addCleanup(self.localCouchMonitor.deleteReplicatorDocs)
self.addCleanup(logging.debug, 'Cleanup called deleteReplicatorDocs()')
self.addCleanup(self.testInit.tearDownCouch)
self.addCleanup(logging.debug, 'Cleanup called tearDownCouch()')
self.workDir = self.testInit.generateWorkDir()
self.addCleanup(self.testInit.delWorkDir)
self.addCleanup(logging.debug, 'Cleanup called delWorkDir()')
return
def setUp(self):
"""
_setUp_
Figure out the location of the XML report produced by CMSSW.
"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase=True)
self.testInit.setupCouch("report_t/fwjrs", "FWJRDump")
self.xmlPath = os.path.join(
WMCore.WMBase.getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml")
self.badxmlPath = os.path.join(
WMCore.WMBase.getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWFailReport2.xml")
self.testDir = self.testInit.generateWorkDir()
return
def setUp(self):
"""
_setUp_
setUp function for unittest
"""
# Set constants
self.couchDB = "config_test"
self.configURL = "RANDOM;;URL;;NAME"
self.configString = "This is a random string"
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(
customModules=["WMComponent.DBS3Buffer", 'WMCore.Agent.Database'],
useDefault=False)
self.testInit.setupCouch(self.couchDB, "GroupUser", "ConfigCache")
myThread = threading.currentThread()
self.bufferFactory = DAOFactory(
package="WMComponent.DBSBuffer.Database",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.buffer3Factory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
locationAction = self.bufferFactory(
classname="DBSBufferFiles.AddLocation")
locationAction.execute(siteName="se1.cern.ch")
locationAction.execute(siteName="se1.fnal.gov")
locationAction.execute(siteName="malpaquet")
# Set heartbeat
self.componentName = 'JobSubmitter'
self.heartbeatAPI = HeartbeatAPI(self.componentName)
self.heartbeatAPI.registerComponent()
# Set up a config cache
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName=self.couchDB)
configCache.createUserGroup(groupname="testGroup", username='******')
self.testDir = self.testInit.generateWorkDir()
psetPath = os.path.join(self.testDir, "PSet.txt")
f = open(psetPath, 'w')
f.write(self.configString)
f.close()
configCache.addConfig(newConfig=psetPath, psetHash=None)
configCache.save()
self.configURL = "%s;;%s;;%s" % (os.environ["COUCHURL"], self.couchDB,
configCache.getCouchID())
return
def setUp(self):
"""
_setUp_
"""
super(WMBSHelperTest, self).setUp()
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase=True)
self.testInit.setupCouch("wmbshelper_t/jobs", "JobDump")
self.testInit.setupCouch("wmbshelper_t/fwjrs", "FWJRDump")
self.testInit.setupCouch("config_test", "GroupUser", "ConfigCache")
os.environ["COUCHDB"] = "wmbshelper_t"
self.testInit.setSchema(customModules=[
"WMCore.WMBS", "WMComponent.DBS3Buffer", "WMCore.BossAir",
"WMCore.ResourceControl"
],
useDefault=False)
self.workDir = self.testInit.generateWorkDir()
self.wmspec = self.createWMSpec()
self.topLevelTask = getFirstTask(self.wmspec)
self.inputDataset = self.topLevelTask.inputDataset()
self.dataset = self.topLevelTask.getInputDatasetPath()
self.dbs = DBSReader(self.inputDataset.dbsurl)
self.rucioAcct = "wmcore_transferor"
self.rucio = Rucio(self.rucioAcct)
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=threading.currentThread().logger,
dbinterface=threading.currentThread().dbi)
self.configFile = EmulatorSetup.setupWMAgentConfig()
self.config = loadConfigurationFile(self.configFile)
self.config.component_("JobSubmitter")
self.config.JobSubmitter.submitDir = self.workDir
self.config.JobSubmitter.submitScript = os.path.join(
getTestBase(), 'WMComponent_t/JobSubmitter_t', 'submit.sh')
return
def setUp(self):
self.testInit = TestInit('CouchWorkQueueTest')
self.testInit.setLogging()
self.testInit.setupCouch('wq_backend_test_inbox', 'WorkQueue')
self.testInit.setupCouch('wq_backend_test', 'WorkQueue')
self.couch_db = self.testInit.couch.couchServer.connectDatabase('wq_backend_test')
self.backend = WorkQueueBackend(db_url = self.testInit.couchUrl,
db_name = 'wq_backend_test',
inbox_name = 'wq_backend_test_inbox')
self.processingSpec = rerecoWorkload('testProcessing', rerecoArgs)
def setUp(self):
"""
_setUp_
"""
EmulatorHelper.setEmulators(phedex=True,
dbs=True,
siteDB=True,
requestMgr=True)
self.specGenerator = WMSpecGenerator("WMSpecs")
#self.configFile = EmulatorSetup.setupWMAgentConfig()
self.schema = []
self.couchApps = ["WorkQueue"]
self.testInit = TestInitCouchApp('WorkQueueServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=self.schema, useDefault=False)
self.testInit.setupCouch('workqueue_t', *self.couchApps)
self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps)
return
def setUp(self):
"""
_setUp_
"""
super(WorkQueueTest, self).setUp()
self.specGenerator = WMSpecGenerator("WMSpecs")
# self.configFile = EmulatorSetup.setupWMAgentConfig()
self.schema = []
self.couchApps = ["WorkQueue"]
self.testInit = TestInitCouchApp('WorkQueueServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=self.schema, useDefault=False)
self.testInit.setupCouch('workqueue_t', *self.couchApps)
self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps)
self.testInit.setupCouch('local_workqueue_t', *self.couchApps)
self.testInit.setupCouch('local_workqueue_t_inbox', *self.couchApps)
self.testInit.generateWorkDir()
return
def setUp(self):
"""
_setUp_
Initialize the database and couch.
"""
super(StepChainTests, self).setUp()
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase=True)
self.testInit.setupCouch("stepchain_t", "ConfigCache")
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
couchServer = CouchServer(os.environ["COUCHURL"])
self.configDatabase = couchServer.connectDatabase("stepchain_t")
self.testInit.generateWorkDir()
self.workload = None
return
class AnalysisCollectionService_t(unittest.TestCase):
"""
Unit tests for AnalysisCollectionService
"""
def setUp(self):
"""Set up test couch instance"""
self.dbsURL = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader'
self.dataset = '/RelValProdTTbar/JobRobot-MC_3XY_V24_JobRobot-v1/GEN-SIM-DIGI-RECO'
"""Set up couch test environment"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setupCouch("wmcore-acdc-acservice", "GroupUser", "ACDC")
self.acService = AnalysisCollectionService(
url=self.testInit.couchUrl, database=self.testInit.couchDbName)
self.collection = self.acService.createCollection(
'ewv-testcoll', 'ewv2', 'Analysis')
def tearDown(self):
"""Clean up couch instance"""
self.testInit.tearDownCouch()
def testCreateFilesetFromDBS(self):
"""Test creating an analysis fileset from DBS"""
rangesMask = Mask()
rangesMask.addRunWithLumiRanges(run=1,
lumiList=[[1, 9], [12, 12], [31, 31],
[38, 39], [49, 49], [51, 52],
[64, 65], [82, 82], [92,
98]])
fs, fl = self.acService.createFilesetFromDBS(self.collection,
filesetName='test_fs',
dbsURL=self.dbsURL,
dataset=self.dataset,
mask=rangesMask)
self.assertTrue(fl['_id'])
self.assertEqual(len(fl['files']), 21)
for file in fl['files']:
self.assertTrue(fl['files'][file]['merged'])
def setUp(self):
"""
_setUp_
Setup a couch database for testing
of produced JSON
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setupCouch("histogram_dump_t")
self.histogramDB = Database(dbname="histogram_dump_t")
def setUp(self):
"""
setUP global values
"""
testname = self.id().split('.')[-1]
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename='service_unittests.log',
filemode='w')
logger_name = 'Service%s' % testname.replace('test', '', 1)
self.logger = logging.getLogger(logger_name)
self.cache_path = tempfile.mkdtemp()
self.testInit = TestInitCouchApp("RegistrationTest")
self.couch_db = "regsvc"
self.testInit.setupCouch(self.couch_db)
def setUp(self):
"""
Basic setUp
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testDir = self.testInit.generateWorkDir()
return
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["WMStats"]
self.testInit = TestInitCouchApp('WorkQueueServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=self.schema, useDefault=False)
dbName = 'wmstats_t'
self.testInit.setupCouch(dbName, "WMStats")
reqDBName = "reqmgrdb_t"
self.testInit.setupCouch(reqDBName, "ReqMgr")
wmstatsURL = "%s/%s" % (self.testInit.couchUrl, dbName)
reqDBURL = "%s/%s" % (self.testInit.couchUrl, reqDBName)
self.reqDBWriter = RequestDBWriter(reqDBURL)
self.wmstatsReader = WMStatsReader(wmstatsURL, reqdbURL=reqDBURL)
self.wmstatsReader.defaultStale = {}
self.wmstatsReader.reqDB.defaultStale = {}
return
def setUp(self):
"""
_setUp_
Initialize the database and couch.
"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setupCouch("stepchain_t", "ConfigCache")
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
couchServer = CouchServer(os.environ["COUCHURL"])
self.configDatabase = couchServer.connectDatabase("stepchain_t")
self.testInit.generateWorkDir()
self.workload = None
self.jsonTemplate = getTestFile(
'data/ReqMgr/requests/DMWM/StepChain_MC.json')
return
class ReproducibleSeedingTests(unittest.TestCase):
def setUp(self):
self.testInit = TestInitCouchApp("ReproducibleSeedingTest")
self.testInit.setupCouch("seeding_config_cache", "GroupUser",
"ConfigCache")
self.database = Database(self.testInit.couchDbName,
self.testInit.couchUrl)
self.documentId = None
def tearDown(self):
self.testInit.tearDownCouch()
return
def testA(self):
"""instantiate"""
document = Document()
document[u'pset_tweak_details'] = {}
document[u'pset_tweak_details'][u'process'] = {}
document[u'pset_tweak_details'][u'process'][
u'RandomNumberGeneratorService'] = {}
document[u'pset_tweak_details'][u'process'][
u'RandomNumberGeneratorService'][u'seed1'] = {}
document[u'pset_tweak_details'][u'process'][
u'RandomNumberGeneratorService'][u'seed2'] = {}
document[u'pset_tweak_details'][u'process'][
u'RandomNumberGeneratorService'][u'seed3'] = {}
document = self.database.commitOne(document)[0]
seeder = ReproducibleSeeding(CouchUrl=self.testInit.couchUrl,
CouchDBName=self.testInit.couchDbName,
ConfigCacheDoc=document[u'id'])
job = Job("testjob")
seeder(job)
baggage = job.getBaggage()
seed1 = getattr(baggage.process.RandomNumberGeneratorService, "seed1",
None)
self.assertTrue(seed1 != None)
def setUp(self):
"""
setup for test.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.reqmonDBName = "wmstat_t"
self.localDBName = "wmstat_t_local"
self.testInit.setupCouch(self.reqmonDBName, "WMStatsAgent")
self.testInit.setupCouch(self.localDBName, "WMStatsAgent")
self.testDir = self.testInit.generateWorkDir()
return
def setUp(self):
"""
Basic setUp
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setupCouch("config_test", "GroupUser", "ConfigCache")
self.testDir = self.testInit.generateWorkDir()
return
def setUp(self, initRoot=True):
# default set
self.schemaModules = []
self.initialize()
if self.schemaModules:
import warnings
warnings.warn("use RESTAndCouchUnitTest instead",
DeprecationWarning)
from WMQuality.TestInitCouchApp import TestInitCouchApp
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging() # logLevel = logging.SQLDEBUG
self.testInit.setDatabaseConnection(destroyAllDatabase=True)
self.testInit.setSchema(customModules=self.schemaModules,
useDefault=False)
# Now pull the dbURL from the factory
# I prefer this method because the factory has better error handling
# Also because then you know everything is the same
myThread = threading.currentThread()
self.config.setDBUrl(myThread.dbFactory.dburl)
logging.info("This is our config: %s" % self.config)
self.initRoot = initRoot
if initRoot:
self.rt = Root(self.config, testName=self._testMethodName)
try:
self.rt.start(blocking=False)
except RuntimeError as e:
# there appears to be worker threads from a previous test
# hanging out. Try to slay them so that we can keep going
print("Failed to load cherrypy with exception: %s\n" % e)
print("The threads are: \n%s\n" % threading.enumerate())
print("The previous test was %s\n" % self.rt.getLastTest())
print(traceback.format_exc())
self.rt.stop()
raise e
return
def setUp(self):
"""
_setUp_
Initialize the database and couch.
"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setupCouch("taskchain_t", "ConfigCache")
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
couchServer = CouchServer(os.environ["COUCHURL"])
self.configDatabase = couchServer.connectDatabase("taskchain_t")
self.testInit.generateWorkDir()
self.workload = None
self.differentNCores = getTestFile(
'data/ReqMgr/requests/Integration/TaskChain_RelVal_Multicore.json')
return
def setUp(self):
"""
setup for test.
"""
myThread = threading.currentThread()
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase=True)
self.testInit.setSchema(
customModules=["WMCore.WMBS", "WMComponent.DBS3Buffer"],
useDefault=False)
self.databaseName = "taskarchiver_t_0"
self.testInit.setupCouch("%s/workloadsummary" % self.databaseName,
"WorkloadSummary")
self.testInit.setupCouch("%s/jobs" % self.databaseName, "JobDump")
self.testInit.setupCouch("%s/fwjrs" % self.databaseName, "FWJRDump")
self.testInit.setupCouch("wmagent_summary_t", "WMStats")
self.testInit.setupCouch("wmagent_summary_central_t", "WMStats")
self.testInit.setupCouch("stat_summary_t", "SummaryStats")
reqmgrdb = "reqmgrdb_t"
self.testInit.setupCouch(reqmgrdb, "ReqMgr")
reqDBURL = "%s/%s" % (self.testInit.couchUrl, reqmgrdb)
self.requestWriter = RequestDBWriter(reqDBURL)
self.requestWriter.defaultStale = {}
self.daofactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.getJobs = self.daofactory(classname="Jobs.GetAllJobs")
self.inject = self.daofactory(
classname="Workflow.MarkInjectedWorkflows")
self.testDir = self.testInit.generateWorkDir()
os.makedirs(os.path.join(self.testDir, 'specDir'))
self.nJobs = 10
self.campaignName = 'aCampaign'
self.uploadPublishInfo = False
self.uploadPublishDir = None
return
def setUp(self):
"""
setup for test.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.reqmonDBName = "wmstat_t"
self.localDBName = "wmstat_t_local"
self.testInit.setupCouch(self.reqmonDBName, "WMStats")
self.testInit.setupCouch(self.localDBName, "WMStats")
self.testDir = self.testInit.generateWorkDir()
EmulatorHelper.setEmulators(localCouch=True,
reqMon=False,
wmagentDB=True)
return
def setUp(self):
"""
_setUp_
Initialize the database and couch.
"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
self.testDir = self.testInit.generateWorkDir()
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.listTasksByWorkflow = self.daoFactory(
classname="Workflow.LoadFromName")
self.listFilesets = self.daoFactory(classname="Fileset.List")
self.listSubsMapping = self.daoFactory(
classname="Subscriptions.ListSubsAndFilesetsFromWorkflow")
return
def setUp(self):
"""
_setUp_
Figure out the location of the XML report produced by CMSSW.
"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase=True)
self.testInit.setupCouch("report_t/fwjrs", "FWJRDump")
testData = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t")
self.xmlPath = os.path.join(testData, "CMSSWProcessingReport.xml")
self.badxmlPath = os.path.join(testData, "CMSSWFailReport2.xml")
self.skippedFilesxmlPath = os.path.join(testData, "CMSSWSkippedNonExistentFile.xml")
self.skippedAllFilesxmlPath = os.path.join(testData, "CMSSWSkippedAll.xml")
self.fallbackXmlPath = os.path.join(testData, "CMSSWInputFallback.xml")
self.twoFileFallbackXmlPath = os.path.join(testData, "CMSSWTwoFileRemote.xml")
self.pileupXmlPath = os.path.join(testData, "CMSSWPileup.xml")
self.withEventsXmlPath = os.path.join(testData, "CMSSWWithEventCounts.xml")
self.noLocationReport = os.path.join(testData, "Report.0.pkl")
self.testDir = self.testInit.generateWorkDir()
return