def setUp(self):
super(DatasetTestCase, self).setUp()
Globals.GlobalParams.resetParams()
EmulatorHelper.setEmulators(phedex=True,
dbs=False,
siteDB=True,
requestMgr=False)
def setUp(self):
"""
_setUp_
Set up test environment
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(
customModules=["WMCore.WMBS", "WMCore.BossAir"], useDefault=False)
self.testInit.setupCouch('workqueue_t', 'WorkQueue')
self.testInit.setupCouch('workqueue_inbox_t', 'WorkQueue')
self.testDir = self.testInit.generateWorkDir(deleteOnDestruction=False)
EmulatorHelper.setEmulators(phedex=True,
dbs=True,
siteDB=True,
requestMgr=True)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=logging,
dbinterface=myThread.dbi)
self.listWorkflows = self.daoFactory(
classname="Workflow.ListForSubmitter")
self.configFile = EmulatorSetup.setupWMAgentConfig()
def setUp(self):
"""
_setUp_
Install the DBSBuffer schema into the database and connect to PhEDEx.
"""
self.phedexURL = "https://bogus.cern.ch/bogus"
self.dbsURL = "https://bogus.cern.ch/bogus"
EmulatorHelper.setEmulators(phedex = True, dbs = True, siteDB = True)
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer",
"WMCore.WMBS"],
useDefault = False)
self.testFilesA = []
self.testFilesB = []
self.testDatasetA = "/BogusPrimary/Run2012Z-PromptReco-v1/RECO"
self.testDatasetB = "/BogusPrimary/CRUZET11-v1/RAW"
return
def setUp(self):
"""
setup for test.
"""
myThread = threading.currentThread()
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
#self.tearDown()
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
self.testInit.setupCouch("jobarchiver_t_0/jobs", "JobDump")
self.testInit.setupCouch("jobarchiver_t_0/fwjrs", "FWJRDump")
self.daofactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.getJobs = self.daofactory(classname="Jobs.GetAllJobs")
self.testDir = self.testInit.generateWorkDir(deleteOnDestruction=False)
self.nJobs = 10
self.alertsReceiver = None
EmulatorHelper.setEmulators(phedex=True,
dbs=True,
siteDB=True,
requestMgr=False)
return
def setUp(self):
WorkQueueTestCase.setUp(self)
EmulatorHelper.setEmulators(phedex = True, dbs = True,
siteDB = True, requestMgr = False)
self.globalQCouchUrl = "%s/%s" % (self.testInit.couchUrl, self.globalQDB)
self.localQCouchUrl = "%s/%s" % (self.testInit.couchUrl,
self.localQInboxDB)
def setUp(self):
"""
_setUp_
Install schema and create a DAO factory for WMBS.
"""
EmulatorHelper.setEmulators(phedex = False, dbs = False,
siteDB = True, requestMgr = False)
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMCore.WMBS",
"WMCore.ResourceControl",
"WMCore.BossAir"],
useDefault = False)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.baDaoFactory = DAOFactory(package = "WMCore.BossAir",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.insertRunJob = self.baDaoFactory(classname = "NewJobs")
self.insertState = self.baDaoFactory(classname = "NewState")
states = ['PEND', 'RUN', 'Idle', 'Running']
self.insertState.execute(states)
self.tempDir = self.testInit.generateWorkDir()
return
def setUp(self):
"""
_setUp_
Install the DBSBuffer schema into the database and connect to PhEDEx.
"""
self.phedexURL = "https://bogus.cern.ch/bogus"
self.dbsURL = "https://bogus.cern.ch/bogus"
EmulatorHelper.setEmulators(phedex=True, dbs=True, siteDB=True)
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(
customModules=["WMComponent.DBS3Buffer", "WMCore.WMBS"],
useDefault=False)
self.testFilesA = []
self.testFilesB = []
self.testDatasetA = "/BogusPrimary/Run2012Z-PromptReco-v1/RECO"
self.testDatasetB = "/BogusPrimary/CRUZET11-v1/RAW"
return
def setUp(self):
"""
_setUp_
Install schema and create a DAO factory for WMBS.
"""
EmulatorHelper.setEmulators(phedex=False,
dbs=False,
siteDB=True,
requestMgr=False)
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=[
"WMCore.WMBS", "WMCore.ResourceControl", "WMCore.BossAir"
],
useDefault=False)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.baDaoFactory = DAOFactory(package="WMCore.BossAir",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.insertRunJob = self.baDaoFactory(classname="NewJobs")
self.insertState = self.baDaoFactory(classname="NewState")
states = ['PEND', 'RUN', 'Idle', 'Running']
self.insertState.execute(states)
self.tempDir = self.testInit.generateWorkDir()
return
def setUp(self):
self.fakeDBS = False
EmulatorHelper.setEmulators(phedex=False,
dbs=self.fakeDBS,
siteDB=True,
requestMgr=False)
Globals.GlobalParams.resetParams()
def setUp(self):
"""
_setUp_
Setup couchdb and the test environment
"""
super(ResubmitBlockTest, self).setUp()
self.group = 'unknown'
self.user = '******'
# Set external test helpers
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setupCouch("resubmitblock_t", "ACDC", "GroupUser")
EmulatorHelper.setEmulators(phedex=False,
dbs=False,
siteDB=True,
requestMgr=False)
# Define test environment
self.couchUrl = os.environ["COUCHURL"]
self.acdcDBName = 'resubmitblock_t'
self.validLocations = ['T2_US_Nebraska', 'T1_US_FNAL', 'T1_UK_RAL']
self.siteWhitelist = ['T2_XX_SiteA']
self.workflowName = 'dballest_ReReco_workflow'
couchServer = CouchServer(dburl=self.couchUrl)
self.acdcDB = couchServer.connectDatabase(self.acdcDBName,
create=False)
user = makeUser(self.group, '*****@*****.**', self.couchUrl,
self.acdcDBName)
user.create()
return
def setUp(self):
"""
Setup for unit tests
"""
super(SiteDBTest, self).setUp()
EmulatorHelper.setEmulators(phedex=False, dbs=False, siteDB=False, requestMgr=True)
self.mySiteDB = SiteDBJSON()
def setUp(self):
"""
_setUp_
Setup couchdb and the test environment
"""
# Set external test helpers
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setupCouch("resubmitblock_t", "ACDC", "GroupUser")
EmulatorHelper.setEmulators(siteDB = True)
# Define test environment
self.couchUrl = os.environ["COUCHURL"]
self.acdcDBName = 'resubmitblock_t'
self.validLocations = ['srm-cms.gridpp.rl.ac.uk', 'cmssrm.fnal.gov', 'srm.unl.edu']
self.validLocationsCMSNames = ['T2_US_Nebraska', 'T1_US_FNAL', 'T1_UK_RAL']
self.siteWhitelist = ['T2_XX_SiteA']
self.workflowName = 'dballest_ReReco_workflow'
couchServer = CouchServer(dburl = self.couchUrl)
self.acdcDB = couchServer.connectDatabase(self.acdcDBName, create = False)
user = makeUser('unknown', '*****@*****.**', self.couchUrl, self.acdcDBName)
user.create()
return
def setUp(self):
"""
setup for test.
"""
myThread = threading.currentThread()
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
#self.tearDown()
self.testInit.setSchema(customModules = ["WMCore.WMBS"],
useDefault = False)
self.testInit.setupCouch("jobarchiver_t_0/jobs", "JobDump")
self.testInit.setupCouch("jobarchiver_t_0/fwjrs", "FWJRDump")
self.daofactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.getJobs = self.daofactory(classname = "Jobs.GetAllJobs")
self.testDir = self.testInit.generateWorkDir(deleteOnDestruction = False)
self.nJobs = 10
EmulatorHelper.setEmulators(phedex = True, dbs = True,
siteDB = True, requestMgr = False)
self.configFile = EmulatorSetup.setupWMAgentConfig()
return
def testEmulator(self):
EmulatorHelper.setEmulators(True, True, True, True)
self.assertEqual(PhEDEx().wrapped.__module__,
'WMQuality.Emulators.PhEDExClient.PhEDEx')
self.assertEqual(DBSReader(self.globalDBS).wrapped.__module__,
'WMQuality.Emulators.DBSClient.DBSReader')
self.assertEqual(SiteDBJSON().wrapped.__module__,
'WMQuality.Emulators.SiteDBClient.SiteDB')
self.assertEqual(RequestManager().wrapped.__module__,
'WMQuality.Emulators.RequestManagerClient.RequestManager')
self.assertEqual(PhEDEx().__class__.__name__, 'PhEDEx')
self.assertEqual(DBSReader(self.globalDBS).__class__.__name__, 'DBSReader')
self.assertEqual(SiteDBJSON().__class__.__name__, 'SiteDBJSON')
self.assertEqual(RequestManager().__class__.__name__, 'RequestManager')
EmulatorHelper.resetEmulators()
self.assertEqual(PhEDEx().wrapped.__module__,
'WMCore.Services.PhEDEx.PhEDEx')
self.assertEqual(DBSReader(self.globalDBS).wrapped.__module__,
'WMCore.Services.DBS.DBS2Reader')
self.assertEqual(SiteDBJSON().wrapped.__module__,
'WMCore.Services.SiteDB.SiteDB')
self.assertEqual(RequestManager().wrapped.__module__,
'WMCore.Services.RequestManager.RequestManager')
self.assertEqual(PhEDEx().__class__.__name__, 'PhEDEx')
self.assertEqual(DBSReader(self.globalDBS).__class__.__name__, 'DBS2Reader')
self.assertEqual(SiteDBJSON().__class__.__name__, 'SiteDBJSON')
self.assertEqual(RequestManager().__class__.__name__, 'RequestManager')
def setUp(self):
"""
_setUp_
Setup couchdb and the test environment
"""
super(ResubmitBlockTest, self).setUp()
self.group = 'unknown'
self.user = '******'
# Set external test helpers
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setupCouch("resubmitblock_t", "ACDC", "GroupUser")
EmulatorHelper.setEmulators(phedex=False, dbs=False, siteDB=True, requestMgr=False)
# Define test environment
self.couchUrl = os.environ["COUCHURL"]
self.acdcDBName = 'resubmitblock_t'
self.validLocations = ['T2_US_Nebraska', 'T1_US_FNAL', 'T1_UK_RAL']
self.siteWhitelist = ['T2_XX_SiteA']
self.workflowName = 'dballest_ReReco_workflow'
couchServer = CouchServer(dburl=self.couchUrl)
self.acdcDB = couchServer.connectDatabase(self.acdcDBName, create=False)
user = makeUser(self.group, '*****@*****.**', self.couchUrl, self.acdcDBName)
user.create()
return
def setUp(self):
"""
Setup for unit tests
"""
super(SiteDBTest, self).setUp()
EmulatorHelper.setEmulators(phedex=False, dbs=False, siteDB=False, requestMgr=True)
self.mySiteDB = SiteDBJSON()
def testEmulator(self):
EmulatorHelper.setEmulators(True, True, True, True)
self.assertEqual(PhEDEx().wrapped.__module__,
'WMQuality.Emulators.PhEDExClient.PhEDEx')
self.assertEqual(DBSReader(self.globalDBS).wrapped.__module__,
'WMQuality.Emulators.DBSClient.DBSReader')
self.assertEqual(SiteDBJSON().wrapped.__module__,
'WMQuality.Emulators.SiteDBClient.SiteDB')
self.assertEqual(RequestManager().wrapped.__module__,
'WMQuality.Emulators.RequestManagerClient.RequestManager')
self.assertEqual(PhEDEx().__class__.__name__, 'PhEDEx')
self.assertEqual(DBSReader(self.globalDBS).__class__.__name__, 'DBSReader')
self.assertEqual(SiteDBJSON().__class__.__name__, 'SiteDBJSON')
self.assertEqual(RequestManager().__class__.__name__, 'RequestManager')
EmulatorHelper.resetEmulators()
self.assertEqual(PhEDEx().wrapped.__module__,
'WMCore.Services.PhEDEx.PhEDEx')
self.assertEqual(DBSReader(self.globalDBS).wrapped.__module__,
'WMCore.Services.DBS.DBS2Reader')
self.assertEqual(SiteDBJSON().wrapped.__module__,
'WMCore.Services.SiteDB.SiteDB')
self.assertEqual(RequestManager().wrapped.__module__,
'WMCore.Services.RequestManager.RequestManager')
self.assertEqual(PhEDEx().__class__.__name__, 'PhEDEx')
self.assertEqual(DBSReader(self.globalDBS).__class__.__name__, 'DBS2Reader')
self.assertEqual(SiteDBJSON().__class__.__name__, 'SiteDBJSON')
self.assertEqual(RequestManager().__class__.__name__, 'RequestManager')
def setUp(self):
WorkQueueTestCase.setUp(self)
EmulatorHelper.setEmulators(phedex = True, dbs = True,
siteDB = True, requestMgr = False)
GlobalParams.resetParams()
self.globalQCouchUrl = "%s/%s" % (self.testInit.couchUrl, self.globalQDB)
self.localQCouchUrl = "%s/%s" % (self.testInit.couchUrl,
self.localQInboxDB)
def testGetEmulators(self):
from WMCore.Services.PhEDEx.PhEDEx import PhEDEx
phedexJSON = PhEDEx(responseType='json')
self.assertTrue( hasattr( phedexJSON, '_testNonExistentInEmulator' ) )
EmulatorHelper.setEmulators(phedex = True, dbs = True,
siteDB = True, requestMgr = False)
phedexJSON2 = PhEDEx(responseType='json')
self.assertFalse( hasattr( phedexJSON2, '_testNonExistentInEmulator' ) )
EmulatorHelper.resetEmulators()
phedexJSON2 = PhEDEx(responseType='json')
self.assertTrue( hasattr( phedexJSON2, '_testNonExistentInEmulator' ) )
def testGetEmulators(self):
from WMCore.Services.PhEDEx.PhEDEx import PhEDEx
phedexJSON = PhEDEx(responseType='json')
self.assertTrue(hasattr(phedexJSON, '_testNonExistentInEmulator'))
EmulatorHelper.setEmulators(phedex=True,
dbs=True,
siteDB=True,
requestMgr=False)
phedexJSON2 = PhEDEx(responseType='json')
self.assertFalse(hasattr(phedexJSON2, '_testNonExistentInEmulator'))
EmulatorHelper.resetEmulators()
phedexJSON2 = PhEDEx(responseType='json')
self.assertTrue(hasattr(phedexJSON2, '_testNonExistentInEmulator'))
def setUp(self):
"""
If we dont have a wmspec file create one
"""
EmulatorHelper.setEmulators(phedex=False, dbs=False, siteDB=True, requestMgr=True)
WorkQueueTestCase.setUp(self)
self.cacheDir = tempfile.mkdtemp()
self.specGenerator = WMSpecGenerator(self.cacheDir)
self.specs = self.createReRecoSpec(1, "file")
# Create queues
self.localQueue = localQueue(DbName=self.queueDB, InboxDbName=self.queueInboxDB, \
NegotiationTimeout=0, QueueURL='global.example.com', CacheDir=self.cacheDir, central_logdb_url="%s/%s" %(self.couchURL, self.logDBName), log_reporter='lq_profile_test')
def setUp(self):
"""
Initialize the database and couch.
"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setupCouch("pileupfetcher_t", "ConfigCache")
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
couchServer = CouchServer(os.environ["COUCHURL"])
self.configDatabase = couchServer.connectDatabase("pileupfetcher_t")
self.testDir = self.testInit.generateWorkDir()
EmulatorHelper.setEmulators(dbs=True)
def setUp(self):
"""
Initialize the database and couch.
"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setupCouch("pileupfetcher_t", "ConfigCache")
self.testInit.setSchema(customModules = ["WMCore.WMBS"],
useDefault = False)
couchServer = CouchServer(os.environ["COUCHURL"])
self.configDatabase = couchServer.connectDatabase("pileupfetcher_t")
self.testDir = self.testInit.generateWorkDir()
EmulatorHelper.setEmulators(dbs = True)
def setUp(self):
"""
If we dont have a wmspec file create one
"""
EmulatorHelper.setEmulators(phedex=False,
dbs=False,
siteDB=True,
requestMgr=True)
WorkQueueTestCase.setUp(self)
self.cacheDir = tempfile.mkdtemp()
self.specGenerator = WMSpecGenerator(self.cacheDir)
self.specs = self.createReRecoSpec(1, "file")
# Create queues
self.localQueue = localQueue(DbName=self.queueDB, InboxDbName=self.queueInboxDB, \
NegotiationTimeout=0, QueueURL='global.example.com', CacheDir=self.cacheDir, central_logdb_url="%s/%s" %(self.couchURL, self.logDBName), log_reporter='lq_profile_test')
def setUp(self):
"""
If we dont have a wmspec file create one
Warning: For the real profiling test including
spec generation. need to use real spec instead of
using emulator generated spec which doesn't include
couchDB access and cmssw access
"""
EmulatorHelper.setEmulators(phedex=True, dbs=True, siteDB=True, requestMgr=True)
WorkQueueTestCase.setUp(self)
self.cacheDir = tempfile.mkdtemp()
self.specGenerator = WMSpecGenerator(self.cacheDir)
self.specNamePrefix = "TestReReco_"
self.specs = self.createReRecoSpec(5, "file")
# Create queues
self.globalQueue = globalQueue(DbName=self.globalQDB, InboxDbName=self.globalQInboxDB, NegotiationTimeout=0)
def setUp(self):
"""
If we dont have a wmspec file create one
"""
EmulatorHelper.setEmulators(phedex = True, dbs = True,
siteDB = True, requestMgr = True)
WorkQueueTestCase.setUp(self)
self.cacheDir = tempfile.mkdtemp()
self.specGenerator = WMSpecGenerator(self.cacheDir)
self.specs = self.createReRecoSpec(1, "file")
# Create queues
self.localQueue = localQueue(DbName = self.queueDB,
InboxDbName = self.queueInboxDB,
NegotiationTimeout = 0,
QueueURL = 'global.example.com',
CacheDir = self.cacheDir)
def setUp(self):
"""
If we dont have a wmspec file create one
"""
EmulatorHelper.setEmulators(phedex = True, dbs = True,
siteDB = True, requestMgr = True)
WorkQueueTestCase.setUp(self)
self.cacheDir = tempfile.mkdtemp()
self.specGenerator = WMSpecGenerator(self.cacheDir)
self.specs = self.createReRecoSpec(1, "file")
# Create queues
self.localQueue = localQueue(DbName = self.queueDB,
InboxDbName = self.queueInboxDB,
NegotiationTimeout = 0,
QueueURL = 'global.example.com',
CacheDir = self.cacheDir)
def setUp(self):
"""
_setUp_
"""
EmulatorHelper.setEmulators(phedex = True, dbs = True,
siteDB = True, requestMgr = True)
self.specGenerator = WMSpecGenerator("WMSpecs")
#self.configFile = EmulatorSetup.setupWMAgentConfig()
self.schema = []
self.couchApps = ["WorkQueue"]
self.testInit = TestInitCouchApp('WorkQueueServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = self.schema,
useDefault = False)
self.testInit.setupCouch('workqueue_t', *self.couchApps)
self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps)
return
def setUp(self):
"""
_setUp_
"""
EmulatorHelper.setEmulators(phedex=True,
dbs=True,
siteDB=True,
requestMgr=True)
self.specGenerator = WMSpecGenerator("WMSpecs")
#self.configFile = EmulatorSetup.setupWMAgentConfig()
self.schema = []
self.couchApps = ["WorkQueue"]
self.testInit = TestInitCouchApp('WorkQueueServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=self.schema, useDefault=False)
self.testInit.setupCouch('workqueue_t', *self.couchApps)
self.testInit.setupCouch('workqueue_t_inbox', *self.couchApps)
return
def setUp(self):
"""
If we dont have a wmspec file create one
Warning: For the real profiling test including
spec generation. need to use real spec instead of
using emulator generated spec which doesn't include
couchDB access and cmssw access
"""
EmulatorHelper.setEmulators(phedex=True,
dbs=True,
siteDB=True,
requestMgr=True)
WorkQueueTestCase.setUp(self)
self.cacheDir = tempfile.mkdtemp()
self.specGenerator = WMSpecGenerator(self.cacheDir)
self.specNamePrefix = "TestReReco_"
self.specs = self.createReRecoSpec(5, "file")
# Create queues
self.globalQueue = globalQueue(DbName=self.globalQDB,
InboxDbName=self.globalQInboxDB,
NegotiationTimeout=0)
def setUp(self):
"""
_setUp_
Set up test environment
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMCore.WMBS",
"WMCore.BossAir"],
useDefault = False)
self.testInit.setupCouch('workqueue_t', 'WorkQueue')
self.testInit.setupCouch('workqueue_inbox_t', 'WorkQueue')
self.testDir = self.testInit.generateWorkDir(deleteOnDestruction = False)
EmulatorHelper.setEmulators(phedex = True, dbs = True,
siteDB = True, requestMgr = True)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = logging,
dbinterface = myThread.dbi)
self.listWorkflows = self.daoFactory(classname = "Workflow.ListForSubmitter")
def setUp(self):
"""
Setup for unit tests
"""
EmulatorHelper.setEmulators(siteDB = True)
self.mySiteDB = SiteDBJSON()
#!/usr/bin/env python
"""
WorkQueue.Policy.Start.Dataset tests
"""
import unittest
from WMCore.WorkQueue.Policy.Start.Dataset import Dataset
from WMCore.WMSpec.StdSpecs.ReReco import rerecoWorkload, getTestArguments
from WMCore.Services.EmulatorSwitch import EmulatorHelper
EmulatorHelper.setEmulators(phedex = True, dbs = True,
siteDB = True, requestMgr = False)
from WMCore.Services.DBS.DBSReader import DBSReader
from WMCore_t.WMSpec_t.samples.MultiTaskProcessingWorkload \
import workload as MultiTaskProcessingWorkload
from WMCore.WorkQueue.WorkQueueExceptions import *
from WMCore_t.WorkQueue_t.WorkQueue_t import getFirstTask
from WMQuality.Emulators.DataBlockGenerator import Globals
rerecoArgs = getTestArguments()
parentProcArgs = getTestArguments()
parentProcArgs.update(IncludeParents = "True")
class DatasetTestCase(unittest.TestCase):
splitArgs = dict(SliceType = 'NumberOfFiles', SliceSize = 5)
def testTier1ReRecoWorkload(self):
"""Tier1 Re-reco workflow"""
Tier1ReRecoWorkload = rerecoWorkload('ReRecoWorkload', rerecoArgs)
inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset()
dataset = "/%s/%s/%s" % (inputDataset.primary,
def setUp(self):
self.fakeDBS = False
EmulatorHelper.setEmulators(phedex = False, dbs = self.fakeDBS,
siteDB = True, requestMgr = False)
Globals.GlobalParams.resetParams()
def setUp(self):
super(DatasetTestCase, self).setUp()
Globals.GlobalParams.resetParams()
EmulatorHelper.setEmulators(phedex=True, dbs=False, siteDB=True, requestMgr=False)
def setUp(self):
EmulatorHelper.setEmulators(phedex = True, dbs = True,
siteDB = True, requestMgr = False)
def setUp(self):
WorkQueueTestCase.setUp(self)
EmulatorHelper.setEmulators(phedex = True, dbs = True,
siteDB = True, requestMgr = False)
def setUp(self):
"""
Setup for unit tests
"""
EmulatorHelper.setEmulators(siteDB = True)
self.mySiteDB = SiteDBJSON()
def setUp(self):
WorkQueueTestCase.setUp(self)
EmulatorHelper.setEmulators(phedex=True,
dbs=True,
siteDB=True,
requestMgr=False)
def setUp(self):
Globals.GlobalParams.resetParams()
EmulatorHelper.setEmulators(phedex = True, dbs = True,
siteDB = True, requestMgr = False)
def setUp(self):
"""
If we dont have a wmspec file create one
"""
EmulatorHelper.setEmulators(phedex = True, dbs = True,
siteDB = True, requestMgr = False)
#set up WMAgent config file for couchdb
self.configFile = EmulatorSetup.setupWMAgentConfig()
WorkQueueTestCase.setUp(self)
# Basic production Spec
self.spec = monteCarloWorkload('testProduction', mcArgs)
getFirstTask(self.spec).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB'])
getFirstTask(self.spec).addProduction(totalevents = 10000)
self.spec.setSpecUrl(os.path.join(self.workDir, 'testworkflow.spec'))
self.spec.save(self.spec.specUrl())
# Sample Tier1 ReReco spec
self.processingSpec = rerecoWorkload('testProcessing', rerecoArgs)
self.processingSpec.setSpecUrl(os.path.join(self.workDir,
'testProcessing.spec'))
self.processingSpec.save(self.processingSpec.specUrl())
# Sample Tier1 ReReco spec
self.parentProcSpec = rerecoWorkload('testParentProcessing', parentProcArgs)
self.parentProcSpec.setSpecUrl(os.path.join(self.workDir,
'testParentProcessing.spec'))
self.parentProcSpec.save(self.parentProcSpec.specUrl())
# ReReco spec with blacklist
self.blacklistSpec = rerecoWorkload('blacklistSpec', rerecoArgs)
self.blacklistSpec.setSpecUrl(os.path.join(self.workDir,
'testBlacklist.spec'))
getFirstTask(self.blacklistSpec).data.constraints.sites.blacklist = ['T2_XX_SiteA']
self.blacklistSpec.save(self.blacklistSpec.specUrl())
# ReReco spec with whitelist
self.whitelistSpec = rerecoWorkload('whitelistlistSpec', rerecoArgs)
self.whitelistSpec.setSpecUrl(os.path.join(self.workDir,
'testWhitelist.spec'))
getFirstTask(self.whitelistSpec).data.constraints.sites.whitelist = ['T2_XX_SiteB']
self.whitelistSpec.save(self.whitelistSpec.specUrl())
# setup Mock DBS and PhEDEx
inputDataset = getFirstTask(self.processingSpec).inputDataset()
self.dataset = "/%s/%s/%s" % (inputDataset.primary,
inputDataset.processed,
inputDataset.tier)
# Create queues
globalCouchUrl = "%s/%s" % (self.testInit.couchUrl, self.globalQDB)
self.globalQueue = globalQueue(DbName = self.globalQDB,
InboxDbName = self.globalQInboxDB,
QueueURL = globalCouchUrl)
# self.midQueue = WorkQueue(SplitByBlock = False, # mid-level queue
# PopulateFilesets = False,
# ParentQueue = self.globalQueue,
# CacheDir = None)
# ignore mid queue as it causes database duplication's
# copy jobStateMachine couchDB configuration here since we don't want/need to pass whole configuration
jobCouchConfig = Configuration()
jobCouchConfig.section_("JobStateMachine")
jobCouchConfig.JobStateMachine.couchurl = os.environ["COUCHURL"]
jobCouchConfig.JobStateMachine.couchDBName = "testcouchdb"
# copy bossAir configuration here since we don't want/need to pass whole configuration
bossAirConfig = Configuration()
bossAirConfig.section_("BossAir")
bossAirConfig.BossAir.pluginDir = "WMCore.BossAir.Plugins"
bossAirConfig.BossAir.pluginNames = ["CondorPlugin"]
bossAirConfig.section_("Agent")
bossAirConfig.Agent.agentName = "TestAgent"
self.localQueue = localQueue(DbName = self.localQDB,
InboxDbName = self.localQInboxDB,
ParentQueueCouchUrl = globalCouchUrl,
JobDumpConfig = jobCouchConfig,
BossAirConfig = bossAirConfig,
CacheDir = self.workDir)
self.localQueue2 = localQueue(DbName = self.localQDB2,
InboxDbName = self.localQInboxDB2,
ParentQueueCouchUrl = globalCouchUrl,
JobDumpConfig = jobCouchConfig,
BossAirConfig = bossAirConfig,
CacheDir = self.workDir)
# configuration for the Alerts messaging framework, work (alerts) and
# control channel addresses to which alerts
# these are destination addresses where AlertProcessor:Receiver listens
config = Configuration()
config.section_("Alert")
config.Alert.address = "tcp://127.0.0.1:5557"
config.Alert.controlAddr = "tcp://127.0.0.1:5559"
# standalone queue for unit tests
self.queue = WorkQueue(JobDumpConfig = jobCouchConfig,
BossAirConfig = bossAirConfig,
DbName = self.queueDB,
InboxDbName = self.queueInboxDB,
CacheDir = self.workDir,
config = config)
# create relevant sites in wmbs
rc = ResourceControl()
for site, se in self.queue.SiteDB.mapping.items():
rc.insertSite(site, 100, se, cmsName = site)
daofactory = DAOFactory(package = "WMCore.WMBS",
logger = threading.currentThread().logger,
dbinterface = threading.currentThread().dbi)
addLocation = daofactory(classname = "Locations.New")
addLocation.execute(siteName = site, seName = se)
def setUp(self):
Globals.GlobalParams.resetParams()
EmulatorHelper.setEmulators(phedex = True, dbs = True,
siteDB = True, requestMgr = False)
def setUp(self):
EmulatorHelper.setEmulators(phedex=True,
dbs=True,
siteDB=True,
requestMgr=False)
