def tearDown(self):
"""
_tearDown_
"""
self.testInit.clearDatabase()
EmulatorSetup.deleteConfig(self.configFile)
return
def tearDown(self):
"""
Database deletion
"""
self.testInit.clearDatabase(modules = ["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl"])
self.testInit.delWorkDir()
self.testInit.tearDownCouch()
EmulatorSetup.deleteConfig(self.configFile)
return
def tearDown(self):
"""
Database deletion
"""
self.testInit.clearDatabase(modules=["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl"])
self.testInit.delWorkDir()
self.testInit.tearDownCouch()
EmulatorSetup.deleteConfig(self.configFile)
return
def tearDown(self):
"""
Database deletion
"""
self.testInit.clearDatabase()
self.testInit.delWorkDir()
self.testInit.tearDownCouch()
EmulatorSetup.deleteConfig(self.configFile)
return
def tearDown(self):
"""
Database deletion
"""
self.testInit.clearDatabase()
self.testInit.delWorkDir()
self.testInit.tearDownCouch()
EmulatorSetup.deleteConfig(self.configFile)
return
def tearDown(self):
"""
_tearDown_
"""
self.testInit.clearDatabase()
EmulatorSetup.deleteConfig(self.configFile)
return
def tearDown(self):
"""
_tearDown_
Tear down the databases
"""
self.testInit.clearDatabase()
self.testInit.tearDownCouch()
self.testInit.delWorkDir()
EmulatorSetup.deleteConfig(self.configFile)
def tearDown(self):
"""
_tearDown_
Tear down the databases
"""
self.testInit.clearDatabase()
self.testInit.tearDownCouch()
self.testInit.delWorkDir()
EmulatorSetup.deleteConfig(self.configFile)
def tearDown(self):
"""
_tearDown_
Tear everything down.
"""
self.testInit.clearDatabase()
self.testInit.delWorkDir()
self.testInit.tearDownCouch()
EmulatorSetup.deleteConfig(self.configFile)
return
def tearDown(self):
"""
_tearDown_
tearDown function for unittest
"""
self.testInit.clearDatabase()
self.testInit.delWorkDir()
EmulatorSetup.deleteConfig(self.configFile)
return
def tearDown(self):
"""
_tearDown_
Tear everything down.
"""
self.testInit.clearDatabase()
self.testInit.delWorkDir()
self.testInit.tearDownCouch()
EmulatorSetup.deleteConfig(self.configFile)
return
def tearDown(self):
"""
Database deletion
"""
self.testInit.clearDatabase(modules=["WMCore.WMBS"])
self.testInit.tearDownCouch()
self.testInit.delWorkDir()
EmulatorSetup.deleteConfig(self.configFile)
super(JobArchiverTest, self).tearDown()
return
def tearDown(self):
"""
_tearDown_
tearDown function for unittest
"""
self.testInit.clearDatabase()
self.testInit.delWorkDir()
EmulatorSetup.deleteConfig(self.configFile)
return
def tearDown(self):
"""
Database deletion
"""
EmulatorHelper.resetEmulators()
self.testInit.clearDatabase(modules = ["WMCore.WMBS"])
self.testInit.tearDownCouch()
self.testInit.delWorkDir()
EmulatorSetup.deleteConfig(self.configFile)
return
def tearDown(self):
"""
Database deletion
"""
self.testInit.clearDatabase(modules=["WMCore.WMBS"])
self.testInit.tearDownCouch()
self.testInit.delWorkDir()
EmulatorSetup.deleteConfig(self.configFile)
super(JobArchiverTest, self).tearDown()
return
def tearDown(self):
"""
_tearDown_
Standard tearDown
"""
myThread = threading.currentThread()
self.testInit.clearDatabase()
self.testInit.delWorkDir()
self.testInit.tearDownCouch()
EmulatorSetup.deleteConfig(self.configFile)
myThread.logdbClient = None
return
def tearDown(self):
"""
_tearDown_
Standard tearDown
"""
myThread = threading.currentThread()
self.testInit.clearDatabase()
self.testInit.delWorkDir()
self.testInit.tearDownCouch()
EmulatorSetup.deleteConfig(self.configFile)
myThread.logdbClient = None
return
def tearDown(self):
"""
_tearDown_
Clear out the database.
"""
self.testInit.clearDatabase()
self.testInit.tearDownCouch()
self.testInit.delWorkDir()
EmulatorSetup.deleteConfig(self.configFile)
super(WMBSHelperTest, self).tearDown()
return
def tearDown(self):
"""
_tearDown_
Clear out the database.
"""
self.testInit.clearDatabase()
self.testInit.tearDownCouch()
self.testInit.delWorkDir()
EmulatorSetup.deleteConfig(self.configFile)
super(WMBSHelperTest, self).tearDown()
return
def tearDown(self):
"""
_tearDown_
Tear down everything and go home.
"""
self.testInit.clearDatabase()
self.testInit.delWorkDir()
EmulatorSetup.deleteConfig(self.configFile)
return
def tearDown(self):
"""
_tearDown_
Tear down everything and go home.
"""
self.testInit.clearDatabase()
self.testInit.delWorkDir()
EmulatorSetup.deleteConfig(self.configFile)
return
def setUp(self):
"""
_setUp_
Set everything up.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMCore.WMBS", "WMCore.BossAir",
"WMCore.ResourceControl"],
useDefault = False)
self.testInit.setupCouch("jobsubmittercaching_t/jobs", "JobDump")
self.testInit.setupCouch("jobsubmittercaching_t/fwjrs", "FWJRDump")
resourceControl = ResourceControl()
for siteName in ["T1_US_FNAL", "T1_UK_RAL"]:
resourceControl.insertSite(siteName = siteName, pnn = "se.%s" % (siteName),
ceName = siteName, plugin = "CondorPlugin", cmsName = siteName)
resourceControl.insertThreshold(siteName = siteName, taskType = "Processing",
maxSlots = 10000, pendingSlots = 10000)
self.testDir = self.testInit.generateWorkDir()
self.configFile = EmulatorSetup.setupWMAgentConfig()
return
def setUp(self):
"""
setup for test.
"""
super(ErrorHandlerTest, self).setUp()
myThread = threading.currentThread()
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
self.testInit.setupCouch("errorhandler_t", "GroupUser", "ACDC")
self.testInit.setupCouch("errorhandler_t_jd/jobs", "JobDump")
self.testInit.setupCouch("errorhandler_t_jd/fwjrs", "FWJRDump")
self.daofactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.getJobs = self.daofactory(classname="Jobs.GetAllJobs")
self.setJobTime = self.daofactory(classname="Jobs.SetStateTime")
locationAction = self.daofactory(classname="Locations.New")
locationAction.execute(siteName="malpaquet", pnn="T2_CH_CERN")
self.testDir = self.testInit.generateWorkDir()
self.configFile = EmulatorSetup.setupWMAgentConfig()
self.nJobs = 10
self.dataCS = DataCollectionService(url=self.testInit.couchUrl,
database="errorhandler_t")
return
def setUp(self):
"""
_setUp_
Set up test environment
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(
customModules=["WMCore.WMBS", "WMCore.BossAir"], useDefault=False)
self.testInit.setupCouch('workqueue_t', 'WorkQueue')
self.testInit.setupCouch('workqueue_inbox_t', 'WorkQueue')
self.testDir = self.testInit.generateWorkDir(deleteOnDestruction=False)
EmulatorHelper.setEmulators(phedex=True,
dbs=True,
siteDB=True,
requestMgr=True)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=logging,
dbinterface=myThread.dbi)
self.listWorkflows = self.daoFactory(
classname="Workflow.ListForSubmitter")
self.configFile = EmulatorSetup.setupWMAgentConfig()
def setUp(self):
"""
setup for test.
"""
super(ErrorHandlerTest, self).setUp()
myThread = threading.currentThread()
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase=True)
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
self.testInit.setupCouch("errorhandler_t", "GroupUser", "ACDC")
self.testInit.setupCouch("errorhandler_t_jd/jobs", "JobDump")
self.testInit.setupCouch("errorhandler_t_jd/fwjrs", "FWJRDump")
self.daofactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.getJobs = self.daofactory(classname="Jobs.GetAllJobs")
self.setJobTime = self.daofactory(classname="Jobs.SetStateTime")
locationAction = self.daofactory(classname="Locations.New")
locationAction.execute(siteName="malpaquet", pnn="T2_CH_CERN")
self.testDir = self.testInit.generateWorkDir()
self.configFile = EmulatorSetup.setupWMAgentConfig()
self.nJobs = 10
self.dataCS = DataCollectionService(url=self.testInit.couchUrl,
database="errorhandler_t")
return
def setUp(self):
"""
setup for test.
"""
super(RetryManagerTest, self).setUp()
myThread = threading.currentThread()
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
self.testInit.setupCouch("retry_manager_t/jobs", "JobDump")
self.testInit.setupCouch("retry_manager_t/fwjrs", "FWJRDump")
self.daofactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.getJobs = self.daofactory(classname="Jobs.GetAllJobs")
self.setJobTime = self.daofactory(classname="Jobs.SetStateTime")
self.increaseRetry = self.daofactory(classname="Jobs.IncrementRetry")
self.testDir = self.testInit.generateWorkDir()
self.configFile = EmulatorSetup.setupWMAgentConfig()
self.nJobs = 10
return
def setUp(self):
"""
setup for test.
"""
myThread = threading.currentThread()
super(JobArchiverTest, self).setUp()
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
# self.tearDown()
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
self.testInit.setupCouch("jobarchiver_t_0/jobs", "JobDump")
self.testInit.setupCouch("jobarchiver_t_0/fwjrs", "FWJRDump")
self.daofactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.getJobs = self.daofactory(classname="Jobs.GetAllJobs")
self.testDir = self.testInit.generateWorkDir(deleteOnDestruction=False)
self.nJobs = 10
self.configFile = EmulatorSetup.setupWMAgentConfig()
return
def __init__(self):
"""
__init__
Init the DB
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase=True)
self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer"],
useDefault=False)
self.configFile = EmulatorSetup.setupWMAgentConfig()
myThread = threading.currentThread()
self.bufferFactory = DAOFactory(
package="WMComponent.DBSBuffer.Database",
logger=myThread.logger,
dbinterface=myThread.dbi)
locationAction = self.bufferFactory(
classname="DBSBufferFiles.AddLocation")
locationAction.execute(siteName="se1.cern.ch")
locationAction.execute(siteName="se1.fnal.gov")
locationAction.execute(siteName="malpaquet")
config = self.getConfig()
self.dbsUploader = DBSUploadPoller(config=config)
return
def setUp(self):
"""
_setUp_
Standard setup: Now with 100% more couch
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=[
"WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl",
"WMCore.Agent.Database"
])
self.testInit.setupCouch("jobsubmitter_t/jobs", "JobDump")
self.testInit.setupCouch("jobsubmitter_t/fwjrs", "FWJRDump")
self.testInit.setupCouch("wmagent_summary_t", "WMStats")
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.baDaoFactory = DAOFactory(package="WMCore.BossAir",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.testDir = self.testInit.generateWorkDir()
# Set heartbeat
self.componentName = 'JobSubmitter'
self.heartbeatAPI = HeartbeatAPI(self.componentName)
self.heartbeatAPI.registerComponent()
self.configFile = EmulatorSetup.setupWMAgentConfig()
return
def setUp(self):
"""
setup for test.
"""
super(JobArchiverTest, self).setUp()
myThread = threading.currentThread()
super(JobArchiverTest, self).setUp()
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
# self.tearDown()
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
self.testInit.setupCouch("jobarchiver_t_0/jobs", "JobDump")
self.testInit.setupCouch("jobarchiver_t_0/fwjrs", "FWJRDump")
self.daofactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.getJobs = self.daofactory(classname="Jobs.GetAllJobs")
self.testDir = self.testInit.generateWorkDir(deleteOnDestruction=False)
self.nJobs = 10
self.configFile = EmulatorSetup.setupWMAgentConfig()
return
def setUp(self):
"""
_setUp_
Set everything up.
"""
super(JobSubmitterCachingTest, self).setUp()
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=[
"WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl"
],
useDefault=False)
self.testInit.setupCouch("jobsubmittercaching_t/jobs", "JobDump")
self.testInit.setupCouch("jobsubmittercaching_t/fwjrs", "FWJRDump")
resourceControl = ResourceControl()
for siteName in ["T1_US_FNAL", "T1_UK_RAL"]:
resourceControl.insertSite(siteName=siteName,
pnn="%s_Disk" % (siteName),
ceName=siteName,
plugin="SimpleCondorPlugin",
cmsName=siteName)
resourceControl.insertThreshold(siteName=siteName,
taskType="Processing",
maxSlots=10000,
pendingSlots=10000)
self.testDir = self.testInit.generateWorkDir()
self.configFile = EmulatorSetup.setupWMAgentConfig()
return
def setUp(self):
"""
_setUp_
Standard setup: Now with 100% more couch
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl", "WMCore.Agent.Database"])
self.testInit.setupCouch("jobsubmitter_t/jobs", "JobDump")
self.testInit.setupCouch("jobsubmitter_t/fwjrs", "FWJRDump")
self.testInit.setupCouch("wmagent_summary_t", "WMStats")
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.baDaoFactory = DAOFactory(package = "WMCore.BossAir",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.testDir = self.testInit.generateWorkDir()
# Set heartbeat
self.componentName = 'JobSubmitter'
self.heartbeatAPI = HeartbeatAPI(self.componentName)
self.heartbeatAPI.registerComponent()
self.configFile = EmulatorSetup.setupWMAgentConfig()
return
def setUp(self):
"""
_setUp_
setUp function for unittest
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer"],
useDefault = False)
self.testDir = self.testInit.generateWorkDir(deleteOnDestruction = False)
self.configFile = EmulatorSetup.setupWMAgentConfig()
myThread = threading.currentThread()
self.bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.buffer3Factory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = self.bufferFactory(classname = "DBSBufferFiles.AddLocation")
locationAction.execute(siteName = "se1.cern.ch")
locationAction.execute(siteName = "se1.fnal.gov")
locationAction.execute(siteName = "malpaquet")
self.dbsUrl = "https://localhost:1443/dbs/dev/global/DBSWriter"
self.dbsApi = None
return
def setUp(self):
"""
_setUp_
setUp function for unittest
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer"],
useDefault = False)
self.testDir = self.testInit.generateWorkDir(deleteOnDestruction = False)
self.configFile = EmulatorSetup.setupWMAgentConfig()
myThread = threading.currentThread()
self.bufferFactory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = self.bufferFactory(classname = "DBSBufferFiles.AddLocation")
locationAction.execute(siteName = "se1.cern.ch")
locationAction.execute(siteName = "se1.fnal.gov")
locationAction.execute(siteName = "malpaquet")
self.dbsUrl = "https://localhost:1443/dbs/dev/global/DBSWriter"
self.dbsApi = None
return
def setUp(self):
"""
setup for test.
"""
super(RetryManagerTest, self).setUp()
myThread = threading.currentThread()
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
self.testInit.setupCouch("retry_manager_t/jobs", "JobDump")
self.testInit.setupCouch("retry_manager_t/fwjrs", "FWJRDump")
self.daofactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.getJobs = self.daofactory(classname="Jobs.GetAllJobs")
self.setJobTime = self.daofactory(classname="Jobs.SetStateTime")
self.increaseRetry = self.daofactory(classname="Jobs.IncrementRetry")
self.testDir = self.testInit.generateWorkDir()
self.configFile = EmulatorSetup.setupWMAgentConfig()
self.nJobs = 10
return
def __init__(self):
"""
__init__
Init the DB
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase = True)
self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer"],
useDefault = False)
self.configFile = EmulatorSetup.setupWMAgentConfig()
myThread = threading.currentThread()
self.bufferFactory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = self.bufferFactory(classname = "DBSBufferFiles.AddLocation")
locationAction.execute(siteName = "se1.cern.ch")
locationAction.execute(siteName = "se1.fnal.gov")
locationAction.execute(siteName = "malpaquet")
config = self.getConfig()
self.dbsUploader = DBSUploadPoller(config = config)
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
self.testInit.clearDatabase(modules=['WMCore.WMBS', 'WMCore.ResourceControl', 'WMCore.Agent.Database'])
self.testInit.delWorkDir()
self._teardown = True
self.testInit.tearDownCouch()
EmulatorSetup.deleteConfig(self.configFile)
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
self.testInit.clearDatabase(modules=['WMCore.WMBS', 'WMCore.ResourceControl', 'WMCore.Agent.Database'])
self.testInit.delWorkDir()
self._teardown = True
self.testInit.tearDownCouch()
EmulatorSetup.deleteConfig(self.configFile)
return
def setUp(self):
"""
_setUp_
Setup the database and logging connection. Try to create all of the
WMBS tables. Also, create some dummy locations.
"""
super(JobCreatorTest, self).setUp()
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=[
'WMCore.WMBS', 'WMCore.ResourceControl', 'WMCore.Agent.Database'
],
useDefault=False)
self.couchdbname = "jobcreator_t"
self.testInit.setupCouch("%s/jobs" % self.couchdbname, "JobDump")
self.testInit.setupCouch("%s/fwjrs" % self.couchdbname, "FWJRDump")
self.configFile = EmulatorSetup.setupWMAgentConfig()
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
locationAction = self.daoFactory(classname="Locations.New")
for site in self.sites:
locationAction.execute(siteName=site, pnn=site)
# Create sites in resourceControl
resourceControl = ResourceControl()
for site in self.sites:
resourceControl.insertSite(siteName=site, pnn=site, ceName=site)
resourceControl.insertThreshold(siteName=site,
taskType='Processing',
maxSlots=10000,
pendingSlots=10000)
self.resourceControl = resourceControl
self._setup = True
self._teardown = False
self.testDir = self.testInit.generateWorkDir()
self.cwd = os.getcwd()
# Set heartbeat
self.componentName = 'JobCreator'
self.heartbeatAPI = HeartbeatAPI(self.componentName)
self.heartbeatAPI.registerComponent()
if PY3:
self.assertItemsEqual = self.assertCountEqual
return
def setUp(self):
"""
_setUp_
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=["WMCore.WMBS"])
self.splitterFactory = SplitterFactory(package="WMCore.JobSplitting")
myThread = threading.currentThread()
self.myThread = myThread
daoFactory = DAOFactory(package="WMCore.WMBS",
logger=logging,
dbinterface=myThread.dbi)
self.WMBSFactory = daoFactory
config = self.getConfig()
self.changer = ChangeState(config)
myResourceControl = ResourceControl()
myResourceControl.insertSite("T1_US_FNAL", 10, 20, "T1_US_FNAL_Disk",
"T1_US_FNAL")
myResourceControl.insertSite("T1_US_FNAL", 10, 20, "T3_US_FNALLPC",
"T1_US_FNAL")
myResourceControl.insertSite("T2_CH_CERN", 10, 20, "T2_CH_CERN",
"T2_CH_CERN")
self.fileset1 = Fileset(name="TestFileset1")
for fileNum in range(11):
newFile = File("/some/file/name%d" % fileNum,
size=1000,
events=100)
newFile.addRun(Run(1, *[1]))
newFile.setLocation('T1_US_FNAL_Disk')
self.fileset1.addFile(newFile)
self.fileset1.create()
workflow1 = Workflow(spec="spec.xml",
owner="hufnagel",
name="TestWorkflow1",
task="Test")
workflow1.create()
self.subscription1 = Subscription(fileset=self.fileset1,
workflow=workflow1,
split_algo="Harvest",
type="Harvesting")
self.subscription1.create()
self.configFile = EmulatorSetup.setupWMAgentConfig()
return
def setUp(self):
"""
_setUp_
Setup the database and logging connection. Try to create all of the
WMBS tables. Also, create some dummy locations.
"""
myThread = threading.currentThread()
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
#self.tearDown()
self.testInit.setSchema(customModules = ['WMCore.WMBS',
'WMCore.ResourceControl',
'WMCore.Agent.Database'], useDefault = False)
self.couchdbname = "jobcreator_t"
self.testInit.setupCouch("%s/jobs" % self.couchdbname, "JobDump")
self.testInit.setupCouch("%s/fwjrs" % self.couchdbname, "FWJRDump")
self.configFile = EmulatorSetup.setupWMAgentConfig()
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = self.daoFactory(classname = "Locations.New")
for site in self.sites:
locationAction.execute(siteName = site, pnn = site)
#Create sites in resourceControl
resourceControl = ResourceControl()
for site in self.sites:
resourceControl.insertSite(siteName = site, pnn = site, ceName = site)
resourceControl.insertThreshold(siteName = site, taskType = 'Processing', \
maxSlots = 10000, pendingSlots = 10000)
self.resourceControl = resourceControl
self._setup = True
self._teardown = False
self.testDir = self.testInit.generateWorkDir()
self.cwd = os.getcwd()
# Set heartbeat
self.componentName = 'JobCreator'
self.heartbeatAPI = HeartbeatAPI(self.componentName)
self.heartbeatAPI.registerComponent()
return
def setUp(self):
"""
_setUp_
Set up vital components
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMCore.WMBS",'WMCore.MsgService',
'WMCore.ResourceControl', 'WMCore.ThreadPool',
'WMCore.Agent.Database'],
useDefault = False)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = self.daoFactory(classname = "Locations.New")
pendingSlots = self.daoFactory(classname = "Locations.SetPendingSlots")
for site in self.sites:
locationAction.execute(siteName = site, pnn = 'se.%s' % (site), ceName = site)
pendingSlots.execute(siteName = site, pendingSlots = 1000)
#Create sites in resourceControl
resourceControl = ResourceControl()
for site in self.sites:
resourceControl.insertSite(siteName = site, pnn = 'se.%s' % (site), ceName = site)
resourceControl.insertThreshold(siteName = site, taskType = 'Processing', \
maxSlots = 10000, pendingSlots = 10000)
self.testDir = self.testInit.generateWorkDir()
# Set heartbeat
for component in self.components:
heartbeatAPI = HeartbeatAPI(component)
heartbeatAPI.registerComponent()
self.configFile = EmulatorSetup.setupWMAgentConfig()
return
def setUp(self):
"""
_setUp_
Set up vital components
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMCore.WMBS",'WMCore.MsgService',
'WMCore.ResourceControl', 'WMCore.ThreadPool',
'WMCore.Agent.Database'],
useDefault = False)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = self.daoFactory(classname = "Locations.New")
pendingSlots = self.daoFactory(classname = "Locations.SetPendingSlots")
for site in self.sites:
locationAction.execute(siteName = site, pnn = 'se.%s' % (site), ceName = site)
pendingSlots.execute(siteName = site, pendingSlots = 1000)
#Create sites in resourceControl
resourceControl = ResourceControl()
for site in self.sites:
resourceControl.insertSite(siteName = site, pnn = 'se.%s' % (site), ceName = site)
resourceControl.insertThreshold(siteName = site, taskType = 'Processing', \
maxSlots = 10000, pendingSlots = 10000)
self.testDir = self.testInit.generateWorkDir()
# Set heartbeat
for component in self.components:
heartbeatAPI = HeartbeatAPI(component)
heartbeatAPI.registerComponent()
self.configFile = EmulatorSetup.setupWMAgentConfig()
return
def testKillWorkflow(self):
"""
_testKillWorkflow_
Verify that workflow killing works correctly.
"""
configFile = EmulatorSetup.setupWMAgentConfig()
config = loadConfigurationFile(configFile)
baAPI = BossAirAPI(config=config)
# Create nine jobs
self.setupForKillTest(baAPI=baAPI)
self.assertEqual(len(baAPI._listRunJobs()), 9)
killWorkflow("Main", config, config)
self.verifyFileKillStatus()
self.verifyJobKillStatus()
self.assertEqual(len(baAPI._listRunJobs()), 8)
EmulatorSetup.deleteConfig(configFile)
return
def testKillWorkflow(self):
"""
_testKillWorkflow_
Verify that workflow killing works correctly.
"""
configFile = EmulatorSetup.setupWMAgentConfig()
config = loadConfigurationFile(configFile)
baAPI = BossAirAPI(config = config)
# Create nine jobs
self.setupForKillTest(baAPI = baAPI)
self.assertEqual(len(baAPI._listRunJobs()), 9)
killWorkflow("Main", config, config)
self.verifyFileKillStatus()
self.verifyJobKillStatus()
self.assertEqual(len(baAPI._listRunJobs()), 8)
EmulatorSetup.deleteConfig(configFile)
return
def setUp(self):
"""
_setUp_
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMCore.WMBS"])
self.splitterFactory = SplitterFactory(package = "WMCore.JobSplitting")
myThread = threading.currentThread()
self.myThread = myThread
daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = logging,
dbinterface = myThread.dbi)
self.WMBSFactory = daoFactory
config = self.getConfig()
self.changer = ChangeState(config)
myResourceControl = ResourceControl()
myResourceControl.insertSite("SomeSite", 10, 20, "SomeSE", "SomeCE")
myResourceControl.insertSite("SomeSite", 10, 20, "SomeSE2", "SomeCE")
myResourceControl.insertSite("SomeSite2", 10, 20, "SomeSE3", "SomeCE2")
self.fileset1 = Fileset(name = "TestFileset1")
for file in range(11):
newFile = File("/some/file/name%d" % file, size = 1000, events = 100)
newFile.addRun(Run(1,*[1]))
newFile.setLocation('SomeSE')
self.fileset1.addFile(newFile)
self.fileset1.create()
workflow1 = Workflow(spec = "spec.xml", owner = "hufnagel", name = "TestWorkflow1", task="Test")
workflow1.create()
self.subscription1 = Subscription(fileset = self.fileset1,
workflow = workflow1,
split_algo = "Harvest",
type = "Harvesting")
self.subscription1.create()
self.configFile = EmulatorSetup.setupWMAgentConfig()
return
def setUp(self):
"""
setup for test.
"""
super(JobTrackerTest, self).setUp()
myThread = threading.currentThread()
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
# self.testInit.clearDatabase(modules = ["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl"])
self.testInit.setSchema(customModules=[
"WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl"
],
useDefault=False)
self.testInit.setupCouch("jobtracker_t/jobs", "JobDump")
self.testInit.setupCouch("jobtracker_t/fwjrs", "FWJRDump")
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.getJobs = self.daoFactory(classname="Jobs.GetAllJobs")
# Create sites in resourceControl
resourceControl = ResourceControl()
resourceControl.insertSite(siteName='malpaquet',
pnn='se.malpaquet',
ceName='malpaquet',
plugin="CondorPlugin")
resourceControl.insertThreshold(siteName='malpaquet', taskType='Processing', \
maxSlots=10000, pendingSlots=10000)
locationAction = self.daoFactory(classname="Locations.New")
locationAction.execute(siteName="malpaquet",
pnn="malpaquet",
ceName="malpaquet",
plugin="CondorPlugin")
# Create user
newuser = self.daoFactory(classname="Users.New")
newuser.execute(dn="jchurchill")
# We actually need the user name
self.user = getpass.getuser()
self.testDir = self.testInit.generateWorkDir()
self.configFile = EmulatorSetup.setupWMAgentConfig()
def setUp(self):
"""
setup for test.
"""
super(JobTrackerTest, self).setUp()
myThread = threading.currentThread()
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
# self.testInit.clearDatabase(modules = ["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl"])
self.testInit.setSchema(customModules=["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl"],
useDefault=False)
self.testInit.setupCouch("jobtracker_t/jobs", "JobDump")
self.testInit.setupCouch("jobtracker_t/fwjrs", "FWJRDump")
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.getJobs = self.daoFactory(classname="Jobs.GetAllJobs")
# Create sites in resourceControl
resourceControl = ResourceControl()
resourceControl.insertSite(siteName='malpaquet', pnn='se.malpaquet',
ceName='malpaquet', plugin="CondorPlugin")
resourceControl.insertThreshold(siteName='malpaquet', taskType='Processing', \
maxSlots=10000, pendingSlots=10000)
locationAction = self.daoFactory(classname="Locations.New")
locationAction.execute(siteName="malpaquet", pnn="malpaquet",
ceName="malpaquet", plugin="CondorPlugin")
# Create user
newuser = self.daoFactory(classname="Users.New")
newuser.execute(dn="jchurchill")
# We actually need the user name
self.user = getpass.getuser()
self.testDir = self.testInit.generateWorkDir()
self.configFile = EmulatorSetup.setupWMAgentConfig()
def setUp(self):
"""
_setUp_
"""
super(WMBSHelperTest, self).setUp()
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase=True)
self.testInit.setupCouch("wmbshelper_t/jobs", "JobDump")
self.testInit.setupCouch("wmbshelper_t/fwjrs", "FWJRDump")
self.testInit.setupCouch("config_test", "GroupUser", "ConfigCache")
os.environ["COUCHDB"] = "wmbshelper_t"
self.testInit.setSchema(customModules=[
"WMCore.WMBS", "WMComponent.DBS3Buffer", "WMCore.BossAir",
"WMCore.ResourceControl"
],
useDefault=False)
self.workDir = self.testInit.generateWorkDir()
self.wmspec = self.createWMSpec()
self.topLevelTask = getFirstTask(self.wmspec)
self.inputDataset = self.topLevelTask.inputDataset()
self.dataset = self.topLevelTask.getInputDatasetPath()
self.dbs = DBSReader(self.inputDataset.dbsurl)
self.rucioAcct = "wmcore_transferor"
self.rucio = Rucio(self.rucioAcct)
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=threading.currentThread().logger,
dbinterface=threading.currentThread().dbi)
self.configFile = EmulatorSetup.setupWMAgentConfig()
self.config = loadConfigurationFile(self.configFile)
self.config.component_("JobSubmitter")
self.config.JobSubmitter.submitDir = self.workDir
self.config.JobSubmitter.submitScript = os.path.join(
getTestBase(), 'WMComponent_t/JobSubmitter_t', 'submit.sh')
return
def setUp(self):
"""
_setUp_
"""
super(WMBSHelperTest, self).setUp()
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase=True)
self.testInit.setupCouch("wmbshelper_t/jobs", "JobDump")
self.testInit.setupCouch("wmbshelper_t/fwjrs", "FWJRDump")
self.testInit.setupCouch("config_test", "GroupUser", "ConfigCache")
os.environ["COUCHDB"] = "wmbshelper_t"
self.testInit.setSchema(customModules=["WMCore.WMBS",
"WMComponent.DBS3Buffer",
"WMCore.BossAir",
"WMCore.ResourceControl"],
useDefault=False)
self.workDir = self.testInit.generateWorkDir()
self.wmspec = self.createWMSpec()
self.topLevelTask = getFirstTask(self.wmspec)
self.inputDataset = self.topLevelTask.inputDataset()
self.dataset = self.topLevelTask.getInputDatasetPath()
self.dbs = DBSReader(self.inputDataset.dbsurl)
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=threading.currentThread().logger,
dbinterface=threading.currentThread().dbi)
self.configFile = EmulatorSetup.setupWMAgentConfig()
self.config = loadConfigurationFile(self.configFile)
self.config.component_("JobSubmitter")
self.config.JobSubmitter.submitDir = self.workDir
self.config.JobSubmitter.submitScript = os.path.join(getTestBase(),
'WMComponent_t/JobSubmitter_t',
'submit.sh')
return
def setUp(self):
"""
_setUp_
Set up test environment
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=["WMCore.WMBS", "WMCore.BossAir"],
useDefault=False)
self.testInit.setupCouch('workqueue_t', 'WorkQueue')
self.testInit.setupCouch('workqueue_inbox_t', 'WorkQueue')
self.testDir = self.testInit.generateWorkDir(deleteOnDestruction=False)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=logging,
dbinterface=myThread.dbi)
self.listWorkflows = self.daoFactory(classname="Workflow.ListForSubmitter")
self.configFile = EmulatorSetup.setupWMAgentConfig()
def tearDown(self):
"""tearDown"""
WorkQueueTestCase.tearDown(self)
#Delete WMBSAgent config file
EmulatorSetup.deleteConfig(self.configFile)
EmulatorHelper.resetEmulators()
def setUp(self):
"""
If we dont have a wmspec file create one
"""
EmulatorHelper.setEmulators(phedex = True, dbs = True,
siteDB = True, requestMgr = False)
#set up WMAgent config file for couchdb
self.configFile = EmulatorSetup.setupWMAgentConfig()
WorkQueueTestCase.setUp(self)
# Basic production Spec
self.spec = monteCarloWorkload('testProduction', mcArgs)
getFirstTask(self.spec).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB'])
getFirstTask(self.spec).addProduction(totalevents = 10000)
self.spec.setSpecUrl(os.path.join(self.workDir, 'testworkflow.spec'))
self.spec.save(self.spec.specUrl())
# Sample Tier1 ReReco spec
self.processingSpec = rerecoWorkload('testProcessing', rerecoArgs)
self.processingSpec.setSpecUrl(os.path.join(self.workDir,
'testProcessing.spec'))
self.processingSpec.save(self.processingSpec.specUrl())
# Sample Tier1 ReReco spec
self.parentProcSpec = rerecoWorkload('testParentProcessing', parentProcArgs)
self.parentProcSpec.setSpecUrl(os.path.join(self.workDir,
'testParentProcessing.spec'))
self.parentProcSpec.save(self.parentProcSpec.specUrl())
# ReReco spec with blacklist
self.blacklistSpec = rerecoWorkload('blacklistSpec', rerecoArgs)
self.blacklistSpec.setSpecUrl(os.path.join(self.workDir,
'testBlacklist.spec'))
getFirstTask(self.blacklistSpec).data.constraints.sites.blacklist = ['T2_XX_SiteA']
self.blacklistSpec.save(self.blacklistSpec.specUrl())
# ReReco spec with whitelist
self.whitelistSpec = rerecoWorkload('whitelistlistSpec', rerecoArgs)
self.whitelistSpec.setSpecUrl(os.path.join(self.workDir,
'testWhitelist.spec'))
getFirstTask(self.whitelistSpec).data.constraints.sites.whitelist = ['T2_XX_SiteB']
self.whitelistSpec.save(self.whitelistSpec.specUrl())
# setup Mock DBS and PhEDEx
inputDataset = getFirstTask(self.processingSpec).inputDataset()
self.dataset = "/%s/%s/%s" % (inputDataset.primary,
inputDataset.processed,
inputDataset.tier)
# Create queues
globalCouchUrl = "%s/%s" % (self.testInit.couchUrl, self.globalQDB)
self.globalQueue = globalQueue(DbName = self.globalQDB,
InboxDbName = self.globalQInboxDB,
QueueURL = globalCouchUrl)
# self.midQueue = WorkQueue(SplitByBlock = False, # mid-level queue
# PopulateFilesets = False,
# ParentQueue = self.globalQueue,
# CacheDir = None)
# ignore mid queue as it causes database duplication's
# copy jobStateMachine couchDB configuration here since we don't want/need to pass whole configuration
jobCouchConfig = Configuration()
jobCouchConfig.section_("JobStateMachine")
jobCouchConfig.JobStateMachine.couchurl = os.environ["COUCHURL"]
jobCouchConfig.JobStateMachine.couchDBName = "testcouchdb"
# copy bossAir configuration here since we don't want/need to pass whole configuration
bossAirConfig = Configuration()
bossAirConfig.section_("BossAir")
bossAirConfig.BossAir.pluginDir = "WMCore.BossAir.Plugins"
bossAirConfig.BossAir.pluginNames = ["CondorPlugin"]
bossAirConfig.section_("Agent")
bossAirConfig.Agent.agentName = "TestAgent"
self.localQueue = localQueue(DbName = self.localQDB,
InboxDbName = self.localQInboxDB,
ParentQueueCouchUrl = globalCouchUrl,
JobDumpConfig = jobCouchConfig,
BossAirConfig = bossAirConfig,
CacheDir = self.workDir)
self.localQueue2 = localQueue(DbName = self.localQDB2,
InboxDbName = self.localQInboxDB2,
ParentQueueCouchUrl = globalCouchUrl,
JobDumpConfig = jobCouchConfig,
BossAirConfig = bossAirConfig,
CacheDir = self.workDir)
# configuration for the Alerts messaging framework, work (alerts) and
# control channel addresses to which alerts
# these are destination addresses where AlertProcessor:Receiver listens
config = Configuration()
config.section_("Alert")
config.Alert.address = "tcp://127.0.0.1:5557"
config.Alert.controlAddr = "tcp://127.0.0.1:5559"
# standalone queue for unit tests
self.queue = WorkQueue(JobDumpConfig = jobCouchConfig,
BossAirConfig = bossAirConfig,
DbName = self.queueDB,
InboxDbName = self.queueInboxDB,
CacheDir = self.workDir,
config = config)
# create relevant sites in wmbs
rc = ResourceControl()
for site, se in self.queue.SiteDB.mapping.items():
rc.insertSite(site, 100, se, cmsName = site)
daofactory = DAOFactory(package = "WMCore.WMBS",
logger = threading.currentThread().logger,
dbinterface = threading.currentThread().dbi)
addLocation = daofactory(classname = "Locations.New")
addLocation.execute(siteName = site, seName = se)