class AlertProcessorTest(unittest.TestCase):
def setUp(self):
self.testInit = TestInit(__file__)
self.testInit.setLogging(logLevel = logging.DEBUG)
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMCore.WMBS",'WMCore.Agent.Database',
"WMCore.ResourceControl"],
useDefault = False)
self.testDir = self.testInit.generateWorkDir()
self.config = Configuration()
self.config.section_("Agent")
self.config.Agent.useMsgService = False
self.config.Agent.useTrigger = False
self.config.component_("AlertProcessor")
self.config.AlertProcessor.componentDir = self.testDir
self.config.AlertProcessor.address = "tcp://127.0.0.1:5557"
self.config.AlertProcessor.controlAddr = "tcp://127.0.0.1:5559"
self.config.section_("CoreDatabase")
self.config.CoreDatabase.socket = os.environ.get("DBSOCK")
self.config.CoreDatabase.connectUrl = os.environ.get("DATABASE")
self.config.AlertProcessor.section_("critical")
self.config.AlertProcessor.section_("soft")
self.config.AlertProcessor.critical.level = 5
self.config.AlertProcessor.soft.level = 0
self.config.AlertProcessor.soft.bufferSize = 3
self.config.AlertProcessor.critical.section_("sinks")
self.config.AlertProcessor.soft.section_("sinks")
def tearDown(self):
self.testInit.clearDatabase()
self.testInit.delWorkDir()
def testAlertProcessorBasic(self):
alertProcessor = AlertProcessor(self.config)
try:
# alertProcessor.startComponent() causes the flow to stop, Harness.py
# the method just calls prepareToStart() and waits for ever
# alertProcessor.startDaemon() no good for this either ... puts everything
# on background
alertProcessor.prepareToStart()
except Exception, ex:
print ex
self.fail(str(ex))
logging.debug("AlertProcessor and its sub-components should be running now ...")
logging.debug("Going to stop the component ...")
# stop via component method
try:
alertProcessor.stopAlertProcessor()
except Exception, ex:
print ex
self.fail(str(ex))
class CouchTest(unittest.TestCase):
def setUp(self):
self.testInit = TestInit(__file__)
self.testInit.setLogging(logLevel=logging.DEBUG)
self.testDir = self.testInit.generateWorkDir()
self.config = getConfig(self.testDir)
# mock generator instance to communicate some configuration values
self.generator = utils.AlertGeneratorMock(self.config)
self.testName = self.id().split('.')[-1]
def tearDown(self):
self.testInit.delWorkDir()
self.generator = None
def testAlertGeneratorCouchDbSizePollerBasic(self):
config = getConfig("/tmp")
try:
poller = CouchDbSizePoller(config.AlertGenerator.couchDbSizePoller,
self.generator)
except Exception, ex:
self.fail("%s: exception: %s" % (self.testName, ex))
poller.check() # -> on real system dir may result in permission denied
poller._dbDirectory = "/dev"
poller.check() # -> OK
# test failing during set up
poller = CouchDbSizePoller(config.AlertGenerator.couchDbSizePoller,
self.generator)
poller._query = "nonsense query"
# this will fail on the above query
self.assertRaises(Exception, poller._getDbDir)
poller.check()
class CouchTest(unittest.TestCase):
def setUp(self):
self.testInit = TestInit(__file__)
self.testInit.setLogging(logLevel = logging.DEBUG)
self.testDir = self.testInit.generateWorkDir()
self.config = getConfig(self.testDir)
# mock generator instance to communicate some configuration values
self.generator = utils.AlertGeneratorMock(self.config)
self.testProcesses = []
self.testName = self.id().split('.')[-1]
def tearDown(self):
self.testInit.delWorkDir()
self.generator = None
utils.terminateProcesses(self.testProcesses)
def testAlertGeneratorCouchDbSizePollerBasic(self):
config = getConfig("/tmp")
try:
poller = CouchDbSizePoller(config.AlertGenerator.couchDbSizePoller, self.generator)
except Exception, ex:
self.fail("%s: exception: %s" % (self.testName, ex))
poller.check() # -> on real system dir may result in permission denied
poller._dbDirectory = "/dev"
poller.check() # -> OK
# test failing during set up
poller = CouchDbSizePoller(config.AlertGenerator.couchDbSizePoller, self.generator)
poller._query = "nonsense query"
poller._dbDirectory = poller._getDbDir()
poller.check()
self.assertEquals(poller._dbDirectory, None)
class AlertProcessorTest(unittest.TestCase):
def setUp(self):
self.testInit = TestInit(__file__)
self.testInit.setLogging(logLevel=logging.DEBUG)
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=[
"WMCore.WMBS", 'WMCore.Agent.Database', "WMCore.ResourceControl"
],
useDefault=False)
self.testDir = self.testInit.generateWorkDir()
self.config = Configuration()
self.config.section_("Agent")
self.config.Agent.useMsgService = False
self.config.Agent.useTrigger = False
self.config.component_("AlertProcessor")
self.config.AlertProcessor.componentDir = self.testDir
self.config.AlertProcessor.address = "tcp://127.0.0.1:5557"
self.config.AlertProcessor.controlAddr = "tcp://127.0.0.1:5559"
self.config.section_("CoreDatabase")
self.config.CoreDatabase.socket = os.environ.get("DBSOCK")
self.config.CoreDatabase.connectUrl = os.environ.get("DATABASE")
self.config.AlertProcessor.section_("critical")
self.config.AlertProcessor.section_("soft")
self.config.AlertProcessor.critical.level = 5
self.config.AlertProcessor.soft.level = 0
self.config.AlertProcessor.soft.bufferSize = 3
self.config.AlertProcessor.critical.section_("sinks")
self.config.AlertProcessor.soft.section_("sinks")
def tearDown(self):
self.testInit.clearDatabase()
self.testInit.delWorkDir()
def testAlertProcessorBasic(self):
alertProcessor = AlertProcessor(self.config)
try:
# alertProcessor.startComponent() causes the flow to stop, Harness.py
# the method just calls prepareToStart() and waits for ever
# alertProcessor.startDaemon() no good for this either ... puts everything
# on background
alertProcessor.prepareToStart()
except Exception as ex:
print ex
self.fail(str(ex))
logging.debug(
"AlertProcessor and its sub-components should be running now ...")
logging.debug("Going to stop the component ...")
# stop via component method
try:
alertProcessor.stopAlertProcessor()
except Exception as ex:
print ex
self.fail(str(ex))
class MockPluginTest(unittest.TestCase):
def setUp(self):
self.testinit = TestInit(__file__)
self.workdir = self.testinit.generateWorkDir()
jobList[0]['cache_dir'] = self.workdir
def tearDown(self):
self.testinit.delWorkDir()
def testInit(self):
wrongconfig = Configuration()
wrongconfig.section_('BossAir')
self.assertRaises( BossAirPluginException, MockPlugin, wrongconfig )
wrongconfig.BossAir.section_('MockPlugin')
self.assertRaises( BossAirPluginException, MockPlugin, wrongconfig )
#The config does not contain fakeReport parameter
self.assertRaises( BossAirPluginException, MockPlugin, wrongconfig )
#The fakeReport does not exist
wrongconfig.BossAir.MockPlugin.fakeReport = 'asdf'
self.assertRaises( BossAirPluginException, MockPlugin, wrongconfig )
def testTrack(self):
mp = MockPlugin(config)
#Check that the job has been scheduled
self.assertEquals({}, mp.jobsScheduledEnd)
# Don't be racy
currentTime = datetime.now()
#id is the only required parameter in the job dictionary
res = mp.track( jobList, currentTime )
self.assertTrue( mp.jobsScheduledEnd.has_key(1L) )
#check scheduled end (N.B. this includes 20% of random time)
scheduledEnd = mp.jobsScheduledEnd[1L]
timeTillJob = scheduledEnd - currentTime
self.assertTrue( timeTillJob >= timedelta(minutes = TEST_JOB_LEN - 1), \
"Time till Job %s !>= Delta %s" % (timeTillJob, \
timedelta(minutes = TEST_JOB_LEN - 1)))
self.assertTrue( timeTillJob <= timedelta(minutes = TEST_JOB_LEN*120/100 + 1), \
"Time till Job %s !<= Delta %s" % (timeTillJob, \
timedelta(minutes = TEST_JOB_LEN * 120/100 + 1)) )
#the job is running
self.assertEquals( 'Running', res[0][0]['status'])
self.assertEquals( 'Running', res[1][0]['status'])
self.assertEquals( [], res[2])
#the job is not running anymore
mp.jobsScheduledEnd[1L] = datetime(1900,1,1)
res = mp.track( jobList )
self.assertEquals( [], res[0])
self.assertEquals( 'Done', res[1][0]['status'])
self.assertEquals( 'Done', res[2][0]['status'])
del mp
class SetupCMSSWPsetTest(unittest.TestCase):
def setUp(self):
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testDir = self.testInit.generateWorkDir()
sys.path.insert(
0,
os.path.join(WMCore.WMBase.getTestBase(),
"WMCore_t/WMRuntime_t/Scripts_t"))
def tearDown(self):
sys.path.remove(
os.path.join(WMCore.WMBase.getTestBase(),
"WMCore_t/WMRuntime_t/Scripts_t"))
del sys.modules["WMTaskSpace"]
self.testInit.delWorkDir()
def createTestStep(self):
"""
_createTestStep_
Create a test step that can be passed to the setup script.
"""
newStep = WMStep("cmsRun1")
newStepHelper = CMSSWStepHelper(newStep)
newStepHelper.setStepType("CMSSW")
newStepHelper.setGlobalTag("SomeGlobalTag")
stepTemplate = StepFactory.getStepTemplate("CMSSW")
stepTemplate(newStep)
newStep.application.command.configuration = "PSet.py"
newStep.application.multicore.numberOfCores = "auto"
return newStepHelper
def loadProcessFromPSet(self):
"""
_loadProcessFromPSet_
This requires changing the working directory,
do so in a safe manner to encapsulate the change to this method only
"""
currentPath = os.getcwd()
loadedProcess = None
try:
if not os.path.isdir(self.testDir):
raise
os.chdir(self.testDir)
testFile = "PSet.py"
pset = imp.load_source('process', testFile)
loadedProcess = pset.process
except Exception, ex:
self.fail(
"An exception was caught while trying to load the PSet, %s" %
str(ex))
finally:
class MockPluginTest(unittest.TestCase):
def setUp(self):
self.testinit = TestInit(__file__)
self.workdir = self.testinit.generateWorkDir()
jobList[0]['cache_dir'] = self.workdir
def tearDown(self):
self.testinit.delWorkDir()
def testInit(self):
wrongconfig = Configuration()
wrongconfig.section_('BossAir')
self.assertRaises( BossAirPluginException, MockPlugin, wrongconfig )
wrongconfig.BossAir.section_('MockPlugin')
self.assertRaises( BossAirPluginException, MockPlugin, wrongconfig )
#The config does not contain fakeReport parameter
self.assertRaises( BossAirPluginException, MockPlugin, wrongconfig )
#The fakeReport does not exist
wrongconfig.BossAir.MockPlugin.fakeReport = 'asdf'
self.assertRaises( BossAirPluginException, MockPlugin, wrongconfig )
def testTrack(self):
mp = MockPlugin(config)
#Check that the job has been scheduled
self.assertEquals({}, mp.jobsScheduledEnd)
# Don't be racy
currentTime = datetime.now()
#id is the only required parameter in the job dictionary
res = mp.track( jobList, currentTime )
self.assertTrue( 1 in mp.jobsScheduledEnd )
#check scheduled end (N.B. this includes 20% of random time)
scheduledEnd = mp.jobsScheduledEnd[1]
timeTillJob = scheduledEnd - currentTime
self.assertTrue( timeTillJob >= timedelta(minutes = TEST_JOB_LEN - 1), \
"Time till Job %s !>= Delta %s" % (timeTillJob, \
timedelta(minutes = TEST_JOB_LEN - 1)))
self.assertTrue( timeTillJob <= timedelta(minutes = TEST_JOB_LEN*120/100 + 1), \
"Time till Job %s !<= Delta %s" % (timeTillJob, \
timedelta(minutes = TEST_JOB_LEN * 120/100 + 1)) )
#the job is running
self.assertEquals( 'Running', res[0][0]['status'])
self.assertEquals( 'Running', res[1][0]['status'])
self.assertEquals( [], res[2])
#the job is not running anymore
mp.jobsScheduledEnd[1] = datetime(1900,1,1)
res = mp.track( jobList )
self.assertEquals( [], res[0])
self.assertEquals( 'Done', res[1][0]['status'])
self.assertEquals( 'Done', res[2][0]['status'])
del mp
class WorkQueueTestCase(unittest.TestCase):
def setSchema(self):
"this can be override if the schema setting is different"
self.schema = ["WMCore.WMBS","WMComponent.DBS3Buffer","WMCore.BossAir"]
self.couchApps = ["WorkQueue"]
def setUp(self):
"""
_setUp_
Setup the database and logging connection. Try to create all of the
WMBS tables. Also add some dummy locations.
"""
self.queueDB = 'workqueue_t'
self.queueInboxDB = 'workqueue_t_inbox'
self.globalQDB = 'workqueue_t_global'
self.globalQInboxDB = 'workqueue_t_global_inbox'
self.localQDB = 'workqueue_t_local'
self.localQInboxDB = 'workqueue_t_local_inbox'
self.localQDB2 = 'workqueue_t_local2'
self.localQInboxDB2 = 'workqueue_t_local2_inbox'
self.configCacheDB = 'workqueue_t_config_cache'
self.setSchema()
self.testInit = TestInit('WorkQueueTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = self.schema,
useDefault = False)
self.testInit.setupCouch(self.queueDB, *self.couchApps)
self.testInit.setupCouch(self.queueInboxDB, *self.couchApps)
self.testInit.setupCouch(self.globalQDB, *self.couchApps)
self.testInit.setupCouch(self.globalQInboxDB , *self.couchApps)
self.testInit.setupCouch(self.localQDB, *self.couchApps)
self.testInit.setupCouch(self.localQInboxDB, *self.couchApps)
self.testInit.setupCouch(self.localQDB2, *self.couchApps)
self.testInit.setupCouch(self.localQInboxDB2, *self.couchApps)
self.testInit.setupCouch(self.configCacheDB, 'ConfigCache')
couchServer = CouchServer(os.environ.get("COUCHURL"))
self.configCacheDBInstance = couchServer.connectDatabase(self.configCacheDB)
self.workDir = self.testInit.generateWorkDir()
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
#self.testInit.tearDownCouch()
self.testInit.clearDatabase()
self.testInit.delWorkDir()
class SetupCMSSWPsetTest(unittest.TestCase):
def setUp(self):
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testDir = self.testInit.generateWorkDir()
sys.path.insert(0, os.path.join(WMCore.WMBase.getTestBase(),
"WMCore_t/WMRuntime_t/Scripts_t"))
def tearDown(self):
sys.path.remove(os.path.join(WMCore.WMBase.getTestBase(),
"WMCore_t/WMRuntime_t/Scripts_t"))
del sys.modules["WMTaskSpace"]
self.testInit.delWorkDir()
def createTestStep(self):
"""
_createTestStep_
Create a test step that can be passed to the setup script.
"""
newStep = WMStep("cmsRun1")
newStepHelper = CMSSWStepHelper(newStep)
newStepHelper.setStepType("CMSSW")
newStepHelper.setGlobalTag("SomeGlobalTag")
stepTemplate = StepFactory.getStepTemplate("CMSSW")
stepTemplate(newStep)
newStep.application.command.configuration = "PSet.py"
newStep.application.multicore.numberOfCores = "auto"
return newStepHelper
def loadProcessFromPSet(self):
"""
_loadProcessFromPSet_
This requires changing the working directory,
do so in a safe manner to encapsulate the change to this method only
"""
currentPath = os.getcwd()
loadedProcess = None
try:
if not os.path.isdir(self.testDir):
raise
os.chdir(self.testDir)
testFile = "PSet.py"
pset = imp.load_source('process', testFile)
loadedProcess = pset.process
except Exception, ex:
self.fail("An exception was caught while trying to load the PSet, %s" % str(ex))
finally:
class AlertGeneratorTest(unittest.TestCase):
def setUp(self):
self.testInit = TestInit(__file__)
self.testInit.setLogging(logLevel = logging.DEBUG)
self.testInit.clearDatabase()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMCore.WMBS",'WMCore.Agent.Database',
"WMCore.ResourceControl"],
useDefault = False)
self.testDir = self.testInit.generateWorkDir()
# AlertGenerator instance
self.generator = None
self.config = getConfig(self.testDir)
self.config.section_("CoreDatabase")
self.config.CoreDatabase.socket = os.environ.get("DBSOCK")
self.config.CoreDatabase.connectUrl = os.environ.get("DATABASE")
self.testProcesses = []
self.testComponentDaemonXml = "/tmp/TestComponent/Daemon.xml"
def tearDown(self):
self.testInit.clearDatabase()
self.testInit.delWorkDir()
self.generator = None
utils.terminateProcesses(self.testProcesses)
# if the directory and file "/tmp/TestComponent/Daemon.xml" after
# ComponentsPoller test exist, then delete it
d = os.path.dirname(self.testComponentDaemonXml)
if os.path.exists(d):
shutil.rmtree(d)
def _startComponent(self):
self.generator = AlertGenerator(self.config)
try:
# self.proc.startComponent() causes the flow to stop, Harness.py
# the method just calls prepareToStart() and waits for ever
# self.proc.startDaemon() no good for this either ... puts everything
# on background
self.generator.prepareToStart()
except Exception, ex:
print ex
self.fail(str(ex))
print "AlertGenerator and its sub-components should be running now ..."
class FileSinkTest(unittest.TestCase):
def setUp(self):
self.testInit = TestInit(__file__)
self.testInit.setLogging(logLevel=logging.DEBUG)
self.testDir = self.testInit.generateWorkDir()
self.config = ConfigSection("file")
self.config.outputfile = os.path.join(self.testDir,
"FileSinkTestNew.json")
def tearDown(self):
self.testInit.delWorkDir()
def testFileSinkBasic(self):
sink = FileSink(self.config)
alerts = []
nAlerts = 10
for i in range(nAlerts):
a = Alert(Source=__file__,
Level=i,
Timestamp=time.time(),
Type="Test")
alerts.append(a)
sink.send(alerts)
# test by reading back
loadAlerts = sink.load()
self.assertEqual(len(loadAlerts), nAlerts)
# Since FileSink implementation depends on line-separated JSONs of
# Alert instance, test handling new lines in the payload
alerts = []
testMsg = "addtional \n message"
for i in range(10, 20):
a = Alert(Source=__file__,
Level=i,
Timestamp=time.time(),
Type="Test",
Details={"message": testMsg})
alerts.append(a)
self.failUnless(os.path.exists(self.config.outputfile))
sink.send(alerts)
# test by reading back
loadAlerts = sink.load()
self.assertEqual(len(loadAlerts), 20)
for a in loadAlerts[10:]:
self.assertEqual(a["Details"]["message"], testMsg)
class MySQLTest(unittest.TestCase):
def setUp(self):
self.testInit = TestInit(__file__)
self.testInit.setLogging(logLevel = logging.DEBUG)
self.testInit.setDatabaseConnection()
self.testDir = self.testInit.generateWorkDir()
self.config = getConfig(self.testDir)
# mock generator instance to communicate some configuration values
self.generator = utils.AlertGeneratorMock(self.config)
self.testProcesses = []
self.testName = self.id().split('.')[-1]
def tearDown(self):
self.testInit.delWorkDir()
self.generator = None
utils.terminateProcesses(self.testProcesses)
def testMySQLPollerBasic(self):
config = getConfig("/tmp")
generator = utils.AlertGeneratorMock(config)
# take for instance mysqlCPUPoller configuration here, just need
# appropriate attributes set
try:
poller = MySQLPoller(config.AlertGenerator.mysqlCPUPoller, generator)
except Exception, ex:
self.fail("%s: exception: %s" % (self.testName, ex))
# this class would not have defined polling sample function, give it one
poller.sample = lambda proc: float(12)
self.assertEqual(len(poller._measurements), 0)
poller.check()
self.assertEqual(len(poller._measurements), 1)
self.assertEqual(poller._measurements[0], 12)
# test handling of a non-existing process
MySQLPoller._getProcessPID = lambda inst: 1212121212
poller = MySQLPoller(config.AlertGenerator.mysqlCPUPoller, generator)
# polling should not even happen so don't have to define sample function
poller.check()
self.assertEqual(poller._measurements, None)
self.assertEqual(poller._dbProcessDetail, None)
class ConfigurationExTest(unittest.TestCase):
"""
test case for Configuration object
"""
def setUp(self):
"""set up"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
self.functionSave = "%s/WMCore_Agent_Configuration_t_function.py" % self.testDir
def tearDown(self):
"""clean up"""
self.testInit.delWorkDir()
def testCallableConfigParams(self):
"""ctor"""
def f():
return True
config = Configuration()
config.section_("SectionF")
#creating field for the following test
config.SectionF.aFunction = ''
#Cannot set a function for plain Configuration objects
#config.SectionF.__setattr__('aFunction', f)
self.assertRaises(RuntimeError, config.SectionF.__setattr__, config.SectionF.aFunction, f)
config = ConfigurationEx()
config.section_("SectionF")
#No failures with configurationEx
config.SectionF.aFunction = f
#However ConfigurationEx instances cannot be saved
self.assertRaises(RuntimeError, saveConfigurationFile, config, self.functionSave)
class FileSinkTest(unittest.TestCase):
def setUp(self):
self.testInit = TestInit(__file__)
self.testInit.setLogging(logLevel=logging.DEBUG)
self.testDir = self.testInit.generateWorkDir()
self.config = ConfigSection("file")
self.config.outputfile = os.path.join(self.testDir, "FileSinkTestNew.json")
def tearDown(self):
self.testInit.delWorkDir()
def testFileSinkBasic(self):
sink = FileSink(self.config)
alerts = []
nAlerts = 10
for i in range(nAlerts):
a = Alert(Source=__file__, Level=i, Timestamp=time.time(), Type="Test")
alerts.append(a)
sink.send(alerts)
# test by reading back
loadAlerts = sink.load()
self.assertEqual(len(loadAlerts), nAlerts)
# Since FileSink implementation depends on line-separated JSONs of
# Alert instance, test handling new lines in the payload
alerts = []
testMsg = "addtional \n message"
for i in range(10, 20):
a = Alert(Source=__file__, Level=i, Timestamp=time.time(), Type="Test", Details={"message": testMsg})
alerts.append(a)
self.failUnless(os.path.exists(self.config.outputfile))
sink.send(alerts)
# test by reading back
loadAlerts = sink.load()
self.assertEqual(len(loadAlerts), 20)
for a in loadAlerts[10:]:
self.assertEqual(a["Details"]["message"], testMsg)
class ConfigurationExTest(unittest.TestCase):
"""
test case for Configuration object
"""
def setUp(self):
"""set up"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
self.functionSave = "%s/WMCore_Agent_Configuration_t_function.py" % self.testDir
def tearDown(self):
"""clean up"""
self.testInit.delWorkDir()
def testCallableConfigParams(self):
"""ctor"""
def f():
return True
config = Configuration()
config.section_("SectionF")
#creating field for the following test
config.SectionF.aFunction = ''
#Cannot set a function for plain Configuration objects
#config.SectionF.__setattr__('aFunction', f)
self.assertRaises(RuntimeError, config.SectionF.__setattr__,
config.SectionF.aFunction, f)
config = ConfigurationEx()
config.section_("SectionF")
#No failures with configurationEx
config.SectionF.aFunction = f
#However ConfigurationEx instances cannot be saved
self.assertRaises(RuntimeError, saveConfigurationFile, config,
self.functionSave)
class MySQLTest(unittest.TestCase):
def setUp(self):
self.testInit = TestInit(__file__)
self.testInit.setLogging(logLevel = logging.DEBUG)
self.testInit.setDatabaseConnection()
self.testDir = self.testInit.generateWorkDir()
self.config = getConfig(self.testDir)
# mock generator instance to communicate some configuration values
self.generator = utils.AlertGeneratorMock(self.config)
self.testName = self.id().split('.')[-1]
def tearDown(self):
self.testInit.delWorkDir()
self.generator = None
def testMySQLPollerBasic(self):
config = getConfig("/tmp")
generator = utils.AlertGeneratorMock(config)
# take for instance mysqlCPUPoller configuration here, just need
# appropriate attributes set
try:
poller = MySQLPoller(config.AlertGenerator.mysqlCPUPoller, generator)
except Exception, ex:
self.fail("%s: exception: %s" % (self.testName, ex))
# this class would not have defined polling sample function, give it one
poller.sample = lambda proc: float(12)
self.assertEqual(len(poller._measurements), 0)
poller.check()
self.assertEqual(len(poller._measurements), 1)
self.assertEqual(poller._measurements[0], 12)
# test handling of a non-existing process
MySQLPoller._getProcessPID = lambda inst: 1212121212
self.assertRaises(Exception, MySQLPoller,
config.AlertGenerator.mysqlCPUPoller, generator)
class RuntimeTest(unittest.TestCase):
"""
_RuntimeTest_
A unittest to test the WMRuntime/WMSpec/Storage/etc tree
"""
# This is an integration test
__integration__ = "Any old bollocks"
def setUp(self):
"""
Basic setUp
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testDir = self.testInit.generateWorkDir()
# Random variables
self.workloadDir = None
self.unpackDir = None
self.initialDir = os.getcwd()
self.origPath = sys.path
# Create some dirs
os.makedirs(os.path.join(self.testDir, 'packages'))
return
def tearDown(self):
"""
_tearDown_
Remove any references you put directly into the modules
"""
self.testInit.delWorkDir()
# Clean up imports
if 'WMSandbox' in sys.modules.keys():
del sys.modules['WMSandbox']
if 'WMSandbox.JobIndex' in sys.modules.keys():
del sys.modules['WMSandbox.JobIndex']
return
def createTestWorkload(self, workloadName = 'Test', emulator = True):
"""
_createTestWorkload_
Creates a test workload for us to run on, hold the basic necessities.
"""
workloadDir = os.path.join(self.testDir, workloadName)
#arguments = getTestArguments()
#workload = rerecoWorkload("Tier1ReReco", arguments)
#rereco = workload.getTask("ReReco")
workload = testWorkload(emulation = emulator)
rereco = workload.getTask("ReReco")
# Set environment and site-local-config
siteConfigPath = os.path.join(workloadDir, 'SITECONF/local/JobConfig/')
if not os.path.exists(siteConfigPath):
os.makedirs(siteConfigPath)
shutil.copy('site-local-config.xml', siteConfigPath)
environment = rereco.data.section_('environment')
environment.CMS_PATH = workloadDir
taskMaker = TaskMaker(workload, workloadDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
workload.save(workloadName)
return workload
def unpackComponents(self, workload):
"""
Run the unpacker to build the directories
IMPORTANT NOTE:
This is not how we do things on the worker node
On the worker node we do not run multiple tasks
So here we create multiple tasks in different directories
To mimic running on multiple systems
"""
listOfTasks = getListOfTasks(workload = workload)
self.unpackDir = os.path.join(self.testDir, 'unpack')
if not os.path.exists(self.unpackDir):
os.makedirs(self.unpackDir)
os.chdir(self.unpackDir)
sandbox = workload.data.sandbox
for task in listOfTasks:
# We have to create a directory, unpack in it, and then get out
taskName = task.name()
taskDir = os.path.join(self.unpackDir, taskName)
if not os.path.exists(taskDir):
# Well then we have to make it
os.makedirs(taskDir)
os.chdir(taskDir)
# Now that we're here, run the unpacker
package = os.path.join(self.testDir, 'packages', '%sJobPackage.pkl' % (taskName))
jobIndex = 1
RunUnpacker(sandbox = sandbox, package = package,
jobIndex = jobIndex, jobname = taskName)
# And go back to where we started
os.chdir(self.unpackDir)
os.chdir(self.initialDir)
return
def createWMBSComponents(self, workload):
"""
Create the WMBS Components for this job
"""
listOfTasks = []
listOfSubs = []
rerecoTask = None
for primeTask in workload.taskIterator():
# There should only be one prime task, and it should be the rerecoTask
rerecoTask = primeTask
for task in primeTask.taskIterator():
listOfTasks.append(task)
for task in listOfTasks:
fileset = self.getFileset()
sub = self.createSubscriptions(task = task,
fileset = fileset)
#listOfSubs.append(sub)
return
def createSubscriptions(self, task, fileset):
"""
Create a subscription based on a task
"""
type = task.taskType()
work = task.makeWorkflow()
sub = Subscription(fileset = fileset,
workflow = work,
split_algo = "FileBased",
type = type)
package = self.createWMBSJobs(subscription = sub,
task = task)
packName = os.path.join(self.testDir, 'packages',
'%sJobPackage.pkl' %(task.name()))
package.save(packName)
return sub
def createWMBSJobs(self, subscription, task):
"""
Create the jobs for WMBS Components
Send a subscription/task, get back a package.
"""
splitter = SplitterFactory()
jobfactory = splitter(subscription = subscription,
package = "WMCore.DataStructs",
generators = makeGenerators(task))
params = task.jobSplittingParameters()
jobGroups = jobfactory(**params)
jobID = 1
package = JobPackage()
for group in jobGroups:
for job in group.jobs:
job['id'] = jobID
jobID += 1
package[job['id']] = job
return package
def getFileset(self):
"""
Get a fileset based on the task
"""
fileset = Fileset(name = 'Merge%s' %(type))
for i in range(0, random.randint(15,25)):
# Use the testDir to generate a random lfn
inpFile = File(lfn = "%s/%s.root" %(self.testDir, makeUUID()),
size = random.randint(200000, 1000000),
events = random.randint(1000,2000) )
inpFile.setLocation('Megiddo')
fileset.addFile(inpFile)
return fileset
def runJobs(self, workload):
"""
This might actually run the job. Who knows?
"""
listOfTasks = []
for primeTask in workload.taskIterator():
listOfTasks.append(primeTask)
# Only run primeTasks for now
for task in listOfTasks:
jobName = task.name()
taskDir = os.path.join(self.unpackDir, jobName, 'job')
os.chdir(taskDir)
sys.path.append(taskDir)
# Scream, run around in panic, blow up machine
print "About to run jobs"
print taskDir
miniStartup(dir = taskDir)
# When exiting, go back to where you started
os.chdir(self.initialDir)
sys.path.remove(taskDir)
return
def testA_CreateWorkload(self):
"""
_CreateWorkload_
Create a workload
Unpack the workload
Check for consistency
"""
workloadName = 'basicWorkload'
workload = self.createTestWorkload(workloadName = workloadName)
self.createWMBSComponents(workload = workload)
taskNames = []
for task in getListOfTasks(workload = workload):
taskNames.append(task.name())
workloadPath = os.path.join(self.testDir, workloadName, "TestWorkload")
siteConfigDir = os.path.join(self.testDir, workloadName, 'SITECONF/local/JobConfig/')
# Pre-run checks
# Does it have the right directories?
dirList = os.listdir(workloadPath)
self.assertEqual(dirList, ['WMSandbox', 'TestWorkload-Sandbox.tar.bz2'])
dirList = os.listdir(os.path.join(workloadPath, 'WMSandbox'))
for taskName in taskNames:
self.assertTrue(taskName in dirList)
# Do we have job packages
for task in taskNames:
self.assertTrue('%sJobPackage.pkl' % (task) in os.listdir(os.path.join(self.testDir, 'packages')))
# Does it have the SITECONF?
self.assertTrue('site-local-config.xml' in os.listdir(siteConfigDir))
# Now actually see if you can unpack it.
self.unpackComponents(workload = workload)
# Check for proper unpacking
# Check the the task has the right directories,
# and that the PSetTweaks and WMSandbox directories
# have the right contents
taskContents = ['WMSandbox', 'WMCore', 'PSetTweaks']
PSetContents = ['PSetTweak.pyc', 'CVS', 'PSetTweak.py',
'__init__.pyc', 'WMTweak.py', '__init__.py']
taskSandbox = ['JobPackage.pcl', 'JobIndex.py', '__init__.py', 'WMWorkload.pkl']
taskSandbox.extend(taskNames) # Should have a directory for each task
for task in taskNames:
self.assertTrue(task in os.listdir(os.path.join(self.testDir, 'unpack')))
taskDir = os.path.join(self.testDir, 'unpack', task, 'job')
self.assertTrue(os.path.isdir(taskDir))
self.assertEqual(os.listdir(taskDir).sort(), taskContents.sort())
self.assertEqual(os.listdir(os.path.join(taskDir, 'WMSandbox')).sort(),
taskSandbox.sort())
self.assertEqual(os.listdir(os.path.join(taskDir, 'PSetTweaks')).sort(),
PSetContents.sort())
# And we're done.
# Assume if we got this far everything is good
# At the end, copy the directory
#if os.path.exists('tmpDir'):
# shutil.rmtree('tmpDir')
#shutil.copytree(self.testDir, 'tmpDir')
return
def testB_EmulatorTest(self):
"""
_EmulatorTest_
This is where things get scary. We need to not only unpack the job,
but also ascertain whether it can run locally in emulator mode.
This requires...uh...emulator emulation.
"""
# Assume all this works, because we tested it in testA
workloadName = 'basicWorkload'
workload = self.createTestWorkload(workloadName = workloadName)
self.createWMBSComponents(workload = workload)
self.unpackComponents(workload = workload)
self.runJobs(workload = workload)
# Check the report
taskDir = os.path.join(self.testDir, 'unpack/ReReco/job/WMTaskSpace')
report = Report()
report.load(os.path.join(taskDir, 'Report.0.pkl'))
cmsReport = report.data.cmsRun1
# Now validate the report
self.assertEqual(report.data.ceName, socket.gethostname())
self.assertEqual(report.data.seName, 'cmssrm.fnal.gov')
self.assertEqual(report.data.siteName, 'T1_US_FNAL')
self.assertEqual(report.data.hostName, socket.gethostname())
self.assertTrue(report.data.completed)
# Should have status 0 (emulator job)
self.assertEqual(cmsReport.status, 0)
# Should have one output module
self.assertEqual(cmsReport.outputModules, ['TestOutputModule'])
# It should have one file for input and output
self.assertEqual(cmsReport.input.PoolSource.files.fileCount, 1)
self.assertEqual(cmsReport.output.TestOutputModule.files.fileCount, 1)
# So, um, I guess we're done
# At the end, copy the directory
#if os.path.exists('tmpDir'):
# shutil.rmtree('tmpDir')
#shutil.copytree(self.testDir, 'tmpDir')
return
class otherLogArchiveTexst:#(unittest.TestCase):
def setUp(self):
# stolen from CMSSWExecutor_t. thanks, dave
# first, delete all the sandboxen and taskspaces
# because of caching, this leaks from other tests in other files
# this sucks because the other tests are using sandboxen that
# are deleted after the test is over, which causes theses tests
# to break
modsToDelete = []
# not sure what happens if you delete from
# an arrey you're iterating over. doing it in
# two steps
for modname in sys.modules.keys():
# need to blow away things in sys.modules, otherwise
# they are cached and we look at old taskspaces
if modname.startswith('WMTaskSpace'):
modsToDelete.append(modname)
if modname.startswith('WMSandbox'):
modsToDelete.append(modname)
for modname in modsToDelete:
try:
reload(sys.modules[modname])
except:
pass
del sys.modules[modname]
self.oldpath = sys.path[:]
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
self.job = Job(name = "/UnitTests/DeleterTask/DeleteTest-test-job")
shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("DeleterTask")
cmsswHelper = self.task.makeStep("cmsRun1")
cmsswHelper.setStepType('CMSSW')
stepHelper = cmsswHelper.addStep("DeleteTest")
stepHelper.setStepType('LogArchive')
self.cmsswstep = cmsswHelper.data
self.cmsswHelper = cmsswHelper
self.stepdata = stepHelper.data
self.stephelp = LogArchiveTemplate.LogArchiveStepHelper(stepHelper.data)
self.task.applyTemplates()
self.executor = StepFactory.getStepExecutor(self.stephelp.stepType())
taskMaker = TaskMaker(self.workload, os.path.join(self.testDir))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.task.build(os.path.join(self.testDir, 'UnitTests'))
sys.path.insert(0, self.testDir)
sys.path.insert(0, os.path.join(self.testDir, 'UnitTests'))
# binDir = inspect.getsourcefile(ModuleLocator)
# binDir = binDir.replace("__init__.py", "bin")
#
# if not binDir in os.environ['PATH']:
# os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
open( os.path.join( self.testDir, 'UnitTests', '__init__.py'),'w').close()
shutil.copyfile( os.path.join( os.path.dirname( __file__ ), 'MergeSuccess.pkl'),
os.path.join( self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
def tearDown(self):
sys.path = self.oldpath[:]
self.testInit.delWorkDir()
# making double sure WMTaskSpace and WMSandbox are gone
modsToDelete = []
# not sure what happens if you delete from
# an arrey you're iterating over. doing it in
# two steps
for modname in sys.modules.keys():
# need to blow away things in sys.modules, otherwise
# they are cached and we look at old taskspaces
if modname.startswith('WMTaskSpace'):
modsToDelete.append(modname)
if modname.startswith('WMSandbox'):
modsToDelete.append(modname)
for modname in modsToDelete:
try:
reload(sys.modules[modname])
except:
pass
del sys.modules[modname]
myThread = threading.currentThread()
if hasattr(myThread, "factory"):
myThread.factory = {}
@attr('integration')
def testCPBackendLogArchiveAgainstReportNew(self):
myReport = Report()
myReport.unpersist(os.path.join( self.testDir, 'UnitTests','WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
myReport.data.cmsRun1.status = 0
myReport.persist(os.path.join( self.testDir,'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
executor = LogArchiveExecutor.LogArchive()
executor.initialise( self.stepdata, self.job)
self.setLocalOverride(self.stepdata)
self.stepdata.override.newLogArchive = True
executor.step = self.stepdata
executor.execute( )
self.assertTrue( os.path.exists( os.path.join( self.testDir, 'hosts' )))
self.assertTrue( os.path.exists( os.path.join( self.testDir, 'test1', 'hosts')))
@attr('integration')
def testCPBackendLogArchiveAgainstReportFailedStepNew(self):
myReport = Report()
myReport.unpersist(os.path.join( self.testDir, 'UnitTests','WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
myReport.data.cmsRun1.status = 1
myReport.persist(os.path.join( self.testDir,'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
executor = LogArchiveExecutor.LogArchive()
executor.initialise( self.stepdata, self.job)
self.setLocalOverride(self.stepdata)
self.stepdata.override.newLogArchive = True
executor.step = self.stepdata
executor.execute( )
self.assertFalse( os.path.exists( os.path.join( self.testDir, 'hosts' )))
self.assertFalse( os.path.exists( os.path.join( self.testDir, 'test1', 'hosts')))
return
@attr('integration')
def testCPBackendLogArchiveAgainstReportOld(self):
myReport = Report()
myReport.unpersist(os.path.join( self.testDir,'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
myReport.data.cmsRun1.status = 0
myReport.persist(os.path.join( self.testDir,'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
executor = LogArchiveExecutor.LogArchive()
executor.initialise( self.stepdata, self.job)
self.setLocalOverride(self.stepdata)
executor.step = self.stepdata
executor.execute( )
self.assertTrue( os.path.exists( os.path.join( self.testDir, 'hosts' )))
self.assertTrue( os.path.exists( os.path.join( self.testDir, 'test1', 'hosts')))
return
@attr('integration')
def testCPBackendLogArchiveAgainstReportFailedStepOld(self):
myReport = Report()
myReport.unpersist(os.path.join( self.testDir,'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
myReport.data.cmsRun1.status = 1
myReport.persist(os.path.join( self.testDir, 'UnitTests','WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
executor = LogArchiveExecutor.LogArchive()
executor.initialise( self.stepdata, self.job)
self.setLocalOverride(self.stepdata)
executor.step = self.stepdata
executor.execute( )
self.assertFalse( os.path.exists( os.path.join( self.testDir, 'hosts' )))
self.assertFalse( os.path.exists( os.path.join( self.testDir, 'test1', 'hosts')))
return
@attr('workerNodeTest')
def testOnWorkerNodes(self):
raise RuntimeError
# Stage a file out, stage it back in, check it, delete it
myReport = Report()
myReport.unpersist(os.path.join( self.testDir,'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
myReport.data.cmsRun1.status = 1
del myReport.data.cmsRun1.output
myReport.data.cmsRun1.section_('output')
myReport.data.cmsRun1.output.section_('stagingTestOutput')
myReport.data.cmsRun1.output.stagingTestOutput.section_('files')
myReport.data.cmsRun1.output.stagingTestOutput.fileCount = 0
targetFiles = [ '/store/temp/WMAgent/storetest-%s' % time.time(),
'/store/unmerged/WMAgent/storetest-%s' % time.time()]
for file in targetFiles:
print("Adding file for LogArchive %s" % file)
self.addLogArchiveFile(myReport, file )
myReport.persist(os.path.join( self.testDir, 'UnitTests','WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
executor = LogArchiveExecutor.LogArchive()
executor.initialise( self.stepdata, self.job)
executor.step = self.stepdata
print("beginning stageout")
executor.execute( )
print("stageout done")
# pull in the report with the stage out info
myReport = Report()
myReport.unpersist(os.path.join( self.testDir,'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
print("Got the stage out data back")
print(myReport.data)
# now, transfer them back
# TODO make a stagein step in the task - Melo
import WMCore.Storage.FileManager as FileManagerModule
fileManager = FileManagerModule.FileManager( numberOfRetries = 10, retryPauseTime = 1)
for file in targetFiles:
print("Staging in %s" % file)
fileManager.stageOut( fileToStage = { 'LFN' : file,
'PFN' : '%s/%s' % (self.testDir, file) },
stageOut = False)
self.assertTrue( os.path.exists( '%s/%s' % (self.testDir, file)))
self.assertEqual( os.path.getsize('/etc/hosts', '%s/%s' % (self.testDir, file)))
# now, should delete the files we made
for file in targetFiles:
print("deleting %s" % file)
fileManager.deleteLFN(file)
# try staging in again to make sure teh files are gone
for file in targetFiles:
print("Staging in (should fail) %s" % file)
self.assertRaises( LogArchiveError, \
FileManagerModule.FileManager.stageOut, \
fileManager,fileToStage = { 'LFN' : file,
'PFN' : '%s/%s' % (self.testDir, file) },
stageOut = False )
# need to make sure files didn't show up
self.assertFalse( os.path.exists( os.path.join( self.testDir, 'hosts' )))
self.assertFalse( os.path.exists( os.path.join( self.testDir, 'test1', 'hosts')))
def addLogArchiveFile(self, myReport, lfn):
myId = myReport.data.cmsRun1.output.stagingTestOutput.fileCount
mySection = myReport.data.cmsRun1.output.stagingTestOutput.section_('file%s' % myId)
mySection.section_('runs')
setattr(mySection.runs,'114475', [33])
mySection.section_('branches')
mySection.lfn = lfn
mySection.dataset = {'applicationName': 'cmsRun', 'primaryDataset': 'Calo', 'processedDataset': 'Commissioning09-PromptReco-v8', 'dataTier': 'ALCARECO', 'applicationVersion': 'CMSSW_3_2_7'}
mySection.module_label = 'ALCARECOStreamCombined'
mySection.parents = []
mySection.location = 'srm-cms.cern.ch'
mySection.checksums = {'adler32': 'bbcf2215', 'cksum': '2297542074'}
mySection.pfn = '/etc/hosts'
mySection.events = 20000
mySection.merged = False
mySection.size = 37556367
myReport.data.cmsRun1.output.stagingTestOutput.fileCount = myId + 1
def setLocalOverride(self, step):
step.section_('override')
step.override.command = 'cp'
step.override.option = ''
step.override.__setattr__('lfn-prefix', self.testDir +"/")
step.override.__setattr__('se-name','DUMMYSE')
step.override.__setattr__('phedex-node','DUMMYPNN')
class RuntimeTest(unittest.TestCase):
"""
_RuntimeTest_
A unittest to test the WMRuntime/WMSpec/Storage/etc tree
"""
# This is an integration test
__integration__ = "Any old bollocks"
def setUp(self):
"""
Basic setUp
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testDir = self.testInit.generateWorkDir()
# Random variables
self.workloadDir = None
self.unpackDir = None
self.initialDir = os.getcwd()
self.origPath = sys.path
# Create some dirs
os.makedirs(os.path.join(self.testDir, 'packages'))
return
def tearDown(self):
"""
_tearDown_
Remove any references you put directly into the modules
"""
self.testInit.delWorkDir()
# Clean up imports
if 'WMSandbox' in sys.modules.keys():
del sys.modules['WMSandbox']
if 'WMSandbox.JobIndex' in sys.modules.keys():
del sys.modules['WMSandbox.JobIndex']
return
def createTestWorkload(self, workloadName = 'Test', emulator = True):
"""
_createTestWorkload_
Creates a test workload for us to run on, hold the basic necessities.
"""
workloadDir = os.path.join(self.testDir, workloadName)
#arguments = getTestArguments()
#workload = rerecoWorkload("Tier1ReReco", arguments)
#rereco = workload.getTask("ReReco")
workload = testWorkload(emulation = emulator)
rereco = workload.getTask("ReReco")
# Set environment and site-local-config
siteConfigPath = os.path.join(workloadDir, 'SITECONF/local/JobConfig/')
if not os.path.exists(siteConfigPath):
os.makedirs(siteConfigPath)
shutil.copy('site-local-config.xml', siteConfigPath)
environment = rereco.data.section_('environment')
environment.CMS_PATH = workloadDir
taskMaker = TaskMaker(workload, workloadDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
workload.save(workloadName)
return workload
def unpackComponents(self, workload):
"""
Run the unpacker to build the directories
IMPORTANT NOTE:
This is not how we do things on the worker node
On the worker node we do not run multiple tasks
So here we create multiple tasks in different directories
To mimic running on multiple systems
"""
listOfTasks = getListOfTasks(workload = workload)
self.unpackDir = os.path.join(self.testDir, 'unpack')
if not os.path.exists(self.unpackDir):
os.makedirs(self.unpackDir)
os.chdir(self.unpackDir)
sandbox = workload.data.sandbox
for task in listOfTasks:
# We have to create a directory, unpack in it, and then get out
taskName = task.name()
taskDir = os.path.join(self.unpackDir, taskName)
if not os.path.exists(taskDir):
# Well then we have to make it
os.makedirs(taskDir)
os.chdir(taskDir)
# Now that we're here, run the unpacker
package = os.path.join(self.testDir, 'packages', '%sJobPackage.pkl' % (taskName))
jobIndex = 1
RunUnpacker(sandbox = sandbox, package = package,
jobIndex = jobIndex, jobname = taskName)
# And go back to where we started
os.chdir(self.unpackDir)
os.chdir(self.initialDir)
return
def createWMBSComponents(self, workload):
"""
Create the WMBS Components for this job
"""
listOfTasks = []
listOfSubs = []
rerecoTask = None
for primeTask in workload.taskIterator():
# There should only be one prime task, and it should be the rerecoTask
rerecoTask = primeTask
for task in primeTask.taskIterator():
listOfTasks.append(task)
for task in listOfTasks:
fileset = self.getFileset()
sub = self.createSubscriptions(task = task,
fileset = fileset)
#listOfSubs.append(sub)
return
def createSubscriptions(self, task, fileset):
"""
Create a subscription based on a task
"""
type = task.taskType()
work = task.makeWorkflow()
sub = Subscription(fileset = fileset,
workflow = work,
split_algo = "FileBased",
type = type)
package = self.createWMBSJobs(subscription = sub,
task = task)
packName = os.path.join(self.testDir, 'packages',
'%sJobPackage.pkl' %(task.name()))
package.save(packName)
return sub
def createWMBSJobs(self, subscription, task):
"""
Create the jobs for WMBS Components
Send a subscription/task, get back a package.
"""
splitter = SplitterFactory()
geneFac = GeneratorFactory()
jobfactory = splitter(subscription = subscription,
package = "WMCore.DataStructs",
generators = geneFac.makeGenerators(task))
params = task.jobSplittingParameters()
jobGroups = jobfactory(**params)
jobID = 1
package = JobPackage()
for group in jobGroups:
for job in group.jobs:
job['id'] = jobID
jobID += 1
package[job['id']] = job
return package
def getFileset(self):
"""
Get a fileset based on the task
"""
fileset = Fileset(name = 'Merge%s' %(type))
for i in range(0, random.randint(15,25)):
# Use the testDir to generate a random lfn
inpFile = File(lfn = "%s/%s.root" %(self.testDir, makeUUID()),
size = random.randint(200000, 1000000),
events = random.randint(1000,2000) )
inpFile.setLocation('Megiddo')
fileset.addFile(inpFile)
return fileset
def runJobs(self, workload):
"""
This might actually run the job. Who knows?
"""
listOfTasks = []
for primeTask in workload.taskIterator():
listOfTasks.append(primeTask)
# Only run primeTasks for now
for task in listOfTasks:
jobName = task.name()
taskDir = os.path.join(self.unpackDir, jobName, 'job')
os.chdir(taskDir)
sys.path.append(taskDir)
# Scream, run around in panic, blow up machine
print "About to run jobs"
print taskDir
miniStartup(dir = taskDir)
# When exiting, go back to where you started
os.chdir(self.initialDir)
sys.path.remove(taskDir)
return
@attr('integration')
def testA_CreateWorkload(self):
"""
_CreateWorkload_
Create a workload
Unpack the workload
Check for consistency
"""
workloadName = 'basicWorkload'
workload = self.createTestWorkload(workloadName = workloadName)
self.createWMBSComponents(workload = workload)
taskNames = []
for task in getListOfTasks(workload = workload):
taskNames.append(task.name())
workloadPath = os.path.join(self.testDir, workloadName, "TestWorkload")
siteConfigDir = os.path.join(self.testDir, workloadName, 'SITECONF/local/JobConfig/')
# Pre-run checks
# Does it have the right directories?
dirList = os.listdir(workloadPath)
self.assertEqual(dirList, ['WMSandbox', 'TestWorkload-Sandbox.tar.bz2'])
dirList = os.listdir(os.path.join(workloadPath, 'WMSandbox'))
for taskName in taskNames:
self.assertTrue(taskName in dirList)
# Do we have job packages
for task in taskNames:
self.assertTrue('%sJobPackage.pkl' % (task) in os.listdir(os.path.join(self.testDir, 'packages')))
# Does it have the SITECONF?
self.assertTrue('site-local-config.xml' in os.listdir(siteConfigDir))
# Now actually see if you can unpack it.
self.unpackComponents(workload = workload)
# Check for proper unpacking
# Check the the task has the right directories,
# and that the PSetTweaks and WMSandbox directories
# have the right contents
taskContents = ['WMSandbox', 'WMCore', 'PSetTweaks']
PSetContents = ['PSetTweak.pyc', 'CVS', 'PSetTweak.py',
'__init__.pyc', 'WMTweak.py', '__init__.py']
taskSandbox = ['JobPackage.pcl', 'JobIndex.py', '__init__.py', 'WMWorkload.pkl']
taskSandbox.extend(taskNames) # Should have a directory for each task
for task in taskNames:
self.assertTrue(task in os.listdir(os.path.join(self.testDir, 'unpack')))
taskDir = os.path.join(self.testDir, 'unpack', task, 'job')
self.assertTrue(os.path.isdir(taskDir))
self.assertEqual(os.listdir(taskDir).sort(), taskContents.sort())
self.assertEqual(os.listdir(os.path.join(taskDir, 'WMSandbox')).sort(),
taskSandbox.sort())
self.assertEqual(os.listdir(os.path.join(taskDir, 'PSetTweaks')).sort(),
PSetContents.sort())
# And we're done.
# Assume if we got this far everything is good
# At the end, copy the directory
#if os.path.exists('tmpDir'):
# shutil.rmtree('tmpDir')
#shutil.copytree(self.testDir, 'tmpDir')
return
@attr('integration')
def testB_EmulatorTest(self):
"""
_EmulatorTest_
This is where things get scary. We need to not only unpack the job,
but also ascertain whether it can run locally in emulator mode.
This requires...uh...emulator emulation.
"""
# Assume all this works, because we tested it in testA
workloadName = 'basicWorkload'
workload = self.createTestWorkload(workloadName = workloadName)
self.createWMBSComponents(workload = workload)
self.unpackComponents(workload = workload)
self.runJobs(workload = workload)
# Check the report
taskDir = os.path.join(self.testDir, 'unpack/ReReco/job/WMTaskSpace')
report = Report()
report.load(os.path.join(taskDir, 'Report.0.pkl'))
cmsReport = report.data.cmsRun1
# Now validate the report
self.assertEqual(report.data.ceName, socket.gethostname())
self.assertEqual(report.data.seName, 'cmssrm.fnal.gov')
self.assertEqual(report.data.siteName, 'T1_US_FNAL')
self.assertEqual(report.data.hostName, socket.gethostname())
self.assertTrue(report.data.completed)
# Should have status 0 (emulator job)
self.assertEqual(cmsReport.status, 0)
# Should have one output module
self.assertEqual(cmsReport.outputModules, ['TestOutputModule'])
# It should have one file for input and output
self.assertEqual(cmsReport.input.PoolSource.files.fileCount, 1)
self.assertEqual(cmsReport.output.TestOutputModule.files.fileCount, 1)
# So, um, I guess we're done
# At the end, copy the directory
#if os.path.exists('tmpDir'):
# shutil.rmtree('tmpDir')
#shutil.copytree(self.testDir, 'tmpDir')
return
class ResourceControlTest(EmulatedUnitTestCase):
def setUp(self):
"""
_setUp_
Install schema and create a DAO factory for WMBS.
"""
super(ResourceControlTest, self).setUp()
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=["WMCore.WMBS",
"WMCore.ResourceControl",
"WMCore.BossAir"],
useDefault=False)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.baDaoFactory = DAOFactory(package="WMCore.BossAir",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.insertRunJob = self.baDaoFactory(classname="NewJobs")
self.insertState = self.baDaoFactory(classname="NewState")
states = ['PEND', 'RUN', 'Idle', 'Running']
self.insertState.execute(states)
self.tempDir = self.testInit.generateWorkDir()
return
def tearDown(self):
"""
_tearDown_
Clear the schema.
"""
super(ResourceControlTest, self).tearDown()
self.testInit.clearDatabase()
return
def createJobs(self):
"""
_createJobs_
Create test jobs in WMBS and BossAir
"""
testWorkflow = Workflow(spec=makeUUID(), owner="tapas",
name=makeUUID(), task="Test")
testWorkflow.create()
testFilesetA = Fileset(name="TestFilesetA")
testFilesetA.create()
testFilesetB = Fileset(name="TestFilesetB")
testFilesetB.create()
testFilesetC = Fileset(name="TestFilesetC")
testFilesetC.create()
testFileA = File(lfn="testFileA", locations=set(["testSE1", "testSE2"]))
testFileA.create()
testFilesetA.addFile(testFileA)
testFilesetA.commit()
testFilesetB.addFile(testFileA)
testFilesetB.commit()
testFilesetC.addFile(testFileA)
testFilesetC.commit()
testSubscriptionA = Subscription(fileset=testFilesetA,
workflow=testWorkflow,
type="Processing")
testSubscriptionA.create()
testSubscriptionA.addWhiteBlackList([{"site_name": "testSite1", "valid": True}])
testSubscriptionB = Subscription(fileset=testFilesetB,
workflow=testWorkflow,
type="Processing")
testSubscriptionB.create()
testSubscriptionB.addWhiteBlackList([{"site_name": "testSite1", "valid": False}])
testSubscriptionC = Subscription(fileset=testFilesetC,
workflow=testWorkflow,
type="Merge")
testSubscriptionC.create()
testJobGroupA = JobGroup(subscription=testSubscriptionA)
testJobGroupA.create()
testJobGroupB = JobGroup(subscription=testSubscriptionB)
testJobGroupB.create()
testJobGroupC = JobGroup(subscription=testSubscriptionC)
testJobGroupC.create()
# Site1, Has been assigned a location and is complete.
testJobA = Job(name="testJobA", files=[testFileA])
testJobA["couch_record"] = makeUUID()
testJobA.create(group=testJobGroupA)
testJobA["state"] = "success"
# Site 1, Has been assigned a location and is incomplete.
testJobB = Job(name="testJobB", files=[testFileA])
testJobB["couch_record"] = makeUUID()
testJobB["cache_dir"] = self.tempDir
testJobB.create(group=testJobGroupA)
testJobB["state"] = "executing"
runJobB = RunJob()
runJobB.buildFromJob(testJobB)
runJobB["status"] = "PEND"
# Does not have a location, white listed to site 1
testJobC = Job(name="testJobC", files=[testFileA])
testJobC["couch_record"] = makeUUID()
testJobC.create(group=testJobGroupA)
testJobC["state"] = "new"
# Site 2, Has been assigned a location and is complete.
testJobD = Job(name="testJobD", files=[testFileA])
testJobD["couch_record"] = makeUUID()
testJobD.create(group=testJobGroupB)
testJobD["state"] = "success"
# Site 2, Has been assigned a location and is incomplete.
testJobE = Job(name="testJobE", files=[testFileA])
testJobE["couch_record"] = makeUUID()
testJobE.create(group=testJobGroupB)
testJobE["state"] = "executing"
runJobE = RunJob()
runJobE.buildFromJob(testJobE)
runJobE["status"] = "RUN"
# Does not have a location, site 1 is blacklisted.
testJobF = Job(name="testJobF", files=[testFileA])
testJobF["couch_record"] = makeUUID()
testJobF.create(group=testJobGroupB)
testJobF["state"] = "new"
# Site 3, Has been assigned a location and is complete.
testJobG = Job(name="testJobG", files=[testFileA])
testJobG["couch_record"] = makeUUID()
testJobG.create(group=testJobGroupC)
testJobG["state"] = "cleanout"
# Site 3, Has been assigned a location and is incomplete.
testJobH = Job(name="testJobH", files=[testFileA])
testJobH["couch_record"] = makeUUID()
testJobH.create(group=testJobGroupC)
testJobH["state"] = "new"
# Site 3, Does not have a location.
testJobI = Job(name="testJobI", files=[testFileA])
testJobI["couch_record"] = makeUUID()
testJobI.create(group=testJobGroupC)
testJobI["state"] = "new"
# Site 3, Does not have a location and is in cleanout.
testJobJ = Job(name="testJobJ", files=[testFileA])
testJobJ["couch_record"] = makeUUID()
testJobJ.create(group=testJobGroupC)
testJobJ["state"] = "cleanout"
changeStateAction = self.daoFactory(classname="Jobs.ChangeState")
changeStateAction.execute(jobs=[testJobA, testJobB, testJobC, testJobD,
testJobE, testJobF, testJobG, testJobH,
testJobI, testJobJ])
self.insertRunJob.execute([runJobB, runJobE])
setLocationAction = self.daoFactory(classname="Jobs.SetLocation")
setLocationAction.execute(testJobA["id"], "testSite1")
setLocationAction.execute(testJobB["id"], "testSite1")
setLocationAction.execute(testJobD["id"], "testSite1")
setLocationAction.execute(testJobE["id"], "testSite2")
setLocationAction.execute(testJobG["id"], "testSite1")
setLocationAction.execute(testJobH["id"], "testSite1")
return
def testInsert(self):
"""
_testInsert_
Verify that inserting sites and thresholds works correctly, even if the
site or threshold already exists.
"""
myResourceControl = ResourceControl()
myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1")
myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1")
myResourceControl.insertSite("testSite2", 100, 200, "testSE2", "testCE2")
myResourceControl.insertThreshold("testSite1", "Processing", 20, 10)
myResourceControl.insertThreshold("testSite1", "Merge", 200, 100)
myResourceControl.insertThreshold("testSite1", "Merge", 250, 150)
myResourceControl.insertThreshold("testSite2", "Processing", 50, 30)
myResourceControl.insertThreshold("testSite2", "Merge", 135, 100)
createThresholds = myResourceControl.listThresholdsForCreate()
self.assertEqual(len(createThresholds.keys()), 2,
"Error: Wrong number of site in Resource Control DB")
self.assertTrue("testSite1" in createThresholds.keys(),
"Error: Test Site 1 missing from thresholds.")
self.assertTrue("testSite2" in createThresholds.keys(),
"Error: Test Site 2 missing from thresholds.")
self.assertEqual(createThresholds["testSite1"]["total_slots"], 10,
"Error: Wrong number of total slots.")
self.assertEqual(createThresholds["testSite1"]["pending_jobs"], {0: 0},
"Error: Wrong number of running jobs: %s" %
createThresholds["testSite1"]["pending_jobs"])
self.assertEqual(createThresholds["testSite2"]["total_slots"], 100,
"Error: Wrong number of total slots.")
self.assertEqual(createThresholds["testSite2"]["pending_jobs"], {0: 0},
"Error: Wrong number of running jobs.")
thresholds = myResourceControl.listThresholdsForSubmit()
self.assertEqual(len(thresholds.keys()), 2,
"Error: Wrong number of sites in Resource Control DB")
self.assertTrue("testSite1" in thresholds.keys(),
"Error: testSite1 missing from thresholds.")
self.assertTrue("testSite2" in thresholds.keys(),
"Error: testSite2 missing from thresholds.")
site1Info = thresholds["testSite1"]
site2Info = thresholds["testSite2"]
site1Thresholds = site1Info["thresholds"]
site2Thresholds = site2Info["thresholds"]
procThreshold1 = None
procThreshold2 = None
mergeThreshold1 = None
mergeThreshold2 = None
for taskType, threshold in site1Thresholds.items():
if taskType == "Merge":
mergeThreshold1 = threshold
elif taskType == "Processing":
procThreshold1 = threshold
for taskType, threshold in site2Thresholds.items():
if taskType == "Merge":
mergeThreshold2 = threshold
elif taskType == "Processing":
procThreshold2 = threshold
self.assertEqual(len(site1Thresholds), 2,
"Error: Wrong number of task types.")
self.assertEqual(len(site2Thresholds), 2,
"Error: Wrong number of task types.")
self.assertNotEqual(procThreshold1, None)
self.assertNotEqual(procThreshold2, None)
self.assertNotEqual(mergeThreshold1, None)
self.assertNotEqual(mergeThreshold2, None)
self.assertEqual(site1Info["total_pending_slots"], 10,
"Error: Site thresholds wrong")
self.assertEqual(site1Info["total_running_slots"], 20,
"Error: Site thresholds wrong")
self.assertEqual(site1Info["total_running_jobs"], 0,
"Error: Site thresholds wrong")
self.assertEqual(site1Info["total_pending_jobs"], 0,
"Error: Site thresholds wrong")
self.assertEqual(procThreshold1["task_running_jobs"], 0,
"Error: Site thresholds wrong")
self.assertEqual(procThreshold1["task_pending_jobs"], 0,
"Error: Site thresholds wrong")
self.assertEqual(procThreshold1["max_slots"], 20,
"Error: Site thresholds wrong")
self.assertEqual(procThreshold1["pending_slots"], 10,
"Error: Site thresholds wrong")
self.assertEqual(mergeThreshold1["task_running_jobs"], 0,
"Error: Site thresholds wrong")
self.assertEqual(mergeThreshold1["task_pending_jobs"], 0,
"Error: Site thresholds wrong")
self.assertEqual(mergeThreshold1["max_slots"], 250,
"Error: Site thresholds wrong")
self.assertEqual(mergeThreshold1["pending_slots"], 150,
"Error: Site thresholds wrong")
self.assertEqual(site2Info["total_pending_slots"], 100,
"Error: Site thresholds wrong")
self.assertEqual(site2Info["total_running_slots"], 200,
"Error: Site thresholds wrong")
self.assertEqual(site2Info["total_running_jobs"], 0,
"Error: Site thresholds wrong")
self.assertEqual(site2Info["total_pending_jobs"], 0,
"Error: Site thresholds wrong")
self.assertEqual(procThreshold2["task_running_jobs"], 0,
"Error: Site thresholds wrong")
self.assertEqual(procThreshold2["task_pending_jobs"], 0,
"Error: Site thresholds wrong")
self.assertEqual(procThreshold2["max_slots"], 50,
"Error: Site thresholds wrong")
self.assertEqual(procThreshold2["pending_slots"], 30,
"Error: Site thresholds wrong")
self.assertEqual(mergeThreshold2["task_running_jobs"], 0,
"Error: Site thresholds wrong")
self.assertEqual(mergeThreshold2["task_pending_jobs"], 0,
"Error: Site thresholds wrong")
self.assertEqual(mergeThreshold2["max_slots"], 135,
"Error: Site thresholds wrong")
self.assertEqual(mergeThreshold2["pending_slots"], 100,
"Error: Site thresholds wrong")
def testList(self):
"""
_testList_
Test the functions that list thresholds for creating jobs and submitting
jobs.
"""
myResourceControl = ResourceControl()
myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1", "T1_US_FNAL", "LsfPlugin")
myResourceControl.insertSite("testSite2", 20, 40, "testSE2", "testCE2", "T3_US_FNAL", "LsfPlugin")
myResourceControl.insertThreshold("testSite1", "Processing", 20, 10)
myResourceControl.insertThreshold("testSite1", "Merge", 200, 100)
myResourceControl.insertThreshold("testSite2", "Processing", 50, 25)
myResourceControl.insertThreshold("testSite2", "Merge", 135, 65)
self.createJobs()
createThresholds = myResourceControl.listThresholdsForCreate()
submitThresholds = myResourceControl.listThresholdsForSubmit()
self.assertEqual(len(createThresholds.keys()), 2,
"Error: Wrong number of sites in create thresholds")
self.assertEqual(createThresholds["testSite1"]["total_slots"], 10,
"Error: Wrong number of slots for site 1")
self.assertEqual(createThresholds["testSite2"]["total_slots"], 20,
"Error: Wrong number of slots for site 2")
# We should have two running jobs with locations at site one,
# two running jobs without locations at site two, and one running
# job without a location at site one and two.
self.assertEqual(createThresholds["testSite1"]["pending_jobs"], {0: 4},
"Error: Wrong number of pending jobs for site 1")
# We should have one running job with a location at site 2 and
# another running job without a location.
self.assertEqual(createThresholds["testSite2"]["pending_jobs"], {0: 2},
"Error: Wrong number of pending jobs for site 2")
# We should also have a phedex_name
self.assertEqual(createThresholds["testSite1"]["cms_name"], "T1_US_FNAL")
self.assertEqual(createThresholds["testSite2"]["cms_name"], "T3_US_FNAL")
mergeThreshold1 = None
mergeThreshold2 = None
procThreshold1 = None
procThreshold2 = None
self.assertEqual(set(submitThresholds.keys()), set(["testSite1", "testSite2"]))
for taskType, threshold in submitThresholds["testSite1"]["thresholds"].items():
if taskType == "Merge":
mergeThreshold1 = threshold
elif taskType == "Processing":
procThreshold1 = threshold
for taskType, threshold in submitThresholds["testSite2"]["thresholds"].items():
if taskType == "Merge":
mergeThreshold2 = threshold
elif taskType == "Processing":
procThreshold2 = threshold
self.assertEqual(submitThresholds["testSite1"]["total_running_jobs"], 0,
"Error: Wrong number of running jobs for submit thresholds.")
self.assertEqual(submitThresholds["testSite2"]["total_running_jobs"], 1,
"Error: Wrong number of running jobs for submit thresholds.")
self.assertEqual(submitThresholds["testSite1"]["total_pending_jobs"], 1,
"Error: Wrong number of pending jobs for submit thresholds.")
self.assertEqual(submitThresholds["testSite2"]["total_pending_jobs"], 0,
"Error: Wrong number of pending jobs for submit thresholds.")
self.assertEqual(mergeThreshold1["task_running_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(mergeThreshold1["task_pending_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(procThreshold1["task_running_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(procThreshold1["task_pending_jobs"], 1,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(mergeThreshold2["task_running_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(mergeThreshold2["task_pending_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(procThreshold2["task_running_jobs"], 1,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(procThreshold2["task_pending_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
return
def testListSiteInfo(self):
"""
_testListSiteInfo_
Verify that the listSiteInfo() methods works properly.
"""
myResourceControl = ResourceControl()
myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1")
myResourceControl.insertSite("testSite2", 100, 200, "testSE2", "testCE2")
siteInfo = myResourceControl.listSiteInfo("testSite1")
self.assertEqual(siteInfo["site_name"], "testSite1",
"Error: Site name is wrong.")
self.assertEqual(siteInfo["pnn"], ["testSE1"],
"Error: SE name is wrong.")
self.assertEqual(siteInfo["ce_name"], "testCE1",
"Error: CE name is wrong.")
self.assertEqual(siteInfo["pending_slots"], 10,
"Error: Pending slots is wrong.")
self.assertEqual(siteInfo["running_slots"], 20,
"Error: Pending slots is wrong.")
return
def testUpdateJobSlots(self):
"""
_testUpdateJobSlots_
Verify that it is possible to update the number of job slots at a site.
"""
myResourceControl = ResourceControl()
myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1")
siteInfo = myResourceControl.listSiteInfo("testSite1")
self.assertEqual(siteInfo["pending_slots"], 10, "Error: Pending slots is wrong.")
self.assertEqual(siteInfo["running_slots"], 20, "Error: Running slots is wrong.")
myResourceControl.setJobSlotsForSite("testSite1", pendingJobSlots=20)
siteInfo = myResourceControl.listSiteInfo("testSite1")
self.assertEqual(siteInfo["pending_slots"], 20, "Error: Pending slots is wrong.")
myResourceControl.setJobSlotsForSite("testSite1", runningJobSlots=40)
siteInfo = myResourceControl.listSiteInfo("testSite1")
self.assertEqual(siteInfo["running_slots"], 40, "Error: Running slots is wrong.")
myResourceControl.setJobSlotsForSite("testSite1", 5, 10)
siteInfo = myResourceControl.listSiteInfo("testSite1")
self.assertEqual(siteInfo["pending_slots"], 5, "Error: Pending slots is wrong.")
self.assertEqual(siteInfo["running_slots"], 10, "Error: Running slots is wrong.")
return
def testThresholdsForSite(self):
"""
_testThresholdsForSite_
Check that we can get the thresholds in intelligible form
for each site
"""
myResourceControl = ResourceControl()
myResourceControl.insertSite("testSite1", 20, 40, "testSE1", "testCE1")
myResourceControl.insertThreshold("testSite1", "Processing", 10, 8)
myResourceControl.insertThreshold("testSite1", "Merge", 5, 3)
result = myResourceControl.thresholdBySite(siteName="testSite1")
procInfo = {}
mergInfo = {}
for res in result:
if res['task_type'] == 'Processing':
procInfo = res
elif res['task_type'] == 'Merge':
mergInfo = res
self.assertEqual(procInfo.get('pending_slots', None), 20)
self.assertEqual(procInfo.get('running_slots', None), 40)
self.assertEqual(procInfo.get('max_slots', None), 10)
self.assertEqual(procInfo.get('task_pending_slots', None), 8)
self.assertEqual(mergInfo.get('pending_slots', None), 20)
self.assertEqual(mergInfo.get('running_slots', None), 40)
self.assertEqual(mergInfo.get('max_slots', None), 5)
self.assertEqual(mergInfo.get('task_pending_slots', None), 3)
return
def testThresholdPriority(self):
"""
_testThresholdPriority_
Test that we get things back in priority order
"""
myResourceControl = ResourceControl()
myResourceControl.insertSite("testSite1", 20, 40, "testSE1", "testCE1")
myResourceControl.insertThreshold("testSite1", "Processing", 10, 8)
myResourceControl.insertThreshold("testSite1", "Merge", 5, 3)
# test default task priorities
result = myResourceControl.listThresholdsForSubmit()
self.assertEqual(result['testSite1']['thresholds']['Merge']['priority'], 4)
self.assertEqual(result['testSite1']['thresholds']['Processing']['priority'], 0)
myResourceControl.changeTaskPriority("Merge", 3)
myResourceControl.changeTaskPriority("Processing", 1)
result = myResourceControl.listThresholdsForSubmit()
self.assertEqual(result['testSite1']['thresholds']['Merge']['priority'], 3)
self.assertEqual(result['testSite1']['thresholds']['Processing']['priority'], 1)
myResourceControl.changeTaskPriority("Merge", 1)
myResourceControl.changeTaskPriority("Processing", 3)
result = myResourceControl.listThresholdsForSubmit()
self.assertEqual(result['testSite1']['thresholds']['Merge']['priority'], 1)
self.assertEqual(result['testSite1']['thresholds']['Processing']['priority'], 3)
return
def testChangeSiteState(self):
"""
_testNewState_
Check that we can change the state between different values and
retrieve it through the threshold methods
"""
self.tempDir = self.testInit.generateWorkDir()
config = self.createConfig()
myResourceControl = ResourceControl(config)
myResourceControl.insertSite("testSite1", 20, 40, "testSE1", "testCE1")
myResourceControl.insertThreshold("testSite1", "Processing", 10, 5)
result = myResourceControl.listThresholdsForCreate()
self.assertEqual(result['testSite1']['state'], 'Normal', 'Error: Wrong site state')
myResourceControl.changeSiteState("testSite1", "Down")
result = myResourceControl.listThresholdsForCreate()
self.assertEqual(result['testSite1']['state'], 'Down', 'Error: Wrong site state')
# If you set the value to 'Normal' instead of 'Down' this test should FAIL
# self.assertEqual(result['testSite1']['state'], 'Normal', 'Error: Wrong site state')
def testAbortedState(self):
"""
_testAbortedState_
Check that we can kill jobs when a site is set to aborted
### We no longer need this test as we are not killing jobs that are running
"""
self.tempDir = self.testInit.generateWorkDir()
config = self.createConfig()
myResourceControl = ResourceControl(config)
myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1", "T1_US_FNAL", "MockPlugin")
myResourceControl.insertSite("testSite2", 20, 40, "testSE2", "testCE2", "T1_IT_CNAF", "MockPlugin")
myResourceControl.insertThreshold("testSite1", "Processing", 20, 10)
myResourceControl.insertThreshold("testSite1", "Merge", 200, 100)
myResourceControl.insertThreshold("testSite2", "Processing", 50, 25)
myResourceControl.insertThreshold("testSite2", "Merge", 135, 65)
self.createJobs()
myResourceControl.changeSiteState("testSite1", "Aborted")
## Now check the tempDir for a FWJR for the killed job
reportPath = os.path.join(self.tempDir, "Report.0.pkl")
report = Report()
report.load(reportPath)
self.assertEqual(report.getExitCode(), 71301)
return
def createConfig(self):
"""
_createConfig_
Create a config and save it to the temp dir. Set the WMAGENT_CONFIG
environment variable so the config gets picked up.
"""
config = Configuration()
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = "resource_control_t"
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = os.getenv('COUCHURL')
config.JobStateMachine.couchDBName = "bossair_t"
config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t'
config.JobStateMachine.summaryStatsDBName = 'stat_summary_t'
config.section_("BossAir")
config.BossAir.pluginDir = "WMCore.BossAir.Plugins"
config.BossAir.pluginNames = ["MockPlugin"]
config.BossAir.section_("MockPlugin")
config.BossAir.MockPlugin.fakeReport = os.path.join(getTestBase(),
'WMComponent_t/JobAccountant_t/fwjrs',
"MergeSuccess.pkl")
configHandle = open(os.path.join(self.tempDir, "config.py"), "w")
configHandle.write(str(config))
configHandle.close()
os.environ["WMAGENT_CONFIG"] = os.path.join(self.tempDir, "config.py")
return config
def testInsertSite(self):
"""
_testInsertSite_
Test to see if we can insert a fake test site alone
with a single option
"""
self.createConfig()
resControlPath = os.path.join(getTestBase(), "../../bin/wmagent-resource-control")
env = os.environ
env['PYTHONPATH'] = ":".join(sys.path)
cmdline = [resControlPath, "--add-Test"]
retval = subprocess.Popen(cmdline,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env)
(_, _) = retval.communicate()
myResourceControl = ResourceControl()
result = myResourceControl.listThresholdsForSubmit()
self.assertEqual(len(result), 1)
self.assertTrue('CERN' in result)
for x in result:
self.assertEqual(len(result[x]['thresholds']), 7)
self.assertEqual(result[x]['total_pending_slots'], 500)
self.assertEqual(result[x]['total_running_slots'], 1)
for taskType, thresh in result[x]['thresholds'].items():
if taskType == 'Processing':
self.assertEqual(thresh['priority'], 0)
self.assertEqual(thresh['max_slots'], 1)
# Verify that sites with more than one SE were added correctly.
cernInfo = myResourceControl.listSiteInfo("CERN")
self.assertTrue(len(cernInfo["pnn"]) == 2)
return
class scaleTestFiller:
"""
_scaleTestFiller_
Initializes the DB and the DBSUploader
On __call__() it creates data and uploads it.
"""
def __init__(self):
"""
__init__
Init the DB
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase = True)
self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer"],
useDefault = False)
self.configFile = EmulatorSetup.setupWMAgentConfig()
myThread = threading.currentThread()
self.bufferFactory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = self.bufferFactory(classname = "DBSBufferFiles.AddLocation")
locationAction.execute(siteName = "se1.cern.ch")
locationAction.execute(siteName = "se1.fnal.gov")
locationAction.execute(siteName = "malpaquet")
config = self.getConfig()
self.dbsUploader = DBSUploadPoller(config = config)
return
def __call__(self):
"""
__call__
Generate some random data
"""
# Generate somewhere between one and a thousand files
name = "ThisIsATest_%s" % (makeUUID())
nFiles = random.randint(10, 2000)
name = name.replace('-', '_')
name = '%s-v0' % name
files = self.getFiles(name = name, nFiles = nFiles)
print("Inserting %i files for dataset %s" % (nFiles * 2, name))
try:
self.dbsUploader.algorithm()
except:
self.dbsUploader.close()
raise
# Repeat just to make sure
try:
self.dbsUploader.algorithm()
except:
self.dbsUploader.close()
raise
return
def getConfig(self):
"""
_getConfig_
This creates the actual config file used by the component
"""
config = self.testInit.getConfiguration()
self.testInit.generateWorkDir(config)
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = 'DBSUpload'
config.Agent.useHeartbeat = False
#Now the CoreDatabase information
#This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("DBS3Upload")
config.DBS3Upload.pollInterval = 10
config.DBS3Upload.logLevel = 'DEBUG'
config.DBS3Upload.DBSBlockMaxFiles = 500
config.DBS3Upload.DBSBlockMaxTime = 600
config.DBS3Upload.DBSBlockMaxSize = 999999999999
config.DBS3Upload.dbsUrl = 'http://cms-xen40.fnal.gov:8787/dbs/prod/global/DBSWriter'
config.DBS3Upload.namespace = 'WMComponent.DBS3Buffer.DBS3Upload'
config.DBS3Upload.componentDir = os.path.join(os.getcwd(), 'Components')
config.DBS3Upload.nProcesses = 1
config.DBS3Upload.dbsWaitTime = 1
return config
def getFiles(self, name, tier = 'RECO', nFiles = 12, site = "malpaquet", nLumis = 1):
"""
Create some quick dummy test files
"""
files = []
for f in range(nFiles):
testFile = DBSBufferFile(lfn = '/data/store/random/random/RANDOM/test/0/%s-%s-%i.root' % (name, site, f), size = 1024,
events = 20, checksums = {'cksum': 1})
testFile.setAlgorithm(appName = name, appVer = "CMSSW_3_1_1",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier))
lumis = []
for i in range(nLumis):
lumis.append((f * 100000) + i)
testFile.addRun(Run( 1, *lumis))
testFile.setAcquisitionEra(name.split('-')[0])
testFile.setProcessingVer("0")
testFile.setGlobalTag("Weird")
testFile.create()
testFile.setLocation(site)
files.append(testFile)
count = 0
for f in files:
count += 1
testFileChild = DBSBufferFile(lfn = '/data/store/random/random/RANDOM/test/0/%s-%s-%i-child.root' %(name, site, count), size = 1024,
events = 10, checksums = {'cksum': 1})
testFileChild.setAlgorithm(appName = name, appVer = "CMSSW_3_1_1",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileChild.setDatasetPath("/%s/%s_2/RECO" %(name, name))
testFileChild.addRun(Run( 1, *[45]))
testFileChild.create()
testFileChild.setLocation(site)
testFileChild.addParents([f['lfn']])
return files
class testJSONRequests(unittest.TestCase):
def setUp(self):
self.testInit = TestInit(__file__)
self.testInit.setLogging()
tmp = self.testInit.generateWorkDir()
self.request = Requests.JSONRequests(idict={'req_cache_path' : tmp})
def roundTrip(self, data):
encoded = self.request.encode(data)
#print encoded
#print encoded.__class__.__name__
decoded = self.request.decode(encoded)
#print decoded.__class__.__name__
self.assertEqual( data, decoded )
def roundTripLax(self, data):
encoded = self.request.encode(data)
decoded = self.request.decode(encoded)
datakeys = data.keys()
for k in decoded.keys():
assert k in datakeys
datakeys.pop(datakeys.index(k))
#print 'the following keys were dropped\n\t',datakeys
def testSet1(self):
self.roundTrip(set([]))
def testSet2(self):
self.roundTrip(set([1, 2, 3, 4, Run(1)]))
def testSet3(self):
self.roundTrip(set(['a', 'b', 'c', 'd']))
def testSet4(self):
self.roundTrip(set([1, 2, 3, 4, 'a', 'b']))
def testRun1(self):
self.roundTrip(Run(1))
def testRun2(self):
self.roundTrip(Run(1, 1))
def testRun3(self):
self.roundTrip(Run(1, 2, 3))
def testMask1(self):
self.roundTrip(Mask())
def testMask2(self):
mymask = Mask()
mymask['FirstEvent'] = 9999
mymask['LastEvent'] = 999
self.roundTrip(mymask)
def testMask3(self):
mymask = Mask()
mymask['FirstEvent'] = 9999
mymask['LastEvent'] = 999
myjob = DataStructsJob()
myjob["mask"] = mymask
self.roundTrip(myjob)
def testMask4(self):
self.roundTrip({'LastRun': None, 'FirstRun': None, 'LastEvent': None,
'FirstEvent': None, 'LastLumi': None, 'FirstLumi': None})
def testMask5(self):
mymask = Mask()
mymask['FirstEvent'] = 9999
mymask['LastEvent'] = 999
myjob = DataStructsJob()
myjob["mask"] = mymask
self.roundTripLax(myjob)
def testMask6(self):
mymask = Mask()
myjob = DataStructsJob()
myjob["mask"] = mymask
self.roundTripLax(myjob)
def testSpecialCharacterPasswords(self):
url = 'http://*****:*****@ssw:rd@localhost:6666'
req = JSONRequests(url)
self.assertEqual(req['host'], 'http://localhost:6666')
self.assertEqual(req.additionalHeaders['Authorization'], 'Basic dXNlcm5hbWU6cEBzc3c6cmQ=')
class HarvestTest(unittest.TestCase):
"""
_HarvestTest_
Test for EndOfRun job splitter
"""
def setUp(self):
"""
_setUp_
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=["WMCore.WMBS"])
self.splitterFactory = SplitterFactory(package="WMCore.JobSplitting")
myThread = threading.currentThread()
self.myThread = myThread
daoFactory = DAOFactory(package="WMCore.WMBS",
logger=logging,
dbinterface=myThread.dbi)
self.WMBSFactory = daoFactory
config = self.getConfig()
self.changer = ChangeState(config)
myResourceControl = ResourceControl()
myResourceControl.insertSite("T1_US_FNAL", 10, 20, "T1_US_FNAL_Disk", "T1_US_FNAL")
myResourceControl.insertSite("T1_US_FNAL", 10, 20, "T3_US_FNALLPC", "T1_US_FNAL")
myResourceControl.insertSite("T2_CH_CERN", 10, 20, "T2_CH_CERN", "T2_CH_CERN")
self.fileset1 = Fileset(name="TestFileset1")
for fileNum in range(11):
newFile = File("/some/file/name%d" % fileNum, size=1000, events=100)
newFile.addRun(Run(1, *[1]))
newFile.setLocation('T1_US_FNAL_Disk')
self.fileset1.addFile(newFile)
self.fileset1.create()
workflow1 = Workflow(spec="spec.xml", owner="hufnagel", name="TestWorkflow1", task="Test")
workflow1.create()
self.subscription1 = Subscription(fileset=self.fileset1,
workflow=workflow1,
split_algo="Harvest",
type="Harvesting")
self.subscription1.create()
self.configFile = EmulatorSetup.setupWMAgentConfig()
return
def tearDown(self):
"""
_tearDown_
"""
self.testInit.clearDatabase()
EmulatorSetup.deleteConfig(self.configFile)
return
def getConfig(self):
"""
_getConfig_
"""
config = self.testInit.getConfiguration()
self.testInit.generateWorkDir(config)
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', None)
config.JobStateMachine.couchDBName = 'wmagent_jobdump'
return config
def finishJobs(self, jobGroups, subscription=None):
"""
_finishJobs_
"""
if not subscription:
subscription = self.subscription1
for f in subscription.acquiredFiles():
subscription.completeFiles(f)
for jobGroup in jobGroups:
self.changer.propagate(jobGroup.jobs, 'executing', 'created')
self.changer.propagate(jobGroup.jobs, 'complete', 'executing')
self.changer.propagate(jobGroup.jobs, 'success', 'complete')
self.changer.propagate(jobGroup.jobs, 'cleanout', 'success')
return
def testHarvestEndOfRunTrigger(self):
"""
_testDQMHarvestEndOfRunTrigger_
Make sure that the basic splitting algo works, which is only, ExpressMerge is ALL done, fire a job against that fileset
"""
self.assertEqual(self.fileset1.open, True, "Fileset is closed. Shouldn't")
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
jobGroups = jobFactory()
self.assertEqual(len(jobGroups), 0,
"We got 1 or more jobGroups with an open fileset and no periodic configuration")
self.fileset1.markOpen(False)
self.assertEqual(self.fileset1.open, False, "Fileset is opened, why?")
# We should also check if there are aqcuired files, if there are, there are jobs,
# we don't want to fire another jobs while previous are running (output is integrating whatever input)
# TODO : The above one we can do when all is done. Not priority
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
jobGroups = jobFactory()
self.assertEqual(len(jobGroups), 1,
"Harvest jobsplitter didn't create a single jobGroup after the fileset was closed")
return
def testPeriodicTrigger(self):
"""
_testPeriodicTrigger_
"""
self.assertEqual(self.fileset1.open, True, "Fileset is not open, not testing periodic here")
# Test timeout (5s for this first test)
# there should be no acquired files, if there are, shouldn't be a job
# self.subscription1.acquireFiles(self.subscription1.availableFiles().pop())
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval=3)
self.assertEqual(len(jobGroups), 1, "Didn't created the first periodic job when there were acquired files")
# For the whole thing to work, faking the first job finishing, and putting the files as complete
self.finishJobs(jobGroups)
# Adding more of files, so we have new stuff to process
for fileNum in range(12, 24):
newFile = File("/some/file/name%d" % fileNum, size=1000, events=100)
newFile.addRun(Run(1, *[1]))
newFile.setLocation('T1_US_FNAL_Disk')
self.fileset1.addFile(newFile)
self.fileset1.commit()
# Testing that it doesn't create a job unless the delay is past
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval=2)
self.assertEqual(len(jobGroups), 0,
"Created one or more job, when there were non-acquired file and the period is not passed by")
time.sleep(2)
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval=2)
self.assertEqual(len(jobGroups), 1,
"Didn't created one or more job, and there weren't and the period is passed by")
# Finishing out previous jobs
self.finishJobs(jobGroups)
# Adding more of files, so we have new stuff to process
for fileNum in range(26, 36):
newFile = File("/some/file/name%d" % fileNum, size=1000, events=100)
newFile.addRun(Run(1, *[1]))
newFile.setLocation('T1_US_FNAL_Disk')
self.fileset1.addFile(newFile)
self.fileset1.commit()
# Trying to create another job just afterwards, it shouldn't, because it should respect the configured delay
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval=2)
self.assertEqual(len(jobGroups), 0, "Created one or more job, there are new files, but the delay is not past")
time.sleep(2)
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval=2)
self.assertEqual(len(jobGroups), 1, "Didn't created one or more job, there are new files and the delay is past")
# Last check is whether the job gets all the files or not
numFilesJob = jobGroups[0].jobs[0].getFiles()
numFilesFileset = self.fileset1.getFiles()
self.assertEqual(numFilesJob, numFilesFileset, "Job didn't got all the files")
# Finishing out previous jobs
self.finishJobs(jobGroups)
# Adding files for the first location
for fileNum in range(38, 48):
newFile = File("/some/file/name%d" % fileNum, size=1000, events=100)
newFile.addRun(Run(1, *[1]))
newFile.setLocation('T1_US_FNAL_Disk')
self.fileset1.addFile(newFile)
self.fileset1.commit()
# Then another location
for fileNum in range(50, 56):
newFile = File("/some/file/name%d" % fileNum, size=1000, events=100)
newFile.addRun(Run(1, *[1]))
newFile.setLocation('T2_CH_CERN')
self.fileset1.addFile(newFile)
self.fileset1.commit()
# We should have jobs in both locations
time.sleep(2)
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval=2)
self.assertEqual(len(jobGroups[0].getJobs()), 2, "We didn't get 2 jobs for 2 locations")
firstJobLocation = jobGroups[0].getJobs()[0].getFileLocations()[0]
secondJobLocation = jobGroups[0].getJobs()[1].getFileLocations()[0]
self.assertEqual(firstJobLocation, 'T2_CH_CERN')
self.assertEqual(secondJobLocation, 'T1_US_FNAL')
self.finishJobs(jobGroups)
for fileNum in range(60, 65):
newFile = File("/some/file/name%d" % fileNum, size=1000, events=100)
newFile.addRun(Run(2, *[2]))
newFile.setLocation('T2_CH_CERN')
self.fileset1.addFile(newFile)
self.fileset1.commit()
for fileNum in range(70, 75):
newFile = File("/some/file/name%d" % fileNum, size=1000, events=100)
newFile.addRun(Run(3, *[3]))
newFile.setLocation('T2_CH_CERN')
self.fileset1.addFile(newFile)
self.fileset1.commit()
time.sleep(2)
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval=2)
# This is one of the most "complicated" tests so worth to comment, 4 jobs should be created
# 1 - all previous files from SomeSE and run = 1 (a lot, like ~45)
# 2 - Few files from SomeSE3, Run = 1
# 3 - Few files from SomeSE3, Run = 2
# 4 - Few files from SomeSE3, Run = 3
self.assertEqual(len(jobGroups[0].getJobs()), 4, "We didn't get 4 jobs for adding 2 different runs to SomeSE3")
return
def testMultipleRunHarvesting(self):
"""
_testMultipleRunHarvesting_
Add some files with multiple runs in each, make sure the jobs
are created by location and run. Verify each job mask afterwards.
Note that in this test run are splitted between sites,
in real life that MUST NOT happen we still don't support that.
"""
multipleFilesFileset = Fileset(name="TestFileset")
newFile = File("/some/file/test1", size=1000, events=100)
newFile.addRun(Run(1, *[1, 3, 4, 5, 6, 7]))
newFile.addRun(Run(2, *[1, 2, 4, 5, 6, 7]))
newFile.setLocation('T1_US_FNAL_Disk')
multipleFilesFileset.addFile(newFile)
newFile = File("/some/file/test2", size=1000, events=100)
newFile.addRun(Run(1, *[2, 8]))
newFile.addRun(Run(2, *[3, 8]))
newFile.setLocation('T2_CH_CERN')
multipleFilesFileset.addFile(newFile)
multipleFilesFileset.create()
harvestingWorkflow = Workflow(spec="spec.xml",
owner="hufnagel",
name="TestWorkflow",
task="Test")
harvestingWorkflow.create()
harvestSub = Subscription(fileset=multipleFilesFileset,
workflow=harvestingWorkflow,
split_algo="Harvest",
type="Harvesting")
harvestSub.create()
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub)
jobGroups = jobFactory(periodic_harvest_interval=2)
self.assertEqual(len(jobGroups), 1, "A single job group was not created")
self.assertEqual(len(jobGroups[0].getJobs()), 4,
"Four jobs were not created")
for job in jobGroups[0].getJobs():
runs = job['mask'].getRunAndLumis()
self.assertEqual(len(runs), 1, "Job has more than one run configured")
ll = LumiList(compactList={1: [[1, 1], [3, 7], [2, 2], [8, 8]],
2: [[1, 2], [4, 7], [3, 3], [8, 8]]})
run = runs.keys()[0]
for lumiPair in runs[run]:
for lumi in range(lumiPair[0], lumiPair[1] + 1):
self.assertTrue((str(run), lumi) in ll, "All of %s not in %s" % (lumiPair, ll))
self.finishJobs(jobGroups, harvestSub)
newFile = File("/some/file/test3", size=1000, events=100)
newFile.addRun(Run(1, *range(9, 15)))
newFile.setLocation('T2_CH_CERN')
multipleFilesFileset.addFile(newFile)
multipleFilesFileset.commit()
time.sleep(2)
jobGroups = jobFactory(periodic_harvest_interval=2)
self.assertEqual(len(jobGroups), 1, "A single job group was not created")
self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created")
for job in jobGroups[0].getJobs():
runs = job['mask'].getRunAndLumis()
self.assertEqual(len(runs), 1, "Job has more than one run configured")
ll = LumiList(compactList={1: [[1, 1], [3, 7], [2, 2], [8, 8], [9, 14]],
2: [[1, 2], [4, 7], [3, 3], [8, 8]]})
run = runs.keys()[0]
for lumiPair in runs[run]:
for lumi in range(lumiPair[0], lumiPair[1] + 1):
self.assertTrue((run, lumi) in ll, "All of %s not in %s" % (lumiPair, ll))
harvestingWorkflowSib = Workflow(spec="spec.xml",
owner="hufnagel",
name="TestWorkflowSib",
task="TestSib")
harvestingWorkflowSib.create()
harvestSubSib = Subscription(fileset=multipleFilesFileset,
workflow=harvestingWorkflowSib,
split_algo="Harvest",
type="Harvesting")
harvestSubSib.create()
jobFactorySib = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSubSib)
multipleFilesFileset.markOpen(False)
jobGroups = jobFactorySib(periodic_harvest_sibling=True)
self.assertEqual(len(jobGroups), 0, "A single job group was created")
self.finishJobs(jobGroups, harvestSub)
jobGroups = jobFactorySib(periodic_harvest_sibling=True)
self.assertEqual(len(jobGroups), 1, "A single job group was not created")
self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created")
for job in jobGroups[0].getJobs():
runs = job['mask'].getRunAndLumis()
self.assertEqual(len(runs), 1, "Job has more than one run configured")
ll = LumiList(compactList={1: [[1, 1], [3, 7], [2, 2], [8, 8], [9, 14]],
2: [[1, 2], [4, 7], [3, 3], [8, 8]]})
run = runs.keys()[0]
for lumiPair in runs[run]:
for lumi in range(lumiPair[0], lumiPair[1] + 1):
self.assertTrue((run, lumi) in ll, "All of %s not in %s" % (lumiPair, ll))
def testMultiRunHarvesting(self):
"""
_testMultiRunHarvesting_
Provided a fileset with a couple of files and different runs, create a
single job for all the runs at a specific location, which also adds a
baggage to the job (True) which is later on looked up by SetupCMSSWPSet.
"""
multipleFilesFileset = createCommonFileset()
self.assertEqual(multipleFilesFileset.open, True)
harvestingWorkflow = Workflow(spec="spec.xml",
owner="amaltaro",
name="TestWorkflow",
task="Test")
harvestingWorkflow.create()
harvestSub = Subscription(fileset=multipleFilesFileset,
workflow=harvestingWorkflow,
split_algo="Harvest",
type="Harvesting")
harvestSub.create()
multipleFilesFileset.markOpen(False)
self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed")
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub)
jobGroups = jobFactory(dqmHarvestUnit="multiRun")
self.assertEqual(len(jobGroups), 1)
for jobGroup in jobGroups:
self.assertEqual(len(jobGroup.jobs), 1)
for job in jobGroup.jobs:
baggage = job.getBaggage()
self.assertTrue(getattr(baggage, "multiRun", False), "It's supposed to be a multiRun job")
self.assertEqual(getattr(baggage, "runLimits", ""), "-1-6")
def testByRunHarvesting(self):
"""
_testByRunHarvesting_
Provided a fileset with a couple of files and 4 different runs, create
one single job per run and location.
The multiRun baggage should be false in this case.
"""
multipleFilesFileset = createCommonFileset()
self.assertEqual(multipleFilesFileset.open, True, "Fileset should be open!")
harvestingWorkflow = Workflow(spec="spec.xml",
owner="amaltaro",
name="TestWorkflow",
task="Test")
harvestingWorkflow.create()
harvestSub = Subscription(fileset=multipleFilesFileset,
workflow=harvestingWorkflow,
split_algo="Harvest",
type="Harvesting")
harvestSub.create()
multipleFilesFileset.markOpen(False)
self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed")
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub)
jobGroups = jobFactory()
self.assertEqual(len(jobGroups), 1, "Should have created 1 job group")
for jobGroup in jobGroups:
self.assertEqual(len(jobGroup.jobs), 6, "Should have created 6 jobs")
for job in jobGroup.jobs:
baggage = job.getBaggage()
self.assertFalse(getattr(baggage, "multiRun", False), "It's supposed to be a byRun job")
def testByRunAndRunWhitelist(self):
"""
_testByRunAndRunWhitelist_
Create harvesting jobs by run for the runs provided in the RunWhitelist
"""
multipleFilesFileset = createCommonFileset()
self.assertEqual(multipleFilesFileset.open, True)
harvestingWorkflow = Workflow(spec="spec.xml", owner="amaltaro",
name="TestWorkflow", task="Test")
harvestingWorkflow.create()
harvestSub = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflow,
split_algo="Harvest", type="Harvesting")
harvestSub.create()
multipleFilesFileset.markOpen(False)
self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed")
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub)
jobGroups = jobFactory(runWhitelist=[1, 3])
self.assertEqual(len(jobGroups), 1, "One jobgroup per location")
for jobGroup in jobGroups:
self.assertEqual(len(jobGroup.jobs), 2)
def testByRunAndRunBlacklist(self):
"""
_testByRunAndRunWhitelist_
Create harvesting jobs by run for the runs provided in the RunWhitelist
"""
multipleFilesFileset = createCommonFileset()
self.assertEqual(multipleFilesFileset.open, True)
harvestingWorkflow = Workflow(spec="spec.xml", owner="amaltaro",
name="TestWorkflow", task="Test")
harvestingWorkflow.create()
harvestSub = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflow,
split_algo="Harvest", type="Harvesting")
harvestSub.create()
multipleFilesFileset.markOpen(False)
self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed")
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub)
jobGroups = jobFactory(runWhitelist=[1, 2, 3, 4, 5], runBlacklist=[1, 3])
self.assertEqual(len(jobGroups), 1, "One jobgroup per location")
for jobGroup in jobGroups:
self.assertEqual(len(jobGroup.jobs), 3)
class SetupCMSSWPsetTest(unittest.TestCase):
def setUp(self):
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testDir = self.testInit.generateWorkDir()
sys.path.insert(
0,
os.path.join(WMCore.WMBase.getTestBase(),
"WMCore_t/WMRuntime_t/Scripts_t"))
def tearDown(self):
sys.path.remove(
os.path.join(WMCore.WMBase.getTestBase(),
"WMCore_t/WMRuntime_t/Scripts_t"))
if 'WMTaskSpace' in sys.modules:
del sys.modules["WMTaskSpace"]
self.testInit.delWorkDir()
os.unsetenv("WMAGENT_SITE_CONFIG_OVERRIDE")
def createTestStep(self):
"""
_createTestStep_
Create a test step that can be passed to the setup script.
"""
newStep = WMStep("cmsRun1")
stepTemplate = StepFactory.getStepTemplate("CMSSW")
stepTemplate.install(newStep)
newStepHelper = stepTemplate.helper(newStep)
newStepHelper.setStepType("CMSSW")
newStepHelper.setGlobalTag("SomeGlobalTag")
newStepHelper.data.application.section_("setup")
newStepHelper.cmsswSetup("CMSSW_11_0_2",
scramArch=['slc7_amd64_gcc820'])
return newStepHelper
def createTestJob(self):
"""
_createTestJob_
Create a test job that has parents for each input file.
"""
newJob = Job(name="TestJob")
newJob.addFile(
File(lfn="/some/file/one",
parents=set([File(lfn="/some/parent/one")])))
newJob.addFile(
File(lfn="/some/file/two",
parents=set([File(lfn="/some/parent/two")])))
return newJob
def loadProcessFromPSet(self, psetPath=None):
"""
_loadProcessFromPSet_
This requires changing the working directory,
do so in a safe manner to encapsulate the change to this method only
"""
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import Unpickler
currentPath = os.getcwd()
loadedProcess = None
if psetPath is None:
psetPath = self.testDir
with open(os.path.join(psetPath, "PSet.pkl")) as f:
pset = Unpickler(f).load()
os.chdir(currentPath)
return pset
def testPSetFixup(self):
"""
_testPSetFixup_
Verify that all necessary parameters are set in the PSet.
"""
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.stepSpace = ConfigSection(name="stepSpace")
setupScript.stepSpace.location = self.testDir
shutil.copyfile(
os.path.join(os.path.dirname(__file__), "WMTaskSpace", "cmsRun1",
"PSet.py"),
os.path.join(setupScript.stepSpace.location, "PSet.py"))
setupScript.job = self.createTestJob()
setupScript()
fixedPSet = self.loadProcessFromPSet(setupScript.stepSpace.location)
self.assertTrue(hasattr(fixedPSet.source, 'fileNames'))
self.assertTrue(hasattr(fixedPSet.source, 'secondaryFileNames'))
self.assertEqual(fixedPSet.maxEvents.input._value, -1,
"Error: Wrong maxEvents.")
def testEventsPerLumi(self):
"""
_testEventsPerLumi_
Verify that you can put in events per lumi in the process.
"""
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.step.setEventsPerLumi(500)
setupScript.stepSpace = ConfigSection(name="stepSpace")
setupScript.stepSpace.location = self.testDir
shutil.copyfile(
os.path.join(os.path.dirname(__file__), "WMTaskSpace", "cmsRun1",
"PSet.py"),
os.path.join(setupScript.stepSpace.location, "PSet.py"))
setupScript.job = self.createTestJob()
setupScript()
fixedPSet = self.loadProcessFromPSet(setupScript.stepSpace.location)
self.assertTrue(hasattr(fixedPSet.source, 'fileNames'))
self.assertTrue(hasattr(fixedPSet.source, 'secondaryFileNames'))
self.assertEqual(fixedPSet.source.numberEventsInLuminosityBlock._value,
500,
"Error: Wrong number of events per luminosity block")
self.assertEqual(fixedPSet.maxEvents.input._value, -1,
"Error: Wrong maxEvents.")
def testChainedProcesing(self):
"""
test for chained CMSSW processing - check the overriden TFC, its values
and that input files is / are set correctly.
"""
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.stepSpace = ConfigSection(name="stepSpace")
setupScript.stepSpace.location = self.testDir
shutil.copyfile(
os.path.join(os.path.dirname(__file__), "WMTaskSpace", "cmsRun1",
"PSet.py"),
os.path.join(setupScript.stepSpace.location, "PSet.py"))
setupScript.job = self.createTestJob()
setupScript.step.setupChainedProcessing("my_first_step",
"my_input_module")
setupScript()
fixedPSet = self.loadProcessFromPSet(setupScript.stepSpace.location)
# test if the overriden TFC is right
print("DEBUG override: {0}".format(
setupScript.step.data.application.overrideCatalog))
self.assertTrue(
hasattr(setupScript.step.data.application, "overrideCatalog"),
"Error: overriden TFC was not set")
tfc = loadTFC(setupScript.step.data.application.overrideCatalog)
inputFile = "../my_first_step/my_input_module.root"
self.assertEqual(tfc.matchPFN("direct", inputFile), inputFile)
self.assertEqual(tfc.matchLFN("direct", inputFile), inputFile)
self.assertTrue(hasattr(fixedPSet.source, 'fileNames'))
def testPileupSetup(self):
"""
Test the pileup setting.
reference (setupScript.process instance):
in test/python/WMCore_t/WMRuntime_t/Scripts_t/WMTaskSpace/cmsRun1/PSet.py
"""
try:
from dbs.apis.dbsClient import DbsApi
except ImportError as ex:
raise unittest.SkipTest
# this is modified and shortened version of
# WMCore/test/python/WMCore_t/Misc_t/site-local-config.xml
# since the dataset name in question (below) is only present at
# storm-fe-cms.cr.cnaf.infn.it, need to make the test think it's its local SE
siteLocalConfigContent = \
"""
<site-local-config>
<site name="-SOME-SITE-NAME-">
<event-data>
<catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/>
</event-data>
<local-stage-out>
<!-- original cmssrm.fnal.gov -->
<phedex-node value="T2_CH_CERN"/>
<command value="test-copy"/>
<catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/>
</local-stage-out>
<calib-data>
<frontier-connect>
<load balance="proxies"/>
<proxy url="http://cmsfrontier1.fnal.gov:3128"/>
<proxy url="http://cmsfrontier2.fnal.gov:3128"/>
</frontier-connect>
</calib-data>
</site>
</site-local-config>
"""
siteLocalConfig = os.path.join(self.testDir,
"test-site-local-config.xml")
f = open(siteLocalConfig, 'w')
f.write(siteLocalConfigContent)
f.close()
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.stepSpace = ConfigSection(name="stepSpace")
setupScript.stepSpace.location = os.path.join(self.testDir, "cmsRun1")
setupScript.job = self.createTestJob()
# define pileup configuration
# despite of the implementation considering whichever type of pileup,
# only "data" and "mc" types are eventually considered and lead to any
# modifications of job input files
pileupConfig = {
"data": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
],
"mc": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
]
}
dbsUrl = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader"
setupScript.step.setupPileup(pileupConfig, dbsUrl)
# SetupCMSSWPset pileup handling will be consulting SiteLocalConfig
# to determine StorageElement (SE) name the job is running on
# SiteLocalConfig loads the site-local-config.xml file from env.
# variable defined location ; if the variable is not defined already, set it
# obviously, if "WMAGENT_SITE_CONFIG_OVERRIDE" is already set here, the above
# thick with SE name is not effective
if not os.getenv("WMAGENT_SITE_CONFIG_OVERRIDE", None):
os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = siteLocalConfig
# find out local site name from the testing local site config,
# will be needed later
siteConfig = loadSiteLocalConfig()
seLocalName = siteConfig.localStageOut["phedex-node"]
print("Running on site '%s', local SE name: '%s'" %
(siteConfig.siteName, seLocalName))
# before calling the script, SetupCMSSWPset will try to load JSON
# pileup configuration file, need to create it in self.testDir
fetcher = PileupFetcher()
fetcher.setWorkingDirectory(self.testDir)
fetcher.createPileupConfigFile(setupScript.step)
setupScript()
# now test all modifications carried out in SetupCMSSWPset.__call__
# which will also test that CMSSWStepHelper.setupPileup run correctly
mixModules, dataMixModules = setupScript._getPileupMixingModules()
# load in the pileup configuration in the form of dict which
# PileupFetcher previously saved in a JSON file
pileupDict = setupScript._getPileupConfigFromJson()
# get the sub dict for particular pileup type
# for pileupDict structure description - see PileupFetcher._queryDbsAndGetPileupConfig
for pileupType, modules in zip(("data", "mc"),
(dataMixModules, mixModules)):
# getting KeyError here - above pileupConfig is not correct - need
# to have these two types of pile type
d = pileupDict[pileupType]
self._mixingModulesInputFilesTest(modules, d, seLocalName)
def _mixingModulesInputFilesTest(self, modules, pileupSubDict,
seLocalName):
"""
pileupSubDic - contains only dictionary for particular pile up type
"""
# consider only locally available files
filesInConfigDict = []
for v in viewvalues(pileupSubDict):
if seLocalName in v["phedexNodeNames"]:
filesInConfigDict.extend(v["FileList"])
for m in modules:
inputTypeAttrib = getattr(m, "input", None) or getattr(
m, "secsource", None)
fileNames = inputTypeAttrib.fileNames.value
if fileNames == None:
fileNames = []
m = (
"Pileup configuration file list '%s' and mixing modules input "
"filelist '%s' are not identical." %
(filesInConfigDict, fileNames))
self.assertEqual(sorted(filesInConfigDict), sorted(fileNames), m)
class StoreResultsTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Initialize the database.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
self.testDir = self.testInit.generateWorkDir()
return
def tearDown(self):
"""
_tearDown_
Clear out the database.
"""
self.testInit.clearDatabase()
self.testInit.delWorkDir()
return
def testStoreResults(self):
"""
_testStoreResults_
Create a StoreResults workflow and verify it installs into WMBS
correctly.
"""
arguments = StoreResultsWorkloadFactory.getTestArguments()
arguments.update({'CmsPath': "/uscmst1/prod/sw/cms"})
factory = StoreResultsWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", arguments)
testWMBSHelper = WMBSHelper(testWorkload,
"StoreResults",
"SomeBlock",
cachepath=self.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
testWorkflow = Workflow(name="TestWorkload",
task="/TestWorkload/StoreResults")
testWorkflow.load()
self.assertEqual(len(testWorkflow.outputMap.keys()), 2,
"Error: Wrong number of WF outputs.")
goldenOutputMods = ["Merged"]
for goldenOutputMod in goldenOutputMods:
mergedOutput = testWorkflow.outputMap[goldenOutputMod][0][
"merged_output_fileset"]
unmergedOutput = testWorkflow.outputMap[goldenOutputMod][0][
"output_fileset"]
mergedOutput.loadData()
unmergedOutput.loadData()
self.assertEqual(
mergedOutput.name,
"/TestWorkload/StoreResults/merged-%s" % goldenOutputMod,
"Error: Merged output fileset is wrong: %s" %
mergedOutput.name)
self.assertEqual(
unmergedOutput.name,
"/TestWorkload/StoreResults/merged-%s" % goldenOutputMod,
"Error: Unmerged output fileset is wrong: %s." %
unmergedOutput.name)
logArchOutput = testWorkflow.outputMap["logArchive"][0][
"merged_output_fileset"]
unmergedLogArchOutput = testWorkflow.outputMap["logArchive"][0][
"output_fileset"]
logArchOutput.loadData()
unmergedLogArchOutput.loadData()
self.assertEqual(logArchOutput.name,
"/TestWorkload/StoreResults/merged-logArchive",
"Error: LogArchive output fileset is wrong.")
self.assertEqual(unmergedLogArchOutput.name,
"/TestWorkload/StoreResults/merged-logArchive",
"Error: LogArchive output fileset is wrong.")
topLevelFileset = Fileset(name="TestWorkload-StoreResults-SomeBlock")
topLevelFileset.loadData()
procSubscription = Subscription(fileset=topLevelFileset,
workflow=testWorkflow)
procSubscription.loadData()
self.assertEqual(procSubscription["type"], "Merge",
"Error: Wrong subscription type.")
self.assertEqual(procSubscription["split_algo"],
"ParentlessMergeBySize", "Error: Wrong split algo.")
return
class ReportIntegrationTest(unittest.TestCase):
"""
_ReportIntegrationTest_
"""
def setUp(self):
"""
_setUp_
Setup the database and WMBS for the test.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer",
"WMCore.WMBS"],
useDefault = False)
myThread = threading.currentThread()
self.daofactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.dbsfactory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = self.daofactory(classname = "Locations.New")
locationAction.execute(siteName = "site1", seName = "cmssrm.fnal.gov")
inputFile = File(lfn = "/path/to/some/lfn", size = 10, events = 10,
locations = "cmssrm.fnal.gov")
inputFile.create()
inputFileset = Fileset(name = "InputFileset")
inputFileset.create()
inputFileset.addFile(inputFile)
inputFileset.commit()
unmergedFileset = Fileset(name = "UnmergedFileset")
unmergedFileset.create()
mergedFileset = Fileset(name = "MergedFileset")
mergedFileset.create()
procWorkflow = Workflow(spec = "wf001.xml", owner = "Steve",
name = "TestWF", task = "/TestWF/None")
procWorkflow.create()
procWorkflow.addOutput("outputRECORECO", unmergedFileset)
mergeWorkflow = Workflow(spec = "wf002.xml", owner = "Steve",
name = "MergeWF", task = "/MergeWF/None")
mergeWorkflow.create()
mergeWorkflow.addOutput("Merged", mergedFileset)
insertWorkflow = self.dbsfactory(classname = "InsertWorkflow")
insertWorkflow.execute("TestWF", "/TestWF/None", 0, 0, 0, 0)
insertWorkflow.execute("MergeWF", "/MergeWF/None", 0, 0, 0, 0)
self.procSubscription = Subscription(fileset = inputFileset,
workflow = procWorkflow,
split_algo = "FileBased",
type = "Processing")
self.procSubscription.create()
self.procSubscription.acquireFiles()
self.mergeSubscription = Subscription(fileset = unmergedFileset,
workflow = mergeWorkflow,
split_algo = "WMBSMergeBySize",
type = "Merge")
self.mergeSubscription.create()
self.procJobGroup = JobGroup(subscription = self.procSubscription)
self.procJobGroup.create()
self.mergeJobGroup = JobGroup(subscription = self.mergeSubscription)
self.mergeJobGroup.create()
self.testJob = Job(name = "testJob", files = [inputFile])
self.testJob.create(group = self.procJobGroup)
self.testJob["state"] = "complete"
myThread = threading.currentThread()
self.daofactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.stateChangeAction = self.daofactory(classname = "Jobs.ChangeState")
self.setFWJRAction = self.daofactory(classname = "Jobs.SetFWJRPath")
self.getJobTypeAction = self.daofactory(classname = "Jobs.GetType")
locationAction = self.daofactory(classname = "Locations.New")
locationAction.execute(siteName = "cmssrm.fnal.gov")
self.stateChangeAction.execute(jobs = [self.testJob])
self.tempDir = tempfile.mkdtemp()
return
def tearDown(self):
"""
_tearDown_
Clear out the database and the pickled report file.
"""
self.testInit.clearDatabase()
try:
os.remove(os.path.join(self.tempDir, "ProcReport.pkl"))
os.remove(os.path.join(self.tempDir, "MergeReport.pkl"))
except Exception as ex:
pass
try:
os.rmdir(self.tempDir)
except Exception as ex:
pass
return
def createConfig(self, workerThreads):
"""
_createConfig_
Create a config for the JobAccountant with the given number of worker
threads. This config needs to include information for connecting to the
database as the component will create it's own database connections.
These parameters are still pulled from the environment.
"""
config = self.testInit.getConfiguration()
self.testInit.generateWorkDir(config)
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = os.getenv("COUCHURL")
config.JobStateMachine.couchDBName = "report_integration_t"
config.JobStateMachine.jobSummaryDBName = "report_integration_wmagent_summary_t"
config.component_("JobAccountant")
config.JobAccountant.pollInterval = 60
config.JobAccountant.workerThreads = workerThreads
config.JobAccountant.componentDir = os.getcwd()
config.JobAccountant.logLevel = 'SQLDEBUG'
config.component_("TaskArchiver")
config.TaskArchiver.localWMStatsURL = "%s/%s" % (config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName)
return config
def verifyJobSuccess(self, jobID):
"""
_verifyJobSuccess_
Verify that the metadata for a successful job is correct. This will
check the outcome, retry count and state.
"""
testJob = Job(id = jobID)
testJob.load()
assert testJob["state"] == "success", \
"Error: test job in wrong state: %s" % testJob["state"]
assert testJob["retry_count"] == 0, \
"Error: test job has wrong retry count: %s" % testJob["retry_count"]
assert testJob["outcome"] == "success", \
"Error: test job has wrong outcome: %s" % testJob["outcome"]
return
def verifyFileMetaData(self, jobID, fwkJobReportFiles):
"""
_verifyFileMetaData_
Verify that all the files that were output by a job made it into WMBS
correctly. Compare the contents of WMBS to the files in the frameworks
job report.
Note that fwkJobReportFiles is a list of DataStructs File objects.
"""
testJob = Job(id = jobID)
testJob.loadData()
inputLFNs = []
for inputFile in testJob["input_files"]:
inputLFNs.append(inputFile["lfn"])
for fwkJobReportFile in fwkJobReportFiles:
outputFile = File(lfn = fwkJobReportFile["lfn"])
outputFile.loadData(parentage = 1)
assert outputFile["events"] == int(fwkJobReportFile["events"]), \
"Error: Output file has wrong events: %s, %s" % \
(outputFile["events"], fwkJobReportFile["events"])
assert outputFile["size"] == int(fwkJobReportFile["size"]), \
"Error: Output file has wrong size: %s, %s" % \
(outputFile["size"], fwkJobReportFile["size"])
for ckType in fwkJobReportFile["checksums"].keys():
assert ckType in outputFile["checksums"].keys(), \
"Error: Output file is missing checksums: %s" % ckType
assert outputFile["checksums"][ckType] == fwkJobReportFile["checksums"][ckType], \
"Error: Checksums don't match."
assert len(fwkJobReportFile["checksums"].keys()) == \
len(outputFile["checksums"].keys()), \
"Error: Wrong number of checksums."
jobType = self.getJobTypeAction.execute(jobID = jobID)
if jobType == "Merge":
assert str(outputFile["merged"]) == "True", \
"Error: Merge jobs should output merged files."
else:
assert outputFile["merged"] == fwkJobReportFile["merged"], \
"Error: Output file merged output is wrong: %s, %s" % \
(outputFile["merged"], fwkJobReportFile["merged"])
assert len(outputFile["locations"]) == 1, \
"Error: outputfile should have one location: %s" % outputFile["locations"]
assert list(outputFile["locations"])[0] == list(fwkJobReportFile["locations"])[0], \
"Error: wrong location for file."
assert len(outputFile["parents"]) == len(inputLFNs), \
"Error: Output file has wrong number of parents."
for outputParent in outputFile["parents"]:
assert outputParent["lfn"] in inputLFNs, \
"Error: Unknown parent file: %s" % outputParent["lfn"]
fwjrRuns = {}
for run in fwkJobReportFile["runs"]:
fwjrRuns[run.run] = run.lumis
for run in outputFile["runs"]:
assert run.run in fwjrRuns, \
"Error: Extra run in output: %s" % run.run
for lumi in run:
assert lumi in fwjrRuns[run.run], \
"Error: Extra lumi: %s" % lumi
fwjrRuns[run.run].remove(lumi)
if len(fwjrRuns[run.run]) == 0:
del fwjrRuns[run.run]
assert len(fwjrRuns.keys()) == 0, \
"Error: Missing runs, lumis: %s" % fwjrRuns
testJobGroup = JobGroup(id = testJob["jobgroup"])
testJobGroup.loadData()
jobGroupFileset = testJobGroup.output
jobGroupFileset.loadData()
assert outputFile["id"] in jobGroupFileset.getFiles(type = "id"), \
"Error: output file not in jobgroup fileset."
if testJob["mask"]["FirstEvent"] == None:
assert outputFile["first_event"] == 0, \
"Error: first event not set correctly: 0, %s" % \
outputFile["first_event"]
else:
assert testJob["mask"]["FirstEvent"] == outputFile["first_event"], \
"Error: last event not set correctly: %s, %s" % \
(testJob["mask"]["FirstEvent"], outputFile["first_event"])
return
def testReportHandling(self):
"""
_testReportHandling_
Verify that we're able to parse a CMSSW report, convert it to a Report()
style report, pickle it and then have the accountant process it.
"""
self.procPath = os.path.join(WMCore.WMBase.getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml")
myReport = Report("cmsRun1")
myReport.parse(self.procPath)
# Fake some metadata that should be added by the stageout scripts.
for fileRef in myReport.getAllFileRefsFromStep("cmsRun1"):
fileRef.size = 1024
fileRef.location = "cmssrm.fnal.gov"
fwjrPath = os.path.join(self.tempDir, "ProcReport.pkl")
cmsRunStep = myReport.retrieveStep("cmsRun1")
cmsRunStep.status = 0
myReport.setTaskName('/TestWF/None')
myReport.persist(fwjrPath)
self.setFWJRAction.execute(jobID = self.testJob["id"], fwjrPath = fwjrPath)
pFile = DBSBufferFile(lfn = "/path/to/some/lfn", size = 600000, events = 60000)
pFile.setAlgorithm(appName = "cmsRun", appVer = "UNKNOWN",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
pFile.setDatasetPath("/bogus/dataset/path")
#pFile.addRun(Run(1, *[45]))
pFile.create()
config = self.createConfig(workerThreads = 1)
accountant = JobAccountantPoller(config)
accountant.setup()
accountant.algorithm()
self.verifyJobSuccess(self.testJob["id"])
self.verifyFileMetaData(self.testJob["id"], myReport.getAllFilesFromStep("cmsRun1"))
inputFile = File(lfn = "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR09_R_34X_V5_All_v1/0000/outputRECORECO.root")
inputFile.load()
self.testMergeJob = Job(name = "testMergeJob", files = [inputFile])
self.testMergeJob.create(group = self.mergeJobGroup)
self.testMergeJob["state"] = "complete"
self.stateChangeAction.execute(jobs = [self.testMergeJob])
self.mergePath = os.path.join(WMCore.WMBase.getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWMergeReport.xml")
myReport = Report("mergeReco")
myReport.parse(self.mergePath)
# Fake some metadata that should be added by the stageout scripts.
for fileRef in myReport.getAllFileRefsFromStep("mergeReco"):
fileRef.size = 1024
fileRef.location = "cmssrm.fnal.gov"
fileRef.dataset = {"applicationName": "cmsRun", "applicationVersion": "CMSSW_3_4_2_patch1",
"primaryDataset": "MinimumBias", "processedDataset": "Rereco-v1",
"dataTier": "RECO"}
fwjrPath = os.path.join(self.tempDir, "MergeReport.pkl")
myReport.setTaskName('/MergeWF/None')
cmsRunStep = myReport.retrieveStep("mergeReco")
cmsRunStep.status = 0
myReport.persist(fwjrPath)
self.setFWJRAction.execute(jobID = self.testMergeJob["id"], fwjrPath = fwjrPath)
accountant.algorithm()
self.verifyJobSuccess(self.testMergeJob["id"])
self.verifyFileMetaData(self.testMergeJob["id"], myReport.getAllFilesFromStep("mergeReco"))
return
class deleteFileTest(unittest.TestCase):
def setUp(self):
# stolen from CMSSWExecutor_t. thanks, dave
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("DeleterTask")
stepHelper = step = self.task.makeStep("DeleteTest")
self.step = stepHelper.data
self.actualStep = stepHelper
template = DeleteTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.executor = StepFactory.getStepExecutor(self.actualStep.stepType())
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.sandboxDir = "%s/UnitTests" % self.testDir
self.task.build(self.testDir)
sys.path.insert(0, self.testDir)
sys.path.insert(0, self.sandboxDir)
self.job = Job(name = "/UnitTest/DeleterTask/DeleteTest-test-job")
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def tearDown(self):
self.testInit.delWorkDir()
sys.path.remove(self.testDir)
sys.path.remove(self.sandboxDir)
def setLocalOverride(self, step):
step.section_('override')
step.override.command = 'test-copy'
step.override.option = ''
step.override.__setattr__('lfn-prefix', '')
step.override.__setattr__('phedex-node','DUMMYPNN')
@attr('integration')
def testManualDeleteOld(self):
self.assertTrue(os.path.exists( os.path.join(self.testDir, 'testfile')))
self.step.section_('filesToDelete')
self.step.filesToDelete.file1 = os.path.join(self.testDir, 'testfile')
self.setLocalOverride(self.step)
self.executor.initialise(self.step, self.job)
self.executor.execute()
self.assertFalse(os.path.exists( os.path.join(self.testDir, 'testfile')))
return
@attr('integration')
def testManualDeleteNew(self):
self.assertTrue(os.path.exists( os.path.join(self.testDir, 'testfile')))
self.step.section_('filesToDelete')
self.step.filesToDelete.file1 = os.path.join(self.testDir, 'testfile')
self.setLocalOverride(self.step)
self.step.override.newStageOut = True
self.executor.initialise(self.step, self.job)
self.executor.execute()
self.assertFalse(os.path.exists( os.path.join(self.testDir, 'testfile')))
return
@attr('integration')
def testJobDeleteOld(self):
self.assertTrue(os.path.exists( os.path.join(self.testDir, 'testfile')))
self.setLocalOverride(self.step)
self.job['input_files'] = [ {'lfn': os.path.join(self.testDir, 'testfile') } ]
self.executor.initialise(self.step, self.job)
self.executor.execute()
self.assertFalse(os.path.exists( os.path.join(self.testDir, 'testfile')))
return
@attr('integration')
def testJobDeleteNew(self):
self.assertTrue(os.path.exists( os.path.join(self.testDir, 'testfile')))
self.setLocalOverride(self.step)
self.step.override.newStageOut = True
self.job['input_files'] = [ {'lfn': os.path.join(self.testDir, 'testfile') } ]
self.executor.initialise(self.step, self.job)
self.executor.execute()
self.assertFalse(os.path.exists( os.path.join(self.testDir, 'testfile')))
return
class ResourceControlTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Install schema and create a DAO factory for WMBS.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=[
"WMCore.WMBS", "WMCore.ResourceControl", "WMCore.BossAir"
],
useDefault=False)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.baDaoFactory = DAOFactory(package="WMCore.BossAir",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.insertRunJob = self.baDaoFactory(classname="NewJobs")
self.insertState = self.baDaoFactory(classname="NewState")
states = ['PEND', 'RUN', 'Idle', 'Running']
self.insertState.execute(states)
self.tempDir = self.testInit.generateWorkDir()
return
def tearDown(self):
"""
_tearDown_
Clear the schema.
"""
self.testInit.clearDatabase()
return
def testInsert(self):
"""
_testInsert_
Verify that inserting sites and thresholds works correctly, even if the
site or threshold already exists.
"""
myResourceControl = ResourceControl()
myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1")
myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1")
myResourceControl.insertSite("testSite2", 100, 200, "testSE2",
"testCE2")
myResourceControl.insertThreshold("testSite1", "Processing", 20, 10)
myResourceControl.insertThreshold("testSite1", "Merge", 200, 100)
myResourceControl.insertThreshold("testSite1", "Merge", 250, 150)
myResourceControl.insertThreshold("testSite2", "Processing", 50, 30)
myResourceControl.insertThreshold("testSite2", "Merge", 135, 100)
createThresholds = myResourceControl.listThresholdsForCreate()
self.assertEqual(len(createThresholds.keys()), 2,
"Error: Wrong number of site in Resource Control DB")
self.assertTrue("testSite1" in createThresholds.keys(),
"Error: Test Site 1 missing from thresholds.")
self.assertTrue("testSite2" in createThresholds.keys(),
"Error: Test Site 2 missing from thresholds.")
self.assertEqual(createThresholds["testSite1"]["total_slots"], 10,
"Error: Wrong number of total slots.")
self.assertEqual(
createThresholds["testSite1"]["pending_jobs"], 0,
"Error: Wrong number of running jobs: %s" %
createThresholds["testSite1"]["pending_jobs"])
self.assertEqual(createThresholds["testSite2"]["total_slots"], 100,
"Error: Wrong number of total slots.")
self.assertEqual(createThresholds["testSite2"]["pending_jobs"], 0,
"Error: Wrong number of running jobs.")
thresholds = myResourceControl.listThresholdsForSubmit()
self.assertEqual(
len(thresholds.keys()), 2,
"Error: Wrong number of sites in Resource Control DB")
self.assertTrue("testSite1" in thresholds.keys(),
"Error: testSite1 missing from thresholds.")
self.assertTrue("testSite2" in thresholds.keys(),
"Error: testSite2 missing from thresholds.")
site1Info = thresholds["testSite1"]
site2Info = thresholds["testSite2"]
site1Thresholds = site1Info["thresholds"]
site2Thresholds = site2Info["thresholds"]
procThreshold1 = None
procThreshold2 = None
mergeThreshold1 = None
mergeThreshold2 = None
for threshold in site1Thresholds:
if threshold["task_type"] == "Merge":
mergeThreshold1 = threshold
elif threshold["task_type"] == "Processing":
procThreshold1 = threshold
for threshold in site2Thresholds:
if threshold["task_type"] == "Merge":
mergeThreshold2 = threshold
elif threshold["task_type"] == "Processing":
procThreshold2 = threshold
self.assertEqual(len(site1Thresholds), 2,
"Error: Wrong number of task types.")
self.assertEqual(len(site2Thresholds), 2,
"Error: Wrong number of task types.")
self.assertNotEqual(procThreshold1, None)
self.assertNotEqual(procThreshold2, None)
self.assertNotEqual(mergeThreshold1, None)
self.assertNotEqual(mergeThreshold2, None)
self.assertEqual(site1Info["total_pending_slots"], 10,
"Error: Site thresholds wrong")
self.assertEqual(site1Info["total_running_slots"], 20,
"Error: Site thresholds wrong")
self.assertEqual(site1Info["total_running_jobs"], 0,
"Error: Site thresholds wrong")
self.assertEqual(site1Info["total_pending_jobs"], 0,
"Error: Site thresholds wrong")
self.assertEqual(procThreshold1["task_running_jobs"], 0,
"Error: Site thresholds wrong")
self.assertEqual(procThreshold1["task_pending_jobs"], 0,
"Error: Site thresholds wrong")
self.assertEqual(procThreshold1["max_slots"], 20,
"Error: Site thresholds wrong")
self.assertEqual(procThreshold1["pending_slots"], 10,
"Error: Site thresholds wrong")
self.assertEqual(mergeThreshold1["task_running_jobs"], 0,
"Error: Site thresholds wrong")
self.assertEqual(mergeThreshold1["task_pending_jobs"], 0,
"Error: Site thresholds wrong")
self.assertEqual(mergeThreshold1["max_slots"], 250,
"Error: Site thresholds wrong")
self.assertEqual(mergeThreshold1["pending_slots"], 150,
"Error: Site thresholds wrong")
self.assertEqual(site2Info["total_pending_slots"], 100,
"Error: Site thresholds wrong")
self.assertEqual(site2Info["total_running_slots"], 200,
"Error: Site thresholds wrong")
self.assertEqual(site2Info["total_running_jobs"], 0,
"Error: Site thresholds wrong")
self.assertEqual(site2Info["total_pending_jobs"], 0,
"Error: Site thresholds wrong")
self.assertEqual(procThreshold2["task_running_jobs"], 0,
"Error: Site thresholds wrong")
self.assertEqual(procThreshold2["task_pending_jobs"], 0,
"Error: Site thresholds wrong")
self.assertEqual(procThreshold2["max_slots"], 50,
"Error: Site thresholds wrong")
self.assertEqual(procThreshold2["pending_slots"], 30,
"Error: Site thresholds wrong")
self.assertEqual(mergeThreshold2["task_running_jobs"], 0,
"Error: Site thresholds wrong")
self.assertEqual(mergeThreshold2["task_pending_jobs"], 0,
"Error: Site thresholds wrong")
self.assertEqual(mergeThreshold2["max_slots"], 135,
"Error: Site thresholds wrong")
self.assertEqual(mergeThreshold2["pending_slots"], 100,
"Error: Site thresholds wrong")
def testList(self):
"""
_testList_
Test the functions that list thresholds for creating jobs and submitting
jobs.
"""
myResourceControl = ResourceControl()
myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1",
"T1_US_FNAL", "LsfPlugin")
myResourceControl.insertSite("testSite2", 20, 40, "testSE2", "testCE2")
myResourceControl.insertThreshold("testSite1", "Processing", 20, 10)
myResourceControl.insertThreshold("testSite1", "Merge", 200, 100)
myResourceControl.insertThreshold("testSite2", "Processing", 50, 25)
myResourceControl.insertThreshold("testSite2", "Merge", 135, 65)
testWorkflow = Workflow(spec=makeUUID(),
owner="Steve",
name=makeUUID(),
task="Test")
testWorkflow.create()
testFilesetA = Fileset(name="TestFilesetA")
testFilesetA.create()
testFilesetB = Fileset(name="TestFilesetB")
testFilesetB.create()
testFilesetC = Fileset(name="TestFilesetC")
testFilesetC.create()
testFileA = File(lfn="testFileA",
locations=set(["testSE1", "testSE2"]))
testFileA.create()
testFilesetA.addFile(testFileA)
testFilesetA.commit()
testFilesetB.addFile(testFileA)
testFilesetB.commit()
testFilesetC.addFile(testFileA)
testFilesetC.commit()
testSubscriptionA = Subscription(fileset=testFilesetA,
workflow=testWorkflow,
type="Processing")
testSubscriptionA.create()
testSubscriptionA.addWhiteBlackList([{
"site_name": "testSite1",
"valid": True
}])
testSubscriptionB = Subscription(fileset=testFilesetB,
workflow=testWorkflow,
type="Processing")
testSubscriptionB.create()
testSubscriptionB.addWhiteBlackList([{
"site_name": "testSite1",
"valid": False
}])
testSubscriptionC = Subscription(fileset=testFilesetC,
workflow=testWorkflow,
type="Merge")
testSubscriptionC.create()
testJobGroupA = JobGroup(subscription=testSubscriptionA)
testJobGroupA.create()
testJobGroupB = JobGroup(subscription=testSubscriptionB)
testJobGroupB.create()
testJobGroupC = JobGroup(subscription=testSubscriptionC)
testJobGroupC.create()
# Site1, Has been assigned a location and is complete.
testJobA = Job(name="testJobA", files=[testFileA])
testJobA["couch_record"] = makeUUID()
testJobA.create(group=testJobGroupA)
testJobA["state"] = "success"
# Site 1, Has been assigned a location and is incomplete.
testJobB = Job(name="testJobB", files=[testFileA])
testJobB["couch_record"] = makeUUID()
testJobB.create(group=testJobGroupA)
testJobB["state"] = "executing"
runJobB = RunJob()
runJobB.buildFromJob(testJobB)
runJobB["status"] = "PEND"
# Does not have a location, white listed to site 1
testJobC = Job(name="testJobC", files=[testFileA])
testJobC["couch_record"] = makeUUID()
testJobC.create(group=testJobGroupA)
testJobC["state"] = "new"
# Site 2, Has been assigned a location and is complete.
testJobD = Job(name="testJobD", files=[testFileA])
testJobD["couch_record"] = makeUUID()
testJobD.create(group=testJobGroupB)
testJobD["state"] = "success"
# Site 2, Has been assigned a location and is incomplete.
testJobE = Job(name="testJobE", files=[testFileA])
testJobE["couch_record"] = makeUUID()
testJobE.create(group=testJobGroupB)
testJobE["state"] = "executing"
runJobE = RunJob()
runJobE.buildFromJob(testJobE)
runJobE["status"] = "RUN"
# Does not have a location, site 1 is blacklisted.
testJobF = Job(name="testJobF", files=[testFileA])
testJobF["couch_record"] = makeUUID()
testJobF.create(group=testJobGroupB)
testJobF["state"] = "new"
# Site 3, Has been assigned a location and is complete.
testJobG = Job(name="testJobG", files=[testFileA])
testJobG["couch_record"] = makeUUID()
testJobG.create(group=testJobGroupC)
testJobG["state"] = "cleanout"
# Site 3, Has been assigned a location and is incomplete.
testJobH = Job(name="testJobH", files=[testFileA])
testJobH["couch_record"] = makeUUID()
testJobH.create(group=testJobGroupC)
testJobH["state"] = "new"
# Site 3, Does not have a location.
testJobI = Job(name="testJobI", files=[testFileA])
testJobI["couch_record"] = makeUUID()
testJobI.create(group=testJobGroupC)
testJobI["state"] = "new"
# Site 3, Does not have a location and is in cleanout.
testJobJ = Job(name="testJobJ", files=[testFileA])
testJobJ["couch_record"] = makeUUID()
testJobJ.create(group=testJobGroupC)
testJobJ["state"] = "cleanout"
changeStateAction = self.daoFactory(classname="Jobs.ChangeState")
changeStateAction.execute(jobs=[
testJobA, testJobB, testJobC, testJobD, testJobE, testJobF,
testJobG, testJobH, testJobI, testJobJ
])
self.insertRunJob.execute([runJobB, runJobE])
setLocationAction = self.daoFactory(classname="Jobs.SetLocation")
setLocationAction.execute(testJobA["id"], "testSite1")
setLocationAction.execute(testJobB["id"], "testSite1")
setLocationAction.execute(testJobD["id"], "testSite1")
setLocationAction.execute(testJobE["id"], "testSite1")
setLocationAction.execute(testJobG["id"], "testSite1")
setLocationAction.execute(testJobH["id"], "testSite1")
createThresholds = myResourceControl.listThresholdsForCreate()
submitThresholds = myResourceControl.listThresholdsForSubmit()
self.assertEqual(len(createThresholds.keys()), 2,
"Error: Wrong number of sites in create thresholds")
self.assertEqual(createThresholds["testSite1"]["total_slots"], 10,
"Error: Wrong number of slots for site 1")
self.assertEqual(createThresholds["testSite2"]["total_slots"], 20,
"Error: Wrong number of slots for site 2")
# We should have two running jobs with locations at site one,
# two running jobs without locations at site two, and one running
# job without a location at site one and two.
self.assertEqual(createThresholds["testSite1"]["pending_jobs"], 4,
"Error: Wrong number of pending jobs for site 1")
# We should have one running job with a location at site 2 and
# another running job without a location.
self.assertEqual(createThresholds["testSite2"]["pending_jobs"], 2,
"Error: Wrong number of pending jobs for site 2")
# We should also have a phedex_name
self.assertEqual(createThresholds["testSite1"]["cms_name"],
"T1_US_FNAL")
self.assertEqual(createThresholds["testSite2"]["cms_name"], None)
mergeThreshold1 = None
mergeThreshold2 = None
procThreshold1 = None
procThreshold2 = None
self.assertEqual(submitThresholds["testSite1"]['cms_name'],
'T1_US_FNAL')
for threshold in submitThresholds["testSite1"]["thresholds"]:
if threshold['task_type'] == "Merge":
mergeThreshold1 = threshold
elif threshold['task_type'] == "Processing":
procThreshold1 = threshold
self.assertEqual(submitThresholds["testSite2"]['cms_name'], None)
for threshold in submitThresholds["testSite2"]["thresholds"]:
if threshold['task_type'] == "Merge":
mergeThreshold2 = threshold
elif threshold['task_type'] == "Processing":
procThreshold2 = threshold
self.assertEqual(
submitThresholds["testSite1"]["total_running_jobs"], 1,
"Error: Wrong number of running jobs for submit thresholds.")
self.assertEqual(
submitThresholds["testSite2"]["total_running_jobs"], 0,
"Error: Wrong number of running jobs for submit thresholds.")
self.assertEqual(
submitThresholds["testSite1"]["total_pending_jobs"], 1,
"Error: Wrong number of pending jobs for submit thresholds.")
self.assertEqual(
submitThresholds["testSite2"]["total_pending_jobs"], 0,
"Error: Wrong number of pending jobs for submit thresholds.")
self.assertEqual(
mergeThreshold1["task_running_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(
mergeThreshold1["task_pending_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(
procThreshold1["task_running_jobs"], 1,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(
procThreshold1["task_pending_jobs"], 1,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(
mergeThreshold2["task_running_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(
mergeThreshold2["task_pending_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(
procThreshold2["task_running_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(
procThreshold2["task_pending_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
return
def testListSiteInfo(self):
"""
_testListSiteInfo_
Verify that the listSiteInfo() methods works properly.
"""
myResourceControl = ResourceControl()
myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1")
myResourceControl.insertSite("testSite2", 100, 200, "testSE2",
"testCE2")
siteInfo = myResourceControl.listSiteInfo("testSite1")
self.assertEqual(siteInfo["site_name"], "testSite1",
"Error: Site name is wrong.")
self.assertEqual(siteInfo["se_name"], ["testSE1"],
"Error: SE name is wrong.")
self.assertEqual(siteInfo["ce_name"], "testCE1",
"Error: CE name is wrong.")
self.assertEqual(siteInfo["pending_slots"], 10,
"Error: Pending slots is wrong.")
self.assertEqual(siteInfo["running_slots"], 20,
"Error: Pending slots is wrong.")
return
def testUpdateJobSlots(self):
"""
_testUpdateJobSlots_
Verify that it is possible to update the number of job slots at a site.
"""
myResourceControl = ResourceControl()
myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1")
siteInfo = myResourceControl.listSiteInfo("testSite1")
self.assertEqual(siteInfo["pending_slots"], 10,
"Error: Pending slots is wrong.")
self.assertEqual(siteInfo["running_slots"], 20,
"Error: Running slots is wrong.")
myResourceControl.setJobSlotsForSite("testSite1", pendingJobSlots=20)
siteInfo = myResourceControl.listSiteInfo("testSite1")
self.assertEqual(siteInfo["pending_slots"], 20,
"Error: Pending slots is wrong.")
myResourceControl.setJobSlotsForSite("testSite1", runningJobSlots=40)
siteInfo = myResourceControl.listSiteInfo("testSite1")
self.assertEqual(siteInfo["running_slots"], 40,
"Error: Running slots is wrong.")
myResourceControl.setJobSlotsForSite("testSite1", 5, 10)
siteInfo = myResourceControl.listSiteInfo("testSite1")
self.assertEqual(siteInfo["pending_slots"], 5,
"Error: Pending slots is wrong.")
self.assertEqual(siteInfo["running_slots"], 10,
"Error: Running slots is wrong.")
return
def testThresholdsForSite(self):
"""
_testThresholdsForSite_
Check that we can get the thresholds in intelligible form
for each site
"""
myResourceControl = ResourceControl()
myResourceControl.insertSite("testSite1", 20, 40, "testSE1", "testCE1")
myResourceControl.insertThreshold("testSite1", "Processing", 10, 8)
myResourceControl.insertThreshold("testSite1", "Merge", 5, 3)
result = myResourceControl.thresholdBySite(siteName="testSite1")
procInfo = {}
mergInfo = {}
for res in result:
if res['task_type'] == 'Processing':
procInfo = res
elif res['task_type'] == 'Merge':
mergInfo = res
self.assertEqual(procInfo.get('pending_slots', None), 20)
self.assertEqual(procInfo.get('running_slots', None), 40)
self.assertEqual(procInfo.get('max_slots', None), 10)
self.assertEqual(procInfo.get('task_pending_slots', None), 8)
self.assertEqual(mergInfo.get('pending_slots', None), 20)
self.assertEqual(mergInfo.get('running_slots', None), 40)
self.assertEqual(mergInfo.get('max_slots', None), 5)
self.assertEqual(mergInfo.get('task_pending_slots', None), 3)
return
def testThresholdPriority(self):
"""
_testThresholdPriority_
Test that we get things back in priority order
"""
myResourceControl = ResourceControl()
myResourceControl.insertSite("testSite1", 20, 40, "testSE1", "testCE1")
myResourceControl.insertThreshold("testSite1",
"Processing",
10,
8,
priority=1)
myResourceControl.insertThreshold("testSite1",
"Merge",
5,
3,
priority=2)
result = myResourceControl.listThresholdsForSubmit()
self.assertEqual(result['testSite1']['thresholds'][0]['task_type'],
'Merge')
self.assertEqual(result['testSite1']['thresholds'][1]['task_type'],
'Processing')
myResourceControl.insertThreshold("testSite1",
"Processing",
10,
8,
priority=2)
myResourceControl.insertThreshold("testSite1",
"Merge",
5,
3,
priority=1)
# Should now be in reverse order
result = myResourceControl.listThresholdsForSubmit()
self.assertEqual(result['testSite1']['thresholds'][1]['task_type'],
'Merge')
self.assertEqual(result['testSite1']['thresholds'][0]['task_type'],
'Processing')
myResourceControl.insertSite("testSite2", 20, 40, "testSE2", "testCE2")
myResourceControl.insertThreshold("testSite2",
"Processing",
10,
8,
priority=1)
myResourceControl.insertThreshold("testSite2",
"Merge",
5,
3,
priority=2)
# Should be in proper order for site 2
result = myResourceControl.listThresholdsForSubmit()
self.assertEqual(result['testSite2']['thresholds'][0]['task_type'],
'Merge')
self.assertEqual(result['testSite2']['thresholds'][1]['task_type'],
'Processing')
# Should now be in reverse order for site 1
self.assertEqual(result['testSite1']['thresholds'][1]['task_type'],
'Merge')
self.assertEqual(result['testSite1']['thresholds'][0]['task_type'],
'Processing')
myResourceControl.insertThreshold("testSite2", "Merge", 20, 10)
result = myResourceControl.listThresholdsForSubmit()
self.assertEqual(result['testSite2']['thresholds'][0]['priority'], 2)
return
def testChangeState(self):
"""
_testChangeState_
Check that we can change the state between different values and
retrieve it through the threshold methods
"""
myResourceControl = ResourceControl()
myResourceControl.insertSite("testSite1", 20, 40, "testSE1", "testCE1")
myResourceControl.insertThreshold("testSite1",
"Processing",
10,
5,
priority=1)
result = myResourceControl.listThresholdsForCreate()
self.assertEqual(result['testSite1']['state'], 'Normal',
'Error: Wrong site state')
myResourceControl.changeSiteState("testSite1", "Down")
result = myResourceControl.listThresholdsForCreate()
self.assertEqual(result['testSite1']['state'], 'Down',
'Error: Wrong site state')
def createConfig(self):
"""
_createConfig_
Create a config and save it to the temp dir. Set the WMAGENT_CONFIG
environment variable so the config gets picked up.
"""
config = Configuration()
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = "resource_control_t"
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
configHandle = open(os.path.join(self.tempDir, "config.py"), "w")
configHandle.write(str(config))
configHandle.close()
os.environ["WMAGENT_CONFIG"] = os.path.join(self.tempDir, "config.py")
return
def testInsertAllSEs(self):
"""
_testInsertAllSEs_
Test to see if we can insert all SEs and Thresholds at once
Depending on the WMCore.Services.SiteDB interface
"""
self.createConfig()
resControlPath = os.path.join(WMCore.WMBase.getTestBase(),
"../../bin/wmagent-resource-control")
env = os.environ
env['PYTHONPATH'] = ":".join(sys.path)
cmdline = [
resControlPath, "--add-all-sites", "--plugin=CondorPlugin",
"--pending-slots=100", "--running-slots=500"
]
retval = subprocess.Popen(cmdline,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env)
(output, _) = retval.communicate()
myResourceControl = ResourceControl()
result = myResourceControl.listThresholdsForSubmit()
self.assertTrue('T1_US_FNAL' in result.keys())
for x in result.keys():
self.assertEqual(len(result[x]['thresholds']), 8)
self.assertEqual(result[x]['total_pending_slots'], 100)
self.assertEqual(result[x]['total_running_slots'], 500)
for thresh in result[x]['thresholds']:
if thresh['task_type'] == 'Processing':
self.assertEqual(thresh['priority'], 1)
self.assertEqual(thresh['max_slots'], 500)
# Verify that sites with more than one SE were added correctly.
nebInfo = myResourceControl.listSiteInfo("T2_US_Nebraska")
self.assertTrue(len(nebInfo["se_name"]) == 3)
return
def testInsertAllSEs2(self):
"""
_testInsertAllSEs2_
Test to see if we can insert all SEs and Thresholds at once
Depending on the WMCore.Services.SiteDB interface
"""
myResourceControl = ResourceControl()
taskList = [{
'taskType': 'Processing',
'maxSlots': 100,
'pendingSlots': 80,
'priority': 1
}, {
'taskType': 'Merge',
'maxSlots': 50,
'pendingSlots': 30,
'priority': 2
}]
myResourceControl.insertAllSEs(siteName='test',
pendingSlots=200,
runningSlots=400,
ceName='glidein-ce.fnal.gov',
plugin='CondorPlugin',
taskList=taskList)
result = myResourceControl.listThresholdsForSubmit()
self.assertTrue('test_cmssrm.fnal.gov' in result.keys())
self.assertEqual(result['test_cmssrm.fnal.gov']['cms_name'],
'T1_US_FNAL')
for x in result.keys():
self.assertEqual(len(result[x]['thresholds']), 2)
self.assertEqual(result[x]['total_pending_slots'], 200)
self.assertEqual(result[x]['total_running_slots'], 400)
for thresh in result[x]['thresholds']:
if thresh['task_type'] == 'Processing':
self.assertEqual(thresh['priority'], 1)
self.assertEqual(thresh['max_slots'], 100)
self.assertEqual(thresh['pending_slots'], 80)
else:
self.assertEqual(thresh['priority'], 2)
self.assertEqual(thresh['max_slots'], 50)
self.assertEqual(thresh['pending_slots'], 30)
return
def testInsertT0(self):
"""
_testInsertT0_
Test to see if we can insert the Tier-0 alone
with a single option
"""
self.createConfig()
resControlPath = os.path.join(WMCore.WMBase.getTestBase(),
"../../bin/wmagent-resource-control")
env = os.environ
env['PYTHONPATH'] = ":".join(sys.path)
cmdline = [resControlPath, "--add-T0"]
retval = subprocess.Popen(cmdline,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env)
(_, _) = retval.communicate()
myResourceControl = ResourceControl()
result = myResourceControl.listThresholdsForSubmit()
self.assertTrue(len(result), 1)
self.assertTrue('CERN' in result)
for x in result:
self.assertEqual(len(result[x]['thresholds']), 10)
self.assertEqual(result[x]['total_pending_slots'], 500)
self.assertEqual(result[x]['total_running_slots'], -1)
for thresh in result[x]['thresholds']:
if thresh['task_type'] == 'Processing':
self.assertEqual(thresh['priority'], 1)
self.assertEqual(thresh['max_slots'], -1)
# Verify that sites with more than one SE were added correctly.
cernInfo = myResourceControl.listSiteInfo("CERN")
self.assertTrue(len(cernInfo["se_name"]) == 2)
return
class LogArchiveTest(unittest.TestCase):
def setUp(self):
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
# shut up SiteLocalConfig
os.environ['CMS_PATH'] = os.getcwd()
workload = copy.deepcopy(testWorkloads.workload)
task = workload.getTask("Production")
step = task.getStep("stageOut1")
# want to get the cmsstep so I can make the Report
cmsstep = task.getStep('cmsRun1')
self.cmsstepdir = os.path.join( self.testDir, 'cmsRun1')
os.mkdir( self.cmsstepdir )
open( os.path.join( self.cmsstepdir, '__init__.py'),'w').close()
open( os.path.join( self.cmsstepdir, 'Report.pkl'),'w').close()
cmsbuilder = CMSSWBuilder.CMSSW()
cmsbuilder( cmsstep.data, 'Production', self.cmsstepdir )
realstep = LogArchiveTemplate.LogArchiveStepHelper(step.data)
realstep.disableRetries()
self.realstep = realstep
self.stepDir = os.path.join( self.testDir, 'stepdir')
os.mkdir( self.stepDir )
builder = LogArchiveBuilder.LogArchive()
builder( step.data, 'Production', self.stepDir)
# stolen from CMSSWExecutor_t. thanks, dave
# first, delete all the sandboxen and taskspaces
# because of caching, this leaks from other tests in other files
# this sucks because the other tests are using sandboxen that
# are deleted after the test is over, which causes theses tests
# to break
modsToDelete = []
# not sure what happens if you delete from
# an arrey you're iterating over. doing it in
# two steps
for modname in sys.modules.keys():
# need to blow away things in sys.modules, otherwise
# they are cached and we look at old taskspaces
if modname.startswith('WMTaskSpace'):
modsToDelete.append(modname)
if modname.startswith('WMSandbox'):
modsToDelete.append(modname)
for modname in modsToDelete:
try:
reload(sys.modules[modname])
except:
pass
del sys.modules[modname]
self.oldpath = sys.path[:]
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
self.job = Job(name = "/UnitTests/DeleterTask/DeleteTest-test-job")
shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("DeleterTask")
cmsswHelper = self.task.makeStep("cmsRun1")
cmsswHelper.setStepType('CMSSW')
stepHelper = cmsswHelper.addStep("DeleteTest")
stepHelper.setStepType('LogArchive')
self.cmsswstep = cmsswHelper.data
self.cmsswHelper = cmsswHelper
self.stepdata = stepHelper.data
self.stephelp = LogArchiveTemplate.LogArchiveStepHelper(stepHelper.data)
self.task.applyTemplates()
self.executor = StepFactory.getStepExecutor(self.stephelp.stepType())
taskMaker = TaskMaker(self.workload, os.path.join(self.testDir))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.task.build(os.path.join(self.testDir, 'UnitTests'))
sys.path.insert(0, self.testDir)
sys.path.insert(0, os.path.join(self.testDir, 'UnitTests'))
# binDir = inspect.getsourcefile(ModuleLocator)
# binDir = binDir.replace("__init__.py", "bin")
#
# if not binDir in os.environ['PATH']:
# os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
open( os.path.join( self.testDir, 'UnitTests', '__init__.py'),'w').close()
shutil.copyfile( os.path.join( os.path.dirname( __file__ ), 'MergeSuccess.pkl'),
os.path.join( self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
def tearDown(self):
sys.path = self.oldpath[:]
self.testInit.delWorkDir()
# making double sure WMTaskSpace and WMSandbox are gone
modsToDelete = []
# not sure what happens if you delete from
# an arrey you're iterating over. doing it in
# two steps
for modname in sys.modules.keys():
# need to blow away things in sys.modules, otherwise
# they are cached and we look at old taskspaces
if modname.startswith('WMTaskSpace'):
modsToDelete.append(modname)
if modname.startswith('WMSandbox'):
modsToDelete.append(modname)
for modname in modsToDelete:
try:
reload(sys.modules[modname])
except:
pass
del sys.modules[modname]
myThread = threading.currentThread()
if hasattr(myThread, "factory"):
myThread.factory = {}
def makeReport(self, fileName):
myReport = Report('oneitem')
myReport.addStep('stageOut1')
mod1 = myReport.addOutputModule('module1')
mod2 = myReport.addOutputModule('module2')
file1 = myReport.addOutputFile('module1', {'lfn': 'FILE1', 'size' : 1, 'events' : 1})
file2 = myReport.addOutputFile('module2', {'lfn': 'FILE2', 'size' : 1, 'events' : 1})
file3 = myReport.addOutputFile('module2', {'lfn': 'FILE3', 'size' : 1, 'events' : 1})
myReport.persist( fileName )
def testExecutorDoesntDetonate(self):
myReport = Report()
myReport.unpersist(os.path.join( self.testDir,'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
myReport.data.cmsRun1.status = 1
myReport.persist(os.path.join( self.testDir, 'UnitTests','WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
executor = LogArchiveExecutor.LogArchive()
executor.initialise( self.stepdata, self.job)
self.setLocalOverride(self.stepdata)
executor.step = self.stepdata
executor.execute( )
self.assertFalse( os.path.exists( os.path.join( self.testDir, 'hosts' )))
self.assertFalse( os.path.exists( os.path.join( self.testDir, 'test1', 'hosts')))
return
def testUnitTestBackend(self):
myReport = Report()
myReport.unpersist(os.path.join( self.testDir,'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
myReport.data.cmsRun1.status = 1
myReport.persist(os.path.join( self.testDir, 'UnitTests','WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
executor = LogArchiveExecutor.LogArchive()
helper = LogArchiveTemplate.LogArchiveStepHelper(self.stepdata)
helper.addOverride(override = 'command', overrideValue='test-win')
helper.addOverride(override = 'option', overrideValue='')
helper.addOverride(override = 'se-name', overrideValue='charlie.sheen.biz')
helper.addOverride(override = 'lfn-prefix', overrideValue='test-win')
executor.initialise( self.stepdata, self.job)
self.setLocalOverride(self.stepdata)
executor.step = self.stepdata
executor.execute( )
self.assertFalse( os.path.exists( os.path.join( self.testDir, 'hosts' )))
self.assertFalse( os.path.exists( os.path.join( self.testDir, 'test1', 'hosts')))
def testUnitTestBackendNew(self):
myReport = Report()
myReport.unpersist(os.path.join( self.testDir,'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
myReport.data.cmsRun1.status = 1
myReport.persist(os.path.join( self.testDir, 'UnitTests','WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
executor = LogArchiveExecutor.LogArchive()
helper = LogArchiveTemplate.LogArchiveStepHelper(self.stepdata)
helper.addOverride(override = 'command', overrideValue='test-win')
helper.addOverride(override = 'option', overrideValue='')
helper.addOverride(override = 'se-name', overrideValue='charlie.sheen.biz')
helper.addOverride(override = 'lfn-prefix', overrideValue='test-win')
helper.setNewStageoutOverride( True )
executor.initialise( self.stepdata, self.job)
self.setLocalOverride(self.stepdata)
executor.step = self.stepdata
executor.execute( )
self.assertFalse( os.path.exists( os.path.join( self.testDir, 'hosts' )))
self.assertFalse( os.path.exists( os.path.join( self.testDir, 'test1', 'hosts')))
def setLocalOverride(self, step):
step.section_('override')
step.override.command = 'cp'
step.override.option = ''
step.override.__setattr__('lfn-prefix', self.testDir +"/")
step.override.__setattr__('se-name','DUMMYSE')
step.override.__setattr__('phedex-node','DUMMYPNN')
class otherStageOutTexst(unittest.TestCase):
def setUp(self):
# stolen from CMSSWExecutor_t. thanks, dave
# first, delete all the sandboxen and taskspaces
# because of caching, this leaks from other tests in other files
# this sucks because the other tests are using sandboxen that
# are deleted after the test is over, which causes theses tests
# to break
modsToDelete = []
# not sure what happens if you delete from
# an arrey you're iterating over. doing it in
# two steps
for modname in sys.modules.keys():
# need to blow away things in sys.modules, otherwise
# they are cached and we look at old taskspaces
if modname.startswith('WMTaskSpace'):
modsToDelete.append(modname)
if modname.startswith('WMSandbox'):
modsToDelete.append(modname)
for modname in modsToDelete:
try:
reload(sys.modules[modname])
except Exception:
pass
del sys.modules[modname]
self.oldpath = sys.path[:]
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
self.job = Job(name="/UnitTests/DeleterTask/DeleteTest-test-job")
shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("DeleterTask")
cmsswHelper = self.task.makeStep("cmsRun1")
cmsswHelper.setStepType('CMSSW')
stepHelper = cmsswHelper.addStep("DeleteTest")
stepHelper.setStepType('StageOut')
self.cmsswstep = cmsswHelper.data
self.cmsswHelper = cmsswHelper
self.stepdata = stepHelper.data
self.stephelp = StageOutTemplate.StageOutStepHelper(stepHelper.data)
self.task.applyTemplates()
self.executor = StepFactory.getStepExecutor(self.stephelp.stepType())
taskMaker = TaskMaker(self.workload, os.path.join(self.testDir))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.task.build(os.path.join(self.testDir, 'UnitTests'))
sys.path.insert(0, self.testDir)
sys.path.insert(0, os.path.join(self.testDir, 'UnitTests'))
# binDir = inspect.getsourcefile(ModuleLocator)
# binDir = binDir.replace("__init__.py", "bin")
#
# if not binDir in os.environ['PATH']:
# os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
open(os.path.join(self.testDir, 'UnitTests', '__init__.py'),
'w').close()
shutil.copyfile(
os.path.join(os.path.dirname(__file__), 'MergeSuccess.pkl'),
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
def tearDown(self):
sys.path = self.oldpath[:]
self.testInit.delWorkDir()
# making double sure WMTaskSpace and WMSandbox are gone
modsToDelete = []
# not sure what happens if you delete from
# an arrey you're iterating over. doing it in
# two steps
for modname in sys.modules.keys():
# need to blow away things in sys.modules, otherwise
# they are cached and we look at old taskspaces
if modname.startswith('WMTaskSpace'):
modsToDelete.append(modname)
if modname.startswith('WMSandbox'):
modsToDelete.append(modname)
for modname in modsToDelete:
try:
reload(sys.modules[modname])
except Exception:
pass
del sys.modules[modname]
myThread = threading.currentThread()
if hasattr(myThread, "factory"):
myThread.factory = {}
@attr('integration')
def testCPBackendStageOutAgainstReportNew(self):
myReport = Report()
myReport.unpersist(
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
myReport.data.cmsRun1.status = 0
myReport.persist(
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
executor = StageOutExecutor.StageOut()
executor.initialise(self.stepdata, self.job)
self.setLocalOverride(self.stepdata)
self.stepdata.override.newStageOut = True
executor.step = self.stepdata
executor.execute()
self.assertTrue(os.path.exists(os.path.join(self.testDir, 'hosts')))
self.assertTrue(
os.path.exists(os.path.join(self.testDir, 'test1', 'hosts')))
@attr('integration')
def testCPBackendStageOutAgainstReportFailedStepNew(self):
myReport = Report()
myReport.unpersist(
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
myReport.data.cmsRun1.status = 1
myReport.persist(
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
executor = StageOutExecutor.StageOut()
executor.initialise(self.stepdata, self.job)
self.setLocalOverride(self.stepdata)
self.stepdata.override.newStageOut = True
executor.step = self.stepdata
executor.execute()
self.assertFalse(os.path.exists(os.path.join(self.testDir, 'hosts')))
self.assertFalse(
os.path.exists(os.path.join(self.testDir, 'test1', 'hosts')))
return
@attr('integration')
def testCPBackendStageOutAgainstReportOld(self):
myReport = Report()
myReport.unpersist(
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
myReport.data.cmsRun1.status = 0
myReport.persist(
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
executor = StageOutExecutor.StageOut()
executor.initialise(self.stepdata, self.job)
self.setLocalOverride(self.stepdata)
executor.step = self.stepdata
executor.execute()
self.assertTrue(os.path.exists(os.path.join(self.testDir, 'hosts')))
self.assertTrue(
os.path.exists(os.path.join(self.testDir, 'test1', 'hosts')))
return
@attr('integration')
def testCPBackendStageOutAgainstReportFailedStepOld(self):
myReport = Report()
myReport.unpersist(
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
myReport.data.cmsRun1.status = 1
myReport.persist(
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
executor = StageOutExecutor.StageOut()
executor.initialise(self.stepdata, self.job)
self.setLocalOverride(self.stepdata)
executor.step = self.stepdata
executor.execute()
self.assertFalse(os.path.exists(os.path.join(self.testDir, 'hosts')))
self.assertFalse(
os.path.exists(os.path.join(self.testDir, 'test1', 'hosts')))
return
@attr('workerNodeTest')
def testOnWorkerNodes(self):
raise RuntimeError
# Stage a file out, stage it back in, check it, delete it
myReport = Report()
myReport.unpersist(
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
myReport.data.cmsRun1.status = 1
del myReport.data.cmsRun1.output
myReport.data.cmsRun1.section_('output')
myReport.data.cmsRun1.output.section_('stagingTestOutput')
myReport.data.cmsRun1.output.stagingTestOutput.section_('files')
myReport.data.cmsRun1.output.stagingTestOutput.fileCount = 0
targetFiles = [
'/store/temp/WMAgent/storetest-%s' % time.time(),
'/store/unmerged/WMAgent/storetest-%s' % time.time()
]
for file in targetFiles:
print("Adding file for StageOut %s" % file)
self.addStageOutFile(myReport, file)
myReport.persist(
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
executor = StageOutExecutor.StageOut()
executor.initialise(self.stepdata, self.job)
executor.step = self.stepdata
print("beginning stageout")
executor.execute()
print("stageout done")
# pull in the report with the stage out info
myReport = Report()
myReport.unpersist(
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
print("Got the stage out data back")
print(myReport.data)
# now, transfer them back
# TODO make a stagein step in the task - Melo
import WMCore.Storage.FileManager as FileManagerModule
fileManager = FileManagerModule.FileManager(numberOfRetries=10,
retryPauseTime=1)
for file in targetFiles:
print("Staging in %s" % file)
fileManager.stageOut(fileToStage={
'LFN': file,
'PFN': '%s/%s' % (self.testDir, file)
})
self.assertTrue(os.path.exists('%s/%s' % (self.testDir, file)))
# self.assertEqual(os.path.getsize('/etc/hosts', '%s/%s' % (self.testDir, file))) This makes no sense - EWV
# now, should delete the files we made
for file in targetFiles:
print("deleting %s" % file)
fileManager.deleteLFN(file)
# try staging in again to make sure teh files are gone
for file in targetFiles:
print("Staging in (should fail) %s" % file)
self.assertRaises(StageOutError,
FileManagerModule.FileManager.stageOut,
fileManager,
fileToStage={
'LFN': file,
'PFN': '%s/%s' % (self.testDir, file)
},
stageOut=False)
# need to make sure files didn't show up
self.assertFalse(os.path.exists(os.path.join(self.testDir, 'hosts')))
self.assertFalse(
os.path.exists(os.path.join(self.testDir, 'test1', 'hosts')))
def addStageOutFile(self, myReport, lfn):
myId = myReport.data.cmsRun1.output.stagingTestOutput.fileCount
mySection = myReport.data.cmsRun1.output.stagingTestOutput.section_(
'file%s' % myId)
mySection.section_('runs')
setattr(mySection.runs, '114475', [33])
mySection.section_('branches')
mySection.lfn = lfn
mySection.dataset = {
'applicationName': 'cmsRun',
'primaryDataset': 'Calo',
'processedDataset': 'Commissioning09-PromptReco-v8',
'dataTier': 'ALCARECO',
'applicationVersion': 'CMSSW_3_2_7'
}
mySection.module_label = 'ALCARECOStreamCombined'
mySection.parents = []
mySection.location = 'srm-cms.cern.ch'
mySection.checksums = {'adler32': 'bbcf2215', 'cksum': '2297542074'}
mySection.pfn = '/etc/hosts'
mySection.events = 20000
mySection.merged = False
mySection.size = 37556367
myReport.data.cmsRun1.output.stagingTestOutput.fileCount = myId + 1
def setLocalOverride(self, step):
step.section_('override')
step.override.command = 'cp'
step.override.option = ''
step.override.__setattr__('lfn-prefix', self.testDir + "/")
step.override.__setattr__('phedex-node', 'DUMMYPNN')
class scaleTestFiller:
"""
_scaleTestFiller_
Initializes the DB and the DBSUploader
On __call__() it creates data and uploads it.
"""
def __init__(self):
"""
__init__
Init the DB
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase=True)
self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer"],
useDefault=False)
self.configFile = EmulatorSetup.setupWMAgentConfig()
myThread = threading.currentThread()
self.bufferFactory = DAOFactory(
package="WMComponent.DBSBuffer.Database",
logger=myThread.logger,
dbinterface=myThread.dbi)
locationAction = self.bufferFactory(
classname="DBSBufferFiles.AddLocation")
locationAction.execute(siteName="se1.cern.ch")
locationAction.execute(siteName="se1.fnal.gov")
locationAction.execute(siteName="malpaquet")
config = self.getConfig()
self.dbsUploader = DBSUploadPoller(config=config)
return
def __call__(self):
"""
__call__
Generate some random data
"""
# Generate somewhere between one and a thousand files
name = "ThisIsATest_%s" % (makeUUID())
nFiles = random.randint(10, 2000)
name = name.replace('-', '_')
name = '%s-v0' % name
files = self.getFiles(name=name, nFiles=nFiles)
print "Inserting %i files for dataset %s" % (nFiles * 2, name)
try:
self.dbsUploader.algorithm()
except:
self.dbsUploader.close()
raise
# Repeat just to make sure
try:
self.dbsUploader.algorithm()
except:
self.dbsUploader.close()
raise
return
def getConfig(self):
"""
_getConfig_
This creates the actual config file used by the component
"""
config = self.testInit.getConfiguration()
self.testInit.generateWorkDir(config)
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = 'DBSUpload'
config.Agent.useHeartbeat = False
#Now the CoreDatabase information
#This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("DBS3Upload")
config.DBS3Upload.pollInterval = 10
config.DBS3Upload.logLevel = 'DEBUG'
config.DBS3Upload.DBSBlockMaxFiles = 500
config.DBS3Upload.DBSBlockMaxTime = 600
config.DBS3Upload.DBSBlockMaxSize = 999999999999
config.DBS3Upload.dbsUrl = 'http://cms-xen40.fnal.gov:8787/dbs/prod/global/DBSWriter'
config.DBS3Upload.namespace = 'WMComponent.DBS3Buffer.DBSUpload'
config.DBS3Upload.componentDir = os.path.join(os.getcwd(),
'Components')
config.DBS3Upload.nProcesses = 1
config.DBS3Upload.dbsWaitTime = 1
return config
def getFiles(self,
name,
tier='RECO',
nFiles=12,
site="malpaquet",
nLumis=1):
"""
Create some quick dummy test files
"""
files = []
for f in range(nFiles):
testFile = DBSBufferFile(
lfn='/data/store/random/random/RANDOM/test/0/%s-%s-%i.root' %
(name, site, f),
size=1024,
events=20,
checksums={'cksum': 1})
testFile.setAlgorithm(appName=name,
appVer="CMSSW_3_1_1",
appFam="RECO",
psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier))
lumis = []
for i in range(nLumis):
lumis.append((f * 100000) + i)
testFile.addRun(Run(1, *lumis))
testFile.setAcquisitionEra(name.split('-')[0])
testFile.setProcessingVer("0")
testFile.setGlobalTag("Weird")
testFile.create()
testFile.setLocation(site)
files.append(testFile)
count = 0
for f in files:
count += 1
testFileChild = DBSBufferFile(
lfn=
'/data/store/random/random/RANDOM/test/0/%s-%s-%i-child.root' %
(name, site, count),
size=1024,
events=10,
checksums={'cksum': 1})
testFileChild.setAlgorithm(appName=name,
appVer="CMSSW_3_1_1",
appFam="RECO",
psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileChild.setDatasetPath("/%s/%s_2/RECO" % (name, name))
testFileChild.addRun(Run(1, *[45]))
testFileChild.create()
testFileChild.setLocation(site)
testFileChild.addParents([f['lfn']])
return files
class StageOutTest(unittest.TestCase):
def setUp(self):
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
# shut up SiteLocalConfig
os.environ['CMS_PATH'] = os.getcwd()
workload = copy.deepcopy(testWorkloads.workload)
task = workload.getTask("Production")
step = task.getStep("stageOut1")
# want to get the cmsstep so I can make the Report
cmsstep = task.getStep('cmsRun1')
self.cmsstepdir = os.path.join(self.testDir, 'cmsRun1')
os.mkdir(self.cmsstepdir)
open(os.path.join(self.cmsstepdir, '__init__.py'), 'w').close()
open(os.path.join(self.cmsstepdir, 'Report.pkl'), 'w').close()
cmsbuilder = CMSSWBuilder.CMSSW()
cmsbuilder(cmsstep.data, 'Production', self.cmsstepdir)
realstep = StageOutTemplate.StageOutStepHelper(step.data)
realstep.disableRetries()
self.realstep = realstep
self.stepDir = os.path.join(self.testDir, 'stepdir')
os.mkdir(self.stepDir)
builder = StageOutBuilder.StageOut()
builder(step.data, 'Production', self.stepDir)
# stolen from CMSSWExecutor_t. thanks, dave
# first, delete all the sandboxen and taskspaces
# because of caching, this leaks from other tests in other files
# this sucks because the other tests are using sandboxen that
# are deleted after the test is over, which causes theses tests
# to break
modsToDelete = []
# not sure what happens if you delete from
# an arrey you're iterating over. doing it in
# two steps
for modname in sys.modules.keys():
# need to blow away things in sys.modules, otherwise
# they are cached and we look at old taskspaces
if modname.startswith('WMTaskSpace'):
modsToDelete.append(modname)
if modname.startswith('WMSandbox'):
modsToDelete.append(modname)
for modname in modsToDelete:
try:
reload(sys.modules[modname])
except Exception:
pass
del sys.modules[modname]
self.oldpath = sys.path[:]
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
self.job = Job(name="/UnitTests/DeleterTask/DeleteTest-test-job")
shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("DeleterTask")
cmsswHelper = self.task.makeStep("cmsRun1")
cmsswHelper.setStepType('CMSSW')
stepHelper = cmsswHelper.addStep("DeleteTest")
stepHelper.setStepType('StageOut')
self.cmsswstep = cmsswHelper.data
self.cmsswHelper = cmsswHelper
self.stepdata = stepHelper.data
self.stephelp = StageOutTemplate.StageOutStepHelper(stepHelper.data)
self.task.applyTemplates()
self.executor = StepFactory.getStepExecutor(self.stephelp.stepType())
taskMaker = TaskMaker(self.workload, os.path.join(self.testDir))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.task.build(os.path.join(self.testDir, 'UnitTests'))
sys.path.insert(0, self.testDir)
sys.path.insert(0, os.path.join(self.testDir, 'UnitTests'))
# binDir = inspect.getsourcefile(ModuleLocator)
# binDir = binDir.replace("__init__.py", "bin")
#
# if not binDir in os.environ['PATH']:
# os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
open(os.path.join(self.testDir, 'UnitTests', '__init__.py'),
'w').close()
shutil.copyfile(
os.path.join(os.path.dirname(__file__), 'MergeSuccess.pkl'),
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
def tearDown(self):
sys.path = self.oldpath[:]
self.testInit.delWorkDir()
# making double sure WMTaskSpace and WMSandbox are gone
modsToDelete = []
# not sure what happens if you delete from
# an arrey you're iterating over. doing it in
# two steps
for modname in sys.modules.keys():
# need to blow away things in sys.modules, otherwise
# they are cached and we look at old taskspaces
if modname.startswith('WMTaskSpace'):
modsToDelete.append(modname)
if modname.startswith('WMSandbox'):
modsToDelete.append(modname)
for modname in modsToDelete:
try:
reload(sys.modules[modname])
except Exception:
pass
del sys.modules[modname]
myThread = threading.currentThread()
if hasattr(myThread, "factory"):
myThread.factory = {}
def makeReport(self, fileName):
myReport = Report('oneitem')
myReport.addStep('stageOut1')
myReport.addOutputModule('module1')
myReport.addOutputModule('module2')
myReport.addOutputFile('module1', {
'lfn': 'FILE1',
'size': 1,
'events': 1
})
myReport.addOutputFile('module2', {
'lfn': 'FILE2',
'size': 1,
'events': 1
})
myReport.addOutputFile('module2', {
'lfn': 'FILE3',
'size': 1,
'events': 1
})
myReport.persist(fileName)
def testExecutorDoesntDetonate(self):
myReport = Report()
myReport.unpersist(
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
myReport.data.cmsRun1.status = 1
myReport.persist(
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
executor = StageOutExecutor.StageOut()
executor.initialise(self.stepdata, self.job)
self.setLocalOverride(self.stepdata)
executor.step = self.stepdata
executor.execute()
self.assertFalse(os.path.exists(os.path.join(self.testDir, 'hosts')))
self.assertFalse(
os.path.exists(os.path.join(self.testDir, 'test1', 'hosts')))
return
def testUnitTestBackend(self):
myReport = Report()
myReport.unpersist(
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
myReport.data.cmsRun1.status = 1
myReport.persist(
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
executor = StageOutExecutor.StageOut()
helper = StageOutTemplate.StageOutStepHelper(self.stepdata)
helper.addOverride(override='command', overrideValue='test-win')
helper.addOverride(override='option', overrideValue='')
helper.addOverride(override='phedex-node',
overrideValue='charlie.sheen.biz')
helper.addOverride(override='lfn-prefix', overrideValue='test-win')
executor.initialise(self.stepdata, self.job)
self.setLocalOverride(self.stepdata)
executor.step = self.stepdata
executor.execute()
self.assertFalse(os.path.exists(os.path.join(self.testDir, 'hosts')))
self.assertFalse(
os.path.exists(os.path.join(self.testDir, 'test1', 'hosts')))
def testUnitTestBackendNew(self):
myReport = Report()
myReport.unpersist(
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
myReport.data.cmsRun1.status = 1
myReport.persist(
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
executor = StageOutExecutor.StageOut()
helper = StageOutTemplate.StageOutStepHelper(self.stepdata)
helper.addOverride(override='command', overrideValue='test-win')
helper.addOverride(override='option', overrideValue='')
helper.addOverride(override='phedex-node',
overrideValue='charlie.sheen.biz')
helper.addOverride(override='lfn-prefix', overrideValue='test-win')
helper.setNewStageoutOverride(True)
executor.initialise(self.stepdata, self.job)
self.setLocalOverride(self.stepdata)
executor.step = self.stepdata
executor.execute()
self.assertFalse(os.path.exists(os.path.join(self.testDir, 'hosts')))
self.assertFalse(
os.path.exists(os.path.join(self.testDir, 'test1', 'hosts')))
def setLocalOverride(self, step):
step.section_('override')
step.override.command = 'cp'
step.override.option = ''
step.override.__setattr__('lfn-prefix', self.testDir + "/")
step.override.__setattr__('phedex-node', 'DUMMYPNN')
class APITest(unittest.TestCase):
def setUp(self):
"""
This much stuff for simple alerts client API testing is
needed because it's also testing setting up alert fw as
done in the BaseWorkerThread.
"""
myThread = threading.currentThread()
self.testInit = TestInit(__file__)
self.testInit.setLogging(logLevel = logging.DEBUG)
self.testDir = self.testInit.generateWorkDir()
# needed for instantiating BaseWorkerThread
self.testInit.setDatabaseConnection()
self.alertsReceiver = None
def tearDown(self):
self.testInit.clearDatabase()
self.testInit.delWorkDir()
if self.alertsReceiver:
self.alertsReceiver.shutdown()
def testAlertsMessagingBasic(self):
config = getConfig("/tmp")
self.assertTrue(hasattr(config, "Alert"))
# initialization
# sender: instance of Alert messages Sender
# preAlert: pre-defined values for Alert instances generated from this class
self.config = config # needed in setUpAlertsMessaging
preAlert, sender = alertAPI.setUpAlertsMessaging(self,
compName = "testBasic")
sendAlert = alertAPI.getSendAlert(sender = sender,
preAlert = preAlert)
# set up a temporary alert message receiver
handler, receiver = utils.setUpReceiver(config.Alert.address,
config.Alert.controlAddr)
# test sending alert
msg = "this is my message Basic"
sendAlert(100, msg = msg)
# wait for the alert to arrive
while len(handler.queue) == 0:
time.sleep(0.3)
print "%s waiting for alert to arrive ..." % inspect.stack()[0][3]
self.assertEqual(len(handler.queue), 1)
alert = handler.queue[0]
self.assertEqual(alert["Component"], "testBasic")
self.assertEqual(alert["Level"], 100)
self.assertEqual(alert["Source"], self.__class__.__name__)
self.assertEqual(alert["Details"]["msg"], msg)
sender.unregister()
receiver.shutdown()
def testAlertsMessagingNotSetUpViaBaseWorkerThread(self):
# alerts will not be set up if 'config.Alert' etc is not provided
config = Configuration()
self.assertFalse(hasattr(config, "Alert"))
# test the same way alerts are set up in the client code (currently
# all cases via BaseWorkerThread)
# this call creates .sender, but here will be set to None
thread = BaseWorkerThread()
thread.config = config
thread.initAlerts(compName = "test1")
self.assertFalse(thread.sender)
# shall do nothing and not fail
thread.sendAlert("nonsense", msg = "nonsense")
def testAlertsSetUpAndSendingViaBaseWorkerThread(self):
# calls as they are made from child/client classes of BaseWorkerThread
config = getConfig("/tmp")
self.assertTrue(hasattr(config, "Alert"))
# test the same way alerts are set up in the client code (currently
# all cases via BaseWorkerThread)
# this call creates .sender, but here will be set to None
thread = BaseWorkerThread()
thread.config = config
thread.initAlerts(compName = "test2")
self.assertTrue(thread.sender)
# set up a temporary alert message receiver
handler, receiver = utils.setUpReceiver(config.Alert.address,
config.Alert.controlAddr)
# send an alert message
msg = "this is my message 1"
thread.sendAlert(10, msg = msg)
# wait for the alert to arrive
while len(handler.queue) == 0:
time.sleep(0.3)
print "%s waiting for alert to arrive ..." % inspect.stack()[0][3]
self.assertEqual(len(handler.queue), 1)
alert = handler.queue[0]
self.assertEqual(alert["Component"], "test2")
self.assertEqual(alert["Level"], 10)
self.assertEqual(alert["Source"], thread.__class__.__name__)
self.assertEqual(alert["Details"]["msg"], msg)
thread.sender.unregister()
receiver.shutdown()
def testAgentConfigurationRetrieving(self):
"""
Test that getting some agent details (config values from config.Agent
section) will be correctly propagated into Alert instances.
Alert instance is obtained via API.getPredefinedAlert factory.
"""
d = dict(Additional = "detail")
# instantiate just plain Alert, no configuration to take
# into account at this point
a = Alert(**d)
self.assertEqual(a["HostName"], None)
self.assertEqual(a["Contact"], None)
self.assertEqual(a["TeamName"], None)
self.assertEqual(a["AgentName"], None)
self.assertEqual(a["Additional"], "detail")
# instantiate via factory which reads configuration instance
config = Configuration()
config.section_("Agent")
config.Agent.hostName = "some1"
config.Agent.contact = "some2"
config.Agent.teamName = "some3"
config.Agent.agentName = "some4"
a = alertAPI.getPredefinedAlert(**d)
self.assertEqual(a["HostName"], "some1")
self.assertEqual(a["Contact"], "some2")
self.assertEqual(a["TeamName"], "some3")
self.assertEqual(a["AgentName"], "some4")
self.assertEqual(a["Additional"], "detail")
class DataProcessingTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Initialize the database.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMCore.WMBS"],
useDefault = False)
self.testDir = self.testInit.generateWorkDir()
return
def tearDown(self):
"""
_tearDown_
Clear out the database.
"""
self.testInit.clearDatabase()
self.testInit.delWorkDir()
return
def testDataProcessing(self):
"""
_testDataProcessing_
Create a data processing workflow and verify it installs into WMBS
correctly. Check that we can drop an output module.
"""
testArgs = getTestArguments()
testArgs['TransientOutputModules'] = ['RECOoutput']
testWorkload = dataProcessingWorkload("TestWorkload", testArgs)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DMWM")
testWMBSHelper = WMBSHelper(testWorkload, "DataProcessing", "SomeBlock", cachepath = self.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
procWorkflow = Workflow(name = "TestWorkload",
task = "/TestWorkload/DataProcessing")
procWorkflow.load()
self.assertEqual(len(procWorkflow.outputMap.keys()), 3,
"Error: Wrong number of WF outputs.")
goldenOutputMods = ["RECOoutput", "ALCARECOoutput"]
for goldenOutputMod in goldenOutputMods:
mergedOutput = procWorkflow.outputMap[goldenOutputMod][0]["merged_output_fileset"]
unmergedOutput = procWorkflow.outputMap[goldenOutputMod][0]["output_fileset"]
mergedOutput.loadData()
unmergedOutput.loadData()
if goldenOutputMod in testArgs["TransientOutputModules"]:
self.assertEqual(mergedOutput.name, "/TestWorkload/DataProcessing/unmerged-%s" % goldenOutputMod,
"Error: Merged output fileset is wrong: %s" % mergedOutput.name)
else:
self.assertEqual(mergedOutput.name, "/TestWorkload/DataProcessing/DataProcessingMerge%s/merged-Merged" % goldenOutputMod,
"Error: Merged output fileset is wrong: %s" % mergedOutput.name)
self.assertEqual(unmergedOutput.name, "/TestWorkload/DataProcessing/unmerged-%s" % goldenOutputMod,
"Error: Unmerged output fileset is wrong.")
logArchOutput = procWorkflow.outputMap["logArchive"][0]["merged_output_fileset"]
unmergedLogArchOutput = procWorkflow.outputMap["logArchive"][0]["output_fileset"]
logArchOutput.loadData()
unmergedLogArchOutput.loadData()
self.assertEqual(logArchOutput.name, "/TestWorkload/DataProcessing/unmerged-logArchive",
"Error: LogArchive output fileset is wrong.")
self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/DataProcessing/unmerged-logArchive",
"Error: LogArchive output fileset is wrong.")
for goldenOutputMod in goldenOutputMods:
if goldenOutputMod in testArgs["TransientOutputModules"]:
# No merge for this output module
continue
mergeWorkflow = Workflow(name = "TestWorkload",
task = "/TestWorkload/DataProcessing/DataProcessingMerge%s" % goldenOutputMod)
mergeWorkflow.load()
self.assertEqual(len(mergeWorkflow.outputMap.keys()), 2,
"Error: Wrong number of WF outputs.")
mergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["merged_output_fileset"]
unmergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["output_fileset"]
mergedMergeOutput.loadData()
unmergedMergeOutput.loadData()
self.assertEqual(mergedMergeOutput.name, "/TestWorkload/DataProcessing/DataProcessingMerge%s/merged-Merged" % goldenOutputMod,
"Error: Merged output fileset is wrong.")
self.assertEqual(unmergedMergeOutput.name, "/TestWorkload/DataProcessing/DataProcessingMerge%s/merged-Merged" % goldenOutputMod,
"Error: Unmerged output fileset is wrong.")
logArchOutput = mergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"]
unmergedLogArchOutput = mergeWorkflow.outputMap["logArchive"][0]["output_fileset"]
logArchOutput.loadData()
unmergedLogArchOutput.loadData()
self.assertEqual(logArchOutput.name, "/TestWorkload/DataProcessing/DataProcessingMerge%s/merged-logArchive" % goldenOutputMod,
"Error: LogArchive output fileset is wrong: %s" % logArchOutput.name)
self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/DataProcessing/DataProcessingMerge%s/merged-logArchive" % goldenOutputMod,
"Error: LogArchive output fileset is wrong.")
topLevelFileset = Fileset(name = "TestWorkload-DataProcessing-SomeBlock")
topLevelFileset.loadData()
procSubscription = Subscription(fileset = topLevelFileset, workflow = procWorkflow)
procSubscription.loadData()
self.assertEqual(procSubscription["type"], "Processing",
"Error: Wrong subscription type.")
self.assertEqual(procSubscription["split_algo"], "LumiBased",
"Error: Wrong split algo.")
unmergedReco = Fileset(name = "/TestWorkload/DataProcessing/unmerged-RECOoutput")
unmergedReco.loadData()
recoMergeWorkflow = Workflow(name = "TestWorkload",
task = "/TestWorkload/DataProcessing/DataProcessingMergeRECOoutput")
# No merge workflow should exist in WMBS
self.assertRaises(IndexError, recoMergeWorkflow.load)
unmergedAlca = Fileset(name = "/TestWorkload/DataProcessing/unmerged-ALCARECOoutput")
unmergedAlca.loadData()
alcaMergeWorkflow = Workflow(name = "TestWorkload",
task = "/TestWorkload/DataProcessing/DataProcessingMergeALCARECOoutput")
alcaMergeWorkflow.load()
mergeSubscription = Subscription(fileset = unmergedAlca, workflow = alcaMergeWorkflow)
mergeSubscription.loadData()
self.assertEqual(mergeSubscription["type"], "Merge",
"Error: Wrong subscription type.")
self.assertEqual(mergeSubscription["split_algo"], "ParentlessMergeBySize",
"Error: Wrong split algo.")
for procOutput in ["RECOoutput", "ALCARECOoutput"]:
unmerged = Fileset(name = "/TestWorkload/DataProcessing/unmerged-%s" % procOutput)
unmerged.loadData()
cleanupWorkflow = Workflow(name = "TestWorkload",
task = "/TestWorkload/DataProcessing/DataProcessingCleanupUnmerged%s" % procOutput)
cleanupWorkflow.load()
cleanupSubscription = Subscription(fileset = unmerged, workflow = cleanupWorkflow)
cleanupSubscription.loadData()
self.assertEqual(cleanupSubscription["type"], "Cleanup",
"Error: Wrong subscription type.")
self.assertEqual(cleanupSubscription["split_algo"], "SiblingProcessingBased",
"Error: Wrong split algo.")
procLogCollect = Fileset(name = "/TestWorkload/DataProcessing/unmerged-logArchive")
procLogCollect.loadData()
procLogCollectWorkflow = Workflow(name = "TestWorkload",
task = "/TestWorkload/DataProcessing/LogCollect")
procLogCollectWorkflow.load()
logCollectSub = Subscription(fileset = procLogCollect, workflow = procLogCollectWorkflow)
logCollectSub.loadData()
self.assertEqual(logCollectSub["type"], "LogCollect",
"Error: Wrong subscription type.")
self.assertEqual(logCollectSub["split_algo"], "MinFileBased",
"Error: Wrong split algo.")
procLogCollect = Fileset(name = "/TestWorkload/DataProcessing/DataProcessingMergeALCARECOoutput/merged-logArchive")
procLogCollect.loadData()
procLogCollectWorkflow = Workflow(name = "TestWorkload",
task = "/TestWorkload/DataProcessing/DataProcessingMergeALCARECOoutput/DataProcessingALCARECOoutputMergeLogCollect")
procLogCollectWorkflow.load()
logCollectSub = Subscription(fileset = procLogCollect, workflow = procLogCollectWorkflow)
logCollectSub.loadData()
self.assertEqual(logCollectSub["type"], "LogCollect",
"Error: Wrong subscription type.")
self.assertEqual(logCollectSub["split_algo"], "MinFileBased",
"Error: Wrong split algo.")
return
class JobPackageTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Create a temporary file to presist the JobPackage to.
"""
self.testInit = TestInit(__file__)
self.persistFile = os.path.join(self.testInit.generateWorkDir(),
"JobPackage.pkl")
return
def tearDown(self):
pass
def testAddingJobs(self):
"""
_testAddingJobs_
Verify that adding jobs to the package works as expected.
"""
package = JobPackage()
for i in range(100):
newJob = Job("Job%s" % i)
newJob["id"] = i
package[i] = newJob
# There is an extra key for the directory the package is stored in.
assert len(package.keys()) == 101, \
"Error: Wrong number of jobs in package."
for i in range(100):
job = package[i]
assert job["id"] == i, \
"Error: Jobs has wrong ID."
assert job["name"] == "Job%d" % i, \
"Error: Job has wrong name."
return
def testPersist(self):
"""
_testPersist_
Verify that we're able to save and load the job package.
"""
package = JobPackage()
for i in range(100):
newJob = Job("Job%s" % i)
newJob["id"] = i
package[i] = newJob
package.save(self.persistFile)
assert os.path.exists(self.persistFile), \
"Error: Package file was never created."
newPackage = JobPackage()
newPackage.load(self.persistFile)
# There is an extra key for the directory the package is stored in.
assert len(newPackage.keys()) == 101, \
"Error: Wrong number of jobs in package."
for i in range(100):
job = newPackage[i]
assert job["id"] == i, \
"Error: Jobs has wrong ID."
assert job["name"] == "Job%d" % i, \
"Error: Job has wrong name."
return
def testBaggage(self):
"""
_testBaggage_
Verify that job baggage is persisted with the package.
"""
package = JobPackage()
for i in range(100):
newJob = Job("Job%s" % i)
newJob["id"] = i
baggage = newJob.getBaggage()
setattr(baggage, "thisJob", newJob["name"])
setattr(baggage, "seed1", 11111111)
setattr(baggage, "seed2", 22222222)
setattr(baggage, "seed3", 33333333)
setattr(baggage, "seed4", 44444444)
setattr(baggage, "seed5", 55555555)
package[i] = newJob
package.save(self.persistFile)
assert os.path.exists(self.persistFile), \
"Error: Package file was never created."
newPackage = JobPackage()
newPackage.load(self.persistFile)
# There is an extra key for the directory the package is stored in.
assert len(newPackage.keys()) == 101, \
"Error: Wrong number of jobs in package."
for i in range(100):
job = newPackage[i]
assert job["id"] == i, \
"Error: Jobs has wrong ID."
assert job["name"] == "Job%d" % i, \
"Error: Job has wrong name."
jobBaggage = job.getBaggage()
assert jobBaggage.thisJob == "Job%d" % i, \
"Error: Job baggage has wrong name."
assert jobBaggage.seed1 == 11111111, \
"Error: Job baggee has wrong value for seed1."
assert jobBaggage.seed2 == 22222222, \
"Error: Job baggee has wrong value for seed2."
assert jobBaggage.seed3 == 33333333, \
"Error: Job baggee has wrong value for seed3."
assert jobBaggage.seed4 == 44444444, \
"Error: Job baggee has wrong value for seed4."
assert jobBaggage.seed5 == 55555555, \
"Error: Job baggee has wrong value for seed5."
return
class CouchTest(unittest.TestCase):
def setUp(self):
self.testInit = TestInit(__file__)
self.testInit.setLogging(logLevel = logging.DEBUG)
self.testDir = self.testInit.generateWorkDir()
self.config = getConfig(self.testDir)
# mock generator instance to communicate some configuration values
self.generator = utils.AlertGeneratorMock(self.config)
self.testName = self.id().split('.')[-1]
def tearDown(self):
self.testInit.delWorkDir()
self.generator = None
def testAlertGeneratorCouchDbSizePollerBasic(self):
config = getConfig("/tmp")
try:
poller = CouchDbSizePoller(config.AlertGenerator.couchDbSizePoller, self.generator)
except Exception as ex:
self.fail("%s: exception: %s" % (self.testName, ex))
poller.check() # -> on real system dir may result in permission denied
poller._dbDirectory = "/dev"
poller.check() # -> OK
# test failing during set up
poller = CouchDbSizePoller(config.AlertGenerator.couchDbSizePoller, self.generator)
poller._query = "nonsense query"
# this will fail on the above query
self.assertRaises(Exception, poller._getDbDir)
poller.check()
def testAlertGeneratorCouchDbSizePollerSoftThreshold(self):
self.config.AlertGenerator.couchDbSizePoller.couchURL = os.getenv("COUCHURL", None)
self.config.AlertGenerator.couchDbSizePoller.soft = 5
self.config.AlertGenerator.couchDbSizePoller.critical = 10
self.config.AlertGenerator.couchDbSizePoller.pollInterval = 0.2
ti = utils.TestInput() # see attributes comments at the class
ti.pollerClass = CouchDbSizePoller
ti.config = self.config.AlertGenerator.couchDbSizePoller
ti.thresholdToTest = self.config.AlertGenerator.couchDbSizePoller.soft
ti.level = self.config.AlertProcessor.soft.level
ti.expected = 1
ti.thresholdDiff = 4
ti.testCase = self
utils.doGenericValueBasedPolling(ti)
def testAlertGeneratorCouchDbSizePollerCriticalThreshold(self):
self.config.AlertGenerator.couchDbSizePoller.couchURL = os.getenv("COUCHURL", None)
self.config.AlertGenerator.couchDbSizePoller.soft = 5
self.config.AlertGenerator.couchDbSizePoller.critical = 10
self.config.AlertGenerator.couchDbSizePoller.pollInterval = 0.2
ti = utils.TestInput() # see attributes comments at the class
ti.pollerClass = CouchDbSizePoller
ti.config = self.config.AlertGenerator.couchDbSizePoller
ti.thresholdToTest = self.config.AlertGenerator.couchDbSizePoller.critical
ti.level = self.config.AlertProcessor.critical.level
ti.expected = 1
ti.thresholdDiff = 1
ti.testCase = self
utils.doGenericValueBasedPolling(ti)
def testAlertGeneratorCouchDbSizePollerNoAlert(self):
self.config.AlertGenerator.couchDbSizePoller.couchURL = os.getenv("COUCHURL", None)
self.config.AlertGenerator.couchDbSizePoller.soft = 5
self.config.AlertGenerator.couchDbSizePoller.critical = 10
self.config.AlertGenerator.couchDbSizePoller.pollInterval = 0.2
ti = utils.TestInput() # see attributes comments at the class
ti.pollerClass = CouchDbSizePoller
ti.config = self.config.AlertGenerator.couchDbSizePoller
ti.thresholdToTest = self.config.AlertGenerator.couchDbSizePoller.soft - 3
ti.expected = 0
ti.thresholdDiff = 2
ti.testCase = self
utils.doGenericValueBasedPolling(ti)
def testAlertGeneratorCouchPollerBasic(self):
self.config.AlertGenerator.section_("bogusCouchPoller")
self.config.AlertGenerator.bogusCouchPoller.couchURL = os.getenv("COUCHURL", None)
self.config.AlertGenerator.bogusCouchPoller.soft = 1000
self.config.AlertGenerator.bogusCouchPoller.critical = 2000
self.config.AlertGenerator.bogusCouchPoller.pollInterval = 0.2
self.config.AlertGenerator.bogusCouchPoller.period = 1
try:
poller = CouchPoller(self.config.AlertGenerator.bogusCouchPoller,
self.generator)
except Exception as ex:
self.fail("%s: exception: %s" % (self.testName, ex))
# this class would not have defined polling sample function, give it one
poller.sample = lambda proc: float(12)
poller.check()
# assuming CouchDb server is running, check that 1 sensible measurement value was collected
self.assertEqual(len(poller._measurements), 1)
self.assertTrue(isinstance(poller._measurements[0], types.FloatType))
# test handling of a non-existing process
CouchPoller._getProcessPID = lambda inst: 1212121212
self.assertRaises(Exception, CouchPoller,
self.config.AlertGenerator.bogusCouchPoller, self.generator)
def testAlertGeneratorCouchCPUPollerSoftThreshold(self):
self.config.AlertGenerator.couchCPUPoller.couchURL = os.getenv("COUCHURL", None)
self.config.AlertGenerator.couchCPUPoller.soft = 70
self.config.AlertGenerator.couchCPUPoller.critical = 80
self.config.AlertGenerator.couchCPUPoller.pollInterval = 0.2
self.config.AlertGenerator.couchCPUPoller.period = 1
ti = utils.TestInput() # see attributes comments at the class
ti.pollerClass = CouchCPUPoller
ti.config = self.config.AlertGenerator.couchCPUPoller
ti.thresholdToTest = self.config.AlertGenerator.couchCPUPoller.soft
ti.level = self.config.AlertProcessor.soft.level
ti.expected = 1
ti.thresholdDiff = 10
ti.testCase = self
utils.doGenericPeriodAndProcessPolling(ti)
def testAlertGeneratorCouchCPUPollerCriticalThreshold(self):
self.config.AlertGenerator.couchCPUPoller.couchURL = os.getenv("COUCHURL", None)
self.config.AlertGenerator.couchCPUPoller.soft = 70
self.config.AlertGenerator.couchCPUPoller.critical = 80
self.config.AlertGenerator.couchCPUPoller.pollInterval = 0.2
self.config.AlertGenerator.couchCPUPoller.period = 1
ti = utils.TestInput() # see attributes comments at the class
ti.pollerClass = CouchCPUPoller
ti.config = self.config.AlertGenerator.couchCPUPoller
ti.thresholdToTest = self.config.AlertGenerator.couchCPUPoller.critical
ti.level = self.config.AlertProcessor.critical.level
ti.expected = 1
ti.thresholdDiff = 10
ti.testCase = self
utils.doGenericPeriodAndProcessPolling(ti)
def testAlertGeneratorCouchCPUPollerNoAlert(self):
self.config.AlertGenerator.couchCPUPoller.couchURL = os.getenv("COUCHURL", None)
self.config.AlertGenerator.couchCPUPoller.soft = 70
self.config.AlertGenerator.couchCPUPoller.critical = 80
self.config.AlertGenerator.couchCPUPoller.pollInterval = 0.2
self.config.AlertGenerator.couchCPUPoller.period = 1
ti = utils.TestInput() # see attributes comments at the class
ti.pollerClass = CouchCPUPoller
ti.config = self.config.AlertGenerator.couchCPUPoller
# lower the threshold so that the alert is never generated
ti.thresholdToTest = self.config.AlertGenerator.couchCPUPoller.soft - 20
ti.level = 0
ti.expected = 0
ti.thresholdDiff = 10
ti.testCase = self
utils.doGenericPeriodAndProcessPolling(ti)
def testAlertGeneratorCouchMemoryPollerSoftThreshold(self):
self.config.AlertGenerator.couchMemPoller.couchURL = os.getenv("COUCHURL", None)
self.config.AlertGenerator.couchMemPoller.soft = 70
self.config.AlertGenerator.couchMemPoller.critical = 80
self.config.AlertGenerator.couchMemPoller.pollInterval = 0.2
self.config.AlertGenerator.couchMemPoller.period = 1
ti = utils.TestInput() # see attributes comments at the class
ti.pollerClass = CouchMemoryPoller
ti.config = self.config.AlertGenerator.couchMemPoller
ti.thresholdToTest = self.config.AlertGenerator.couchMemPoller.soft
ti.level = self.config.AlertProcessor.soft.level
ti.expected = 1
ti.thresholdDiff = 10
ti.testCase = self
utils.doGenericPeriodAndProcessPolling(ti)
def testAlertGeneratorCouchMemoryPollerCriticalThreshold(self):
self.config.AlertGenerator.couchMemPoller.couchURL = os.getenv("COUCHURL", None)
self.config.AlertGenerator.couchMemPoller.soft = 70
self.config.AlertGenerator.couchMemPoller.critical = 80
self.config.AlertGenerator.couchMemPoller.pollInterval = 0.2
self.config.AlertGenerator.couchMemPoller.period = 1
ti = utils.TestInput() # see attributes comments at the class
ti.pollerClass = CouchMemoryPoller
ti.config = self.config.AlertGenerator.couchMemPoller
ti.thresholdToTest = self.config.AlertGenerator.couchMemPoller.critical
ti.level = self.config.AlertProcessor.critical.level
ti.expected = 1
ti.thresholdDiff = 10
ti.testCase = self
utils.doGenericPeriodAndProcessPolling(ti)
def testAlertGeneratorCouchMemoryPollerNoAlert(self):
self.config.AlertGenerator.couchMemPoller.couchURL = os.getenv("COUCHURL", None)
self.config.AlertGenerator.couchMemPoller.soft = 70
self.config.AlertGenerator.couchMemPoller.critical = 80
self.config.AlertGenerator.couchMemPoller.pollInterval = 0.2
self.config.AlertGenerator.couchMemPoller.period = 1
ti = utils.TestInput() # see attributes comments at the class
ti.pollerClass = CouchMemoryPoller
ti.config = self.config.AlertGenerator.couchMemPoller
# lower the threshold so that the alert is never generated
ti.thresholdToTest = self.config.AlertGenerator.couchMemPoller.soft - 20
ti.level = 0
ti.expected = 0
ti.thresholdDiff = 10
ti.testCase = self
utils.doGenericPeriodAndProcessPolling(ti)
def testAlertGeneratorCouchErrorsPollerBasic(self):
try:
poller = CouchErrorsPoller(self.config.AlertGenerator.couchErrorsPoller, self.generator)
except Exception as ex:
self.fail("Exception, reason: %s" % ex)
# even a single values to observe shall be turned into particular iterables
obs = self.config.AlertGenerator.couchErrorsPoller.observables
self.config.AlertGenerator.couchErrorsPoller.observables = 400
try:
poller = CouchErrorsPoller(self.config.AlertGenerator.couchErrorsPoller, self.generator)
except Exception as ex:
self.fail("Exception, reason: %s" % ex)
#self.assertTrue(isinstance(obs, (types.ListType, types.TupleType)))
self.assertTrue(isinstance(self.config.AlertGenerator.couchErrorsPoller.observables,
(types.ListType, types.TupleType)))
# test return value on non-sense HTTP status code
res = poller.sample("40000")
self.assertFalse(res)
# test definitely existing value
res = poller.sample("200")
# on a freshly started couch, this status code may not exist in the
# stats table, so despite correct and meaningful HTTP status code, the
# query may still return None, hence don't assume any particular response
#self.assertTrue(isinstance(res, types.IntType))
poller.check()
def testAlertGeneratorCouchErrorsPollerSoftThreshold(self):
self.config.AlertGenerator.couchErrorsPoller.couchURL = os.getenv("COUCHURL", None)
self.config.AlertGenerator.couchErrorsPoller.soft = 100
self.config.AlertGenerator.couchErrorsPoller.critical = 200
# shall expect corresponding number of generated alerts for each observable value
self.config.AlertGenerator.couchErrorsPoller.observables = (5, 6, 7)
self.config.AlertGenerator.couchErrorsPoller.pollInterval = 0.2
ti = utils.TestInput() # see attributes comments at the class
ti.pollerClass = CouchErrorsPoller
ti.config = self.config.AlertGenerator.couchErrorsPoller
ti.thresholdToTest = self.config.AlertGenerator.couchErrorsPoller.soft
ti.level = self.config.AlertProcessor.soft.level
ti.expected = len(self.config.AlertGenerator.couchErrorsPoller.observables)
ti.thresholdDiff = 10
ti.testCase = self
utils.doGenericValueBasedPolling(ti)
def testAlertGeneratorCouchErrorsPollerCriticalThreshold(self):
self.config.AlertGenerator.couchErrorsPoller.couchURL = os.getenv("COUCHURL", None)
self.config.AlertGenerator.couchErrorsPoller.soft = 100
self.config.AlertGenerator.couchErrorsPoller.critical = 200
# shall expect corresponding number of generated alerts for each observable value
self.config.AlertGenerator.couchErrorsPoller.observables = (5, 6, 7)
self.config.AlertGenerator.couchErrorsPoller.pollInterval = 0.2
ti = utils.TestInput() # see attributes comments at the class
ti.pollerClass = CouchErrorsPoller
ti.config = self.config.AlertGenerator.couchErrorsPoller
ti.thresholdToTest = self.config.AlertGenerator.couchErrorsPoller.critical
ti.level = self.config.AlertProcessor.critical.level
ti.expected = len(self.config.AlertGenerator.couchErrorsPoller.observables)
ti.thresholdDiff = 50
ti.testCase = self
utils.doGenericValueBasedPolling(ti)
def testAlertGeneratorCouchErrorsPollerNoAlert(self):
self.config.AlertGenerator.couchErrorsPoller.couchURL = os.getenv("COUCHURL", None)
self.config.AlertGenerator.couchErrorsPoller.soft = 100
self.config.AlertGenerator.couchErrorsPoller.critical = 200
# shall expect corresponding number of generated alerts for each observable value
self.config.AlertGenerator.couchErrorsPoller.observables = (5, 6, 7)
self.config.AlertGenerator.couchErrorsPoller.pollInterval = 0.2
ti = utils.TestInput() # see attributes comments at the class
ti.pollerClass = CouchErrorsPoller
ti.config = self.config.AlertGenerator.couchErrorsPoller
ti.thresholdToTest = self.config.AlertGenerator.couchErrorsPoller.soft - 30
ti.expected = 0
ti.thresholdDiff = 20
ti.testCase = self
utils.doGenericValueBasedPolling(ti)
class FeederManagerTest(unittest.TestCase):
"""
TestCase for TestFeederManager module
"""
_maxMessage = 10
def setUp(self):
"""
_setUp_
Setup the database and logging connection. Try to create all needed
tables.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.generateWorkDir()
self.testInit.setSchema(customModules = \
['WMCore.Agent.Database',
'WMComponent.FeederManager.Database',
'WMCore.ThreadPool',
'WMCore.WMBS'],
useDefault = False)
return
def tearDown(self):
"""
_tearDown_
Database deletion
"""
self.testInit.clearDatabase()
return
def getConfig(self):
"""
_createConfig_
Create a config for the JobAccountant. This config needs to include
information for connecting to the database as the component will create
it's own database connections. These parameters are still pulled from
the environment.
"""
config = self.testInit.getConfiguration()
self.testInit.generateWorkDir(config)
config.component_("FeederManager")
config.FeederManager.logLevel = "INFO"
config.FeederManager.componentName = "FeederManager"
config.FeederManager.componentDir = \
os.path.join(os.getenv("TESTDIR"), "FeederManager")
config.FeederManager.addDatasetWatchHandler = \
'WMComponent.FeederManager.Handler.DefaultAddDatasetWatch'
# The maximum number of threads to process each message type
config.FeederManager.maxThreads = 10
# The poll interval at which to look for new fileset/feeder association
config.FeederManager.pollInterval = 60
return config
def testA(self):
"""
_testA_
Handle AddDatasetWatch events
"""
myThread = threading.currentThread()
config = self.getConfig()
testFeederManager = FeederManager(config)
testFeederManager.prepareToStart()
for i in xrange(0, FeederManagerTest._maxMessage):
for j in xrange(0, 3):
feederManagerdict = {'payload':{'FeederType':'NO Feeder',
'dataset' : 'NO DATASET', 'FileType' : 'NO FILE TYPE',
'StartRun' : 'NO START RUN' }}
testFeederManager.handleMessage( type = 'AddDatasetWatch',
payload = feederManagerdict )
time.sleep(30)
myThread.workerThreadManager.terminateWorkers()
while threading.activeCount() > 1:
print('Currently: '+str(threading.activeCount())+\
' Threads. Wait until all our threads have finished')
time.sleep(1)
class JobPackageTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Create a temporary file to presist the JobPackage to.
"""
self.testInit = TestInit(__file__)
self.persistFile = os.path.join(self.testInit.generateWorkDir(),
"JobPackage.pkl")
return
def tearDown(self):
pass
def testAddingJobs(self):
"""
_testAddingJobs_
Verify that adding jobs to the package works as expected.
"""
package = JobPackage()
for i in range(100):
newJob = Job("Job%s" % i)
newJob["id"] = i
package[i] = newJob
# There is an extra key for the directory the package is stored in.
assert len(package.keys()) == 101, \
"Error: Wrong number of jobs in package."
for i in range(100):
job = package[i]
assert job["id"] == i, \
"Error: Jobs has wrong ID."
assert job["name"] == "Job%d" % i, \
"Error: Job has wrong name."
return
def testPersist(self):
"""
_testPersist_
Verify that we're able to save and load the job package.
"""
package = JobPackage()
for i in range(100):
newJob = Job("Job%s" % i)
newJob["id"] = i
package[i] = newJob
package.save(self.persistFile)
assert os.path.exists(self.persistFile), \
"Error: Package file was never created."
newPackage = JobPackage()
newPackage.load(self.persistFile)
# There is an extra key for the directory the package is stored in.
assert len(newPackage.keys()) == 101, \
"Error: Wrong number of jobs in package."
for i in range(100):
job = newPackage[i]
assert job["id"] == i, \
"Error: Jobs has wrong ID."
assert job["name"] == "Job%d" % i, \
"Error: Job has wrong name."
return
def testBaggage(self):
"""
_testBaggage_
Verify that job baggage is persisted with the package.
"""
package = JobPackage()
for i in range(100):
newJob = Job("Job%s" % i)
newJob["id"] = i
baggage = newJob.getBaggage()
setattr(baggage, "thisJob", newJob["name"])
setattr(baggage, "seed1", 11111111)
setattr(baggage, "seed2", 22222222)
setattr(baggage, "seed3", 33333333)
setattr(baggage, "seed4", 44444444)
setattr(baggage, "seed5", 55555555)
package[i] = newJob
package.save(self.persistFile)
assert os.path.exists(self.persistFile), \
"Error: Package file was never created."
newPackage = JobPackage()
newPackage.load(self.persistFile)
# There is an extra key for the directory the package is stored in.
assert len(newPackage.keys()) == 101, \
"Error: Wrong number of jobs in package."
for i in range(100):
job = newPackage[i]
assert job["id"] == i, \
"Error: Jobs has wrong ID."
assert job["name"] == "Job%d" % i, \
"Error: Job has wrong name."
jobBaggage = job.getBaggage()
assert jobBaggage.thisJob == "Job%d" % i, \
"Error: Job baggage has wrong name."
assert jobBaggage.seed1 == 11111111, \
"Error: Job baggee has wrong value for seed1."
assert jobBaggage.seed2 == 22222222, \
"Error: Job baggee has wrong value for seed2."
assert jobBaggage.seed3 == 33333333, \
"Error: Job baggee has wrong value for seed3."
assert jobBaggage.seed4 == 44444444, \
"Error: Job baggee has wrong value for seed4."
assert jobBaggage.seed5 == 55555555, \
"Error: Job baggee has wrong value for seed5."
return
class SetupCMSSWPsetTest(unittest.TestCase):
def setUp(self):
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testDir = self.testInit.generateWorkDir()
sys.path.insert(0, os.path.join(WMCore.WMBase.getTestBase(),
"WMCore_t/WMRuntime_t/Scripts_t"))
def tearDown(self):
sys.path.remove(os.path.join(WMCore.WMBase.getTestBase(), "WMCore_t/WMRuntime_t/Scripts_t"))
if 'WMTaskSpace' in sys.modules:
del sys.modules["WMTaskSpace"]
self.testInit.delWorkDir()
os.unsetenv("WMAGENT_SITE_CONFIG_OVERRIDE")
def createTestStep(self):
"""
_createTestStep_
Create a test step that can be passed to the setup script.
"""
newStep = WMStep("cmsRun1")
newStepHelper = CMSSWStepHelper(newStep)
newStepHelper.setStepType("CMSSW")
newStepHelper.setGlobalTag("SomeGlobalTag")
stepTemplate = StepFactory.getStepTemplate("CMSSW")
stepTemplate(newStep)
newStep.application.command.configuration = "PSet.py"
return newStepHelper
def createTestJob(self):
"""
_createTestJob_
Create a test job that has parents for each input file.
"""
newJob = Job(name = "TestJob")
newJob.addFile(File(lfn = "/some/file/one",
parents = set([File(lfn = "/some/parent/one")])))
newJob.addFile(File(lfn = "/some/file/two",
parents = set([File(lfn = "/some/parent/two")])))
return newJob
def loadProcessFromPSet(self):
"""
_loadProcessFromPSet_
This requires changing the working directory,
do so in a safe manner to encapsulate the change to this method only
"""
currentPath = os.getcwd()
loadedProcess = None
try:
if not os.path.isdir(self.testDir):
raise
os.chdir(self.testDir)
testFile = "PSet.py"
pset = imp.load_source('process', testFile)
loadedProcess = pset.process
except Exception as ex:
self.fail("An exception was caught while trying to load the PSet, %s" % str(ex))
finally:
os.chdir(currentPath)
return loadedProcess
def testPSetFixup(self):
"""
_testPSetFixup_
Verify that all necessary parameters are set in the PSet.
"""
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
if os.environ.get('CMSSW_VERSION', None):
del os.environ['CMSSW_VERSION']
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.stepSpace = ConfigSection(name = "stepSpace")
setupScript.stepSpace.location = self.testDir
setupScript.job = self.createTestJob()
setupScript()
fixedPSet = self.loadProcessFromPSet()
self.assertEqual(len(fixedPSet.source.fileNames.value), 2,
"Error: Wrong number of files.")
self.assertEqual(len(fixedPSet.source.secondaryFileNames.value), 2,
"Error: Wrong number of secondary files.")
self.assertEqual(fixedPSet.source.fileNames.value[0], "/some/file/one",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.source.fileNames.value[1], "/some/file/two",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.source.secondaryFileNames.value[0], "/some/parent/one",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.source.secondaryFileNames.value[1], "/some/parent/two",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.maxEvents.input.value, -1,
"Error: Wrong maxEvents.")
def testEventsPerLumi(self):
"""
_testEventsPerLumi_
Verify that you can put in events per lumi in the process.
"""
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
os.environ['CMSSW_VERSION'] = "CMSSW_7_4_0"
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.step.setEventsPerLumi(500)
setupScript.stepSpace = ConfigSection(name = "stepSpace")
setupScript.stepSpace.location = self.testDir
setupScript.job = self.createTestJob()
setupScript()
fixedPSet = self.loadProcessFromPSet()
self.assertEqual(len(fixedPSet.source.fileNames.value), 2,
"Error: Wrong number of files.")
self.assertEqual(len(fixedPSet.source.secondaryFileNames.value), 2,
"Error: Wrong number of secondary files.")
self.assertEqual(fixedPSet.source.fileNames.value[0], "/some/file/one",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.source.fileNames.value[1], "/some/file/two",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.source.secondaryFileNames.value[0], "/some/parent/one",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.source.secondaryFileNames.value[1], "/some/parent/two",
"Error: Wrong input file.")
self.assertEqual(fixedPSet.source.numberEventsInLuminosityBlock.value,
500, "Error: Wrong number of events per luminosity block")
self.assertEqual(fixedPSet.maxEvents.input.value, -1,
"Error: Wrong maxEvents.")
def testChainedProcesing(self):
"""
test for chained CMSSW processing - check the overriden TFC, its values
and that input files is / are set correctly.
"""
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
os.environ['CMSSW_VERSION'] = "CMSSW_7_6_0"
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.stepSpace = ConfigSection(name = "stepSpace")
setupScript.stepSpace.location = self.testDir
setupScript.job = self.createTestJob()
setupScript.step.setupChainedProcessing("my_first_step", "my_input_module")
setupScript()
# test if the overriden TFC is right
self.assertTrue(hasattr(setupScript.step.data.application, "overrideCatalog"),
"Error: overriden TFC was not set")
tfc = loadTFC(setupScript.step.data.application.overrideCatalog)
inputFile = "../my_first_step/my_input_module.root"
self.assertEqual(tfc.matchPFN("direct", inputFile), inputFile)
self.assertEqual(tfc.matchLFN("direct", inputFile), inputFile)
self.assertEqual(setupScript.process.source.fileNames.value, [inputFile])
def testPileupSetup(self):
"""
Test the pileup setting.
reference (setupScript.process instance):
in test/python/WMCore_t/WMRuntime_t/Scripts_t/WMTaskSpace/cmsRun1/PSet.py
"""
try:
from dbs.apis.dbsClient import DbsApi
except ImportError as ex:
raise nose.SkipTest
# this is modified and shortened version of
# WMCore/test/python/WMCore_t/Misc_t/site-local-config.xml
# since the dataset name in question (below) is only present at
# storm-fe-cms.cr.cnaf.infn.it, need to make the test think it's its local SE
siteLocalConfigContent = \
"""
<site-local-config>
<site name="-SOME-SITE-NAME-">
<event-data>
<catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/>
</event-data>
<local-stage-out>
<!-- original cmssrm.fnal.gov -->
<phedex-node value="T2_CH_CERN"/>
<command value="test-copy"/>
<catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/>
</local-stage-out>
<calib-data>
<frontier-connect>
<load balance="proxies"/>
<proxy url="http://cmsfrontier1.fnal.gov:3128"/>
<proxy url="http://cmsfrontier2.fnal.gov:3128"/>
</frontier-connect>
</calib-data>
</site>
</site-local-config>
"""
siteLocalConfig = os.path.join(self.testDir, "test-site-local-config.xml")
f = open(siteLocalConfig, 'w')
f.write(siteLocalConfigContent)
f.close()
from WMCore.WMRuntime.Scripts.SetupCMSSWPset import SetupCMSSWPset
setupScript = SetupCMSSWPset()
setupScript.step = self.createTestStep()
setupScript.stepSpace = ConfigSection(name = "stepSpace")
setupScript.stepSpace.location = os.path.join(self.testDir, "cmsRun1")
setupScript.job = self.createTestJob()
# define pileup configuration
# despite of the implementation considering whichever type of pileup,
# only "data" and "mc" types are eventually considered and lead to any
# modifications of job input files
pileupConfig = {"data": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"],
"mc": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"]}
dbsUrl = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
setupScript.step.setupPileup(pileupConfig, dbsUrl)
# SetupCMSSWPset pileup handling will be consulting SiteLocalConfig
# to determine StorageElement (SE) name the job is running on
# SiteLocalConfig loads the site-local-config.xml file from env.
# variable defined location ; if the variable is not defined already, set it
# obviously, if "WMAGENT_SITE_CONFIG_OVERRIDE" is already set here, the above
# thick with SE name is not effective
if not os.getenv("WMAGENT_SITE_CONFIG_OVERRIDE", None):
os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = siteLocalConfig
# find out local site name from the testing local site config,
# will be needed later
siteConfig = loadSiteLocalConfig()
seLocalName = siteConfig.localStageOut["phedex-node"]
print("Running on site '%s', local SE name: '%s'" % (siteConfig.siteName, seLocalName))
# before calling the script, SetupCMSSWPset will try to load JSON
# pileup configuration file, need to create it in self.testDir
fetcher = PileupFetcher()
fetcher.setWorkingDirectory(self.testDir)
fetcher._createPileupConfigFile(setupScript.step, fakeSites=['T1_US_FNAL'])
setupScript()
# now test all modifications carried out in SetupCMSSWPset.__call__
# which will also test that CMSSWStepHelper.setupPileup run correctly
mixModules, dataMixModules = setupScript._getPileupMixingModules()
# load in the pileup configuration in the form of dict which
# PileupFetcher previously saved in a JSON file
pileupDict = setupScript._getPileupConfigFromJson()
# get the sub dict for particular pileup type
# for pileupDict structure description - see PileupFetcher._queryDbsAndGetPileupConfig
for pileupType, modules in zip(("data", "mc"), (dataMixModules, mixModules)):
# getting KeyError here - above pileupConfig is not correct - need
# to have these two types of pile type
d = pileupDict[pileupType]
self._mixingModulesInputFilesTest(modules, d, seLocalName)
def _mixingModulesInputFilesTest(self, modules, pileupSubDict, seLocalName):
"""
pileupSubDic - contains only dictionary for particular pile up type
"""
# consider only locally available files
filesInConfigDict = []
for v in pileupSubDict.values():
if seLocalName in v["phedexNodeNames"]:
filesInConfigDict.extend(v["FileList"])
for m in modules:
inputTypeAttrib = getattr(m, "input", None) or getattr(m, "secsource", None)
fileNames = inputTypeAttrib.fileNames.value
if fileNames == None:
fileNames = []
m = ("Pileup configuration file list '%s' and mixing modules input "
"filelist '%s' are not identical." % (filesInConfigDict, fileNames))
self.assertEqual(sorted(filesInConfigDict), sorted(fileNames), m)
class Scram_t(unittest.TestCase):
def setUp(self):
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testDir = self.testInit.generateWorkDir()
self.oldCwd = os.getcwd()
if PY3:
self.assertItemsEqual = self.assertCountEqual
def tearDown(self):
self.testInit.delWorkDir()
def testA(self):
"""
instantiate a Scram instance in test mode.
"""
try:
Scram(initialise="/bin/date",
architecture="slc5_amd64_gcc454",
version="CMSSW_X_Y_Z",
test=True)
except Exception as ex:
msg = "Failed to instantiate Scram in test mode:\n %s " % str(ex)
self.fail(msg)
def testB(self):
"""
instantiante a Scram instance in non-test mode
limited what we can test here since we dont have scram etc in unittest env
"""
try:
Scram(initialise="/bin/date",
architecture="slc5_amd64_gcc454",
version="CMSSW_X_Y_Z")
except Exception as ex:
msg = "Failed to instantiate Scram:\n %s " % str(ex)
self.fail(msg)
def testC(self):
"""
test all method calls in test mode
"""
s = Scram(initialise="/bin/date",
architecture="slc5_amd64_gcc454",
version="CMSSW_X_Y_Z",
directory=self.testDir,
test=True)
try:
status = s.project()
except Exception as ex:
msg = "Error running Scram.project:\n %s" % str(ex)
self.fail(msg)
self.assertEqual(status, 0)
self.assertTrue(os.path.exists(s.projectArea))
self.assertTrue("project" in s.lastExecuted)
self.assertTrue("CMSSW_X_Y_Z" in s.lastExecuted)
try:
status = s.runtime()
except Exception as ex:
msg = "Error running Scram.runtime:\n %s" % str(ex)
self.fail(msg)
self.assertEqual(status, 0)
self.assertTrue("ru -sh" in s.lastExecuted)
self.assertTrue("TEST_MODE" in s.runtimeEnv)
comm = "echo \"Hello World\""
try:
status = s(comm)
except Exception as ex:
msg = "Failed to call Scram object:\n %s" % str(ex)
self.assertEqual(status, 0)
self.assertEqual(s.lastExecuted, comm)
def testArchMap(self):
self.assertItemsEqual(OS_TO_ARCH['rhel6'], ['slc5', 'slc6'])
self.assertItemsEqual(OS_TO_ARCH['rhel7'], ['slc7'])
self.assertItemsEqual(ARCH_TO_OS['slc6'], ['rhel6'])
self.assertItemsEqual(ARCH_TO_OS['slc7'], ['rhel7'])
def testScramArchParsing(self):
"""
Test the various modes of parsing for the scram arch
"""
try:
os.chdir(self.testDir)
with tempfile.NamedTemporaryFile() as tf:
tf.write(b'GLIDEIN_REQUIRED_OS = "rhel6" \n')
tf.write(b'Memory = 2048\n')
tf.flush()
with tmpEnv(_CONDOR_MACHINE_AD=tf.name):
self.assertEqual(getSingleScramArch('slc6_blah_blah'),
'slc6_blah_blah')
self.assertEqual(getSingleScramArch('slc5_blah_blah'),
'slc5_blah_blah')
self.assertEqual(
getSingleScramArch(
['slc6_blah_blah', 'slc7_blah_blah']),
'slc6_blah_blah')
self.assertEqual(
getSingleScramArch(
['slc6_blah_blah', 'slc5_blah_blah']),
'slc6_blah_blah')
self.assertEqual(
getSingleScramArch(
['slc7_blah_blah', 'slc8_blah_blah']), None)
with tempfile.NamedTemporaryFile() as tf:
tf.write(b'GLIDEIN_REQUIRED_OS = "rhel7" \n')
tf.write(b'Memory = 2048\n')
tf.flush()
with tmpEnv(_CONDOR_MACHINE_AD=tf.name):
self.assertEqual(getSingleScramArch('slc6_blah_blah'),
'slc6_blah_blah')
self.assertEqual(getSingleScramArch('slc7_blah_blah'),
'slc7_blah_blah')
self.assertEqual(
getSingleScramArch(
['slc6_blah_blah', 'slc7_blah_blah']),
'slc7_blah_blah')
self.assertEqual(
getSingleScramArch(
['slc6_blah_blah', 'slc5_blah_blah']), None)
self.assertEqual(
getSingleScramArch(
['slc7_blah_blah', 'slc8_blah_blah']),
'slc7_blah_blah')
except Exception:
raise
finally:
os.chdir(self.oldCwd)
return
def testCMSSWSupported(self):
"""
Test the functionality of isCMSSWSupported function
"""
self.assertFalse(isCMSSWSupported('CMSSW_1_2_3', ''))
self.assertFalse(isCMSSWSupported(None, 'a'))
self.assertFalse(isCMSSWSupported('CMSSW_1_2_3', 'CMSSW_2_2_3'))
self.assertFalse(isCMSSWSupported('CMSSW_1_2_3', 'CMSSW_1_3_3'))
self.assertFalse(isCMSSWSupported('CMSSW_1_2_3', 'CMSSW_1_2_4'))
self.assertFalse(isCMSSWSupported('CMSSW_1_2_3_pre1', 'CMSSW_1_2_3'))
self.assertFalse(isCMSSWSupported('CMSSW_1_2_3', 'CMSSW_1_2_3_pre1'))
self.assertFalse(
isCMSSWSupported('CMSSW_1_2_3_pre1', 'CMSSW_1_2_3_pre2'))
self.assertFalse(
isCMSSWSupported('CMSSW_1_2_3_pre2', 'CMSSW_1_2_3_pre1'))
self.assertFalse(isCMSSWSupported('CMSSW_7_1_25_patch2',
'CMSSW_7_6_0'))
self.assertFalse(isCMSSWSupported('CMSSW_7_3_2', 'CMSSW_10_4_0'))
self.assertTrue(
isCMSSWSupported('CMSSW_1_2_3_pre1', 'CMSSW_1_2_3_pre1'))
self.assertTrue(isCMSSWSupported('CMSSW_1_2_3', 'CMSSW_1_2_3'))
self.assertTrue(isCMSSWSupported('CMSSW_2_2_3', 'CMSSW_1_2_3'))
self.assertTrue(isCMSSWSupported('CMSSW_1_3_3', 'CMSSW_1_2_3'))
self.assertTrue(isCMSSWSupported('CMSSW_1_2_4', 'CMSSW_1_2_3'))
def testisEnforceGUIDInFileNameSupported(self):
"""
Test functionality of the `isEnforceGUIDInFileNameSupported` function
"""
### invalid input
self.assertFalse(isEnforceGUIDInFileNameSupported(None))
self.assertFalse(isEnforceGUIDInFileNameSupported(''))
### forever supported
self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_11_0_0'))
self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_11_0_2'))
self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_11_1_0_pre1'))
self.assertTrue(
isEnforceGUIDInFileNameSupported('CMSSW_11_1_0_patch1'))
self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_11_1_1'))
### specific releases
self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_10_2_20_UL'))
self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_9_4_16_UL'))
self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_8_0_34_UL'))
self.assertTrue(
isEnforceGUIDInFileNameSupported('CMSSW_7_1_45_patch3'))
### minor supported releases
self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_10_6_8'))
self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_10_6_9'))
self.assertTrue(
isEnforceGUIDInFileNameSupported('CMSSW_10_6_8_patch1'))
self.assertTrue(
isEnforceGUIDInFileNameSupported('CMSSW_10_6_9_patch1'))
self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_10_2_20'))
self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_9_4_16'))
self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_9_3_17'))
self.assertTrue(isEnforceGUIDInFileNameSupported('CMSSW_8_0_34'))
### releases not supported
self.assertFalse(isEnforceGUIDInFileNameSupported('CMSSW_10_6_7'))
self.assertFalse(isEnforceGUIDInFileNameSupported('CMSSW_10_7_0'))
self.assertFalse(isEnforceGUIDInFileNameSupported('CMSSW_10_2_19'))
self.assertFalse(isEnforceGUIDInFileNameSupported('CMSSW_10_3_10'))
self.assertFalse(isEnforceGUIDInFileNameSupported('CMSSW_5_3_10'))
class CMSSW_t(unittest.TestCase):
def setUp(self):
"""
_setUp_
Build a testing environment similar to a WN
"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
# Build a workload/task/step with the basic required information
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("CMSSWExecutor")
stepHelper = self.task.makeStep("ExecutorTest")
self.step = stepHelper.data
template = CMSSWTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.step.application.setup.scramCommand = "scramulator.py"
self.step.application.command.executable = "cmsRun.py"
self.step.application.setup.scramProject = "CMSSW"
self.step.application.setup.scramArch = "slc5_ia32_gcc434"
self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";"
self.step.output.jobReport = "FrameworkJobReport.xml"
self.helper.addOutputModule("outputRECORECO", primaryDataset="Bogus",
processedDataset="Test-Era-v1",
dataTier="DATA")
self.helper.addOutputModule("outputALCARECORECO", primaryDataset="Bogus",
processedDataset="Test-Era-v1",
dataTier="DATA")
self.helper.setGlobalTag("Bogus")
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
# Build the TaskSpace/StepSpace
self.sandboxDir = os.path.join(self.testDir, "UnitTests")
self.task.build(self.testDir)
sys.path.append(self.testDir)
sys.path.append(self.sandboxDir)
# Copy the files that cmsRun would have generated in the step space
open(os.path.join(self.step.builder.workingDir, "outputRECORECO.root"), "w").close()
open(os.path.join(self.step.builder.workingDir, "outputALCARECORECO.root"), "w").close()
shutil.copy(os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml"),
os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))
# Create a job
self.job = Job(name="/UnitTest/CMSSWExecutor/ExecutorTest-test-job")
self.job["id"] = 1
# Set the PATH
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
self.oldCwd = os.getcwd()
def tearDown(self):
"""
_tearDown_
Clean up
"""
self.testInit.delWorkDir()
sys.path.remove(self.testDir)
sys.path.remove(self.sandboxDir)
try:
sys.modules.pop("WMTaskSpace")
sys.modules.pop("WMTaskSpace.ExecutorTest")
sys.modules.pop("WMTaskSpace.ExecutorTest.WMCore")
sys.modules.pop("WMSandbox")
sys.modules.pop("WMSandbox.CMSSWExecutor")
sys.modules.pop("WMSandbox.CMSSWExecutor.ExecutorTest")
except KeyError:
return
return
def testA_Execute(self):
"""
_Execute_
Test the full path, including execute.
"""
try:
os.chdir(self.step.builder.workingDir)
executor = CMSSWExecutor()
executor.initialise(self.step, self.job)
executor.pre()
# Remove the scram pre-script as it requires an actual SCRAM environment
executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
executor.execute()
executor.post()
# Check that there were no errors
self.assertEqual(0, executor.report.getExitCode())
# Check that we processed the XML
self.assertEqual(2, len(executor.report.getAllFiles()))
# Check that the executable was really executed
self.assertTrue(os.path.isfile(os.path.join(self.step.builder.workingDir, "BogusFile.txt")))
except Exception as ex:
self.fail("Failure encountered, %s" % str(ex))
finally:
os.chdir(self.oldCwd)
return
def testB_ExecuteNonZeroExit(self):
"""
_ExecuteNonZeroExit_
Test the execution of a script
which exits with non-zero code.
"""
self.step.application.command.executable = "brokenCmsRun.py"
shutil.copy(os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWFailReport.xml"),
os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))
try:
os.chdir(self.step.builder.workingDir)
executor = StepFactory.getStepExecutor("CMSSW")
executor.initialise(self.step, self.job)
executor.pre()
executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
try:
executor.execute()
self.fail("An exception should have been raised")
except WMExecutionFailure as ex:
executor.diagnostic(ex.code, executor, ExceptionInstance=ex)
self.assertEqual(8001, executor.report.getExitCode())
report = Report()
report.load("Report.pkl")
self.assertEqual(8001, report.getExitCode())
except Exception as ex:
self.fail("Failure encountered, %s" % str(ex))
finally:
os.chdir(self.oldCwd)
return
def testC_ExecuteSegfault(self):
"""
_ExecuteSegfault_
Test the execution of a script
which raises a ABRT signal which
is the normal CMSSW response
to a SEGFAULT.
"""
self.step.application.command.executable = "test.sh"
# CMSSW leaves an empty FWJR when a SEGFAULT is present
open(os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"), "w").close()
try:
os.chdir(self.step.builder.workingDir)
executor = StepFactory.getStepExecutor("CMSSW")
executor.initialise(self.step, self.job)
executor.pre()
executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
try:
executor.execute()
self.fail("An exception should have been raised")
except WMExecutionFailure as ex:
executor.diagnostic(ex.code, executor, ExceptionInstance=ex)
self.assertEqual(50115, executor.report.getExitCode())
report = Report()
report.load("Report.pkl")
self.assertEqual(50115, report.getExitCode())
except Exception as ex:
self.fail("Failure encountered, %s" % str(ex))
finally:
os.chdir(self.oldCwd)
return
def testD_ExecuteNoOutput(self):
"""
_ExecuteNoOutput_
Test what happens when no output is produced,
the proper error should be included.
"""
self.step.application.command.executable = "cmsRun.py"
shutil.copy(os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWSkippedAll.xml"),
os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))
try:
os.chdir(self.step.builder.workingDir)
executor = StepFactory.getStepExecutor("CMSSW")
executor.initialise(self.step, self.job)
executor.pre()
executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
executor.execute()
executor.post()
self.assertEqual(60450, executor.report.getExitCode())
except Exception as ex:
self.fail("Failure encountered, %s" % str(ex))
finally:
os.chdir(self.oldCwd)
return
class DBSUploadTest(unittest.TestCase):
"""
TestCase for DBSUpload module
"""
def setUp(self):
"""
_setUp_
setUp function for unittest
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer"],
useDefault = False)
self.testDir = self.testInit.generateWorkDir(deleteOnDestruction = False)
self.configFile = EmulatorSetup.setupWMAgentConfig()
myThread = threading.currentThread()
self.bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.buffer3Factory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = self.bufferFactory(classname = "DBSBufferFiles.AddLocation")
locationAction.execute(siteName = "se1.cern.ch")
locationAction.execute(siteName = "se1.fnal.gov")
locationAction.execute(siteName = "malpaquet")
self.dbsUrl = "https://*****:*****@attr("integration")
def testBasicUpload(self):
"""
_testBasicUpload_
Verify that we can successfully upload to DBS3. Also verify that the
uploader correctly handles files parentage when uploading.
"""
self.dbsApi = DbsApi(url = self.dbsUrl)
config = self.getConfig()
dbsUploader = DBSUploadPoller(config = config)
# First test verifies that uploader will poll and then not do anything
# as the database is empty.
dbsUploader.algorithm()
acqEra = "Summer%s" % (int(time.time()))
parentFiles = self.createParentFiles(acqEra)
# The algorithm needs to be run twice. On the first iteration it will
# create all the blocks and upload one. On the second iteration it will
# timeout and upload the second block.
dbsUploader.algorithm()
time.sleep(5)
dbsUploader.algorithm()
time.sleep(5)
# Verify the files made it into DBS3.
self.verifyData(parentFiles[0]["datasetPath"], parentFiles)
# Inject some more parent files and some child files into DBSBuffer.
# Run the uploader twice, only the parent files should be added to DBS3.
(moreParentFiles, childFiles) = \
self.createFilesWithChildren(parentFiles, acqEra)
dbsUploader.algorithm()
time.sleep(5)
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(parentFiles[0]["datasetPath"],
parentFiles + moreParentFiles)
# Run the uploader another two times to upload the child files. Verify
# that the child files were uploaded.
dbsUploader.algorithm()
time.sleep(5)
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(childFiles[0]["datasetPath"], childFiles)
return
@attr("integration")
def testDualUpload(self):
"""
_testDualUpload_
Verify that the dual upload mode works correctly.
"""
self.dbsApi = DbsApi(url = self.dbsUrl)
config = self.getConfig(dbs3UploadOnly = True)
dbsUploader = DBSUploadPoller(config = config)
dbsUtil = DBSBufferUtil()
# First test verifies that uploader will poll and then not do anything
# as the database is empty.
dbsUploader.algorithm()
acqEra = "Summer%s" % (int(time.time()))
parentFiles = self.createParentFiles(acqEra)
(moreParentFiles, childFiles) = \
self.createFilesWithChildren(parentFiles, acqEra)
allFiles = parentFiles + moreParentFiles
allBlocks = []
for i in range(4):
blockName = parentFiles[0]["datasetPath"] + "#" + makeUUID()
dbsBlock = DBSBlock(blockName, "malpaquet", 1)
dbsBlock.status = "Open"
dbsUtil.createBlocks([dbsBlock])
for file in allFiles[i * 5 : (i * 5) + 5]:
dbsBlock.addFile(file)
dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]})
if i < 2:
dbsBlock.status = "InDBS"
dbsUtil.updateBlocks([dbsBlock])
dbsUtil.updateFileStatus([dbsBlock], "InDBS")
allBlocks.append(dbsBlock)
blockName = childFiles[0]["datasetPath"] + "#" + makeUUID()
dbsBlock = DBSBlock(blockName, "malpaquet", 1)
dbsBlock.status = "InDBS"
dbsUtil.createBlocks([dbsBlock])
for file in childFiles:
dbsBlock.addFile(file)
dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]})
dbsUtil.updateFileStatus([dbsBlock], "InDBS")
dbsUploader.algorithm()
time.sleep(5)
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(parentFiles[0]["datasetPath"], parentFiles)
# Change the status of the rest of the parent blocks so we can upload
# them and the children.
for dbsBlock in allBlocks:
dbsBlock.status = "InDBS"
dbsUtil.updateBlocks([dbsBlock])
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(parentFiles[0]["datasetPath"], parentFiles + moreParentFiles)
# Run the uploader one more time to upload the children.
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(childFiles[0]["datasetPath"], childFiles)
return
def testCloseSettingsPerWorkflow(self):
"""
_testCloseSettingsPerWorkflow_
Test the block closing mechanics in the DBS3 uploader,
this uses a fake dbs api to avoid reliance on external services.
"""
# Signal trapExit that we are a friend
os.environ["DONT_TRAP_EXIT"] = "True"
try:
# Monkey patch the imports of DbsApi
from WMComponent.DBS3Buffer import DBSUploadPoller as MockDBSUploadPoller
MockDBSUploadPoller.DbsApi = MockDbsApi
# Set the poller and the dbsUtil for verification
myThread = threading.currentThread()
(_, dbsFilePath) = mkstemp(dir = self.testDir)
self.dbsUrl = dbsFilePath
config = self.getConfig()
dbsUploader = MockDBSUploadPoller.DBSUploadPoller(config = config)
dbsUtil = DBSBufferUtil()
# First test is event based limits and timeout with no new files.
# Set the files and workflow
acqEra = "TropicalSeason%s" % (int(time.time()))
workflowName = 'TestWorkload%s' % (int(time.time()))
taskPath = '/%s/TestProcessing' % workflowName
self.injectWorkflow(workflowName, taskPath,
MaxWaitTime = 2, MaxFiles = 100,
MaxEvents = 150)
self.createParentFiles(acqEra, nFiles = 20,
workflowName = workflowName,
taskPath = taskPath)
# The algorithm needs to be run twice. On the first iteration it will
# create all the blocks and upload one with less than 150 events.
# On the second iteration the second block is uploaded.
dbsUploader.algorithm()
dbsUploader.checkBlocks()
openBlocks = dbsUtil.findOpenBlocks()
self.assertEqual(len(openBlocks), 1)
globalFiles = myThread.dbi.processData("SELECT id FROM dbsbuffer_file WHERE status = 'InDBS'")[0].fetchall()
notUploadedFiles = myThread.dbi.processData("SELECT id FROM dbsbuffer_file WHERE status = 'NOTUPLOADED'")[0].fetchall()
self.assertEqual(len(globalFiles), 14)
self.assertEqual(len(notUploadedFiles), 6)
# Check the fake DBS for data
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 2)
for block in fakeDBSInfo:
self.assertTrue('block_events' not in block['block'])
self.assertEqual(block['block']['file_count'], 7)
self.assertEqual(block['block']['open_for_writing'], 0)
self.assertTrue('close_settings' not in block)
time.sleep(3)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
openBlocks = dbsUtil.findOpenBlocks()
self.assertEqual(len(openBlocks), 0)
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 3)
for block in fakeDBSInfo:
if block['block']['file_count'] != 6:
self.assertEqual(block['block']['file_count'], 7)
self.assertTrue('block_events' not in block['block'])
self.assertEqual(block['block']['open_for_writing'], 0)
self.assertTrue('close_settings' not in block)
# Now check the limit by size and timeout with new files
acqEra = "TropicalSeason%s" % (int(time.time()))
workflowName = 'TestWorkload%s' % (int(time.time()))
taskPath = '/%s/TestProcessing' % workflowName
self.injectWorkflow(workflowName, taskPath,
MaxWaitTime = 2, MaxFiles = 5,
MaxEvents = 200000000)
self.createParentFiles(acqEra, nFiles = 16,
workflowName = workflowName,
taskPath = taskPath)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
openBlocks = dbsUtil.findOpenBlocks()
self.assertEqual(len(openBlocks), 1)
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 6)
for block in fakeDBSInfo:
if acqEra in block['block']['block_name']:
self.assertEqual(block['block']['file_count'], 5)
self.assertTrue('block_events' not in block['block'])
self.assertTrue('close_settings' not in block)
self.assertEqual(block['block']['open_for_writing'], 0)
# Put more files, they will go into the same block and then it will be closed
# after timeout
time.sleep(3)
self.createParentFiles(acqEra, nFiles = 3,
workflowName = workflowName,
taskPath = taskPath)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
openBlocks = dbsUtil.findOpenBlocks()
self.assertEqual(len(openBlocks), 0)
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 7)
for block in fakeDBSInfo:
if acqEra in block['block']['block_name']:
if block['block']['file_count'] < 5:
self.assertEqual(block['block']['file_count'], 4)
else:
self.assertEqual(block['block']['file_count'], 5)
self.assertTrue('block_events' not in block['block'])
self.assertEqual(block['block']['open_for_writing'], 0)
self.assertTrue('close_settings' not in block)
# Finally test size limits
acqEra = "TropicalSeason%s" % (int(time.time()))
workflowName = 'TestWorkload%s' % (int(time.time()))
taskPath = '/%s/TestProcessing' % workflowName
self.injectWorkflow(workflowName, taskPath,
MaxWaitTime = 1, MaxFiles = 500,
MaxEvents = 200000000, MaxSize = 2048)
self.createParentFiles(acqEra, nFiles = 7,
workflowName = workflowName,
taskPath = taskPath)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
time.sleep(2)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
self.assertEqual(len(openBlocks), 0)
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 11)
for block in fakeDBSInfo:
if acqEra in block['block']['block_name']:
if block['block']['file_count'] != 1:
self.assertEqual(block['block']['block_size'], 2048)
self.assertEqual(block['block']['file_count'], 2)
self.assertTrue('block_events' not in block['block'])
self.assertEqual(block['block']['open_for_writing'], 0)
self.assertTrue('close_settings' not in block)
except:
self.fail("We failed at some point in the test")
finally:
# We don't trust anyone else with _exit
del os.environ["DONT_TRAP_EXIT"]
return
class ReportIntegrationTest(unittest.TestCase):
"""
_ReportIntegrationTest_
"""
def setUp(self):
"""
_setUp_
Setup the database and WMBS for the test.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer",
"WMCore.WMBS"],
useDefault = False)
myThread = threading.currentThread()
self.daofactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.dbsfactory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = self.daofactory(classname = "Locations.New")
locationAction.execute(siteName = "site1", pnn = "T1_US_FNAL_Disk")
inputFile = File(lfn = "/path/to/some/lfn", size = 10, events = 10,
locations = "T1_US_FNAL_Disk")
inputFile.create()
inputFileset = Fileset(name = "InputFileset")
inputFileset.create()
inputFileset.addFile(inputFile)
inputFileset.commit()
unmergedFileset = Fileset(name = "UnmergedFileset")
unmergedFileset.create()
mergedFileset = Fileset(name = "MergedFileset")
mergedFileset.create()
procWorkflow = Workflow(spec = "wf001.xml", owner = "Steve",
name = "TestWF", task = "/TestWF/None")
procWorkflow.create()
procWorkflow.addOutput("outputRECORECO", unmergedFileset)
mergeWorkflow = Workflow(spec = "wf002.xml", owner = "Steve",
name = "MergeWF", task = "/MergeWF/None")
mergeWorkflow.create()
mergeWorkflow.addOutput("Merged", mergedFileset)
insertWorkflow = self.dbsfactory(classname = "InsertWorkflow")
insertWorkflow.execute("TestWF", "/TestWF/None", 0, 0, 0, 0)
insertWorkflow.execute("MergeWF", "/MergeWF/None", 0, 0, 0, 0)
self.procSubscription = Subscription(fileset = inputFileset,
workflow = procWorkflow,
split_algo = "FileBased",
type = "Processing")
self.procSubscription.create()
self.procSubscription.acquireFiles()
self.mergeSubscription = Subscription(fileset = unmergedFileset,
workflow = mergeWorkflow,
split_algo = "WMBSMergeBySize",
type = "Merge")
self.mergeSubscription.create()
self.procJobGroup = JobGroup(subscription = self.procSubscription)
self.procJobGroup.create()
self.mergeJobGroup = JobGroup(subscription = self.mergeSubscription)
self.mergeJobGroup.create()
self.testJob = Job(name = "testJob", files = [inputFile])
self.testJob.create(group = self.procJobGroup)
self.testJob["state"] = "complete"
myThread = threading.currentThread()
self.daofactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.stateChangeAction = self.daofactory(classname = "Jobs.ChangeState")
self.setFWJRAction = self.daofactory(classname = "Jobs.SetFWJRPath")
self.getJobTypeAction = self.daofactory(classname = "Jobs.GetType")
locationAction = self.daofactory(classname = "Locations.New")
locationAction.execute(siteName = "cmssrm.fnal.gov")
self.stateChangeAction.execute(jobs = [self.testJob])
self.tempDir = tempfile.mkdtemp()
return
def tearDown(self):
"""
_tearDown_
Clear out the database and the pickled report file.
"""
self.testInit.clearDatabase()
try:
os.remove(os.path.join(self.tempDir, "ProcReport.pkl"))
os.remove(os.path.join(self.tempDir, "MergeReport.pkl"))
except Exception as ex:
pass
try:
os.rmdir(self.tempDir)
except Exception as ex:
pass
return
def createConfig(self, workerThreads):
"""
_createConfig_
Create a config for the JobAccountant with the given number of worker
threads. This config needs to include information for connecting to the
database as the component will create it's own database connections.
These parameters are still pulled from the environment.
"""
config = self.testInit.getConfiguration()
self.testInit.generateWorkDir(config)
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = os.getenv("COUCHURL")
config.JobStateMachine.couchDBName = "report_integration_t"
config.JobStateMachine.jobSummaryDBName = "report_integration_wmagent_summary_t"
config.component_("JobAccountant")
config.JobAccountant.pollInterval = 60
config.JobAccountant.workerThreads = workerThreads
config.JobAccountant.componentDir = os.getcwd()
config.JobAccountant.logLevel = 'SQLDEBUG'
config.component_("TaskArchiver")
config.TaskArchiver.localWMStatsURL = "%s/%s" % (config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName)
return config
def verifyJobSuccess(self, jobID):
"""
_verifyJobSuccess_
Verify that the metadata for a successful job is correct. This will
check the outcome, retry count and state.
"""
testJob = Job(id = jobID)
testJob.load()
assert testJob["state"] == "success", \
"Error: test job in wrong state: %s" % testJob["state"]
assert testJob["retry_count"] == 0, \
"Error: test job has wrong retry count: %s" % testJob["retry_count"]
assert testJob["outcome"] == "success", \
"Error: test job has wrong outcome: %s" % testJob["outcome"]
return
def verifyFileMetaData(self, jobID, fwkJobReportFiles):
"""
_verifyFileMetaData_
Verify that all the files that were output by a job made it into WMBS
correctly. Compare the contents of WMBS to the files in the frameworks
job report.
Note that fwkJobReportFiles is a list of DataStructs File objects.
"""
testJob = Job(id = jobID)
testJob.loadData()
inputLFNs = []
for inputFile in testJob["input_files"]:
inputLFNs.append(inputFile["lfn"])
for fwkJobReportFile in fwkJobReportFiles:
outputFile = File(lfn = fwkJobReportFile["lfn"])
outputFile.loadData(parentage = 1)
assert outputFile["events"] == int(fwkJobReportFile["events"]), \
"Error: Output file has wrong events: %s, %s" % \
(outputFile["events"], fwkJobReportFile["events"])
assert outputFile["size"] == int(fwkJobReportFile["size"]), \
"Error: Output file has wrong size: %s, %s" % \
(outputFile["size"], fwkJobReportFile["size"])
for ckType in fwkJobReportFile["checksums"]:
assert ckType in outputFile["checksums"], \
"Error: Output file is missing checksums: %s" % ckType
assert outputFile["checksums"][ckType] == fwkJobReportFile["checksums"][ckType], \
"Error: Checksums don't match."
assert len(fwkJobReportFile["checksums"]) == \
len(outputFile["checksums"]), \
"Error: Wrong number of checksums."
jobType = self.getJobTypeAction.execute(jobID = jobID)
if jobType == "Merge":
assert str(outputFile["merged"]) == "True", \
"Error: Merge jobs should output merged files."
else:
assert outputFile["merged"] == fwkJobReportFile["merged"], \
"Error: Output file merged output is wrong: %s, %s" % \
(outputFile["merged"], fwkJobReportFile["merged"])
assert len(outputFile["locations"]) == 1, \
"Error: outputfile should have one location: %s" % outputFile["locations"]
assert list(outputFile["locations"])[0] == list(fwkJobReportFile["locations"])[0], \
"Error: wrong location for file."
assert len(outputFile["parents"]) == len(inputLFNs), \
"Error: Output file has wrong number of parents."
for outputParent in outputFile["parents"]:
assert outputParent["lfn"] in inputLFNs, \
"Error: Unknown parent file: %s" % outputParent["lfn"]
fwjrRuns = {}
for run in fwkJobReportFile["runs"]:
fwjrRuns[run.run] = run.lumis
for run in outputFile["runs"]:
assert run.run in fwjrRuns, \
"Error: Extra run in output: %s" % run.run
for lumi in run:
assert lumi in fwjrRuns[run.run], \
"Error: Extra lumi: %s" % lumi
fwjrRuns[run.run].remove(lumi)
if len(fwjrRuns[run.run]) == 0:
del fwjrRuns[run.run]
assert len(fwjrRuns) == 0, \
"Error: Missing runs, lumis: %s" % fwjrRuns
testJobGroup = JobGroup(id = testJob["jobgroup"])
testJobGroup.loadData()
jobGroupFileset = testJobGroup.output
jobGroupFileset.loadData()
assert outputFile["id"] in jobGroupFileset.getFiles(type = "id"), \
"Error: output file not in jobgroup fileset."
if testJob["mask"]["FirstEvent"] == None:
assert outputFile["first_event"] == 0, \
"Error: first event not set correctly: 0, %s" % \
outputFile["first_event"]
else:
assert testJob["mask"]["FirstEvent"] == outputFile["first_event"], \
"Error: last event not set correctly: %s, %s" % \
(testJob["mask"]["FirstEvent"], outputFile["first_event"])
return
def testReportHandling(self):
"""
_testReportHandling_
Verify that we're able to parse a CMSSW report, convert it to a Report()
style report, pickle it and then have the accountant process it.
"""
self.procPath = os.path.join(WMCore.WMBase.getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml")
myReport = Report("cmsRun1")
myReport.parse(self.procPath)
# Fake some metadata that should be added by the stageout scripts.
for fileRef in myReport.getAllFileRefsFromStep("cmsRun1"):
fileRef.size = 1024
fileRef.location = "cmssrm.fnal.gov"
fwjrPath = os.path.join(self.tempDir, "ProcReport.pkl")
cmsRunStep = myReport.retrieveStep("cmsRun1")
cmsRunStep.status = 0
myReport.setTaskName('/TestWF/None')
myReport.persist(fwjrPath)
self.setFWJRAction.execute(jobID = self.testJob["id"], fwjrPath = fwjrPath)
pFile = DBSBufferFile(lfn = "/path/to/some/lfn", size = 600000, events = 60000)
pFile.setAlgorithm(appName = "cmsRun", appVer = "UNKNOWN",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
pFile.setDatasetPath("/bogus/dataset/path")
#pFile.addRun(Run(1, *[45]))
pFile.create()
config = self.createConfig(workerThreads = 1)
accountant = JobAccountantPoller(config)
accountant.setup()
accountant.algorithm()
self.verifyJobSuccess(self.testJob["id"])
self.verifyFileMetaData(self.testJob["id"], myReport.getAllFilesFromStep("cmsRun1"))
inputFile = File(lfn = "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR09_R_34X_V5_All_v1/0000/outputRECORECO.root")
inputFile.load()
self.testMergeJob = Job(name = "testMergeJob", files = [inputFile])
self.testMergeJob.create(group = self.mergeJobGroup)
self.testMergeJob["state"] = "complete"
self.stateChangeAction.execute(jobs = [self.testMergeJob])
self.mergePath = os.path.join(WMCore.WMBase.getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWMergeReport.xml")
myReport = Report("mergeReco")
myReport.parse(self.mergePath)
# Fake some metadata that should be added by the stageout scripts.
for fileRef in myReport.getAllFileRefsFromStep("mergeReco"):
fileRef.size = 1024
fileRef.location = "cmssrm.fnal.gov"
fileRef.dataset = {"applicationName": "cmsRun", "applicationVersion": "CMSSW_3_4_2_patch1",
"primaryDataset": "MinimumBias", "processedDataset": "Rereco-v1",
"dataTier": "RECO"}
fwjrPath = os.path.join(self.tempDir, "MergeReport.pkl")
myReport.setTaskName('/MergeWF/None')
cmsRunStep = myReport.retrieveStep("mergeReco")
cmsRunStep.status = 0
myReport.persist(fwjrPath)
self.setFWJRAction.execute(jobID = self.testMergeJob["id"], fwjrPath = fwjrPath)
accountant.algorithm()
self.verifyJobSuccess(self.testMergeJob["id"])
self.verifyFileMetaData(self.testMergeJob["id"], myReport.getAllFilesFromStep("mergeReco"))
return
class Scram_t(unittest.TestCase):
def setUp(self):
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testDir = self.testInit.generateWorkDir()
self.oldCwd = os.getcwd()
def tearDown(self):
self.testInit.delWorkDir()
def testA(self):
"""
instantiate a Scram instance in test mode.
"""
try:
Scram(
initialise="/bin/date",
architecture="slc5_amd64_gcc454",
version="CMSSW_X_Y_Z",
test=True
)
except Exception as ex:
msg = "Failed to instantiate Scram in test mode:\n %s " % str(ex)
self.fail(msg)
def testB(self):
"""
instantiante a Scram instance in non-test mode
limited what we can test here since we dont have scram etc in unittest env
"""
try:
Scram(
initialise="/bin/date",
architecture="slc5_amd64_gcc454",
version="CMSSW_X_Y_Z"
)
except Exception as ex:
msg = "Failed to instantiate Scram:\n %s " % str(ex)
self.fail(msg)
def testC(self):
"""
test all method calls in test mode
"""
s = Scram(
initialise="/bin/date",
architecture="slc5_amd64_gcc454",
version="CMSSW_X_Y_Z",
directory=self.testDir,
test=True
)
try:
status = s.project()
except Exception as ex:
msg = "Error running Scram.project:\n %s" % str(ex)
self.fail(msg)
self.assertEqual(status, 0)
self.assertTrue(os.path.exists(s.projectArea))
self.assertTrue("project" in s.lastExecuted)
self.assertTrue("CMSSW_X_Y_Z" in s.lastExecuted)
try:
status = s.runtime()
except Exception as ex:
msg = "Error running Scram.runtime:\n %s" % str(ex)
self.fail(msg)
self.assertEqual(status, 0)
self.assertTrue("ru -sh" in s.lastExecuted)
self.assertTrue("TEST_MODE" in s.runtimeEnv)
comm = "echo \"Hello World\""
try:
status = s(comm)
except Exception as ex:
msg = "Failed to call Scram object:\n %s" % str(ex)
self.assertEqual(status, 0)
self.assertEqual(s.lastExecuted, comm)
def testArchMap(self):
self.assertItemsEqual(OS_TO_ARCH['rhel6'], ['slc5', 'slc6'])
self.assertItemsEqual(OS_TO_ARCH['rhel7'], ['slc7'])
self.assertItemsEqual(ARCH_TO_OS['slc6'], ['rhel6'])
self.assertItemsEqual(ARCH_TO_OS['slc7'], ['rhel7'])
def testScramArchParsing(self):
"""
Test the various modes of parsing for the scram arch
"""
try:
os.chdir(self.testDir)
with tempfile.NamedTemporaryFile() as tf:
tf.write('GLIDEIN_REQUIRED_OS = "rhel6" \n')
tf.write('Memory = 2048\n')
tf.flush()
with tmpEnv(_CONDOR_MACHINE_AD=tf.name):
self.assertEqual(getSingleScramArch('slc6_blah_blah'), 'slc6_blah_blah')
self.assertEqual(getSingleScramArch('slc5_blah_blah'), 'slc5_blah_blah')
self.assertEqual(getSingleScramArch(['slc6_blah_blah', 'slc7_blah_blah']),
'slc6_blah_blah')
self.assertEqual(getSingleScramArch(['slc6_blah_blah', 'slc5_blah_blah']),
'slc6_blah_blah')
self.assertEqual(getSingleScramArch(['slc7_blah_blah', 'slc8_blah_blah']), None)
with tempfile.NamedTemporaryFile() as tf:
tf.write('GLIDEIN_REQUIRED_OS = "rhel7" \n')
tf.write('Memory = 2048\n')
tf.flush()
with tmpEnv(_CONDOR_MACHINE_AD=tf.name):
self.assertEqual(getSingleScramArch('slc6_blah_blah'), 'slc6_blah_blah')
self.assertEqual(getSingleScramArch('slc7_blah_blah'), 'slc7_blah_blah')
self.assertEqual(getSingleScramArch(['slc6_blah_blah', 'slc7_blah_blah']),
'slc7_blah_blah')
self.assertEqual(getSingleScramArch(['slc6_blah_blah', 'slc5_blah_blah']), None)
self.assertEqual(getSingleScramArch(['slc7_blah_blah', 'slc8_blah_blah']),
'slc7_blah_blah')
except Exception:
raise
finally:
os.chdir(self.oldCwd)
return
class BaseTest(unittest.TestCase):
"""
Some methods of this class are made static and are used from
other test cases.
"""
def setUp(self):
self.testInit = TestInit(__file__)
self.testInit.setLogging(logLevel=logging.DEBUG)
self.testDir = self.testInit.generateWorkDir()
self.config = getConfig(self.testDir)
self.testComponentDaemonXml = "/tmp/TestComponent/Daemon.xml"
def tearDown(self):
self.testInit.delWorkDir()
self.generator = None
# if the directory and file "/tmp/TestComponent/Daemon.xml" after
# ComponentsPoller test exist, then delete it
d = os.path.dirname(self.testComponentDaemonXml)
if os.path.exists(d):
shutil.rmtree(d)
def testSenderReceiverBasic(self):
sender = Sender(self.config.Alert.address,
self.config.Alert.controlAddr, self.__class__.__name__)
handler, receiver = utils.setUpReceiver(self.config.Alert.address,
self.config.Alert.controlAddr)
a = Alert(Component=inspect.stack()[0][3])
sender(a)
while len(handler.queue) == 0:
time.sleep(0.5)
print "%s waiting for alert to arrive" % inspect.stack()[0][3]
receiver.shutdown()
self.assertEqual(len(handler.queue), 1)
self.assertEqual(handler.queue[0]["Component"], inspect.stack()[0][3])
def testProcessDetailBasic(self):
pid = os.getpid()
name = inspect.stack()[0][3] # test name
pd = ProcessDetail(pid, name)
self.assertEqual(pd.pid, pid)
self.assertEqual(pd.name, name)
self.assertEqual(pd.proc.pid, pid)
numChildren = len(psutil.Process(pid).get_children())
self.assertEqual(len(pd.children), numChildren)
self.assertEqual(len(pd.allProcs), 1 + numChildren)
d = pd.getDetails()
self.assertEqual(d["pid"], pid)
self.assertEqual(d["component"], name)
self.assertEqual(d["numChildrenProcesses"], numChildren)
pd.refresh()
def testMeasurementsBasic(self):
numMes = 10
mes = Measurements(numMes)
self.assertEqual(mes._numOfMeasurements, numMes)
self.assertEqual(len(mes), 0)
mes.append(20)
self.assertEqual(len(mes), 1)
self.assertEqual(mes[0], 20)
mes.append(30)
self.assertEqual(mes[1], 30)
mes.clear()
self.assertEqual(len(mes), 0)
self.assertEqual(mes._numOfMeasurements, numMes)
def testBasePollerBasic(self):
config = getConfig("/tmp")
# create some non-sence config section. just need a bunch of values defined
config.AlertGenerator.section_("bogusPoller")
config.AlertGenerator.bogusPoller.soft = 5 # [percent]
config.AlertGenerator.bogusPoller.critical = 50 # [percent]
config.AlertGenerator.bogusPoller.pollInterval = 2 # [second]
config.AlertGenerator.bogusPoller.period = 10
generator = utils.AlertGeneratorMock(config)
poller = BasePoller(config.AlertGenerator.bogusPoller, generator)
# define dummy check method
poller.check = lambda: 1 + 1
poller.start()
# poller now runs
time.sleep(config.AlertGenerator.bogusPoller.pollInterval * 2)
poller.terminate()
while poller.is_alive():
time.sleep(0.2)
print "%s waiting for test poller to terminate" % inspect.stack(
)[0][3]
def testBasePollerHandleFailedPolling(self):
config = getConfig("/tmp")
# create some non-sence config section. just need a bunch of values defined
config.AlertGenerator.section_("bogusPoller")
config.AlertGenerator.bogusPoller.soft = 5 # [percent]
config.AlertGenerator.bogusPoller.critical = 50 # [percent]
config.AlertGenerator.bogusPoller.pollInterval = 2 # [second]
config.AlertGenerator.bogusPoller.period = 10
generator = utils.AlertGeneratorMock(config)
poller = BasePoller(config.AlertGenerator.bogusPoller, generator)
ex = Exception("test exception")
class Sender(object):
def __call__(self, alert):
self.alert = alert
poller.sender = Sender()
poller._handleFailedPolling(ex)
self.assertEqual(poller.sender.alert["Source"], "BasePoller")
def testPeriodPollerOnRealProcess(self):
config = getConfig("/tmp")
config.component_("AlertProcessor")
config.AlertProcessor.section_("critical")
config.AlertProcessor.section_("soft")
config.AlertProcessor.critical.level = 5
config.AlertProcessor.soft.level = 0
config.component_("AlertGenerator")
config.AlertGenerator.section_("bogusPoller")
config.AlertGenerator.bogusPoller.soft = 5 # [percent]
config.AlertGenerator.bogusPoller.critical = 50 # [percent]
config.AlertGenerator.bogusPoller.pollInterval = 0.2 # [second]
# period during which measurements are collected before evaluating for
# possible alert triggering
config.AlertGenerator.bogusPoller.period = 1
generator = utils.AlertGeneratorMock(config)
poller = PeriodPoller(config.AlertGenerator.bogusPoller, generator)
poller.sender = utils.SenderMock()
# get CPU usage percentage, it's like measuring CPU usage of a real
# component, so use the appropriate poller's method for that
# (PeriodPoller itself is higher-level class so it doesn't define
# a method to provide sampling data)
poller.sample = lambda processDetail: ComponentsCPUPoller.sample(
processDetail)
# get own pid
pid = os.getpid()
name = inspect.stack()[0][3] # test name
pd = ProcessDetail(pid, name)
# need to repeat sampling required number of measurements
numOfMeasurements = int(config.AlertGenerator.bogusPoller.period /
config.AlertGenerator.bogusPoller.pollInterval)
mes = Measurements(numOfMeasurements)
self.assertEqual(len(mes), 0)
for i in range(mes._numOfMeasurements):
poller.check(pd, mes)
# since the whole measurement cycle was done, values should have been nulled
self.assertEqual(len(mes), 0)
def testPeriodPollerCalculationPredefinedInput(self):
config = getConfig("/tmp")
config.component_("AlertProcessor")
config.AlertProcessor.section_("critical")
config.AlertProcessor.section_("soft")
config.AlertProcessor.critical.level = 5
config.AlertProcessor.soft.level = 0
config.component_("AlertGenerator")
config.AlertGenerator.section_("bogusPoller")
# put some threshold numbers, just need to check output calculation
# from check() method
config.AlertGenerator.bogusPoller.soft = 5 # [percent]
config.AlertGenerator.bogusPoller.critical = 50 # [percent]
config.AlertGenerator.bogusPoller.pollInterval = 0.2 # [second]
config.AlertGenerator.bogusPoller.period = 1
generator = utils.AlertGeneratorMock(config)
poller = PeriodPoller(config.AlertGenerator.bogusPoller, generator)
# since poller may trigger an alert, give it mock sender
poller.sender = utils.SenderMock()
# provide sample method with predefined input, float
predefInput = 10.12
poller.sample = lambda processDetail: predefInput
processDetail = None
numOfMeasurements = int(config.AlertGenerator.bogusPoller.period /
config.AlertGenerator.bogusPoller.pollInterval)
mes = Measurements(numOfMeasurements)
for i in range(mes._numOfMeasurements):
poller.check(processDetail, mes)
# the above loop should went 5 times, should reach evaluation of 5 x predefInput
# values, the average should end up 10, which should trigger soft threshold
self.assertEqual(len(poller.sender.queue), 1)
a = poller.sender.queue[0]
self.assertEqual(a["Component"], generator.__class__.__name__)
self.assertEqual(a["Source"], poller.__class__.__name__)
d = a["Details"]
self.assertEqual(d["threshold"],
"%s%%" % config.AlertGenerator.bogusPoller.soft)
self.assertEqual(d["numMeasurements"], mes._numOfMeasurements)
self.assertEqual(d["period"], config.AlertGenerator.bogusPoller.period)
self.assertEqual(d["average"], "%s%%" % predefInput)
# since the whole measurement cycle was done, values should have been nulled
self.assertEqual(len(mes), 0)
class BaseTest(unittest.TestCase):
"""
Some methods of this class are made static and are used from
other test cases.
"""
def setUp(self):
self.testInit = TestInit(__file__)
self.testInit.setLogging(logLevel = logging.DEBUG)
self.testDir = self.testInit.generateWorkDir()
self.config = getConfig(self.testDir)
self.testComponentDaemonXml = "/tmp/TestComponent/Daemon.xml"
def tearDown(self):
self.testInit.delWorkDir()
self.generator = None
# if the directory and file "/tmp/TestComponent/Daemon.xml" after
# ComponentsPoller test exist, then delete it
d = os.path.dirname(self.testComponentDaemonXml)
if os.path.exists(d):
shutil.rmtree(d)
def testSenderReceiverBasic(self):
sender = Sender(self.config.Alert.address,
self.__class__.__name__,
self.config.Alert.controlAddr)
handler, receiver = utils.setUpReceiver(self.config.Alert.address,
self.config.Alert.controlAddr)
a = Alert(Component = inspect.stack()[0][3])
sender(a)
while len(handler.queue) == 0:
time.sleep(0.5)
print "%s waiting for alert to arrive" % inspect.stack()[0][3]
receiver.shutdown()
self.assertEqual(len(handler.queue), 1)
self.assertEqual(handler.queue[0]["Component"], inspect.stack()[0][3])
def testProcessDetailBasic(self):
pid = os.getpid()
name = inspect.stack()[0][3] # test name
pd = ProcessDetail(pid, name)
self.assertEqual(pd.pid, pid)
self.assertEqual(pd.name, name)
self.assertEqual(pd.proc.pid, pid)
numChildren = len(psutil.Process(pid).get_children())
self.assertEqual(len(pd.children), numChildren)
self.assertEqual(len(pd.allProcs), 1 + numChildren)
d = pd.getDetails()
self.assertEqual(d["pid"], pid)
self.assertEqual(d["component"], name)
self.assertEqual(d["numChildrenProcesses"], numChildren)
def testMeasurementsBasic(self):
numMes = 10
mes = Measurements(numMes)
self.assertEqual(mes._numOfMeasurements, numMes)
self.assertEqual(len(mes), 0)
mes.append(20)
self.assertEqual(len(mes), 1)
self.assertEqual(mes[0], 20)
mes.append(30)
self.assertEqual(mes[1], 30)
mes.clear()
self.assertEqual(len(mes), 0)
self.assertEqual(mes._numOfMeasurements, numMes)
def testBasePollerBasic(self):
config = getConfig("/tmp")
# create some non-sence config section. just need a bunch of values defined
config.AlertGenerator.section_("bogusPoller")
config.AlertGenerator.bogusPoller.soft = 5 # [percent]
config.AlertGenerator.bogusPoller.critical = 50 # [percent]
config.AlertGenerator.bogusPoller.pollInterval = 2 # [second]
config.AlertGenerator.bogusPoller.period = 10
generator = utils.AlertGeneratorMock(config)
poller = BasePoller(config.AlertGenerator.bogusPoller, generator)
# define dummy check method
poller.check = lambda: 1+1
poller.start()
# poller now runs
time.sleep(config.AlertGenerator.bogusPoller.pollInterval * 2)
poller.terminate()
while poller.is_alive():
time.sleep(0.2)
print "%s waiting for test poller to terminate" % inspect.stack()[0][3]
def testPeriodPollerOnRealProcess(self):
config = getConfig("/tmp")
config.component_("AlertProcessor")
config.AlertProcessor.section_("critical")
config.AlertProcessor.section_("soft")
config.AlertProcessor.critical.level = 5
config.AlertProcessor.soft.level = 0
config.component_("AlertGenerator")
config.AlertGenerator.section_("bogusPoller")
config.AlertGenerator.bogusPoller.soft = 5 # [percent]
config.AlertGenerator.bogusPoller.critical = 50 # [percent]
config.AlertGenerator.bogusPoller.pollInterval = 0.2 # [second]
# period during which measurements are collected before evaluating for
# possible alert triggering
config.AlertGenerator.bogusPoller.period = 1
generator = utils.AlertGeneratorMock(config)
poller = PeriodPoller(config.AlertGenerator.bogusPoller, generator)
poller.sender = utils.SenderMock()
# get CPU usage percentage, it's like measuring CPU usage of a real
# component, so use the appropriate poller's method for that
# (PeriodPoller itself is higher-level class so it doesn't define
# a method to provide sampling data)
poller.sample = lambda processDetail: ComponentsCPUPoller.sample(processDetail)
# get own pid
pid = os.getpid()
name = inspect.stack()[0][3] # test name
pd = ProcessDetail(pid, name)
# need to repeat sampling required number of measurements
numOfMeasurements = int(config.AlertGenerator.bogusPoller.period /
config.AlertGenerator.bogusPoller.pollInterval)
mes = Measurements(numOfMeasurements)
self.assertEqual(len(mes), 0)
for i in range(mes._numOfMeasurements):
poller.check(pd, mes)
# since the whole measurement cycle was done, values should have been nulled
self.assertEqual(len(mes), 0)
def testPeriodPollerCalculationPredefinedInput(self):
config = getConfig("/tmp")
config.component_("AlertProcessor")
config.AlertProcessor.section_("critical")
config.AlertProcessor.section_("soft")
config.AlertProcessor.critical.level = 5
config.AlertProcessor.soft.level = 0
config.component_("AlertGenerator")
config.AlertGenerator.section_("bogusPoller")
# put some threshold numbers, just need to check output calculation
# from check() method
config.AlertGenerator.bogusPoller.soft = 5 # [percent]
config.AlertGenerator.bogusPoller.critical = 50 # [percent]
config.AlertGenerator.bogusPoller.pollInterval = 0.2 # [second]
config.AlertGenerator.bogusPoller.period = 1
generator = utils.AlertGeneratorMock(config)
poller = PeriodPoller(config.AlertGenerator.bogusPoller, generator)
# since poller may trigger an alert, give it mock sender
poller.sender = utils.SenderMock()
# provide sample method with predefined input, float
predefInput = 10.12
poller.sample = lambda processDetail: predefInput
processDetail = None
numOfMeasurements = int(config.AlertGenerator.bogusPoller.period /
config.AlertGenerator.bogusPoller.pollInterval)
mes = Measurements(numOfMeasurements)
for i in range(mes._numOfMeasurements):
poller.check(processDetail, mes)
# the above loop should went 5 times, should reach evaluation of 5 x predefInput
# values, the average should end up 10, which should trigger soft threshold
self.assertEqual(len(poller.sender.queue), 1)
a = poller.sender.queue[0]
self.assertEqual(a["Component"], generator.__class__.__name__)
self.assertEqual(a["Source"], poller.__class__.__name__)
d = a["Details"]
self.assertEqual(d["threshold"], "%s%%" % config.AlertGenerator.bogusPoller.soft)
self.assertEqual(d["numMeasurements"], mes._numOfMeasurements)
self.assertEqual(d["period"], config.AlertGenerator.bogusPoller.period)
self.assertEqual(d["average"], "%s%%" % predefInput)
# since the whole measurement cycle was done, values should have been nulled
self.assertEqual(len(mes), 0)
class DashboardInterfaceTest(unittest.TestCase):
"""
Test for the dashboard interface and its monitoring interaction
Well, once I've written them it will be
"""
def setUp(self):
"""
Basically, do nothing
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testDir = self.testInit.generateWorkDir()
return
def tearDown(self):
"""
Clean up the test directory
"""
self.testInit.delWorkDir()
return
def createWorkload(self):
"""
Create a workload in order to test things
"""
generator = WMSpecGenerator()
workload = generator.createReRecoSpec("Tier1ReReco")
return workload
def createTestJob(self):
"""
Create a test job to pass to the DashboardInterface
"""
job = Job(name = "ThisIsASillyName")
testFileA = File(lfn = "/this/is/a/lfnA", size = 1024, events = 10)
testFileA.addRun(Run(1, *[45]))
testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10)
testFileB.addRun(Run(1, *[46]))
job.addFile(testFileA)
job.addFile(testFileB)
job['id'] = 1
return job
def createReport(self, outcome = 0):
"""
Create a test report
"""
jobReport = Report()
jobReport.addStep('cmsRun1')
jobReport.setStepStartTime(stepName = 'cmsRun1')
jobReport.setStepStopTime(stepName = 'cmsRun1')
if outcome:
jobReport.addError('cmsRun1', 200, 'FakeError', 'FakeError')
return jobReport
def setupJobEnvironment(self, name = 'test'):
"""
_setupJobEnvironment_
Make some sort of environment in which to run tests
"""
os.environ['WMAGENT_SITE_CONFIG_OVERRIDE'] = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T1_US_FNAL_SiteLocalConfig.xml")
return
def testASuccessfulJobMonitoring(self):
"""
_testASuccessfulJobMonitoring_
Check that the data packets make sense when a job completes successfully
"""
# Get the necessary objects
name = 'testA'
job = self.createTestJob()
workload = self.createWorkload()
task = workload.getTask(taskName = "DataProcessing")
report = self.createReport()
# Fill the job environment
self.setupJobEnvironment(name = name)
# Instantiate DBInfo
dbInfo = DashboardInfo(job = job, task = task)
dbInfo.addDestination('127.0.0.1', 8884)
# Check jobStart information
data = dbInfo.jobStart()
self.assertEqual(data['MessageType'], 'JobStatus')
self.assertEqual(data['StatusValue'], 'running')
self.assertEqual(data['StatusDestination'], "T1_US_FNAL")
self.assertEqual(data['taskId'], 'wmagent_Tier1ReReco')
# Do the first step
step = task.getStep(stepName = "cmsRun1")
# Do the step start
data = dbInfo.stepStart(step = step.data)
self.assertNotEqual(data['jobStart'], None)
self.assertEqual(data['jobStart']['ExeStart'], step.name())
self.assertEqual(data['jobStart']['WNHostName'], socket.gethostname())
self.assertEqual(data['1_ExeStart'], step.name())
#Do the step end
data = dbInfo.stepEnd(step = step.data, stepReport = report)
self.assertEqual(data['1_ExeEnd'], step.name())
self.assertEqual(data['1_ExeExitCode'], 0)
self.assertTrue(data['1_ExeWCTime'] >= 0)
self.assertEqual(report.retrieveStep("cmsRun1").counter, 1)
#Do a second step
step = task.getStep(stepName = "cmsRun1")
#Do the step start (It's not the first step)
data = dbInfo.stepStart(step = step.data)
self.assertEqual(data['jobStart'], None)
self.assertEqual(data['2_ExeStart'], step.name())
#Do the step end
data = dbInfo.stepEnd(step = step.data, stepReport = report)
self.assertEqual(data['2_ExeEnd'], step.name())
self.assertEqual(data['2_ExeExitCode'], 0)
self.assertTrue(data['2_ExeWCTime'] >= 0)
self.assertEqual(report.retrieveStep("cmsRun1").counter, 2)
# End the job!
data = dbInfo.jobEnd()
self.assertEqual(data['ExeEnd'], "cmsRun1")
self.assertEqual(data['JobExitCode'], 0)
self.assertEqual(data['WrapperCPUTime'], 0)
self.assertTrue(data['WrapperWCTime'] >= 0)
self.assertNotEqual(data['JobExitReason'], "")
return
def testAFailedJobMonitoring(self):
"""
_TestAFailedJobMonitoring_
Simulate a job that completes but fails, check that the data sent is
correct
"""
# Get the necessary objects
name = 'testB'
job = self.createTestJob()
workload = self.createWorkload()
task = workload.getTask(taskName = "DataProcessing")
report = self.createReport(outcome = 1)
# Fill the job environment
self.setupJobEnvironment(name = name)
# Instantiate DBInfo
dbInfo = DashboardInfo(job = job, task = task)
dbInfo.addDestination('127.0.0.1', 8884)
# Check jobStart information
data = dbInfo.jobStart()
self.assertEqual(data['MessageType'], 'JobStatus')
self.assertEqual(data['StatusValue'], 'running')
self.assertEqual(data['StatusDestination'], "T1_US_FNAL")
self.assertEqual(data['taskId'], 'wmagent_Tier1ReReco')
# Do the first step
step = task.getStep(stepName = "cmsRun1")
# Do the step start
data = dbInfo.stepStart(step = step.data)
self.assertNotEqual(data['jobStart'], None)
self.assertEqual(data['jobStart']['ExeStart'], step.name())
self.assertEqual(data['jobStart']['WNHostName'], socket.gethostname())
self.assertEqual(data['1_ExeStart'], step.name())
#Do the step end
data = dbInfo.stepEnd(step = step.data, stepReport = report)
self.assertEqual(data['1_ExeEnd'], step.name())
self.assertNotEqual(data['1_ExeExitCode'], 0)
self.assertTrue(data['1_ExeWCTime'] >= 0)
self.assertEqual(report.retrieveStep("cmsRun1").counter, 1)
# End the job!
data = dbInfo.jobEnd()
self.assertEqual(data['ExeEnd'], "cmsRun1")
self.assertNotEqual(data['JobExitCode'], 0)
self.assertEqual(data['WrapperCPUTime'], 0)
self.assertTrue(data['WrapperWCTime'] >= 0)
self.assertNotEqual(data['JobExitReason'].find('cmsRun1'), -1)
return
def testAKilledJobMonitoring(self):
"""
_TestAKilledJobMonitoring_
Simulate a job that is killed check that the data sent is
correct
"""
# Get the necessary objects
name = 'testC'
job = self.createTestJob()
workload = self.createWorkload()
task = workload.getTask(taskName = "DataProcessing")
report = self.createReport(outcome = 1)
# Fill the job environment
self.setupJobEnvironment(name = name)
# Instantiate DBInfo
dbInfo = DashboardInfo(job = job, task = task)
dbInfo.addDestination('127.0.0.1', 8884)
# Check jobStart information
data = dbInfo.jobStart()
self.assertEqual(data['MessageType'], 'JobStatus')
self.assertEqual(data['StatusValue'], 'running')
self.assertEqual(data['StatusDestination'], "T1_US_FNAL")
self.assertEqual(data['taskId'], 'wmagent_Tier1ReReco')
# Do the first step
step = task.getStep(stepName = "cmsRun1")
# Do the step start
data = dbInfo.stepStart(step = step.data)
self.assertNotEqual(data['jobStart'], None)
self.assertEqual(data['jobStart']['ExeStart'], step.name())
self.assertEqual(data['jobStart']['WNHostName'], socket.gethostname())
self.assertEqual(data['1_ExeStart'], step.name())
#Do the step end
data = dbInfo.stepEnd(step = step.data, stepReport = report)
self.assertEqual(data['1_ExeEnd'], step.name())
self.assertNotEqual(data['1_ExeExitCode'], 0)
self.assertTrue(data['1_ExeWCTime'] >= 0)
# Kill the job!
data = dbInfo.jobKilled()
self.assertEqual(data['ExeEnd'], "cmsRun1")
self.assertNotEqual(data['JobExitCode'], 0)
self.assertEqual(data['WrapperCPUTime'], 0)
self.assertTrue(data['WrapperWCTime'] >= 0)
self.assertNotEqual(data['JobExitReason'].find('killed'), -1)
return
@attr('integration')
def testGetDN(self):
"""
_testGetDN_
Checks that we can get a DN
"""
dn = getUserProxyDN()
if 'X509_USER_PROXY' in os.environ:
self.assertNotEqual(dn, None, 'Error: This should get a DN, if you have set one')
else:
self.assertEqual(dn, None, 'Error: There is no proxy in the environment, it should not get one')
class DBSUploadTest(unittest.TestCase):
"""
TestCase for DBSUpload module
"""
def setUp(self):
"""
_setUp_
setUp function for unittest
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer"],
useDefault = False)
self.testDir = self.testInit.generateWorkDir(deleteOnDestruction = False)
self.configFile = EmulatorSetup.setupWMAgentConfig()
myThread = threading.currentThread()
self.bufferFactory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = self.bufferFactory(classname = "DBSBufferFiles.AddLocation")
locationAction.execute(siteName = "se1.cern.ch")
locationAction.execute(siteName = "se1.fnal.gov")
locationAction.execute(siteName = "malpaquet")
self.dbsUrl = "https://*****:*****@attr("integration")
def testBasicUpload(self):
"""
_testBasicUpload_
Verify that we can successfully upload to DBS3. Also verify that the
uploader correctly handles files parentage when uploading.
"""
self.dbsApi = DbsApi(url = self.dbsUrl)
config = self.getConfig()
dbsUploader = DBSUploadPoller(config = config)
# First test verifies that uploader will poll and then not do anything
# as the database is empty.
dbsUploader.algorithm()
acqEra = "Summer%s" % (int(time.time()))
parentFiles = self.createParentFiles(acqEra)
# The algorithm needs to be run twice. On the first iteration it will
# create all the blocks and upload one. On the second iteration it will
# timeout and upload the second block.
dbsUploader.algorithm()
time.sleep(5)
dbsUploader.algorithm()
time.sleep(5)
# Verify the files made it into DBS3.
self.verifyData(parentFiles[0]["datasetPath"], parentFiles)
# Inject some more parent files and some child files into DBSBuffer.
# Run the uploader twice, only the parent files should be added to DBS3.
(moreParentFiles, childFiles) = \
self.createFilesWithChildren(parentFiles, acqEra)
dbsUploader.algorithm()
time.sleep(5)
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(parentFiles[0]["datasetPath"],
parentFiles + moreParentFiles)
# Run the uploader another two times to upload the child files. Verify
# that the child files were uploaded.
dbsUploader.algorithm()
time.sleep(5)
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(childFiles[0]["datasetPath"], childFiles)
return
@attr("integration")
def testDualUpload(self):
"""
_testDualUpload_
Verify that the dual upload mode works correctly.
"""
self.dbsApi = DbsApi(url = self.dbsUrl)
config = self.getConfig()
dbsUploader = DBSUploadPoller(config = config)
dbsUtil = DBSBufferUtil()
# First test verifies that uploader will poll and then not do anything
# as the database is empty.
dbsUploader.algorithm()
acqEra = "Summer%s" % (int(time.time()))
parentFiles = self.createParentFiles(acqEra)
(moreParentFiles, childFiles) = \
self.createFilesWithChildren(parentFiles, acqEra)
allFiles = parentFiles + moreParentFiles
allBlocks = []
for i in range(4):
DBSBufferDataset(parentFiles[0]["datasetPath"]).create()
blockName = parentFiles[0]["datasetPath"] + "#" + makeUUID()
dbsBlock = DBSBufferBlock(blockName,
location = "malpaquet",
datasetpath = None)
dbsBlock.status = "Open"
dbsBlock.setDataset(parentFiles[0]["datasetPath"], 'data', 'VALID')
dbsUtil.createBlocks([dbsBlock])
for file in allFiles[i * 5 : (i * 5) + 5]:
dbsBlock.addFile(file, 'data', 'VALID')
dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]})
if i < 2:
dbsBlock.status = "InDBS"
dbsUtil.updateBlocks([dbsBlock])
dbsUtil.updateFileStatus([dbsBlock], "InDBS")
allBlocks.append(dbsBlock)
DBSBufferDataset(childFiles[0]["datasetPath"]).create()
blockName = childFiles[0]["datasetPath"] + "#" + makeUUID()
dbsBlock = DBSBufferBlock(blockName,
location = "malpaquet",
datasetpath = None)
dbsBlock.status = "InDBS"
dbsBlock.setDataset(childFiles[0]["datasetPath"], 'data', 'VALID')
dbsUtil.createBlocks([dbsBlock])
for file in childFiles:
dbsBlock.addFile(file, 'data', 'VALID')
dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]})
dbsUtil.updateFileStatus([dbsBlock], "InDBS")
dbsUploader.algorithm()
time.sleep(5)
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(parentFiles[0]["datasetPath"], parentFiles)
# Change the status of the rest of the parent blocks so we can upload
# them and the children.
for dbsBlock in allBlocks:
dbsBlock.status = "InDBS"
dbsUtil.updateBlocks([dbsBlock])
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(parentFiles[0]["datasetPath"], parentFiles + moreParentFiles)
# Run the uploader one more time to upload the children.
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(childFiles[0]["datasetPath"], childFiles)
return
def testCloseSettingsPerWorkflow(self):
"""
_testCloseSettingsPerWorkflow_
Test the block closing mechanics in the DBS3 uploader,
this uses a fake dbs api to avoid reliance on external services.
"""
# Signal trapExit that we are a friend
os.environ["DONT_TRAP_EXIT"] = "True"
try:
# Monkey patch the imports of DbsApi
from WMComponent.DBS3Buffer import DBSUploadPoller as MockDBSUploadPoller
MockDBSUploadPoller.DbsApi = MockDbsApi
# Set the poller and the dbsUtil for verification
myThread = threading.currentThread()
(_, dbsFilePath) = mkstemp(dir = self.testDir)
self.dbsUrl = dbsFilePath
config = self.getConfig()
dbsUploader = MockDBSUploadPoller.DBSUploadPoller(config = config)
dbsUtil = DBSBufferUtil()
# First test is event based limits and timeout with no new files.
# Set the files and workflow
acqEra = "TropicalSeason%s" % (int(time.time()))
workflowName = 'TestWorkload%s' % (int(time.time()))
taskPath = '/%s/TestProcessing' % workflowName
self.injectWorkflow(workflowName, taskPath,
MaxWaitTime = 2, MaxFiles = 100,
MaxEvents = 150)
self.createParentFiles(acqEra, nFiles = 20,
workflowName = workflowName,
taskPath = taskPath)
# The algorithm needs to be run twice. On the first iteration it will
# create all the blocks and upload one with less than 150 events.
# On the second iteration the second block is uploaded.
dbsUploader.algorithm()
dbsUploader.checkBlocks()
openBlocks = dbsUtil.findOpenBlocks()
self.assertEqual(len(openBlocks), 1)
globalFiles = myThread.dbi.processData("SELECT id FROM dbsbuffer_file WHERE status = 'InDBS'")[0].fetchall()
notUploadedFiles = myThread.dbi.processData("SELECT id FROM dbsbuffer_file WHERE status = 'NOTUPLOADED'")[0].fetchall()
self.assertEqual(len(globalFiles), 14)
self.assertEqual(len(notUploadedFiles), 6)
# Check the fake DBS for data
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 2)
for block in fakeDBSInfo:
self.assertTrue('block_events' not in block['block'])
self.assertEqual(block['block']['file_count'], 7)
self.assertEqual(block['block']['open_for_writing'], 0)
self.assertTrue('close_settings' not in block)
time.sleep(3)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
openBlocks = dbsUtil.findOpenBlocks()
self.assertEqual(len(openBlocks), 0)
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 3)
for block in fakeDBSInfo:
if block['block']['file_count'] != 6:
self.assertEqual(block['block']['file_count'], 7)
self.assertTrue('block_events' not in block['block'])
self.assertEqual(block['block']['open_for_writing'], 0)
self.assertTrue('close_settings' not in block)
# Now check the limit by size and timeout with new files
acqEra = "TropicalSeason%s" % (int(time.time()))
workflowName = 'TestWorkload%s' % (int(time.time()))
taskPath = '/%s/TestProcessing' % workflowName
self.injectWorkflow(workflowName, taskPath,
MaxWaitTime = 2, MaxFiles = 5,
MaxEvents = 200000000)
self.createParentFiles(acqEra, nFiles = 16,
workflowName = workflowName,
taskPath = taskPath)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
openBlocks = dbsUtil.findOpenBlocks()
self.assertEqual(len(openBlocks), 1)
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 6)
for block in fakeDBSInfo:
if acqEra in block['block']['block_name']:
self.assertEqual(block['block']['file_count'], 5)
self.assertTrue('block_events' not in block['block'])
self.assertTrue('close_settings' not in block)
self.assertEqual(block['block']['open_for_writing'], 0)
# Put more files, they will go into the same block and then it will be closed
# after timeout
time.sleep(3)
self.createParentFiles(acqEra, nFiles = 3,
workflowName = workflowName,
taskPath = taskPath)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
openBlocks = dbsUtil.findOpenBlocks()
self.assertEqual(len(openBlocks), 0)
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 7)
for block in fakeDBSInfo:
if acqEra in block['block']['block_name']:
if block['block']['file_count'] < 5:
self.assertEqual(block['block']['file_count'], 4)
else:
self.assertEqual(block['block']['file_count'], 5)
self.assertTrue('block_events' not in block['block'])
self.assertEqual(block['block']['open_for_writing'], 0)
self.assertTrue('close_settings' not in block)
# Finally test size limits
acqEra = "TropicalSeason%s" % (int(time.time()))
workflowName = 'TestWorkload%s' % (int(time.time()))
taskPath = '/%s/TestProcessing' % workflowName
self.injectWorkflow(workflowName, taskPath,
MaxWaitTime = 1, MaxFiles = 500,
MaxEvents = 200000000, MaxSize = 2048)
self.createParentFiles(acqEra, nFiles = 7,
workflowName = workflowName,
taskPath = taskPath)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
time.sleep(2)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
self.assertEqual(len(openBlocks), 0)
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 11)
for block in fakeDBSInfo:
if acqEra in block['block']['block_name']:
if block['block']['file_count'] != 1:
self.assertEqual(block['block']['block_size'], 2048)
self.assertEqual(block['block']['file_count'], 2)
self.assertTrue('block_events' not in block['block'])
self.assertEqual(block['block']['open_for_writing'], 0)
self.assertTrue('close_settings' not in block)
except:
self.fail("We failed at some point in the test")
finally:
# We don't trust anyone else with _exit
del os.environ["DONT_TRAP_EXIT"]
return
class ProcessPoolTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase = True)
self.testInit.setSchema(customModules = ["WMCore.Agent.Database"],
useDefault = False)
return
def tearDown(self):
"""
_tearDown_
"""
self.testInit.clearDatabase()
return
def testA_ProcessPool(self):
"""
_testProcessPool_
"""
raise nose.SkipTest
config = self.testInit.getConfiguration()
config.Agent.useHeartbeat = False
self.testInit.generateWorkDir(config)
processPool = ProcessPool("ProcessPool_t.ProcessPoolTestWorker",
totalSlaves = 1,
componentDir = config.General.workDir,
config = config,
namespace = "WMCore_t")
processPool.enqueue(["One", "Two", "Three"])
result = processPool.dequeue(3)
self.assertEqual(len(result), 3, "Error: Expected three items back.")
self.assertTrue( "One" in result)
self.assertTrue( "Two" in result)
self.assertTrue( "Three" in result)
return
def testB_ProcessPoolStress(self):
"""
_testProcessPoolStress_
"""
raise nose.SkipTest
config = self.testInit.getConfiguration()
config.Agent.useHeartbeat = False
self.testInit.generateWorkDir(config)
processPool = ProcessPool("ProcessPool_t.ProcessPoolTestWorker",
totalSlaves = 1,
componentDir = config.General.workDir,
namespace = "WMCore_t",
config = config)
result = None
input = None
for i in range(1000):
input = []
while i > 0:
input.append("COMMAND%s" % i)
i -= 1
processPool.enqueue(input)
result = processPool.dequeue(len(input))
self.assertEqual(len(result), len(input),
"Error: Wrong number of results returned.")
for k in result:
self.assertTrue(k in input)
return
def testC_MultiPool(self):
"""
_testMultiPool_
Run a test with multiple workers
"""
raise nose.SkipTest
config = self.testInit.getConfiguration()
config.Agent.useHeartbeat = False
self.testInit.generateWorkDir(config)
processPool = ProcessPool("ProcessPool_t.ProcessPoolTestWorker",
totalSlaves = 3,
componentDir = config.General.workDir,
namespace = "WMCore_t",
config = config)
for i in range(100):
input = []
while i > 0:
input.append("COMMAND%s" % i)
i -= 1
processPool.enqueue(input)
result = processPool.dequeue(len(input))
self.assertEqual(len(result), len(input),
"Error: Wrong number of results returned.")
class StoreResultsTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Initialize the database.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
self.testDir = self.testInit.generateWorkDir()
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.listTasksByWorkflow = self.daoFactory(
classname="Workflow.LoadFromName")
self.listFilesets = self.daoFactory(classname="Fileset.List")
self.listSubsMapping = self.daoFactory(
classname="Subscriptions.ListSubsAndFilesetsFromWorkflow")
return
def tearDown(self):
"""
_tearDown_
Clear out the database.
"""
self.testInit.clearDatabase()
self.testInit.delWorkDir()
return
def testStoreResults(self):
"""
_testStoreResults_
Create a StoreResults workflow and verify it installs into WMBS
correctly.
"""
arguments = StoreResultsWorkloadFactory.getTestArguments()
factory = StoreResultsWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", arguments)
testWMBSHelper = WMBSHelper(testWorkload,
"StoreResults",
"SomeBlock",
cachepath=self.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
testWorkflow = Workflow(name="TestWorkload",
task="/TestWorkload/StoreResults")
testWorkflow.load()
self.assertEqual(len(testWorkflow.outputMap.keys()), 2,
"Error: Wrong number of WF outputs.")
goldenOutputMods = {"Merged": "USER"}
for goldenOutputMod, tier in goldenOutputMods.items():
fset = goldenOutputMod + tier
mergedOutput = testWorkflow.outputMap[fset][0][
"merged_output_fileset"]
unmergedOutput = testWorkflow.outputMap[fset][0]["output_fileset"]
mergedOutput.loadData()
unmergedOutput.loadData()
self.assertEqual(
mergedOutput.name,
"/TestWorkload/StoreResults/merged-%s" % fset,
"Error: Merged output fileset is wrong: %s" %
mergedOutput.name)
self.assertEqual(
unmergedOutput.name,
"/TestWorkload/StoreResults/merged-%s" % fset,
"Error: Unmerged output fileset is wrong: %s." %
unmergedOutput.name)
logArchOutput = testWorkflow.outputMap["logArchive"][0][
"merged_output_fileset"]
unmergedLogArchOutput = testWorkflow.outputMap["logArchive"][0][
"output_fileset"]
logArchOutput.loadData()
unmergedLogArchOutput.loadData()
self.assertEqual(logArchOutput.name,
"/TestWorkload/StoreResults/merged-logArchive",
"Error: LogArchive output fileset is wrong.")
self.assertEqual(unmergedLogArchOutput.name,
"/TestWorkload/StoreResults/merged-logArchive",
"Error: LogArchive output fileset is wrong.")
topLevelFileset = Fileset(name="TestWorkload-StoreResults-SomeBlock")
topLevelFileset.loadData()
procSubscription = Subscription(fileset=topLevelFileset,
workflow=testWorkflow)
procSubscription.loadData()
self.assertEqual(procSubscription["type"], "Merge",
"Error: Wrong subscription type.")
self.assertEqual(procSubscription["split_algo"],
"ParentlessMergeBySize", "Error: Wrong split algo.")
return
def testFilesets(self):
"""
Test workflow tasks, filesets and subscriptions creation
"""
# expected tasks, filesets, subscriptions, etc
expOutTasks = ['/TestWorkload/StoreResults']
expWfTasks = [
'/TestWorkload/StoreResults',
'/TestWorkload/StoreResults/StoreResultsLogCollect'
]
expFsets = [
'TestWorkload-StoreResults-/MinimumBias/ComissioningHI-v1/RAW',
'/TestWorkload/StoreResults/merged-MergedUSER',
'/TestWorkload/StoreResults/merged-logArchive'
]
subMaps = [
(2, '/TestWorkload/StoreResults/merged-logArchive',
'/TestWorkload/StoreResults/StoreResultsLogCollect',
'MinFileBased', 'LogCollect'),
(1, 'TestWorkload-StoreResults-/MinimumBias/ComissioningHI-v1/RAW',
'/TestWorkload/StoreResults', 'ParentlessMergeBySize', 'Merge')
]
testArguments = StoreResultsWorkloadFactory.getTestArguments()
factory = StoreResultsWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", testArguments)
testWMBSHelper = WMBSHelper(testWorkload,
"StoreResults",
blockName=testArguments['InputDataset'],
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self.assertItemsEqual(testWorkload.listOutputProducingTasks(),
expOutTasks)
workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload")
self.assertItemsEqual([item['task'] for item in workflows], expWfTasks)
# returns a tuple of id, name, open and last_update
filesets = self.listFilesets.execute()
self.assertItemsEqual([item[1] for item in filesets], expFsets)
subscriptions = self.listSubsMapping.execute(workflow="TestWorkload",
returnTuple=True)
self.assertItemsEqual(subscriptions, subMaps)
class AgentTest(unittest.TestCase):
def setUp(self):
self.testInit = TestInit(__file__)
self.testInit.setLogging(logLevel = logging.DEBUG)
self.testDir = self.testInit.generateWorkDir()
self.config = getConfig(self.testDir)
# mock generator instance to communicate some configuration values
self.generator = utils.AlertGeneratorMock(self.config)
self.testComponentDaemonXml = os.path.join(self.testDir, "Daemon.xml")
def tearDown(self):
self.testInit.delWorkDir()
self.generator = None
def testComponentsPollerBasic(self):
"""
Test ComponentsPoller class.
Beware of different process context in real running.
"""
config = getConfig("/tmp")
config.component_("AlertProcessor")
config.AlertProcessor.section_("critical")
config.AlertProcessor.section_("soft")
config.AlertProcessor.critical.level = 5
config.AlertProcessor.soft.level = 0
config.component_("AlertGenerator")
config.AlertGenerator.section_("bogusPoller")
config.AlertGenerator.bogusPoller.soft = 5 # [percent]
config.AlertGenerator.bogusPoller.critical = 90 # [percent]
config.AlertGenerator.bogusPoller.pollInterval = 2 # [second]
# period during which measurements are collected before evaluating for possible alert triggering
config.AlertGenerator.bogusPoller.period = 10
# need to create some temp directory, real process and it's
# Daemon.xml so that is looks like agents component process
# and check back the information
pid = os.getpid()
config.component_("TestComponent")
d = os.path.dirname(self.testComponentDaemonXml)
config.TestComponent.componentDir = d
if not os.path.exists(d):
os.mkdir(d)
f = open(self.testComponentDaemonXml, 'w')
f.write(utils.daemonXmlContent % dict(PID_TO_PUT = pid))
f.close()
generator = utils.AlertGeneratorMock(config)
poller = ComponentsPoller(config.AlertGenerator.bogusPoller, generator)
# only 1 component should have valid workDir with proper Daemon.xml content
# other components present in the configuration (AlertProcessor, AlertGenerator)
# should have been ignored
self.assertEqual(len(poller._components), 1)
pd = poller._components[0]
self.assertEqual(pd.pid, pid)
self.assertEqual(pd.name, "TestComponent")
self.assertEqual(len(pd.children), 0)
self.assertEqual(len(poller._compMeasurements), 1)
mes = poller._compMeasurements[0]
numMeasurements = round(config.AlertGenerator.bogusPoller.period / config.AlertGenerator.bogusPoller.pollInterval, 0)
self.assertEqual(mes._numOfMeasurements, numMeasurements)
shutil.rmtree(d)
def _doComponentsPoller(self, thresholdToTest, level, config,
pollerClass, expected = 0):
"""
Components pollers have array of Measurements and ProcessDetails
which make it more difficult to factory with test methods from the
utils module.
"""
handler, receiver = utils.setUpReceiver(self.generator.config.Alert.address,
self.generator.config.Alert.controlAddr)
# need some real process to poll, give itself
pid = os.getpid()
# the input configuration doesn't have component work directories set right, rectify:
# the configuration will have, see with what _doComponentsPoller is called
# two components: AlertGenerator and AlertProcessor defined
configInstance = Configuration.getInstance()
for comp in Configuration.getInstance().listComponents_():
compDir = getattr(configInstance, comp).componentDir
compDir = os.path.join(compDir, comp)
setattr(getattr(configInstance, comp), "componentDir", compDir)
os.makedirs(compDir)
f = open(os.path.join(compDir, "Daemon.xml"), 'w')
f.write(utils.daemonXmlContent % dict(PID_TO_PUT = pid))
f.close()
numMeasurements = config.period / config.pollInterval
poller = pollerClass(config, self.generator)
# inject own input sample data provider
# there is in fact input argument in this case which needs be ignored
poller.sample = lambda proc_: random.randint(thresholdToTest, thresholdToTest + 10)
# the poller will run upon components (as defined in the configuration)
# and poll them. the PID, etc will be run from the compomentsDir
poller.start()
self.assertTrue(poller.is_alive())
if expected != 0:
# watch so that the test can't take for ever, fail in 2mins
timeLimitExceeded = False
startTime = datetime.datetime.now()
limitTime = 2 * 60 # seconds
while len(handler.queue) == 0:
time.sleep(config.pollInterval / 5)
if (datetime.datetime.now() - startTime).seconds > limitTime:
timeLimitExceeded = True
break
else:
time.sleep(config.period * 2)
poller.terminate()
receiver.shutdown()
self.assertFalse(poller.is_alive())
if expected != 0:
if timeLimitExceeded:
self.fail("No alert received in %s seconds." % limitTime)
# there should be just one alert received, poller should have the
# change to send a second
self.assertEqual(len(handler.queue), expected)
a = handler.queue[0]
# soft threshold - alert should have 'soft' level
self.assertEqual(a["Level"], level)
self.assertEqual(a["Component"], self.generator.__class__.__name__)
self.assertEqual(a["Source"], poller.__class__.__name__)
def testComponentsCPUPollerSoftThreshold(self):
self.config.AlertGenerator.componentsCPUPoller.soft = 70
self.config.AlertGenerator.componentsCPUPoller.critical = 80
self.config.AlertGenerator.componentsCPUPoller.pollInterval = 0.2
self.config.AlertGenerator.componentsCPUPoller.period = 1
level = self.config.AlertProcessor.soft.level
thresholdToTest = self.config.AlertGenerator.componentsCPUPoller.soft
# expected 2: 2 components are defined by the configuration, an alert
# will be sent for each of them
self._doComponentsPoller(thresholdToTest, level,
self.config.AlertGenerator.componentsCPUPoller,
ComponentsCPUPoller, expected = 2)
def testComponentsCPUPollerCriticalThreshold(self):
self.config.AlertGenerator.componentsCPUPoller.soft = 70
self.config.AlertGenerator.componentsCPUPoller.critical = 80
self.config.AlertGenerator.componentsCPUPoller.pollInterval = 0.2
self.config.AlertGenerator.componentsCPUPoller.period = 1
level = self.config.AlertProcessor.critical.level
thresholdToTest = self.config.AlertGenerator.componentsCPUPoller.critical
# expected 2: 2 components are defined by the configuration, an alert
# will be sent for each of them
self._doComponentsPoller(thresholdToTest, level,
self.config.AlertGenerator.componentsCPUPoller,
ComponentsCPUPoller, expected = 2)
def testComponentsCPUPollerNoAlert(self):
self.config.AlertGenerator.componentsCPUPoller.soft = 70
self.config.AlertGenerator.componentsCPUPoller.critical = 80
self.config.AlertGenerator.componentsCPUPoller.pollInterval = 0.2
self.config.AlertGenerator.componentsCPUPoller.period = 1
level = 0
# lower the threshold so that the alert never happens
thresholdToTest = self.config.AlertGenerator.componentsCPUPoller.soft - 10
self._doComponentsPoller(thresholdToTest, level,
self.config.AlertGenerator.componentsCPUPoller,
ComponentsCPUPoller, expected = 0)
def testComponentsMemoryPollerSoftThreshold(self):
self.config.AlertGenerator.componentsMemPoller.soft = 70
self.config.AlertGenerator.componentsMemPoller.critical = 80
self.config.AlertGenerator.componentsMemPoller.pollInterval = 0.2
self.config.AlertGenerator.componentsMemPoller.period = 1
level = self.config.AlertProcessor.soft.level
thresholdToTest = self.config.AlertGenerator.componentsMemPoller.soft
# expected 2: 2 components are defined by the configuration, an alert
# will be sent for each of them
self._doComponentsPoller(thresholdToTest, level,
self.config.AlertGenerator.componentsMemPoller,
ComponentsMemoryPoller, expected = 2)
def testComponentsMemoryPollerCriticalThreshold(self):
self.config.AlertGenerator.componentsMemPoller.soft = 70
self.config.AlertGenerator.componentsMemPoller.critical = 80
self.config.AlertGenerator.componentsMemPoller.pollInterval = 0.2
self.config.AlertGenerator.componentsMemPoller.period = 1
level = self.config.AlertProcessor.critical.level
thresholdToTest = self.config.AlertGenerator.componentsMemPoller.critical
# expected 2: 2 components are defined by the configuration, an alert
# will be sent for each of them
self._doComponentsPoller(thresholdToTest, level,
self.config.AlertGenerator.componentsMemPoller,
ComponentsMemoryPoller, expected = 2)
def testComponentsMemoryPollerNoAlert(self):
self.config.AlertGenerator.componentsMemPoller.soft = 70
self.config.AlertGenerator.componentsMemPoller.critical = 80
self.config.AlertGenerator.componentsMemPoller.pollInterval = 0.2
self.config.AlertGenerator.componentsMemPoller.period = 1
level = 0
# lower the threshold so that the alert never happens
thresholdToTest = self.config.AlertGenerator.componentsCPUPoller.soft - 10
self._doComponentsPoller(thresholdToTest, level,
self.config.AlertGenerator.componentsMemPoller,
ComponentsMemoryPoller, expected = 0)
def testComponentsCPUPollerPossiblyOnLiveAgent(self):
"""
If there is currently running agent upon WMAGENT_CONFIG
configuration, then the test will pick up live processes
and poll them.
"""
# check if the live agent configuration was loaded (above this class)
if globals().has_key("config"):
self.config = config
# AlertProcessor values - values for Level soft, resp. critical
# are also needed by this AlertGenerator test
self.config.component_("AlertProcessor")
self.config.AlertProcessor.componentDir = "/tmp"
self.config.AlertProcessor.section_("critical")
self.config.AlertProcessor.section_("soft")
self.config.AlertProcessor.critical.level = 5
self.config.AlertProcessor.soft.level = 0
self.config.component_("AlertGenerator")
self.config.AlertGenerator.componentDir = "/tmp"
self.config.section_("Alert")
self.config.Alert.address = "tcp://127.0.0.1:6557"
self.config.Alert.controlAddr = "tcp://127.0.0.1:6559"
self.config.AlertGenerator.section_("componentsCPUPoller")
else:
self.config = getConfig("/tmp")
self.config.AlertGenerator.componentsCPUPoller.soft = 70
self.config.AlertGenerator.componentsCPUPoller.critical = 80
self.config.AlertGenerator.componentsCPUPoller.pollInterval = 0.2
self.config.AlertGenerator.componentsCPUPoller.period = 0.3
# generator has already been instantiated, but need another one
# with just defined configuration
# mock generator instance to communicate some configuration values
self.generator = utils.AlertGeneratorMock(self.config)
handler, receiver = utils.setUpReceiver(self.generator.config.Alert.address,
self.generator.config.Alert.controlAddr)
numMeasurements = self.config.AlertGenerator.componentsCPUPoller.period / self.config.AlertGenerator.componentsCPUPoller.pollInterval
poller = ComponentsCPUPoller(self.config.AlertGenerator.componentsCPUPoller, self.generator)
# inject own input sample data provider
thresholdToTest = self.config.AlertGenerator.componentsCPUPoller.soft
# there is in fact input argument in this case which needs be ignored
poller.sample = lambda proc_: random.randint(thresholdToTest - 10, thresholdToTest)
poller.start()
self.assertTrue(poller.is_alive())
# no alert shall arrive
time.sleep(5 * self.config.AlertGenerator.componentsCPUPoller.period)
poller.terminate()
receiver.shutdown()
self.assertFalse(poller.is_alive())
class WMAgentTest(unittest.TestCase):
"""
_WMAgentTest_
Global unittest for all WMAgent components
"""
# This is an integration test
__integration__ = "Any old bollocks"
sites = ['T2_US_Florida', 'T2_US_UCSD', 'T2_TW_Taiwan', 'T1_CH_CERN']
components = ['JobCreator', 'JobSubmitter', 'JobTracker',
'JobAccountant', 'JobArchiver', 'TaskArchiver',
'RetryManager', 'ErrorHandler']
def setUp(self):
"""
_setUp_
Set up vital components
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMCore.WMBS",'WMCore.MsgService',
'WMCore.ResourceControl', 'WMCore.ThreadPool',
'WMCore.Agent.Database'],
useDefault = False)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = self.daoFactory(classname = "Locations.New")
pendingSlots = self.daoFactory(classname = "Locations.SetPendingSlots")
for site in self.sites:
locationAction.execute(siteName = site, seName = 'se.%s' % (site), ceName = site)
pendingSlots.execute(siteName = site, pendingSlots = 1000)
#Create sites in resourceControl
resourceControl = ResourceControl()
for site in self.sites:
resourceControl.insertSite(siteName = site, seName = 'se.%s' % (site), ceName = site)
resourceControl.insertThreshold(siteName = site, taskType = 'Processing', \
maxSlots = 10000, pendingSlots = 10000)
self.testDir = self.testInit.generateWorkDir()
# Set heartbeat
for component in self.components:
heartbeatAPI = HeartbeatAPI(component)
heartbeatAPI.registerComponent()
return
def tearDown(self):
"""
_tearDown_
Tear down everything and go home.
"""
self.testInit.clearDatabase()
self.testInit.delWorkDir()
return
def createTestWorkload(self, workloadName = 'Test', emulator = True):
"""
_createTestWorkload_
Creates a test workload for us to run on, hold the basic necessities.
"""
workload = testWorkload("TestWorkload")
rereco = workload.getTask("ReReco")
taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest'))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
workload.save(workloadName)
return workload
def getConfig(self):
"""
_getConfig_
This is the global test configuration object
"""
config = Configuration()
config.component_("Agent")
config.Agent.WMSpecDirectory = self.testDir
config.Agent.agentName = 'testAgent'
config.Agent.componentName = 'test'
# First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
# Now the CoreDatabase information
# This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# JobCreator
config.component_("JobCreator")
config.JobCreator.namespace = 'WMComponent.JobCreator.JobCreator'
config.JobCreator.logLevel = 'DEBUG'
config.JobCreator.maxThreads = 1
config.JobCreator.UpdateFromResourceControl = True
config.JobCreator.pollInterval = 10
config.JobCreator.jobCacheDir = self.testDir
config.JobCreator.defaultJobType = 'processing' #Type of jobs that we run, used for resource control
config.JobCreator.workerThreads = 2
config.JobCreator.componentDir = os.path.join(os.getcwd(), 'Components')
# JobSubmitter
config.component_("JobSubmitter")
config.JobSubmitter.namespace = 'WMComponent.JobSubmitter.JobSubmitter'
config.JobSubmitter.logLevel = 'INFO'
config.JobSubmitter.maxThreads = 1
config.JobSubmitter.pollInterval = 10
config.JobSubmitter.pluginName = 'CondorGlobusPlugin'
config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins'
config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit')
config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov')
config.JobSubmitter.submitScript = os.path.join(getWMBASE(),
'test/python/WMComponent_t/JobSubmitter_t',
'submit.sh')
config.JobSubmitter.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
# JobTracker
config.component_("JobTracker")
config.JobTracker.logLevel = 'DEBUG'
config.JobTracker.pollInterval = 10
config.JobTracker.trackerName = 'CondorTracker'
config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins'
config.JobTracker.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days
config.JobTracker.idleTimeLimit = 7776000
config.JobTracker.heldTimeLimit = 7776000
config.JobTracker.unknTimeLimit = 7776000
# JobAccountant
config.component_("JobAccountant")
config.JobAccountant.pollInterval = 60
config.JobAccountant.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobAccountant.logLevel = 'INFO'
# JobArchiver
config.component_("JobArchiver")
config.JobArchiver.pollInterval = 60
config.JobArchiver.logLevel = 'INFO'
config.JobArchiver.logDir = os.path.join(self.testDir, 'logs')
config.JobArchiver.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobArchiver.numberOfJobsToCluster = 1000
# Task Archiver
config.component_("TaskArchiver")
config.TaskArchiver.componentDir = self.testInit.generateWorkDir()
config.TaskArchiver.WorkQueueParams = {}
config.TaskArchiver.pollInterval = 60
config.TaskArchiver.logLevel = 'INFO'
config.TaskArchiver.timeOut = 0
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL',
'mnorman:[email protected]:5984')
config.JobStateMachine.couchDBName = "mnorman_test"
# Needed, because this is a test
os.makedirs(config.JobSubmitter.submitDir)
return config
def createFileCollection(self, name, nSubs, nFiles, workflowURL = 'test', site = None):
"""
_createFileCollection_
Create a collection of files for splitting into jobs
"""
myThread = threading.currentThread()
testWorkflow = Workflow(spec = workflowURL, owner = "mnorman",
name = name, task="/TestWorkload/ReReco")
testWorkflow.create()
for sub in range(nSubs):
nameStr = '%s-%i' % (name, sub)
testFileset = Fileset(name = nameStr)
testFileset.create()
for f in range(nFiles):
# pick a random site
if not site:
tmpSite = 'se.%s' % (random.choice(self.sites))
else:
tmpSite = 'se.%s' % (site)
testFile = File(lfn = "/lfn/%s/%i" % (nameStr, f), size = 1024, events = 10)
testFile.setLocation(tmpSite)
testFile.create()
testFileset.addFile(testFile)
testFileset.commit()
testFileset.markOpen(isOpen = 0)
testSubscription = Subscription(fileset = testFileset,
workflow = testWorkflow,
type = "Processing",
split_algo = "FileBased")
testSubscription.create()
return
def createReports(self, jobs, retryCount = 0):
"""
_createReports_
Create some dummy job reports for each job
"""
report = Report()
report.addStep('testStep', 0)
for job in jobs:
#reportPath = os.path.join(job['cache_dir'], 'Report.%i.pkl' % (retryCount))
reportPath = job['fwjr_path']
if os.path.exists(reportPath):
os.remove(reportPath)
report.save(reportPath)
return
def testA_StraightThrough(self):
"""
_StraightThrough_
Just run everything straight through without any variations
"""
# Do pre-submit job check
nRunning = getCondorRunningJobs()
self.assertEqual(nRunning, 0, "User currently has %i running jobs. Test will not continue" % (nRunning))
myThread = threading.currentThread()
workload = self.createTestWorkload()
config = self.getConfig()
name = 'WMAgent_Test1'
site = self.sites[0]
nSubs = 5
nFiles = 10
workloadPath = os.path.join(self.testDir, 'workloadTest',
'TestWorkload', 'WMSandbox',
'WMWorkload.pkl')
# Create a collection of files
self.createFileCollection(name = name, nSubs = nSubs,
nFiles = nFiles,
workflowURL = workloadPath,
site = site)
############################################################
# Test the JobCreator
config.Agent.componentName = 'JobCreator'
testJobCreator = JobCreatorPoller(config = config)
testJobCreator.algorithm()
time.sleep(5)
# Did all jobs get created?
getJobsAction = self.daoFactory(classname = "Jobs.GetAllJobs")
result = getJobsAction.execute(state = 'Created', jobType = "Processing")
self.assertEqual(len(result), nSubs*nFiles)
# Count database objects
result = myThread.dbi.processData('SELECT * FROM wmbs_sub_files_acquired')[0].fetchall()
self.assertEqual(len(result), nSubs * nFiles)
# Find the test directory
testDirectory = os.path.join(self.testDir, 'TestWorkload', 'ReReco')
self.assertTrue('JobCollection_1_0' in os.listdir(testDirectory))
self.assertTrue(len(os.listdir(testDirectory)) <= 20)
groupDirectory = os.path.join(testDirectory, 'JobCollection_1_0')
# First job should be in here
self.assertTrue('job_1' in os.listdir(groupDirectory))
jobFile = os.path.join(groupDirectory, 'job_1', 'job.pkl')
self.assertTrue(os.path.isfile(jobFile))
f = open(jobFile, 'r')
job = cPickle.load(f)
f.close()
self.assertEqual(job['workflow'], name)
self.assertEqual(len(job['input_files']), 1)
self.assertEqual(os.path.basename(job['sandbox']), 'TestWorkload-Sandbox.tar.bz2')
###############################################################
# Now test the JobSubmitter
config.Agent.componentName = 'JobSubmitter'
testJobSubmitter = JobSubmitterPoller(config = config)
testJobSubmitter.algorithm()
# Check that jobs are in the right state
result = getJobsAction.execute(state = 'Created', jobType = "Processing")
self.assertEqual(len(result), 0)
result = getJobsAction.execute(state = 'Executing', jobType = "Processing")
self.assertEqual(len(result), nSubs * nFiles)
# Check assigned locations
getLocationAction = self.daoFactory(classname = "Jobs.GetLocation")
for id in result:
loc = getLocationAction.execute(jobid = id)
self.assertEqual(loc, [[site]])
# Check to make sure we have running jobs
nRunning = getCondorRunningJobs()
self.assertEqual(nRunning, nFiles * nSubs)
#################################################################
# Now the JobTracker
config.Agent.componentName = 'JobTracker'
testJobTracker = JobTrackerPoller(config = config)
testJobTracker.setup()
testJobTracker.algorithm()
# Running the algo without removing the jobs should do nothing
result = getJobsAction.execute(state = 'Executing', jobType = "Processing")
self.assertEqual(len(result), nSubs * nFiles)
condorRM()
time.sleep(1)
# All jobs gone?
nRunning = getCondorRunningJobs()
self.assertEqual(nRunning, 0)
testJobTracker.algorithm()
time.sleep(5)
# Running the algo without removing the jobs should do nothing
result = getJobsAction.execute(state = 'Executing', jobType = "Processing")
self.assertEqual(len(result), 0)
result = getJobsAction.execute(state = 'Complete', jobType = "Processing")
self.assertEqual(len(result), nSubs * nFiles)
#################################################################
# Now the JobAccountant
# First you need to load all jobs
self.getFWJRAction = self.daoFactory(classname = "Jobs.GetFWJRByState")
completeJobs = self.getFWJRAction.execute(state = "complete")
# Create reports for all jobs
self.createReports(jobs = completeJobs, retryCount = 0)
config.Agent.componentName = 'JobAccountant'
testJobAccountant = JobAccountantPoller(config = config)
testJobAccountant.setup()
# It should do something with the jobs
testJobAccountant.algorithm()
# All the jobs should be done now
result = getJobsAction.execute(state = 'Complete', jobType = "Processing")
self.assertEqual(len(result), 0)
result = getJobsAction.execute(state = 'Success', jobType = "Processing")
self.assertEqual(len(result), nSubs * nFiles)
#######################################################################
# Now the JobArchiver
config.Agent.componentName = 'JobArchiver'
testJobArchiver = JobArchiverPoller(config = config)
testJobArchiver.algorithm()
# All the jobs should be cleaned up
result = getJobsAction.execute(state = 'Success', jobType = "Processing")
self.assertEqual(len(result), 0)
result = getJobsAction.execute(state = 'Cleanout', jobType = "Processing")
self.assertEqual(len(result), nSubs * nFiles)
logDir = os.path.join(self.testDir, 'logs')
for job in completeJobs:
self.assertFalse(os.path.exists(job['fwjr_path']))
jobFolder = 'JobCluster_%i' \
% (int(job['id']/config.JobArchiver.numberOfJobsToCluster))
jobPath = os.path.join(logDir, jobFolder, 'Job_%i.tar' %(job['id']))
self.assertTrue(os.path.isfile(jobPath))
self.assertTrue(os.path.getsize(jobPath) > 0)
###########################################################################
# Now the TaskAchiver
config.Agent.componentName = 'TaskArchiver'
testTaskArchiver = TaskArchiverPoller(config = config)
testTaskArchiver.algorithm()
result = getJobsAction.execute(state = 'Cleanout', jobType = "Processing")
self.assertEqual(len(result), 0)
for jdict in completeJobs:
job = Job(id = jdict['id'])
self.assertFalse(job.exists())
if os.path.isdir('testDir'):
shutil.rmtree('testDir')
shutil.copytree('%s' %self.testDir, os.path.join(os.getcwd(), 'testDir'))
return
class HarvestTest(unittest.TestCase):
"""
_HarvestTest_
Test for EndOfRun job splitter
"""
def setUp(self):
"""
_setUp_
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMCore.WMBS"])
self.splitterFactory = SplitterFactory(package = "WMCore.JobSplitting")
myThread = threading.currentThread()
self.myThread = myThread
daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = logging,
dbinterface = myThread.dbi)
self.WMBSFactory = daoFactory
config = self.getConfig()
self.changer = ChangeState(config)
myResourceControl = ResourceControl()
myResourceControl.insertSite("SomeSite", 10, 20, "SomeSE", "SomeCE")
myResourceControl.insertSite("SomeSite", 10, 20, "SomeSE2", "SomeCE")
myResourceControl.insertSite("SomeSite2", 10, 20, "SomeSE3", "SomeCE2")
self.fileset1 = Fileset(name = "TestFileset1")
for file in range(11):
newFile = File("/some/file/name%d" % file, size = 1000, events = 100)
newFile.addRun(Run(1,*[1]))
newFile.setLocation('SomeSE')
self.fileset1.addFile(newFile)
self.fileset1.create()
workflow1 = Workflow(spec = "spec.xml", owner = "hufnagel", name = "TestWorkflow1", task="Test")
workflow1.create()
self.subscription1 = Subscription(fileset = self.fileset1,
workflow = workflow1,
split_algo = "Harvest",
type = "Harvesting")
self.subscription1.create()
self.configFile = EmulatorSetup.setupWMAgentConfig()
return
def tearDown(self):
"""
_tearDown_
"""
self.testInit.clearDatabase()
EmulatorSetup.deleteConfig(self.configFile)
return
def getConfig(self):
"""
_getConfig_
"""
config = self.testInit.getConfiguration()
self.testInit.generateWorkDir(config)
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', None)
config.JobStateMachine.couchDBName = 'wmagent_jobdump'
return config
def finishJobs(self, jobGroups, subscription = None):
"""
_finishJobs_
"""
if not subscription:
subscription = self.subscription1
for f in subscription.acquiredFiles():
subscription.completeFiles(f)
for jobGroup in jobGroups:
self.changer.propagate(jobGroup.jobs, 'executing', 'created')
self.changer.propagate(jobGroup.jobs, 'complete', 'executing')
self.changer.propagate(jobGroup.jobs, 'success', 'complete')
self.changer.propagate(jobGroup.jobs, 'cleanout', 'success')
return
def testHarvestEndOfRunTrigger(self):
"""
_testDQMHarvestEndOfRunTrigger_
Make sure that the basic splitting algo works, which is only, ExpressMerge is ALL done, fire a job against that fileset
"""
self.assertEqual(self.fileset1.open, True, "Fileset is closed. Shouldn't")
jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
jobGroups = jobFactory()
self.assertEqual(len(jobGroups), 0 , "We got 1 or more jobGroups with an open fileset and no periodic configuration")
self.fileset1.markOpen(False)
self.assertEqual(self.fileset1.open, False, "Fileset is opened, why?")
# We should also check if there are aqcuired files, if there are, there are jobs,
# we don't want to fire another jobs while previous are running (output is integrating whatever input)
# TODO : The above one we can do when all is done. Not priority
jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
jobGroups = jobFactory()
self.assertEqual(len(jobGroups), 1 , "Harvest jobsplitter didn't create a single jobGroup after the fileset was closed")
return
def testPeriodicTrigger(self):
"""
_testPeriodicTrigger_
"""
self.assertEqual(self.fileset1.open, True, "Fileset is not open, not testing periodic here")
# Test timeout (5s for this first test)
# there should be no acquired files, if there are, shouldn't be a job
#self.subscription1.acquireFiles(self.subscription1.availableFiles().pop())
jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval = 3)
self.assertEqual(len(jobGroups), 1 , "Didn't created the first periodic job when there were acquired files")
# For the whole thing to work, faking the first job finishing, and putting the files as complete
self.finishJobs(jobGroups)
# Adding more of files, so we have new stuff to process
for file in range(12,24):
newFile = File("/some/file/name%d" % file, size = 1000, events = 100)
newFile.addRun(Run(1,*[1]))
newFile.setLocation('SomeSE')
self.fileset1.addFile(newFile)
self.fileset1.commit()
# Testing that it doesn't create a job unless the delay is past
jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval = 2)
self.assertEqual(len(jobGroups), 0 , "Created one or more job, when there were non-acquired file and the period is not passed by")
time.sleep(2)
jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval = 2)
self.assertEqual(len(jobGroups), 1 , "Didn't created one or more job, and there weren't and the period is passed by")
# Finishing out previous jobs
self.finishJobs(jobGroups)
# Adding more of files, so we have new stuff to process
for file in range(26,36):
newFile = File("/some/file/name%d" % file, size = 1000, events = 100)
newFile.addRun(Run(1,*[1]))
newFile.setLocation('SomeSE')
self.fileset1.addFile(newFile)
self.fileset1.commit()
# Trying to create another job just afterwards, it shouldn't, because it should respect the configured delay
jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval = 2)
self.assertEqual(len(jobGroups), 0 , "Created one or more job, there are new files, but the delay is not past")
time.sleep(2)
jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval = 2)
self.assertEqual(len(jobGroups), 1 , "Didn't created one or more job, there are new files and the delay is past")
# Last check is whether the job gets all the files or not
numFilesJob = jobGroups[0].jobs[0].getFiles()
numFilesFileset = self.fileset1.getFiles()
self.assertEqual(numFilesJob, numFilesFileset, "Job didn't got all the files")
# Finishing out previous jobs
self.finishJobs(jobGroups)
# Adding files for the first location
for file in range(38,48):
newFile = File("/some/file/name%d" % file, size = 1000, events = 100)
newFile.addRun(Run(1,*[1]))
newFile.setLocation('SomeSE')
self.fileset1.addFile(newFile)
self.fileset1.commit()
# Then another location
for file in range(50,56):
newFile = File("/some/file/name%d" % file, size = 1000, events = 100)
newFile.addRun(Run(1,*[1]))
newFile.setLocation('SomeSE3')
self.fileset1.addFile(newFile)
self.fileset1.commit()
# We should have jobs in both locations
time.sleep(2)
jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval = 2)
self.assertEqual(len(jobGroups[0].getJobs()), 2 , "We didn't get 2 jobs for 2 locations")
firstJobLocation = jobGroups[0].getJobs()[0].getFileLocations()[0]
secondJobLocation = jobGroups[0].getJobs()[1].getFileLocations()[0]
self.assertEqual(firstJobLocation, 'SomeSite', "First job location is not SomeSite")
self.assertEqual(secondJobLocation, 'SomeSite2', "Second job location is not SomeSite2")
self.finishJobs(jobGroups)
for file in range(60,65):
newFile = File("/some/file/name%d" % file, size = 1000, events = 100)
newFile.addRun(Run(2,*[2]))
newFile.setLocation('SomeSE3')
self.fileset1.addFile(newFile)
self.fileset1.commit()
for file in range(70,75):
newFile = File("/some/file/name%d" % file, size = 1000, events = 100)
newFile.addRun(Run(3,*[3]))
newFile.setLocation('SomeSE3')
self.fileset1.addFile(newFile)
self.fileset1.commit()
time.sleep(2)
jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval = 2)
# This is one of the most "complicated" tests so worth to comment, 4 jobs should be created
# 1 - all previous files from SomeSE and run = 1 (a lot, like ~45)
# 2 - Few files from SomeSE3, Run = 1
# 3 - Few files from SomeSE3, Run = 2
# 4 - Few files from SomeSE3, Run = 3
self.assertEqual(len(jobGroups[0].getJobs()), 4 , "We didn't get 4 jobs for adding 2 different runs to SomeSE3")
return
def testMultipleRunHarvesting(self):
"""
_testMultipleRunHarvesting_
Add some files with multiple runs in each, make sure the jobs
are created by location and run. Verify each job mask afterwards.
Note that in this test run are splitted between sites,
in real life that MUST NOT happen we still don't support that.
"""
multipleFilesFileset = Fileset(name = "TestFileset")
newFile = File("/some/file/test1", size = 1000, events = 100)
newFile.addRun(Run(1,*[1,3,4,5,6,7]))
newFile.addRun(Run(2,*[1,2,4,5,6,7]))
newFile.setLocation('SomeSE')
multipleFilesFileset.addFile(newFile)
newFile = File("/some/file/test2", size = 1000, events = 100)
newFile.addRun(Run(1,*[2,8]))
newFile.addRun(Run(2,*[3,8]))
newFile.setLocation('SomeSE3')
multipleFilesFileset.addFile(newFile)
multipleFilesFileset.create()
harvestingWorkflow = Workflow(spec = "spec.xml",
owner = "hufnagel",
name = "TestWorkflow",
task="Test")
harvestingWorkflow.create()
harvestSub = Subscription(fileset = multipleFilesFileset,
workflow = harvestingWorkflow,
split_algo = "Harvest",
type = "Harvesting")
harvestSub.create()
jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = harvestSub)
jobGroups = jobFactory(periodic_harvest_interval = 2)
self.assertEqual(len(jobGroups), 1, "A single job group was not created")
self.assertEqual(len(jobGroups[0].getJobs()), 4,
"Four jobs were not created")
for job in jobGroups[0].getJobs():
runs = job['mask'].getRunAndLumis()
self.assertEqual(len(runs), 1, "Job has more than one run configured")
possibleLumiPairs = {1 : [[1,1],[3,7],[2,2],[8,8]],
2 : [[1,2],[4,7],[3,3],[8,8]]}
run = runs.keys()[0]
for lumiPair in runs[run]:
self.assertTrue(lumiPair in possibleLumiPairs[run], "Strange lumi pair in the job mask")
self.finishJobs(jobGroups, harvestSub)
newFile = File("/some/file/test3", size = 1000, events = 100)
newFile.addRun(Run(1,*range(9,15)))
newFile.setLocation('SomeSE3')
multipleFilesFileset.addFile(newFile)
multipleFilesFileset.commit()
time.sleep(2)
jobGroups = jobFactory(periodic_harvest_interval = 2)
self.assertEqual(len(jobGroups), 1, "A single job group was not created")
self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created")
for job in jobGroups[0].getJobs():
runs = job['mask'].getRunAndLumis()
self.assertEqual(len(runs), 1, "Job has more than one run configured")
possibleLumiPairs = {1 : [[1,1],[3,7],[2,2],[8,8],[9,14]],
2 : [[1,2],[4,7],[3,3],[8,8]]}
run = runs.keys()[0]
for lumiPair in runs[run]:
self.assertTrue(lumiPair in possibleLumiPairs[run], "Strange lumi pair in the job mask")
harvestingWorkflowSib = Workflow(spec = "spec.xml",
owner = "hufnagel",
name = "TestWorkflowSib",
task="TestSib")
harvestingWorkflowSib.create()
harvestSubSib = Subscription(fileset = multipleFilesFileset,
workflow = harvestingWorkflowSib,
split_algo = "Harvest",
type = "Harvesting")
harvestSubSib.create()
jobFactorySib = self.splitterFactory(package = "WMCore.WMBS", subscription = harvestSubSib)
multipleFilesFileset.markOpen(False)
jobGroups = jobFactorySib(periodic_harvest_sibling = True)
self.assertEqual(len(jobGroups), 0, "A single job group was created")
self.finishJobs(jobGroups, harvestSub)
jobGroups = jobFactorySib(periodic_harvest_sibling = True)
self.assertEqual(len(jobGroups), 1, "A single job group was not created")
self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created")
for job in jobGroups[0].getJobs():
runs = job['mask'].getRunAndLumis()
self.assertEqual(len(runs), 1, "Job has more than one run configured")
possibleLumiPairs = {1 : [[1,1],[3,7],[2,2],[8,8],[9,14]],
2 : [[1,2],[4,7],[3,3],[8,8]]}
run = runs.keys()[0]
for lumiPair in runs[run]:
self.assertTrue(lumiPair in possibleLumiPairs[run], "Strange lumi pair in the job mask")