def atestTaskArchiverPollerAlertsSending_notifyWorkQueue(self):
"""
Cause exception (alert-worthy situation) in
the TaskArchiverPoller notifyWorkQueue method.
"""
return
myThread = threading.currentThread()
config = self.getConfig()
testTaskArchiver = TaskArchiverPoller(config=config)
# shall later be called directly from utils module
handler, self.alertsReceiver = \
utils.setUpReceiver(config.Alert.address, config.Alert.controlAddr)
# prepare input such input which will go until where it expectantly
# fails and shall send an alert
# this will currently fail in the TaskArchiverPoller killSubscriptions
# on trying to access .load() method which items of below don't have.
# should anything change in the TaskArchiverPoller without modifying this
# test accordingly, it may be failing ...
print "failures 'AttributeError: 'dict' object has no attribute 'load' expected ..."
subList = [{'id': 1}, {'id': 2}, {'id': 3}]
testTaskArchiver.notifyWorkQueue(subList)
# wait for the generated alert to arrive
while len(handler.queue) < len(subList):
time.sleep(0.3)
print "%s waiting for alert to arrive ..." % inspect.stack()[0][3]
self.alertsReceiver.shutdown()
self.alertsReceiver = None
# now check if the alert was properly sent (expect this many failures)
self.assertEqual(len(handler.queue), len(subList))
alert = handler.queue[0]
self.assertEqual(alert["Source"], "TaskArchiverPoller")
def testJobArchiverPollerAlertsSending_cleanJobCache(self):
"""
Cause exception (alert-worthy situation) in
the cleanJobCache method.
"""
myThread = threading.currentThread()
config = self.getConfig()
handler, self.alertsReceiver = \
utils.setUpReceiver(config.Alert.address, config.Alert.controlAddr)
testJobArchiver = JobArchiverPoller(config=config)
# invoke the problem and thus Alert message
job = dict(cache_dir=None)
testJobArchiver.cleanJobCache(job)
# wait for the generated alert to arrive
while len(handler.queue) == 0:
time.sleep(0.3)
print "%s waiting for alert to arrive ..." % inspect.stack()[0][3]
self.alertsReceiver.shutdown()
self.alertsReceiver = None
# now check if the alert was properly sent
self.assertEqual(len(handler.queue), 1)
alert = handler.queue[0]
self.assertEqual(alert["Source"], testJobArchiver.__class__.__name__)
def atestTaskArchiverPollerAlertsSending_killSubscriptions(self):
"""
Cause exception (alert-worthy situation) in
the TaskArchiverPoller killSubscriptions method.
(only 1 situation out of two tested).
"""
return
myThread = threading.currentThread()
config = self.getConfig()
testTaskArchiver = TaskArchiverPoller(config=config)
# shall later be called directly from utils module
handler, self.alertsReceiver = \
utils.setUpReceiver(config.Alert.address, config.Alert.controlAddr)
# will fail on calling .load() - regardless, the same except block
numAlerts = 3
doneList = [{'id': x} for x in range(numAlerts)]
# final re-raise is currently commented, so don't expect Exception here
testTaskArchiver.killSubscriptions(doneList)
# wait for the generated alert to arrive
while len(handler.queue) < numAlerts:
time.sleep(0.3)
print "%s waiting for alert to arrive ..." % inspect.stack()[0][3]
self.alertsReceiver.shutdown()
self.alertsReceiver = None
# now check if the alert was properly sent
self.assertEqual(len(handler.queue), numAlerts)
alert = handler.queue[0]
self.assertEqual(alert["Source"], "TaskArchiverPoller")
return
def testAlertsSetUpAndSendingViaBaseWorkerThread(self):
# calls as they are made from child/client classes of BaseWorkerThread
config = getConfig("/tmp")
self.assertTrue(hasattr(config, "Alert"))
# test the same way alerts are set up in the client code (currently
# all cases via BaseWorkerThread)
# this call creates .sender, but here will be set to None
thread = BaseWorkerThread()
thread.config = config
thread.initAlerts(compName="test2")
self.assertTrue(thread.sender)
# set up a temporary alert message receiver
handler, receiver = utils.setUpReceiver(config.Alert.address,
config.Alert.controlAddr)
# send an alert message
msg = "this is my message 1"
thread.sendAlert(10, msg=msg)
# wait for the alert to arrive
while len(handler.queue) == 0:
time.sleep(0.3)
print "%s waiting for alert to arrive ..." % inspect.stack()[0][3]
self.assertEqual(len(handler.queue), 1)
alert = handler.queue[0]
self.assertEqual(alert["Component"], "test2")
self.assertEqual(alert["Level"], 10)
self.assertEqual(alert["Source"], thread.__class__.__name__)
self.assertEqual(alert["Details"]["msg"], msg)
thread.sender.unregister()
receiver.shutdown()
def testAlertsMessagingBasic(self):
config = getConfig("/tmp")
self.assertTrue(hasattr(config, "Alert"))
# initialization
# sender: instance of Alert messages Sender
# preAlert: pre-defined values for Alert instances generated from this class
self.config = config # needed in setUpAlertsMessaging
preAlert, sender = alertAPI.setUpAlertsMessaging(self,
compName="testBasic")
sendAlert = alertAPI.getSendAlert(sender=sender, preAlert=preAlert)
# set up a temporary alert message receiver
handler, receiver = utils.setUpReceiver(config.Alert.address,
config.Alert.controlAddr)
# test sending alert
msg = "this is my message Basic"
sendAlert(100, msg=msg)
# wait for the alert to arrive
while len(handler.queue) == 0:
time.sleep(0.3)
print "%s waiting for alert to arrive ..." % inspect.stack()[0][3]
self.assertEqual(len(handler.queue), 1)
alert = handler.queue[0]
self.assertEqual(alert["Component"], "testBasic")
self.assertEqual(alert["Level"], 100)
self.assertEqual(alert["Source"], self.__class__.__name__)
self.assertEqual(alert["Details"]["msg"], msg)
sender.unregister()
receiver.shutdown()
def testJobArchiverPollerAlertsSending_constructor(self):
"""
Cause exception (alert-worthy situation) in
the JobArchiverPoller constructor.
"""
myThread = threading.currentThread()
config = self.getConfig()
handler, self.alertsReceiver = \
utils.setUpReceiver(config.Alert.address, config.Alert.controlAddr)
config.JobArchiver.logDir = ""
config.JobArchiver.componentDir = ""
# invoke exception and thus Alert message
self.assertRaises(JobArchiverPollerException,
JobArchiverPoller,
config=config)
# wait for the generated alert to arrive
while len(handler.queue) == 0:
time.sleep(0.3)
print "%s waiting for alert to arrive ..." % inspect.stack()[0][3]
self.alertsReceiver.shutdown()
self.alertsReceiver = None
# now check if the alert was properly sent
self.assertEqual(len(handler.queue), 1)
alert = handler.queue[0]
self.assertEqual(alert["Source"], "JobArchiverPoller")
def testAlertsMessagingBasic(self):
config = getConfig("/tmp")
self.assertTrue(hasattr(config, "Alert"))
# initialization
# sender: instance of Alert messages Sender
# preAlert: pre-defined values for Alert instances generated from this class
self.config = config # needed in setUpAlertsMessaging
preAlert, sender = alertAPI.setUpAlertsMessaging(self,
compName = "testBasic")
sendAlert = alertAPI.getSendAlert(sender = sender,
preAlert = preAlert)
# set up a temporary alert message receiver
handler, receiver = utils.setUpReceiver(config.Alert.address,
config.Alert.controlAddr)
# test sending alert
msg = "this is my message Basic"
sendAlert(100, msg = msg)
# wait for the alert to arrive
while len(handler.queue) == 0:
time.sleep(0.3)
print "%s waiting for alert to arrive ..." % inspect.stack()[0][3]
self.assertEqual(len(handler.queue), 1)
alert = handler.queue[0]
self.assertEqual(alert["Component"], "testBasic")
self.assertEqual(alert["Level"], 100)
self.assertEqual(alert["Source"], self.__class__.__name__)
self.assertEqual(alert["Details"]["msg"], msg)
sender.unregister()
receiver.shutdown()
def testJobArchiverPollerAlertsSending_constructor(self):
"""
Cause exception (alert-worthy situation) in
the JobArchiverPoller constructor.
"""
myThread = threading.currentThread()
config = self.getConfig()
handler, self.alertsReceiver = \
utils.setUpReceiver(config.Alert.address, config.Alert.controlAddr)
config.JobArchiver.logDir = ""
config.JobArchiver.componentDir = ""
# invoke exception and thus Alert message
self.assertRaises(JobArchiverPollerException, JobArchiverPoller, config = config)
# wait for the generated alert to arrive
while len(handler.queue) == 0:
time.sleep(0.3)
print "%s waiting for alert to arrive ..." % inspect.stack()[0][3]
self.alertsReceiver.shutdown()
self.alertsReceiver = None
# now check if the alert was properly sent
self.assertEqual(len(handler.queue), 1)
alert = handler.queue[0]
self.assertEqual(alert["Source"], "JobArchiverPoller")
def testAlertsSetUpAndSendingViaBaseWorkerThread(self):
# calls as they are made from child/client classes of BaseWorkerThread
config = getConfig("/tmp")
self.assertTrue(hasattr(config, "Alert"))
# test the same way alerts are set up in the client code (currently
# all cases via BaseWorkerThread)
# this call creates .sender, but here will be set to None
thread = BaseWorkerThread()
thread.config = config
thread.initAlerts(compName = "test2")
self.assertTrue(thread.sender)
# set up a temporary alert message receiver
handler, receiver = utils.setUpReceiver(config.Alert.address,
config.Alert.controlAddr)
# send an alert message
msg = "this is my message 1"
thread.sendAlert(10, msg = msg)
# wait for the alert to arrive
while len(handler.queue) == 0:
time.sleep(0.3)
print "%s waiting for alert to arrive ..." % inspect.stack()[0][3]
self.assertEqual(len(handler.queue), 1)
alert = handler.queue[0]
self.assertEqual(alert["Component"], "test2")
self.assertEqual(alert["Level"], 10)
self.assertEqual(alert["Source"], thread.__class__.__name__)
self.assertEqual(alert["Details"]["msg"], msg)
thread.sender.unregister()
receiver.shutdown()
def testComponentsCPUPollerPossiblyOnLiveAgent(self):
"""
If there is currently running agent upon WMAGENT_CONFIG
configuration, then the test will pick up live processes
and poll them.
"""
# check if the live agent configuration was loaded (above this class)
if globals().has_key("config"):
self.config = config
# AlertProcessor values - values for Level soft, resp. critical
# are also needed by this AlertGenerator test
self.config.component_("AlertProcessor")
self.config.AlertProcessor.componentDir = "/tmp"
self.config.AlertProcessor.section_("critical")
self.config.AlertProcessor.section_("soft")
self.config.AlertProcessor.critical.level = 5
self.config.AlertProcessor.soft.level = 0
self.config.component_("AlertGenerator")
self.config.AlertGenerator.componentDir = "/tmp"
self.config.section_("Alert")
self.config.Alert.address = "tcp://127.0.0.1:6557"
self.config.Alert.controlAddr = "tcp://127.0.0.1:6559"
self.config.AlertGenerator.section_("componentsCPUPoller")
else:
self.config = getConfig("/tmp")
self.config.AlertGenerator.componentsCPUPoller.soft = 70
self.config.AlertGenerator.componentsCPUPoller.critical = 80
self.config.AlertGenerator.componentsCPUPoller.pollInterval = 0.2
self.config.AlertGenerator.componentsCPUPoller.period = 0.3
# generator has already been instantiated, but need another one
# with just defined configuration
# mock generator instance to communicate some configuration values
self.generator = utils.AlertGeneratorMock(self.config)
handler, receiver = utils.setUpReceiver(self.generator.config.Alert.address,
self.generator.config.Alert.controlAddr)
numMeasurements = self.config.AlertGenerator.componentsCPUPoller.period / self.config.AlertGenerator.componentsCPUPoller.pollInterval
poller = ComponentsCPUPoller(self.config.AlertGenerator.componentsCPUPoller, self.generator)
# inject own input sample data provider
thresholdToTest = self.config.AlertGenerator.componentsCPUPoller.soft
# there is in fact input argument in this case which needs be ignored
poller.sample = lambda proc_: random.randint(thresholdToTest - 10, thresholdToTest)
proc = multiprocessing.Process(target = poller.poll, args = ())
proc.start()
self.assertTrue(proc.is_alive())
# no alert shall arrive
time.sleep(5 * self.config.AlertGenerator.componentsCPUPoller.period)
proc.terminate()
poller.shutdown()
receiver.shutdown()
self.assertFalse(proc.is_alive())
def testJobArchiverPollerAlertsSending_cleanJobCache(self):
"""
Cause exception (alert-worthy situation) in
the cleanJobCache method.
"""
myThread = threading.currentThread()
config = self.getConfig()
handler, self.alertsReceiver = \
utils.setUpReceiver(config.Alert.address, config.Alert.controlAddr)
testJobArchiver = JobArchiverPoller(config = config)
# invoke the problem and thus Alert message
job = dict(cache_dir = None)
testJobArchiver.cleanJobCache(job)
# wait for the generated alert to arrive
while len(handler.queue) == 0:
time.sleep(0.3)
print "%s waiting for alert to arrive ..." % inspect.stack()[0][3]
self.alertsReceiver.shutdown()
self.alertsReceiver = None
# now check if the alert was properly sent
self.assertEqual(len(handler.queue), 1)
alert = handler.queue[0]
self.assertEqual(alert["Source"], testJobArchiver.__class__.__name__)
def atestTaskArchiverPollerAlertsSending_notifyWorkQueue(self):
"""
Cause exception (alert-worthy situation) in
the TaskArchiverPoller notifyWorkQueue method.
"""
return
myThread = threading.currentThread()
config = self.getConfig()
testTaskArchiver = TaskArchiverPoller(config = config)
# shall later be called directly from utils module
handler, self.alertsReceiver = \
utils.setUpReceiver(config.Alert.address, config.Alert.controlAddr)
# prepare input such input which will go until where it expectantly
# fails and shall send an alert
# this will currently fail in the TaskArchiverPoller killSubscriptions
# on trying to access .load() method which items of below don't have.
# should anything change in the TaskArchiverPoller without modifying this
# test accordingly, it may be failing ...
print "failures 'AttributeError: 'dict' object has no attribute 'load' expected ..."
subList = [{'id': 1}, {'id': 2}, {'id': 3}]
testTaskArchiver.notifyWorkQueue(subList)
# wait for the generated alert to arrive
while len(handler.queue) < len(subList):
time.sleep(0.3)
print "%s waiting for alert to arrive ..." % inspect.stack()[0][3]
self.alertsReceiver.shutdown()
self.alertsReceiver = None
# now check if the alert was properly sent (expect this many failures)
self.assertEqual(len(handler.queue), len(subList))
alert = handler.queue[0]
self.assertEqual(alert["Source"], "TaskArchiverPoller")
def atestTaskArchiverPollerAlertsSending_killSubscriptions(self):
"""
Cause exception (alert-worthy situation) in
the TaskArchiverPoller killSubscriptions method.
(only 1 situation out of two tested).
"""
return
myThread = threading.currentThread()
config = self.getConfig()
testTaskArchiver = TaskArchiverPoller(config = config)
# shall later be called directly from utils module
handler, self.alertsReceiver = \
utils.setUpReceiver(config.Alert.address, config.Alert.controlAddr)
# will fail on calling .load() - regardless, the same except block
numAlerts = 3
doneList = [{'id': x} for x in range(numAlerts)]
# final re-raise is currently commented, so don't expect Exception here
testTaskArchiver.killSubscriptions(doneList)
# wait for the generated alert to arrive
while len(handler.queue) < numAlerts:
time.sleep(0.3)
print "%s waiting for alert to arrive ..." % inspect.stack()[0][3]
self.alertsReceiver.shutdown()
self.alertsReceiver = None
# now check if the alert was properly sent
self.assertEqual(len(handler.queue), numAlerts)
alert = handler.queue[0]
self.assertEqual(alert["Source"], "TaskArchiverPoller")
return
def testC_DashboardReporterPollerAlertSending_algorithm(self):
"""
Cause exception (alert-worthy situation) in the algorithm()
method.
"""
myThread = threading.currentThread()
config = self.getConfig()
handler, self.alertsReceiver = \
utils.setUpReceiver(config.Alert.address, config.Alert.controlAddr)
# emulate exception behaviour
def raiseException():
raise Exception("My test exception.")
dashboardReporter = DashboardReporterPoller(config = config)
dashboardReporter.pollCouch = raiseException
self.assertRaises(Exception, dashboardReporter.algorithm)
# wait for the generated alert to arrive
while len(handler.queue) == 0:
time.sleep(0.3)
print "%s waiting for alert to arrive ..." % inspect.stack()[0][3]
self.alertsReceiver.shutdown()
self.alertsReceiver = None
# now check if the alert was properly sent
self.assertEqual(len(handler.queue), 1)
alert = handler.queue[0]
self.assertEqual(alert["Source"], dashboardReporter.__class__.__name__)
self.assertEqual(alert["Component"], "DashboardReporter")
return
def testComponentsCPUPollerPossiblyOnLiveAgent(self):
"""
If there is currently running agent upon WMAGENT_CONFIG
configuration, then the test will pick up live processes
and poll them.
"""
# check if the live agent configuration was loaded (above this class)
if globals().has_key("config"):
self.config = config
# AlertProcessor values - values for Level soft, resp. critical
# are also needed by this AlertGenerator test
self.config.component_("AlertProcessor")
self.config.AlertProcessor.componentDir = "/tmp"
self.config.AlertProcessor.section_("critical")
self.config.AlertProcessor.section_("soft")
self.config.AlertProcessor.critical.level = 5
self.config.AlertProcessor.soft.level = 0
self.config.component_("AlertGenerator")
self.config.AlertGenerator.componentDir = "/tmp"
self.config.section_("Alert")
self.config.Alert.address = "tcp://127.0.0.1:6557"
self.config.Alert.controlAddr = "tcp://127.0.0.1:6559"
self.config.AlertGenerator.section_("componentsCPUPoller")
else:
self.config = getConfig("/tmp")
self.config.AlertGenerator.componentsCPUPoller.soft = 70
self.config.AlertGenerator.componentsCPUPoller.critical = 80
self.config.AlertGenerator.componentsCPUPoller.pollInterval = 0.2
self.config.AlertGenerator.componentsCPUPoller.period = 0.3
# generator has already been instantiated, but need another one
# with just defined configuration
# mock generator instance to communicate some configuration values
self.generator = utils.AlertGeneratorMock(self.config)
handler, receiver = utils.setUpReceiver(self.generator.config.Alert.address,
self.generator.config.Alert.controlAddr)
numMeasurements = self.config.AlertGenerator.componentsCPUPoller.period / self.config.AlertGenerator.componentsCPUPoller.pollInterval
poller = ComponentsCPUPoller(self.config.AlertGenerator.componentsCPUPoller, self.generator)
# inject own input sample data provider
thresholdToTest = self.config.AlertGenerator.componentsCPUPoller.soft
# there is in fact input argument in this case which needs be ignored
poller.sample = lambda proc_: random.randint(thresholdToTest - 10, thresholdToTest)
poller.start()
self.assertTrue(poller.is_alive())
# no alert shall arrive
time.sleep(5 * self.config.AlertGenerator.componentsCPUPoller.period)
poller.terminate()
receiver.shutdown()
self.assertFalse(poller.is_alive())
def testSenderReceiverBasic(self):
sender = Sender(self.config.Alert.address,
self.__class__.__name__,
self.config.Alert.controlAddr)
handler, receiver = utils.setUpReceiver(self.config.Alert.address,
self.config.Alert.controlAddr)
a = Alert(Component = "testSenderReceiverBasic")
sender(a)
time.sleep(0.5)
self.assertEqual(len(handler.queue), 1)
self.assertEqual(handler.queue[0]["Component"], "testSenderReceiverBasic")
receiver.shutdown()
def testSenderReceiverBasic(self):
sender = Sender(self.config.Alert.address,
self.config.Alert.controlAddr, self.__class__.__name__)
handler, receiver = utils.setUpReceiver(self.config.Alert.address,
self.config.Alert.controlAddr)
a = Alert(Component=inspect.stack()[0][3])
sender(a)
while len(handler.queue) == 0:
time.sleep(0.5)
print "%s waiting for alert to arrive" % inspect.stack()[0][3]
receiver.shutdown()
self.assertEqual(len(handler.queue), 1)
self.assertEqual(handler.queue[0]["Component"], inspect.stack()[0][3])
def testSenderReceiverBasic(self):
sender = Sender(self.config.Alert.address,
self.__class__.__name__,
self.config.Alert.controlAddr)
handler, receiver = utils.setUpReceiver(self.config.Alert.address,
self.config.Alert.controlAddr)
a = Alert(Component = inspect.stack()[0][3])
sender(a)
while len(handler.queue) == 0:
time.sleep(0.5)
print "%s waiting for alert to arrive" % inspect.stack()[0][3]
receiver.shutdown()
self.assertEqual(len(handler.queue), 1)
self.assertEqual(handler.queue[0]["Component"], inspect.stack()[0][3])
def _doPeriodPoller(self, thresholdToTest, level, config,
pollerClass, expected = 0):
handler, receiver = utils.setUpReceiver(self.generator.config.Alert.address,
self.generator.config.Alert.controlAddr)
numMeasurements = config.period / config.pollInterval
poller = pollerClass(config, self.generator)
# inject own input sample data provider, there will be 1 input argument we don't want here
poller.sample = lambda _: random.randint(thresholdToTest, thresholdToTest + 10)
proc = multiprocessing.Process(target = poller.poll, args = ())
proc.start()
self.assertTrue(proc.is_alive())
if expected != 0:
# wait to poller to work now ... wait for alert to arrive
# #2238 AlertGenerator test can take 1 hour+ (and fail)
# fail 2mins anyway if alert is not received
timeLimitExceeded = False
startTime = datetime.datetime.now()
limitTime = 2 * 60 # seconds
while len(handler.queue) == 0:
time.sleep(config.pollInterval)
if (datetime.datetime.now() - startTime).seconds > limitTime:
timeLimitExceeded = True
break
else:
# no alert shall arrive
time.sleep(config.period * 2)
proc.terminate()
poller.shutdown()
receiver.shutdown()
self.assertFalse(proc.is_alive())
if expected != 0:
# #2238 AlertGenerator test can take 1 hour+ (and fail)
# temporary measure from above loop:
if timeLimitExceeded:
self.fail("No alert received in %s seconds." % limitTime)
# there should be just one alert received, poller should have the
# change to send a second
self.assertEqual(len(handler.queue), expected)
a = handler.queue[0]
# soft threshold - alert should have soft level
self.assertEqual(a["Level"], level)
self.assertEqual(a["Component"], self.generator.__class__.__name__)
self.assertEqual(a["Source"], poller.__class__.__name__)
self.assertEqual(a["Details"]["numMeasurements"], numMeasurements)
else:
self.assertEqual(len(handler.queue), expected)
def _doDiskPoller(self, thresholdToTest, level, config, expected = 0):
poller = DiskSpacePoller(config, self.generator)
# inject own input sample data provider
poller.sample = lambda: self._dfCommandOutputGenerator(thresholdToTest,
thresholdToTest + 10)
handler, receiver = utils.setUpReceiver(self.generator.config.Alert.address,
self.generator.config.Alert.controlAddr)
proc = multiprocessing.Process(target = poller.poll, args = ())
proc.start()
self.assertTrue(proc.is_alive())
# wait to poller to work now ... wait for alert to arrive
if expected != 0:
# #2238 AlertGenerator test can take 1 hour+ (and fail)
# fail 2mins anyway if alert is not received
timeLimitExceeded = False
startTime = datetime.datetime.now()
limitTime = 2 * 60 # seconds
while len(handler.queue) == 0:
time.sleep(config.pollInterval / 5)
if (datetime.datetime.now() - startTime).seconds > limitTime:
timeLimitExceeded = True
break
else:
time.sleep(config.pollInterval * 2)
proc.terminate()
poller.shutdown()
receiver.shutdown()
self.assertFalse(proc.is_alive())
if expected != 0:
# #2238 AlertGenerator test can take 1 hour+ (and fail)
# temporary measure from above loop:
if timeLimitExceeded:
self.fail("No alert received in %s seconds." % limitTime)
# there should be just one alert received, poller should have the
# change to send a second
self.assertEqual(len(handler.queue), expected)
a = handler.queue[0]
# soft threshold - alert should have soft level
self.assertEqual(a["Level"], level)
self.assertEqual(a["Component"], self.generator.__class__.__name__)
self.assertEqual(a["Source"], poller.__class__.__name__)
d = a["Details"]
self.assertEqual(d["mountPoint"], "/data")
self.assertEqual(d["threshold"], "%s%%" % thresholdToTest)
else:
self.assertEqual(len(handler.queue), 0)
def _doDiskPoller(self, thresholdToTest, level, config, expected=0):
poller = DiskSpacePoller(config, self.generator)
# inject own input sample data provider
poller.sample = lambda: self._dfCommandOutputGenerator(
thresholdToTest, thresholdToTest + 10)
handler, receiver = utils.setUpReceiver(
self.generator.config.Alert.address,
self.generator.config.Alert.controlAddr)
poller.start()
self.assertTrue(poller.is_alive())
# wait to poller to work now ... wait for alert to arrive
if expected != 0:
# #2238 AlertGenerator test can take 1 hour+ (and fail)
# fail 2mins anyway if alert is not received
timeLimitExceeded = False
startTime = datetime.datetime.now()
limitTime = 2 * 60 # seconds
while len(handler.queue) == 0:
time.sleep(config.pollInterval / 5)
if (datetime.datetime.now() - startTime).seconds > limitTime:
timeLimitExceeded = True
break
else:
time.sleep(config.pollInterval * 2)
poller.terminate()
receiver.shutdown()
self.assertFalse(poller.is_alive())
if expected != 0:
# #2238 AlertGenerator test can take 1 hour+ (and fail)
# temporary measure from above loop:
if timeLimitExceeded:
self.fail("No alert received in %s seconds." % limitTime)
# there should be just one alert received, poller should have the
# change to send a second
self.assertEqual(len(handler.queue), expected)
a = handler.queue[0]
# soft threshold - alert should have soft level
self.assertEqual(a["Level"], level)
self.assertEqual(a["Component"], self.generator.__class__.__name__)
self.assertEqual(a["Source"], poller.__class__.__name__)
d = a["Details"]
self.assertEqual(d["mountPoint"], "/data")
self.assertEqual(d["threshold"], "%s%%" % thresholdToTest)
else:
self.assertEqual(len(handler.queue), 0)
def _doComponentsPoller(self, thresholdToTest, level, config,
pollerClass, expected = 0):
handler, receiver = utils.setUpReceiver(self.generator.config.Alert.address,
self.generator.config.Alert.controlAddr)
procWorker = multiprocessing.Process(target = utils.worker, args = ())
procWorker.start()
self.testProcesses.append(procWorker)
numMeasurements = config.period / config.pollInterval
poller = pollerClass(config, self.generator)
# inject own input sample data provider
# there is in fact input argument in this case which needs be ignored
poller.sample = lambda proc_: random.randint(thresholdToTest, thresholdToTest + 10)
# have sample process to run upon but sample date will be fooled by random
pd = ProcessDetail(procWorker.pid, "TestProcess")
mes = Measurements(numMeasurements)
poller._components.append(pd)
poller._compMeasurements.append(mes)
proc = multiprocessing.Process(target = poller.poll, args = ())
proc.start()
self.assertTrue(proc.is_alive())
if expected != 0:
while len(handler.queue) == 0:
time.sleep(config.pollInterval / 5)
else:
time.sleep(config.period * 2)
procWorker.terminate()
proc.terminate()
poller.shutdown()
receiver.shutdown()
self.assertFalse(proc.is_alive())
if expected != 0:
# there should be just one alert received, poller should have the
# change to send a second
self.assertEqual(len(handler.queue), expected)
a = handler.queue[0]
# soft threshold - alert should have 'soft' level
self.assertEqual(a["Level"], level)
self.assertEqual(a["Component"], self.generator.__class__.__name__)
self.assertEqual(a["Source"], poller.__class__.__name__)
def testAlertProcessorBasic(self):
"""
Just tests starting and stopping the component machinery.
Should start and stop all configured pollers.
"""
# the generator will run full-fledged pollers that may get triggered
# to send some alerts. need to consume such in order to avoid clashes
# further tests
handler, receiver = utils.setUpReceiver(self.config.Alert.address, self.config.Alert.controlAddr)
self._startComponent()
# test that all poller processes are running
for poller in self.generator._pollers:
self.assertTrue(poller.is_alive())
# just give the pollers some time to run
time.sleep(5)
self._stopComponent()
receiver.shutdown()
print "%s alerts captured by the way (test %s)." % (len(handler.queue), inspect.stack()[0][3])
def testAlertProcessorBasic(self):
"""
Just tests starting and stopping the component machinery.
Should start and stop all configured pollers.
"""
# the generator will run full-fledged pollers that may get triggered
# to send some alerts. need to consume such in order to avoid clashes
# further tests
handler, receiver = utils.setUpReceiver(self.config.Alert.address,
self.config.Alert.controlAddr)
self._startComponent()
# test that all poller processes are running
for poller in self.generator._pollers:
self.assertTrue(poller.is_alive())
# just give the pollers some time to run
time.sleep(5)
self._stopComponent()
receiver.shutdown()
print "%s alerts captured by the way (test %s)." % (len(handler.queue),
inspect.stack()[0][3])
def testJobArchiverPollerAlertsSending_constructor(self):
"""
Cause exception (alert-worthy situation) in
the JobArchiverPoller constructor.
"""
myThread = threading.currentThread()
config = self.getConfig()
handler, self.alertsReceiver = \
utils.setUpReceiver(config.Alert.address, config.Alert.controlAddr)
config.JobArchiver.logDir = ""
config.JobArchiver.componentDir = ""
# invoke exception and thus Alert message
self.assertRaises(JobArchiverPollerException, JobArchiverPoller, config = config)
self.alertsReceiver.shutdown()
self.alertsReceiver = None
# now check if the alert was properly sent
self.assertEqual(len(handler.queue), 1)
alert = handler.queue[0]
self.assertEqual(alert["Source"], "JobArchiverPoller")
def _doComponentsPoller(self, thresholdToTest, level, config,
pollerClass, expected = 0):
"""
Components pollers have array of Measurements and ProcessDetails
which make it more difficult to factory with test methods from the
utils module.
"""
handler, receiver = utils.setUpReceiver(self.generator.config.Alert.address,
self.generator.config.Alert.controlAddr)
# need some real process to poll, give itself
pid = os.getpid()
# the input configuration doesn't have component work directories set right, rectify:
# the configuration will have, see with what _doComponentsPoller is called
# two components: AlertGenerator and AlertProcessor defined
configInstance = Configuration.getInstance()
for comp in Configuration.getInstance().listComponents_():
compDir = getattr(configInstance, comp).componentDir
compDir = os.path.join(compDir, comp)
setattr(getattr(configInstance, comp), "componentDir", compDir)
os.makedirs(compDir)
f = open(os.path.join(compDir, "Daemon.xml"), 'w')
f.write(utils.daemonXmlContent % dict(PID_TO_PUT = pid))
f.close()
numMeasurements = config.period / config.pollInterval
poller = pollerClass(config, self.generator)
# inject own input sample data provider
# there is in fact input argument in this case which needs be ignored
poller.sample = lambda proc_: random.randint(thresholdToTest, thresholdToTest + 10)
# the poller will run upon components (as defined in the configuration)
# and poll them. the PID, etc will be run from the compomentsDir
poller.start()
self.assertTrue(poller.is_alive())
if expected != 0:
# watch so that the test can't take for ever, fail in 2mins
timeLimitExceeded = False
startTime = datetime.datetime.now()
limitTime = 2 * 60 # seconds
while len(handler.queue) == 0:
time.sleep(config.pollInterval / 5)
if (datetime.datetime.now() - startTime).seconds > limitTime:
timeLimitExceeded = True
break
else:
time.sleep(config.period * 2)
poller.terminate()
receiver.shutdown()
self.assertFalse(poller.is_alive())
if expected != 0:
if timeLimitExceeded:
self.fail("No alert received in %s seconds." % limitTime)
# there should be just one alert received, poller should have the
# change to send a second
self.assertEqual(len(handler.queue), expected)
a = handler.queue[0]
# soft threshold - alert should have 'soft' level
self.assertEqual(a["Level"], level)
self.assertEqual(a["Component"], self.generator.__class__.__name__)
self.assertEqual(a["Source"], poller.__class__.__name__)
def _doComponentsPoller(self,
thresholdToTest,
level,
config,
pollerClass,
expected=0):
"""
Components pollers have array of Measurements and ProcessDetails
which make it more difficult to factory with test methods from the
utils module.
"""
handler, receiver = utils.setUpReceiver(
self.generator.config.Alert.address,
self.generator.config.Alert.controlAddr)
# need some real process to poll, give itself
pid = os.getpid()
# the input configuration doesn't have component work directories set right, rectify:
# the configuration will have, see with what _doComponentsPoller is called
# two components: AlertGenerator and AlertProcessor defined
configInstance = Configuration.getInstance()
for comp in Configuration.getInstance().listComponents_():
compDir = getattr(configInstance, comp).componentDir
compDir = os.path.join(compDir, comp)
setattr(getattr(configInstance, comp), "componentDir", compDir)
os.makedirs(compDir)
f = open(os.path.join(compDir, "Daemon.xml"), 'w')
f.write(utils.daemonXmlContent % dict(PID_TO_PUT=pid))
f.close()
numMeasurements = config.period / config.pollInterval
poller = pollerClass(config, self.generator)
# inject own input sample data provider
# there is in fact input argument in this case which needs be ignored
poller.sample = lambda proc_: random.randint(thresholdToTest,
thresholdToTest + 10)
# the poller will run upon components (as defined in the configuration)
# and poll them. the PID, etc will be run from the compomentsDir
poller.start()
self.assertTrue(poller.is_alive())
if expected != 0:
# watch so that the test can't take for ever, fail in 2mins
timeLimitExceeded = False
startTime = datetime.datetime.now()
limitTime = 2 * 60 # seconds
while len(handler.queue) == 0:
time.sleep(config.pollInterval / 5)
if (datetime.datetime.now() - startTime).seconds > limitTime:
timeLimitExceeded = True
break
else:
time.sleep(config.period * 2)
poller.terminate()
receiver.shutdown()
self.assertFalse(poller.is_alive())
if expected != 0:
if timeLimitExceeded:
self.fail("No alert received in %s seconds." % limitTime)
# there should be just one alert received, poller should have the
# change to send a second
self.assertEqual(len(handler.queue), expected)
a = handler.queue[0]
# soft threshold - alert should have 'soft' level
self.assertEqual(a["Level"], level)
self.assertEqual(a["Component"], self.generator.__class__.__name__)
self.assertEqual(a["Source"], poller.__class__.__name__)
