def testReceiverBuferring(self):
"""
Test sending alerts that are buffered into a queue.
"""
# alerts received will be added to the queue
alertList = []
handler = lambda x: alertList.append(x)
rec = Receiver(self.addr, handler, self.ctrl)
rec.startReceiver() # non blocking call
numAlertMsgs = 7
thread = AlertsSender(self.addr, self.ctrl, numAlertMsgs)
# thread will send numAlertMsgs and eventually shut the receiver
# down by a shutdown control message
thread.start()
# wait here until the Receiver is not shut
while rec.isReady():
time.sleep(0.2)
# worker will send shutdown message and execution will resume here
# check the content of the queue - 5 alert messages
self.assertEqual(len(alertList), numAlertMsgs)
for alert in alertList:
self.failUnless(alert.has_key("Type"))
self.assertEqual(alert["Type"], "Alert")
def testReceiverBuferring(self):
"""
Test sending alerts that are buffered into a queue.
"""
# alerts received will be added to the queue
alertList = []
handler = lambda x: alertList.append(x)
rec = Receiver(self.addr, handler, self.ctrl)
rec.startReceiver() # non blocking call
numAlertMsgs = 7
thread = AlertsSender(self.addr, self.ctrl, numAlertMsgs)
# thread will send numAlertMsgs and eventually shut the receiver
# down by a shutdown control message
thread.start()
# wait here until the Receiver is not shut
while rec.isReady():
time.sleep(0.2)
# worker will send shutdown message and execution will resume here
# check the content of the queue - 5 alert messages
self.assertEqual(len(alertList), numAlertMsgs)
for alert in alertList:
self.failUnless("Type" in alert)
self.assertEqual(alert["Type"], "Alert")
def testProcessorWithReceiverAndCouchSink(self):
# set up couch first
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
dbName = "couch_sink"
self.testInit.setupCouch(dbName)
# add corresponding part of the configuration for CouchSink(s)
config = self.config.AlertProcessor
config.critical.sinks.section_("couch")
config.critical.sinks.couch.url = self.testInit.couchUrl
config.critical.sinks.couch.database = self.testInit.couchDbName
# just send the Alert into couch
processor = Processor(config)
rec = Receiver(self.addr, processor, self.ctrl)
rec.startReceiver() # non blocking call
# run worker(), this time directly without Process as above,
# worker will send 10 Alerts to Receiver
worker(self.addr, self.ctrl, 10)
# wait until the Receiver is shut by worker
while rec.isReady():
time.sleep(0.3)
print "%s waiting for Receiver to shut ..." % inspect.stack()[0][3]
def testReceiverBuferring(self):
"""
Test sending alerts that are buffered into a queue.
"""
numAlertMsgs = 7
q = Queue()
# start a process that generates 5 alerts, worker can run
# as a different process, though is not necessary here
self.p = Process(target = worker, args = (self.addr, self.ctrl, q, numAlertMsgs))
self.p.start()
# alerts received will be added to the queue
handler = lambda x: q.put(x)
rec = Receiver(self.addr, handler, self.ctrl)
rec.startReceiver() # non blocking call
# wait here until the Receiver is not shut
while rec.isReady():
time.sleep(0.2)
# worker will send shutdown message and execution will resume here
# check the content of the queue - 5 alert messages
alertCount = 0
while not q.empty():
alert = q.get()
self.failUnless(alert.has_key("Type"))
self.assertEqual(alert["Type"], "Alert")
alertCount += 1
self.assertEqual(alertCount, numAlertMsgs)
def setUpReceiver(address, controlAddr):
"""
Return set up handler, receiver pair.
Receiver starts two channels on the address and controlAddr addresses.
"""
handler = ReceiverHandler()
receiver = Receiver(address, handler, controlAddr)
receiver.startReceiver() # non blocking call
return handler, receiver
def setUpReceiver(address, controlAddr):
"""
Return set up handler, receiver pair.
Receiver starts two channels on the address and controlAddr addresses.
"""
handler = ReceiverHandler()
receiver = Receiver(address, handler, controlAddr)
receiver.startReceiver() # non blocking call
return handler, receiver
def testReceiverShutdownByCall(self):
# start a Receiver
rec = Receiver(self.addr, self.printer, self.ctrl)
rec.startReceiver() # non blocking call
workChann, contChann = self._getSenderChannels()
# send some messages to the receiver and shut it eventually
contChann.send_json(RegisterMsg("Receiver_t"))
workChann.send_json(Alert(Type="Alert", Level=20))
contChann.send_json(UnregisterMsg("Receiver_t"))
# now messages are sent so shutdown the Receiver by a convenience
# call, should block until the Receiver finishes, don't have to wait
rec.shutdown()
def testReceiverShutdownByCall(self):
# start a Receiver
rec = Receiver(self.addr, self.printer, self.ctrl)
rec.startReceiver() # non blocking call
workChann, contChann = self._getSenderChannels()
# send some messages to the receiver and shut it eventually
contChann.send_json(RegisterMsg("Receiver_t"))
workChann.send_json(Alert(Type = "Alert", Level = 20))
contChann.send_json(UnregisterMsg("Receiver_t"))
# now messages are sent so shutdown the Receiver by a convenience
# call, should block until the Receiver finishes, don't have to wait
rec.shutdown()
def testProcessorWithReceiver(self):
"""
Test startup and shutdown of processor in receiver.
"""
processor = Processor(self.config.AlertProcessor)
rec = Receiver(self.addr, processor, self.ctrl)
rec.startReceiver()
# since the above is non-blocking, could send the messages from here
# directly, yet running via Process doesn't harm
sender = Process(target=simpleWorker, args=(self.addr, self.ctrl))
sender.start()
# wait until the Receiver is shut by simpleWorker
while rec.isReady():
time.sleep(0.1)
class AlertProcessor(Harness):
def __init__(self, config):
Harness.__init__(self, config)
self._myName = self.__class__.__name__
self.config = config
# instance of processor
self._processor = None
# instance of Receiver which owns Processor (self._processor)
# and runs on background
self._receiver = None
logging.info("%s initialized." % self._myName)
def preInitialization(self):
"""
Start up the ZMQ Receiver + Processor.
"""
logging.info("%s starting ..." % self._myName)
self._processor = Processor(self.config.AlertProcessor)
# Receiver listens on work channel (address) and on control
# channel (controlAddr)
self._receiver = Receiver(self.config.AlertProcessor.address,
self._processor,
self.config.AlertProcessor.controlAddr)
self._receiver.startReceiver()
logging.info("%s started, Receiver should be listening." % self._myName)
def stopProcessor(self):
"""
Method to shutdown the Alert Processor.
"""
logging.info("%s shutting down, waiting for Receiver ..." % self._myName)
self._receiver.shutdown()
def prepareToStop(self, wait = False, stopPayload = ""):
"""
Override prepareToStop to include call to stopProcessor.
Ugly, but seems no other way to do this...
"""
self.stopProcessor()
Harness.prepareToStop(self, wait, stopPayload)
def testReceiverShutdownByMessage(self):
# start a Receiver
rec = Receiver(self.addr, self.printer, self.ctrl)
rec.startReceiver() # non blocking call
workChann, contChann = self._getSenderChannels()
# send some messages to the receiver and shut it eventually
contChann.send_json(RegisterMsg("Receiver_t"))
workChann.send_json(Alert(Type="Alert", Level=10))
contChann.send_json(UnregisterMsg("Receiver_t"))
# terminate the Receiver
contChann.send_json(ShutdownMsg())
# wait until the Receiver is properly shut
# this will not be necessary when shutting down by a call
while rec.isReady():
time.sleep(0.1)
def testReceiverShutdownByMessage(self):
# start a Receiver
rec = Receiver(self.addr, self.printer, self.ctrl)
rec.startReceiver() # non blocking call
workChann, contChann = self._getSenderChannels()
# send some messages to the receiver and shut it eventually
contChann.send_json(RegisterMsg("Receiver_t"))
workChann.send_json(Alert(Type = "Alert", Level = 10))
contChann.send_json(UnregisterMsg("Receiver_t"))
# terminate the Receiver
contChann.send_json(ShutdownMsg())
# wait until the Receiver is properly shut
# this will not be necessary when shutting down by a call
while rec.isReady():
time.sleep(0.1)
def testProcessorWithReceiver(self):
"""
Test startup and shutdown of processor in receiver.
"""
processor = Processor(self.config.AlertProcessor)
rec = Receiver(self.addr, processor, self.ctrl)
rec.startReceiver() # non-blocking call
# now sender tests control messages (register, unregister, shutdown)
s = Sender(self.addr, self.ctrl, "Processor_t")
s.register()
s.unregister()
s.sendShutdown()
# wait until the Receiver is shut by sending the above control messages
while rec.isReady():
time.sleep(0.3)
print "%s waiting for Receiver to shut ..." % inspect.stack()[0][3]
def testProcessorWithReceiverAndFileSink(self):
# add corresponding part of the configuration for FileSink(s)
config = self.config.AlertProcessor
config.critical.sinks.section_("file")
config.critical.sinks.file.outputfile = self.criticalOutputFile
config.soft.sinks.section_("file")
config.soft.sinks.file.outputfile = self.softOutputFile
processor = Processor(config)
rec = Receiver(self.addr, processor, self.ctrl)
rec.startReceiver() # non blocking call
# run worker(), this time directly without Process as above,
# worker will send 10 Alerts to Receiver
worker(self.addr, self.ctrl, 10)
# wait until the Receiver is shut by worker
while rec.isReady():
time.sleep(0.3)
print "%s waiting for Receiver to shut ..." % inspect.stack()[0][3]
# now check the FileSink output files for content:
# the soft Alerts has threshold level set to 0 so Alerts
# with level 1 and higher, resp. for critical the level
# was above set to 5 so 6 and higher out of worker's 0 .. 9
# (10 Alerts altogether) shall be present
softSink = FileSink(config.soft.sinks.file)
criticalSink = FileSink(config.critical.sinks.file)
softList = softSink.load()
criticalList = criticalSink.load()
# check soft level alerts
# levels 1 .. 4 went in (level 0 is, according to the config not considered)
self.assertEqual(len(softList), 3)
for a, level in zip(softList, range(1, 4)):
self.assertEqual(a["Level"], level)
# check 'critical' levels
# only levels 5 .. 9 went in
self.assertEqual(len(criticalList), 5)
for a, level in zip(criticalList, range(5, 10)):
self.assertEqual(a["Level"], level)
class ProcessorTest(unittest.TestCase):
"""
TestCase for Processor.
"""
def setUp(self):
"""
Set up for tests.
"""
l = logging.getLogger()
l.setLevel(logging.DEBUG)
self.addr = "tcp://127.0.0.1:5557"
self.ctrl = "tcp://127.0.0.1:5559"
self.softOutputFile = "/tmp/ProcessorTestSoftAlerts.json"
self.criticalOutputFile = "/tmp/ProcessorTestCriticalAlerts.json"
self.config = Configuration()
self.config.component_("AlertProcessor")
self.config.AlertProcessor.section_("critical")
self.config.AlertProcessor.section_("soft")
self.config.AlertProcessor.critical.level = 5
self.config.AlertProcessor.soft.level = 1
self.config.AlertProcessor.soft.bufferSize = 3
self.config.AlertProcessor.critical.section_("sinks")
self.config.AlertProcessor.soft.section_("sinks")
def tearDown(self):
for f in (self.criticalOutputFile, self.softOutputFile):
if os.path.exists(f):
os.remove(f)
if hasattr(self, "testInit"):
self.testInit.tearDownCouch()
if hasattr(self, "receiver"):
# wait until the Receiver is shut by the Shutdown control
# message which the worker() function should have sent
while self.receiver.isReady():
logging.info("tearDown: Waiting for Receiver shutdown ...")
time.sleep(ReceiverLogic.TIMEOUT_AFTER_SHUTDOWN * 1.1)
if self.receiver.isReady():
self.receiver.shutdown()
logging.info("tearDown: Is the Receiver shut down: %s" % (not self.receiver.isReady()))
def testProcessorBasic(self):
str(self.config.AlertProcessor)
p = Processor(self.config.AlertProcessor)
def testProcessorWithReceiver(self):
"""
Test startup and shutdown of processor in receiver.
"""
processor = Processor(self.config.AlertProcessor)
# Receiver is waited for shutdown / shutdown explicitly in tearDown()
self.receiver = Receiver(self.addr, processor, self.ctrl)
self.receiver.startReceiver() # non-blocking call
# now sender tests control messages (register, unregister, shutdown)
s = Sender(self.addr, self.ctrl, "Processor_t")
s.register()
s.unregister()
s.sendShutdown()
# give some time so that the previous call shuts down the receiver
time.sleep(ReceiverLogic.TIMEOUT_AFTER_SHUTDOWN * 1.1)
def testProcessorWithReceiverAndFileSink(self):
# add corresponding part of the configuration for FileSink(s)
config = self.config.AlertProcessor
config.critical.sinks.section_("file")
config.critical.sinks.file.outputfile = self.criticalOutputFile
config.soft.sinks.section_("file")
config.soft.sinks.file.outputfile = self.softOutputFile
processor = Processor(config)
# Receiver is waited for shutdown / shutdown explicitly in tearDown()
self.receiver = Receiver(self.addr, processor, self.ctrl)
self.receiver.startReceiver() # non blocking call
# run worker(), this time directly without Process as above,
# worker will send 10 Alerts to Receiver
worker(self.addr, self.ctrl, 10)
# wait until Receiver is shut down (by a message from worker(), then all
# alerts shall be delivered and could proceed to check if successfully delivered
while self.receiver.isReady():
time.sleep(ReceiverLogic.TIMEOUT_AFTER_SHUTDOWN * 1.5)
logging.info("%s: Waiting for Receiver shutdown ..." % inspect.stack()[0][3])
# check the FileSink output files for content:
# the soft Alerts has threshold level set to 0 so Alerts
# with level 1 and higher, resp. for critical the level
# was above set to 5 so 6 and higher out of worker's 0 .. 9
# (10 Alerts altogether) shall be present
softSink = FileSink(config.soft.sinks.file)
criticalSink = FileSink(config.critical.sinks.file)
softList = softSink.load()
criticalList = criticalSink.load()
# check soft level alerts
# levels 1 .. 4 went in (level 0 is, according to the config not considered)
self.assertEqual(len(softList), 3)
for a, level in zip(softList, range(1, 4)):
self.assertEqual(a["Level"], level)
# check 'critical' levels
# only levels 5 .. 9 went in
self.assertEqual(len(criticalList), 5)
for a, level in zip(criticalList, range(5, 10)):
self.assertEqual(a["Level"], level)
def testProcessorWithReceiverAndCouchSink(self):
# set up couch first
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
dbName = "couch_sink"
self.testInit.setupCouch(dbName)
# add corresponding part of the configuration for CouchSink(s)
config = self.config.AlertProcessor
config.critical.sinks.section_("couch")
config.critical.sinks.couch.url = self.testInit.couchUrl
config.critical.sinks.couch.database = self.testInit.couchDbName
# just send the Alert into couch
processor = Processor(config)
# Receiver is waited for shutdown / shutdown explicitly in tearDown()
self.receiver = Receiver(self.addr, processor, self.ctrl)
self.receiver.startReceiver() # non blocking call
# run worker(), this time directly without Process as above,
# worker will send 10 Alerts to Receiver
worker(self.addr, self.ctrl, 10)
# wait until Receiver is shut down (by a message from worker()
# also need to wait, otherwise tearDown kicks off and scrapes the
# couch so half of the alerts will be undelivered
while self.receiver.isReady():
time.sleep(ReceiverLogic.TIMEOUT_AFTER_SHUTDOWN * 1.5)
logging.info("%s: Waiting for Receiver shutdown ..." % inspect.stack()[0][3])
def testForwardSinkEntireChain(self):
"""
The test chain looks as follows:
worker -> Receiver1(+its Processor configured to do ForwardSink) -> Receiver2 whose
address as the destination the ForwardSink is configured with -> Receiver2 will
do FileSink so that it's possible to verify the chain.
"""
# configuration for the Receiver+Processor+ForwardSink 1 (group)
config1 = Configuration()
config1.component_("AlertProcessor")
config1.AlertProcessor.section_("critical")
config1.AlertProcessor.section_("soft")
config1.AlertProcessor.critical.level = 5
config1.AlertProcessor.soft.level = 0
config1.AlertProcessor.soft.bufferSize = 0
config1.AlertProcessor.critical.section_("sinks")
config1.AlertProcessor.soft.section_("sinks")
config1.AlertProcessor.critical.sinks.section_("forward")
config1.AlertProcessor.soft.sinks.section_("forward")
# address of the Receiver2
config1.AlertProcessor.critical.sinks.forward.address = self.address2
config1.AlertProcessor.critical.sinks.forward.controlAddr = self.controlAddr2
config1.AlertProcessor.critical.sinks.forward.label = "ForwardSinkTest"
config1.AlertProcessor.soft.sinks.forward.address = self.address2
config1.AlertProcessor.soft.sinks.forward.controlAddr = self.controlAddr2
config1.AlertProcessor.soft.sinks.forward.label = "ForwardSinkTest"
# 1) first item of the chain is source of Alerts: worker()
# 2) second item is Receiver1 + its Processor + its ForwardSink
processor1 = Processor(config1.AlertProcessor)
# ForwardSink will be created automatically by the Processor
receiver1 = Receiver(self.address1, processor1, self.controlAddr1)
receiver1.startReceiver() # non blocking call
# 3) third group is Receiver2 with its Processor and final FileSink
config2 = Configuration()
config2.component_("AlertProcessor")
config2.AlertProcessor.section_("critical")
config2.AlertProcessor.section_("soft")
config2.AlertProcessor.critical.level = 5
config2.AlertProcessor.soft.level = 0
config2.AlertProcessor.soft.bufferSize = 0
config2.AlertProcessor.critical.section_("sinks")
config2.AlertProcessor.soft.section_("sinks")
config2.AlertProcessor.critical.sinks.section_("file")
config2.AlertProcessor.soft.sinks.section_("file")
# configuration of the final sink
config2.AlertProcessor.critical.sinks.file.outputfile = self.outputfileCritical
config2.AlertProcessor.soft.sinks.file.outputfile = self.outputfileSoft
processor2 = Processor(config2.AlertProcessor)
# final FileSink will be automatically created by the Processor
receiver2 = Receiver(self.address2, processor2, self.controlAddr2)
receiver2.startReceiver() # non blocking call
# now send the Alert messages via worker() and eventually shut the receiver1
worker(self.address1, self.controlAddr1, 10)
# wait until receiver1 shuts
while receiver1.isReady():
time.sleep(0.4)
print "%s waiting for Receiver1 to shut ..." % inspect.stack(
)[0][3]
# shut down receiver2 - need to sendShutdown() to it
s = Sender(self.address2, self.controlAddr2, "some_id")
s.sendShutdown()
# wait until receiver2 shuts
while receiver2.isReady():
time.sleep(0.4)
print "%s waiting for Receiver2 to shut ..." % inspect.stack(
)[0][3]
# check the result in the files
# the bufferSize for soft-level Alerts was set to 0 so all
# Alerts should be present also in the soft-level type file
# initial 10 Alerts (Level 0 .. 9) gets distributed though a cascade
# of two Receivers. soft alerts with level 0 .. 4 are considered
# so Receiver1 forwards through its ForwardSink 0 .. 4 Alerts as soft and
# 5 .. 9 level Alerts through 'critical'. order is not guaranteed
# critical Alerts
fileConfig = ConfigSection("file")
fileConfig.outputfile = self.outputfileCritical
sink = FileSink(fileConfig)
expectedLevels = range(5, 10) # that is 5 .. 9
loadAlerts = sink.load()
self.assertEqual(len(loadAlerts), len(expectedLevels))
d = dict(very="interesting")
for a in loadAlerts:
self.assertEqual(a["Details"], d)
# soft Alerts
fileConfig = ConfigSection("file")
fileConfig.outputfile = self.outputfileSoft
sink = FileSink(fileConfig)
expectedLevels = range(0, 5) # that is 0 .. 4
loadAlerts = sink.load()
self.assertEqual(len(loadAlerts), len(expectedLevels))
for a in loadAlerts:
self.assertEqual(a["Details"], d)
def testForwardSinkEntireChain(self):
"""
The test chain looks as follows:
worker -> Receiver1(+its Processor configured to do ForwardSink) -> Receiver2 whose
address as the destination the ForwardSink is configured with -> Receiver2 will
do FileSink so that it's possible to verify the chain.
"""
# configuration for the Receiver+Processor+ForwardSink 1 (group)
config1 = Configuration()
config1.component_("AlertProcessor")
config1.AlertProcessor.section_("critical")
config1.AlertProcessor.section_("soft")
config1.AlertProcessor.critical.level = 5
config1.AlertProcessor.soft.level = 0
config1.AlertProcessor.soft.bufferSize = 0
config1.AlertProcessor.critical.section_("sinks")
config1.AlertProcessor.soft.section_("sinks")
config1.AlertProcessor.critical.sinks.section_("forward")
config1.AlertProcessor.soft.sinks.section_("forward")
# address of the Receiver2
config1.AlertProcessor.critical.sinks.forward.address = self.address2
config1.AlertProcessor.critical.sinks.forward.controlAddr = self.controlAddr2
config1.AlertProcessor.critical.sinks.forward.label = "ForwardSinkTest"
config1.AlertProcessor.soft.sinks.forward.address = self.address2
config1.AlertProcessor.soft.sinks.forward.controlAddr = self.controlAddr2
config1.AlertProcessor.soft.sinks.forward.label = "ForwardSinkTest"
# 1) first item of the chain is source of Alerts: worker()
# 2) second item is Receiver1 + its Processor + its ForwardSink
processor1 = Processor(config1.AlertProcessor)
# ForwardSink will be created automatically by the Processor
receiver1 = Receiver(self.address1, processor1, self.controlAddr1)
receiver1.startReceiver() # non blocking call
# 3) third group is Receiver2 with its Processor and final FileSink
config2 = Configuration()
config2.component_("AlertProcessor")
config2.AlertProcessor.section_("critical")
config2.AlertProcessor.section_("soft")
config2.AlertProcessor.critical.level = 5
config2.AlertProcessor.soft.level = 0
config2.AlertProcessor.soft.bufferSize = 0
config2.AlertProcessor.critical.section_("sinks")
config2.AlertProcessor.soft.section_("sinks")
config2.AlertProcessor.critical.sinks.section_("file")
config2.AlertProcessor.soft.sinks.section_("file")
# configuration of the final sink
config2.AlertProcessor.critical.sinks.file.outputfile = self.outputfileCritical
config2.AlertProcessor.soft.sinks.file.outputfile = self.outputfileSoft
processor2 = Processor(config2.AlertProcessor)
# final FileSink will be automatically created by the Processor
receiver2 = Receiver(self.address2, processor2, self.controlAddr2)
receiver2.startReceiver() # non blocking call
# now send the Alert messages via worker() and eventually shut the receiver1
worker(self.address1, self.controlAddr1, 10)
# wait until receiver1 shuts
while receiver1.isReady():
time.sleep(0.4)
print "%s waiting for Receiver1 to shut ..." % inspect.stack()[0][3]
# shut down receiver2 - need to sendShutdown() to it
s = Sender(self.address2, self.controlAddr2, "some_id")
s.sendShutdown()
# wait until receiver2 shuts
while receiver2.isReady():
time.sleep(0.4)
print "%s waiting for Receiver2 to shut ..." % inspect.stack()[0][3]
# check the result in the files
# the bufferSize for soft-level Alerts was set to 0 so all
# Alerts should be present also in the soft-level type file
# initial 10 Alerts (Level 0 .. 9) gets distributed though a cascade
# of two Receivers. soft alerts with level 0 .. 4 are considered
# so Receiver1 forwards through its ForwardSink 0 .. 4 Alerts as soft and
# 5 .. 9 level Alerts through 'critical'. order is not guaranteed
# critical Alerts
fileConfig = ConfigSection("file")
fileConfig.outputfile = self.outputfileCritical
sink = FileSink(fileConfig)
expectedLevels = range(5, 10) # that is 5 .. 9
loadAlerts = sink.load()
self.assertEqual(len(loadAlerts), len(expectedLevels))
d = dict(very = "interesting")
for a in loadAlerts:
self.assertEqual(a["Details"], d)
# soft Alerts
fileConfig = ConfigSection("file")
fileConfig.outputfile = self.outputfileSoft
sink = FileSink(fileConfig)
expectedLevels = range(0, 5) # that is 0 .. 4
loadAlerts = sink.load()
self.assertEqual(len(loadAlerts), len(expectedLevels))
for a in loadAlerts:
self.assertEqual(a["Details"], d)
class SenderTest(unittest.TestCase):
def setUp(self):
self.addr = "tcp://127.0.0.1:5557"
self.control = "tcp://127.0.0.1:5559"
logging.basicConfig(logLevel = logging.DEBUG)
# real Receiver instance, do test real stuff rather than with
# mock Receiver
self.receiver = None
def tearDown(self):
"""
Clean up.
"""
if self.receiver:
print "Receiver should be stopped now."
print "Receiver running: (isReady()): %s" % self.receiver.isReady()
if self.receiver.isReady():
print "Receiver shutdown ..."
self.receiver.shutdown()
print "Receiver running: (isReady()): %s" % self.receiver.isReady()
self.receiver = None
def testSenderBasic(self):
"""
Immediate testing register, unregister messages.
Alert messages tested as saved in the queue.
"""
nAlerts = 10
# start Receiver, handler is list for alerts
# wait for control messages to arrive and test immediately
alertsQueue = []
handler = lambda x: alertsQueue.append(x)
self.receiver = Receiver(self.addr, handler, self.control)
self.receiver.startReceiver() # non blocking call
# instantiate sender and send ...
s = Sender(self.addr, "Sender_t", self.control)
# nothing is registered up to now with the Receiver
self.assertEqual(len(self.receiver._receiver._registSenders), 0)
s.register()
# test that RegisterMsg arrived, consider delay
while len(self.receiver._receiver._registSenders) == 0:
time.sleep(0.2)
self.assertEqual(len(self.receiver._receiver._registSenders), 1)
# send some alerts
for i in range(0, nAlerts):
a = Alert(Level = i, Type = "Alert")
s(a) # actual alert message sending
s.unregister()
while len(self.receiver._receiver._registSenders) == 1:
time.sleep(0.2)
self.assertEqual(len(self.receiver._receiver._registSenders), 0)
# this makes sure that Receiver waits certain delay even after shutdown
# is received if there is no more messages coming
self.receiver.shutdown()
self.assertEqual(nAlerts, len(alertsQueue))
def testSenderNonBlockingWhenReceiverNotAvailable(self):
"""
Repeatedly instantiate Sender, register, send alerts, etc
and test that the Sender is not blocking due to undelivered
messages since no Receiver is available.
This test shall wait (between iterations) only delay specified
in the Sender (was hanging indefinitely due to -1 default value).
"""
iterations = 2
nAlerts = 3
for i in range(iterations):
# instantiate sender and send ...
s = Sender(self.addr, "Sender_t", self.control)
s.register()
# send some alerts
for i in range(0, nAlerts):
a = Alert(Level = 10, Type = "Alert")
s(a) # actual alert message sending
s.unregister()
# call destructor explicitly, the hanging should not occur here
del s
class ProcessorTest(unittest.TestCase):
"""
TestCase for Processor.
"""
def setUp(self):
"""
Set up for tests.
"""
l = logging.getLogger()
l.setLevel(logging.DEBUG)
self.addr = "tcp://127.0.0.1:5557"
self.ctrl = "tcp://127.0.0.1:5559"
self.softOutputFile = "/tmp/ProcessorTestSoftAlerts.json"
self.criticalOutputFile = "/tmp/ProcessorTestCriticalAlerts.json"
self.config = Configuration()
self.config.component_("AlertProcessor")
self.config.AlertProcessor.section_("critical")
self.config.AlertProcessor.section_("soft")
self.config.AlertProcessor.critical.level = 5
self.config.AlertProcessor.soft.level = 1
self.config.AlertProcessor.soft.bufferSize = 3
self.config.AlertProcessor.critical.section_("sinks")
self.config.AlertProcessor.soft.section_("sinks")
def tearDown(self):
for f in (self.criticalOutputFile, self.softOutputFile):
if os.path.exists(f):
os.remove(f)
if hasattr(self, "testInit"):
self.testInit.tearDownCouch()
if hasattr(self, "receiver"):
# wait until the Receiver is shut by the Shutdown control
# message which the worker() function should have sent
while self.receiver.isReady():
logging.info("tearDown: Waiting for Receiver shutdown ...")
time.sleep(ReceiverLogic.TIMEOUT_AFTER_SHUTDOWN * 1.1)
if self.receiver.isReady():
self.receiver.shutdown()
logging.info("tearDown: Is the Receiver shut down: %s" % (not self.receiver.isReady()))
def testProcessorBasic(self):
str(self.config.AlertProcessor)
p = Processor(self.config.AlertProcessor)
def testProcessorWithReceiver(self):
"""
Test startup and shutdown of processor in receiver.
"""
processor = Processor(self.config.AlertProcessor)
# Receiver is waited for shutdown / shutdown explicitly in tearDown()
self.receiver = Receiver(self.addr, processor, self.ctrl)
self.receiver.startReceiver() # non-blocking call
# now sender tests control messages (register, unregister, shutdown)
s = Sender(self.addr, self.ctrl, "Processor_t")
s.register()
s.unregister()
s.sendShutdown()
# give some time so that the previous call shuts down the receiver
time.sleep(ReceiverLogic.TIMEOUT_AFTER_SHUTDOWN * 1.1)
def testProcessorWithReceiverAndFileSink(self):
# add corresponding part of the configuration for FileSink(s)
config = self.config.AlertProcessor
config.critical.sinks.section_("file")
config.critical.sinks.file.outputfile = self.criticalOutputFile
config.soft.sinks.section_("file")
config.soft.sinks.file.outputfile = self.softOutputFile
processor = Processor(config)
# Receiver is waited for shutdown / shutdown explicitly in tearDown()
self.receiver = Receiver(self.addr, processor, self.ctrl)
self.receiver.startReceiver() # non blocking call
# run worker(), this time directly without Process as above,
# worker will send 10 Alerts to Receiver
worker(self.addr, self.ctrl, 10)
# wait until Receiver is shut down (by a message from worker(), then all
# alerts shall be delivered and could proceed to check if successfully delivered
while self.receiver.isReady():
time.sleep(ReceiverLogic.TIMEOUT_AFTER_SHUTDOWN * 1.5)
logging.info("%s: Waiting for Receiver shutdown ..." % inspect.stack()[0][3])
# check the FileSink output files for content:
# the soft Alerts has threshold level set to 0 so Alerts
# with level 1 and higher, resp. for critical the level
# was above set to 5 so 6 and higher out of worker's 0 .. 9
# (10 Alerts altogether) shall be present
softSink = FileSink(config.soft.sinks.file)
criticalSink = FileSink(config.critical.sinks.file)
softList = softSink.load()
criticalList = criticalSink.load()
# check soft level alerts
# levels 1 .. 4 went in (level 0 is, according to the config not considered)
self.assertEqual(len(softList), 3)
for a, level in zip(softList, range(1, 4)):
self.assertEqual(a["Level"], level)
# check 'critical' levels
# only levels 5 .. 9 went in
self.assertEqual(len(criticalList), 5)
for a, level in zip(criticalList, range(5, 10)):
self.assertEqual(a["Level"], level)
def testProcessorWithReceiverAndCouchSink(self):
# set up couch first
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
dbName = "couch_sink"
self.testInit.setupCouch(dbName)
# add corresponding part of the configuration for CouchSink(s)
config = self.config.AlertProcessor
config.critical.sinks.section_("couch")
config.critical.sinks.couch.url = self.testInit.couchUrl
config.critical.sinks.couch.database = self.testInit.couchDbName
# just send the Alert into couch
processor = Processor(config)
# Receiver is waited for shutdown / shutdown explicitly in tearDown()
self.receiver = Receiver(self.addr, processor, self.ctrl)
self.receiver.startReceiver() # non blocking call
# run worker(), this time directly without Process as above,
# worker will send 10 Alerts to Receiver
worker(self.addr, self.ctrl, 10)
# wait until Receiver is shut down (by a message from worker()
# also need to wait, otherwise tearDown kicks off and scrapes the
# couch so half of the alerts will be undelivered
while self.receiver.isReady():
time.sleep(ReceiverLogic.TIMEOUT_AFTER_SHUTDOWN * 1.5)
logging.info("%s: Waiting for Receiver shutdown ..." % inspect.stack()[0][3])
class SenderTest(unittest.TestCase):
def setUp(self):
self.addr = "tcp://127.0.0.1:5557"
self.control = "tcp://127.0.0.1:5559"
logging.basicConfig(logLevel=logging.DEBUG)
# real Receiver instance, do test real stuff rather than with
# mock Receiver
self.receiver = None
def tearDown(self):
"""
Clean up.
"""
if self.receiver:
print "Receiver should be stopped now."
print "Receiver running: (isReady()): %s" % self.receiver.isReady()
if self.receiver.isReady():
print "Receiver shutdown ..."
self.receiver.shutdown()
print "Receiver running: (isReady()): %s" % self.receiver.isReady(
)
self.receiver = None
def testSenderBasic(self):
"""
Immediate testing register, unregister messages.
Alert messages tested as saved in the queue.
"""
nAlerts = 10
# start Receiver, handler is list for alerts
# wait for control messages to arrive and test immediately
alertsQueue = []
handler = lambda x: alertsQueue.append(x)
self.receiver = Receiver(self.addr, handler, self.control)
self.receiver.startReceiver() # non blocking call
# instantiate sender and send ...
s = Sender(self.addr, self.control, "Sender_t")
# nothing is registered up to now with the Receiver
self.assertEqual(len(self.receiver._receiver._registSenders), 0)
s.register()
# test that RegisterMsg arrived, consider delay
while len(self.receiver._receiver._registSenders) == 0:
time.sleep(0.2)
self.assertEqual(len(self.receiver._receiver._registSenders), 1)
# send some alerts
for i in range(0, nAlerts):
a = Alert(Level=i, Type="Alert")
s(a) # actual alert message sending
s.unregister()
while len(self.receiver._receiver._registSenders) == 1:
time.sleep(0.2)
self.assertEqual(len(self.receiver._receiver._registSenders), 0)
# this makes sure that Receiver waits certain delay even after shutdown
# is received if there is no more messages coming
self.receiver.shutdown()
self.assertEqual(nAlerts, len(alertsQueue))
def testSenderNonBlockingWhenReceiverNotAvailable(self):
"""
Repeatedly instantiate Sender, register, send alerts, etc
and test that the Sender is not blocking due to undelivered
messages since no Receiver is available.
This test shall wait (between iterations) only delay specified
in the Sender.
"""
iterations = 2
nAlerts = 3
for i in range(iterations):
# instantiate sender and send ...
s = Sender(self.addr, self.control, "Sender_t")
s.register()
# send some alerts
for i in range(0, nAlerts):
a = Alert(Level=10, Type="Alert")
s(a) # actual alert message sending
s.unregister()
# call destructor explicitly, the hanging should not occur here
del s
class AlertProcessor(Harness):
def __init__(self, config):
Harness.__init__(self, config)
self.config = config
# instance of processor
self._processor = None
# instance of Receiver which owns Processor (self._processor)
# and runs on background
self._receiver = None
#3602 related:
# Harness, nor the components, handle signal.SIGTERM which
# is used by wmcoreD --shutdown, hence shutdown sequence is not called
# this shall later be moved into (hopefully largely improved) Harness
signal.signal(signal.SIGTERM, self._signalHandler)
def _signalHandler(self, signalNumber, frame):
logging.info("Signal number %s caught." % signalNumber)
self.prepareToStop()
def preInitialization(self):
"""
Start up the ZMQ Receiver + Processor.
"""
logging.info("preInitialization ...")
# something fishy (again) going on in Harness, wmcoreD
# component may fail, still will be considered as running (#3602)
# this is why #3320 is difficult to fix ... wmcoreD would happily
# continue even after raising an exception even from this very method directly
self._processor = Processor(self.config.AlertProcessor)
# Receiver listens on work channel (address) and on control
# channel (controlAddr)
self._receiver = Receiver(self.config.AlertProcessor.address,
self._processor,
self.config.AlertProcessor.controlAddr)
self._receiver.startReceiver()
logging.info("preInitialization - finished.")
def stopAlertProcessor(self):
"""
Method to shutdown the AlertProcessor.
"""
logging.info("stopAlertProcessor - stopping Receiver ...")
self._receiver.shutdown()
logging.info("stopAlertProcessor finished.")
def prepareToStop(self, wait = False, stopPayload = ""):
"""
Override prepareToStop to include call to stopProcessor.
Ugly, but seems no other way to do this...
"""
logging.info("Shutting down the component - prepareToStop ...")
self.stopAlertProcessor()
Harness.prepareToStop(self, wait, stopPayload)
logging.info("prepareToStop finished.")