def testReceiverBuferring(self):
"""
Test sending alerts that are buffered into a queue.
"""
# alerts received will be added to the queue
alertList = []
handler = lambda x: alertList.append(x)
rec = Receiver(self.addr, handler, self.ctrl)
rec.startReceiver() # non blocking call
numAlertMsgs = 7
thread = AlertsSender(self.addr, self.ctrl, numAlertMsgs)
# thread will send numAlertMsgs and eventually shut the receiver
# down by a shutdown control message
thread.start()
# wait here until the Receiver is not shut
while rec.isReady():
time.sleep(0.2)
# worker will send shutdown message and execution will resume here
# check the content of the queue - 5 alert messages
self.assertEqual(len(alertList), numAlertMsgs)
for alert in alertList:
self.failUnless(alert.has_key("Type"))
self.assertEqual(alert["Type"], "Alert")
def testProcessorWithReceiverAndCouchSink(self):
# set up couch first
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
dbName = "couch_sink"
self.testInit.setupCouch(dbName)
# add corresponding part of the configuration for CouchSink(s)
config = self.config.AlertProcessor
config.critical.sinks.section_("couch")
config.critical.sinks.couch.url = self.testInit.couchUrl
config.critical.sinks.couch.database = self.testInit.couchDbName
# just send the Alert into couch
processor = Processor(config)
# Receiver is waited for shutdown / shutdown explicitly in tearDown()
self.receiver = Receiver(self.addr, processor, self.ctrl)
self.receiver.startReceiver() # non blocking call
# run worker(), this time directly without Process as above,
# worker will send 10 Alerts to Receiver
worker(self.addr, self.ctrl, 10)
# wait until Receiver is shut down (by a message from worker()
# also need to wait, otherwise tearDown kicks off and scrapes the
# couch so half of the alerts will be undelivered
while self.receiver.isReady():
time.sleep(ReceiverLogic.TIMEOUT_AFTER_SHUTDOWN * 1.5)
logging.info("%s: Waiting for Receiver shutdown ..." % inspect.stack()[0][3])
def setUpReceiver(address, controlAddr):
"""
Return set up handler, receiver pair.
Receiver starts two channels on the address and controlAddr addresses.
"""
handler = ReceiverHandler()
receiver = Receiver(address, handler, controlAddr)
receiver.startReceiver() # non blocking call
return handler, receiver
def testReceiverShutdownByCall(self):
# start a Receiver
rec = Receiver(self.addr, self.printer, self.ctrl)
rec.startReceiver() # non blocking call
workChann, contChann = self._getSenderChannels()
# send some messages to the receiver and shut it eventually
contChann.send_json(RegisterMsg("Receiver_t"))
workChann.send_json(Alert(Type="Alert", Level=20))
contChann.send_json(UnregisterMsg("Receiver_t"))
# now messages are sent so shutdown the Receiver by a convenience
# call, should block until the Receiver finishes, don't have to wait
rec.shutdown()
def testProcessorWithReceiver(self):
"""
Test startup and shutdown of processor in receiver.
"""
processor = Processor(self.config.AlertProcessor)
# Receiver is waited for shutdown / shutdown explicitly in tearDown()
self.receiver = Receiver(self.addr, processor, self.ctrl)
self.receiver.startReceiver() # non-blocking call
# now sender tests control messages (register, unregister, shutdown)
s = Sender(self.addr, self.ctrl, "Processor_t")
s.register()
s.unregister()
s.sendShutdown()
# give some time so that the previous call shuts down the receiver
time.sleep(ReceiverLogic.TIMEOUT_AFTER_SHUTDOWN * 1.1)
def testReceiverShutdownByMessage(self):
# start a Receiver
rec = Receiver(self.addr, self.printer, self.ctrl)
rec.startReceiver() # non blocking call
workChann, contChann = self._getSenderChannels()
# send some messages to the receiver and shut it eventually
contChann.send_json(RegisterMsg("Receiver_t"))
workChann.send_json(Alert(Type="Alert", Level=10))
contChann.send_json(UnregisterMsg("Receiver_t"))
# terminate the Receiver
contChann.send_json(ShutdownMsg())
# wait until the Receiver is properly shut
# this will not be necessary when shutting down by a call
while rec.isReady():
time.sleep(0.1)
def preInitialization(self):
"""
Start up the ZMQ Receiver + Processor.
"""
logging.info("preInitialization ...")
# something fishy (again) going on in Harness, wmcoreD
# component may fail, still will be considered as running (#3602)
# this is why #3320 is difficult to fix ... wmcoreD would happily
# continue even after raising an exception even from this very method directly
self._processor = Processor(self.config.AlertProcessor)
# Receiver listens on work channel (address) and on control
# channel (controlAddr)
self._receiver = Receiver(self.config.AlertProcessor.address,
self._processor,
self.config.AlertProcessor.controlAddr)
self._receiver.startReceiver()
logging.info("preInitialization - finished.")
def testProcessorWithReceiverAndFileSink(self):
# add corresponding part of the configuration for FileSink(s)
config = self.config.AlertProcessor
config.critical.sinks.section_("file")
config.critical.sinks.file.outputfile = self.criticalOutputFile
config.soft.sinks.section_("file")
config.soft.sinks.file.outputfile = self.softOutputFile
processor = Processor(config)
# Receiver is waited for shutdown / shutdown explicitly in tearDown()
self.receiver = Receiver(self.addr, processor, self.ctrl)
self.receiver.startReceiver() # non blocking call
# run worker(), this time directly without Process as above,
# worker will send 10 Alerts to Receiver
worker(self.addr, self.ctrl, 10)
# wait until Receiver is shut down (by a message from worker(), then all
# alerts shall be delivered and could proceed to check if successfully delivered
while self.receiver.isReady():
time.sleep(ReceiverLogic.TIMEOUT_AFTER_SHUTDOWN * 1.5)
logging.info("%s: Waiting for Receiver shutdown ..." % inspect.stack()[0][3])
# check the FileSink output files for content:
# the soft Alerts has threshold level set to 0 so Alerts
# with level 1 and higher, resp. for critical the level
# was above set to 5 so 6 and higher out of worker's 0 .. 9
# (10 Alerts altogether) shall be present
softSink = FileSink(config.soft.sinks.file)
criticalSink = FileSink(config.critical.sinks.file)
softList = softSink.load()
criticalList = criticalSink.load()
# check soft level alerts
# levels 1 .. 4 went in (level 0 is, according to the config not considered)
self.assertEqual(len(softList), 3)
for a, level in zip(softList, range(1, 4)):
self.assertEqual(a["Level"], level)
# check 'critical' levels
# only levels 5 .. 9 went in
self.assertEqual(len(criticalList), 5)
for a, level in zip(criticalList, range(5, 10)):
self.assertEqual(a["Level"], level)
def testSenderBasic(self):
"""
Immediate testing register, unregister messages.
Alert messages tested as saved in the queue.
"""
nAlerts = 10
# start Receiver, handler is list for alerts
# wait for control messages to arrive and test immediately
alertsQueue = []
handler = lambda x: alertsQueue.append(x)
self.receiver = Receiver(self.addr, handler, self.control)
self.receiver.startReceiver() # non blocking call
# instantiate sender and send ...
s = Sender(self.addr, self.control, "Sender_t")
# nothing is registered up to now with the Receiver
self.assertEqual(len(self.receiver._receiver._registSenders), 0)
s.register()
# test that RegisterMsg arrived, consider delay
while len(self.receiver._receiver._registSenders) == 0:
time.sleep(0.2)
self.assertEqual(len(self.receiver._receiver._registSenders), 1)
# send some alerts
for i in range(0, nAlerts):
a = Alert(Level=i, Type="Alert")
s(a) # actual alert message sending
s.unregister()
while len(self.receiver._receiver._registSenders) == 1:
time.sleep(0.2)
self.assertEqual(len(self.receiver._receiver._registSenders), 0)
# this makes sure that Receiver waits certain delay even after shutdown
# is received if there is no more messages coming
self.receiver.shutdown()
self.assertEqual(nAlerts, len(alertsQueue))
def testForwardSinkEntireChain(self):
"""
The test chain looks as follows:
worker -> Receiver1(+its Processor configured to do ForwardSink) -> Receiver2 whose
address as the destination the ForwardSink is configured with -> Receiver2 will
do FileSink so that it's possible to verify the chain.
"""
# configuration for the Receiver+Processor+ForwardSink 1 (group)
config1 = Configuration()
config1.component_("AlertProcessor")
config1.AlertProcessor.section_("critical")
config1.AlertProcessor.section_("soft")
config1.AlertProcessor.critical.level = 5
config1.AlertProcessor.soft.level = 0
config1.AlertProcessor.soft.bufferSize = 0
config1.AlertProcessor.critical.section_("sinks")
config1.AlertProcessor.soft.section_("sinks")
config1.AlertProcessor.critical.sinks.section_("forward")
config1.AlertProcessor.soft.sinks.section_("forward")
# address of the Receiver2
config1.AlertProcessor.critical.sinks.forward.address = self.address2
config1.AlertProcessor.critical.sinks.forward.controlAddr = self.controlAddr2
config1.AlertProcessor.critical.sinks.forward.label = "ForwardSinkTest"
config1.AlertProcessor.soft.sinks.forward.address = self.address2
config1.AlertProcessor.soft.sinks.forward.controlAddr = self.controlAddr2
config1.AlertProcessor.soft.sinks.forward.label = "ForwardSinkTest"
# 1) first item of the chain is source of Alerts: worker()
# 2) second item is Receiver1 + its Processor + its ForwardSink
processor1 = Processor(config1.AlertProcessor)
# ForwardSink will be created automatically by the Processor
receiver1 = Receiver(self.address1, processor1, self.controlAddr1)
receiver1.startReceiver() # non blocking call
# 3) third group is Receiver2 with its Processor and final FileSink
config2 = Configuration()
config2.component_("AlertProcessor")
config2.AlertProcessor.section_("critical")
config2.AlertProcessor.section_("soft")
config2.AlertProcessor.critical.level = 5
config2.AlertProcessor.soft.level = 0
config2.AlertProcessor.soft.bufferSize = 0
config2.AlertProcessor.critical.section_("sinks")
config2.AlertProcessor.soft.section_("sinks")
config2.AlertProcessor.critical.sinks.section_("file")
config2.AlertProcessor.soft.sinks.section_("file")
# configuration of the final sink
config2.AlertProcessor.critical.sinks.file.outputfile = self.outputfileCritical
config2.AlertProcessor.soft.sinks.file.outputfile = self.outputfileSoft
processor2 = Processor(config2.AlertProcessor)
# final FileSink will be automatically created by the Processor
receiver2 = Receiver(self.address2, processor2, self.controlAddr2)
receiver2.startReceiver() # non blocking call
# now send the Alert messages via worker() and eventually shut the receiver1
worker(self.address1, self.controlAddr1, 10)
# wait until receiver1 shuts
while receiver1.isReady():
time.sleep(0.4)
print "%s waiting for Receiver1 to shut ..." % inspect.stack(
)[0][3]
# shut down receiver2 - need to sendShutdown() to it
s = Sender(self.address2, self.controlAddr2, "some_id")
s.sendShutdown()
# wait until receiver2 shuts
while receiver2.isReady():
time.sleep(0.4)
print "%s waiting for Receiver2 to shut ..." % inspect.stack(
)[0][3]
# check the result in the files
# the bufferSize for soft-level Alerts was set to 0 so all
# Alerts should be present also in the soft-level type file
# initial 10 Alerts (Level 0 .. 9) gets distributed though a cascade
# of two Receivers. soft alerts with level 0 .. 4 are considered
# so Receiver1 forwards through its ForwardSink 0 .. 4 Alerts as soft and
# 5 .. 9 level Alerts through 'critical'. order is not guaranteed
# critical Alerts
fileConfig = ConfigSection("file")
fileConfig.outputfile = self.outputfileCritical
sink = FileSink(fileConfig)
expectedLevels = range(5, 10) # that is 5 .. 9
loadAlerts = sink.load()
self.assertEqual(len(loadAlerts), len(expectedLevels))
d = dict(very="interesting")
for a in loadAlerts:
self.assertEqual(a["Details"], d)
# soft Alerts
fileConfig = ConfigSection("file")
fileConfig.outputfile = self.outputfileSoft
sink = FileSink(fileConfig)
expectedLevels = range(0, 5) # that is 0 .. 4
loadAlerts = sink.load()
self.assertEqual(len(loadAlerts), len(expectedLevels))
for a in loadAlerts:
self.assertEqual(a["Details"], d)
