def testEventChunking(self):
opts = self.createOptions(eventflushchunksize=5)
eqm = EventQueueManager(opts, log)
events = []
for i in range(10):
events.append(createTestEvent(device='dev%d' % i))
eqm.addEvent(events[-1])
perf_events = []
for i in range(10):
perf_events.append(createTestEvent(device='perfdev%d' % i))
eqm.addPerformanceEvent(perf_events[-1])
heartbeat_events = [createTestEvent(eventClass='/Heartbeat')]
eqm.addHeartbeatEvent(heartbeat_events[-1])
sent_event_chunks = []
def send_events(events):
sent_event_chunks.append(events)
eqm.sendEvents(send_events)
self.assertEquals(5, len(sent_event_chunks))
# First chunk should be heartbeat + 4 perf
self.assertEquals(heartbeat_events + perf_events[:4],
sent_event_chunks[0])
# Second chunk should be 5 perf
self.assertEquals(perf_events[4:9], sent_event_chunks[1])
# Third chunk should be 1 perf + 4 events
self.assertEquals(perf_events[9:] + events[0:4], sent_event_chunks[2])
# Fourth chunk should be 5 events
self.assertEquals(events[4:9], sent_event_chunks[3])
# Fifth chunk should be 1 event
self.assertEquals(events[9:], sent_event_chunks[4])
def testEventChunking(self):
opts = self.createOptions(eventflushchunksize=5)
eqm = EventQueueManager(opts, log)
events = []
for i in range(10):
events.append(createTestEvent(device="dev%d" % i))
eqm.addEvent(events[-1])
perf_events = []
for i in range(10):
perf_events.append(createTestEvent(device="perfdev%d" % i))
eqm.addPerformanceEvent(perf_events[-1])
heartbeat_events = [createTestEvent(eventClass="/Heartbeat")]
eqm.addHeartbeatEvent(heartbeat_events[-1])
sent_event_chunks = []
def send_events(events):
sent_event_chunks.append(events)
eqm.sendEvents(send_events)
self.assertEquals(5, len(sent_event_chunks))
# First chunk should be heartbeat + 4 perf
self.assertEquals(heartbeat_events + perf_events[:4], sent_event_chunks[0])
# Second chunk should be 5 perf
self.assertEquals(perf_events[4:9], sent_event_chunks[1])
# Third chunk should be 1 perf + 4 events
self.assertEquals(perf_events[9:] + events[0:4], sent_event_chunks[2])
# Fourth chunk should be 5 events
self.assertEquals(events[4:9], sent_event_chunks[3])
# Fifth chunk should be 1 event
self.assertEquals(events[9:], sent_event_chunks[4])
def testRestoreEvents(self):
opts = self.createOptions(eventflushchunksize=5)
eqm = EventQueueManager(opts, log)
events = []
for i in range(10):
events.append(createTestEvent(device='dev%d' % i))
eqm.addEvent(events[-1])
perf_events = []
for i in range(7):
perf_events.append(createTestEvent(device='perfdev%d' % i))
eqm.addPerformanceEvent(perf_events[-1])
sent_event_chunks = []
def send_events(events):
# Send an exception on the second batch
if len(sent_event_chunks) == 1:
raise Exception('Test Exception')
sent_event_chunks.append(events)
results = []
def finished_events(result):
results.append(result)
d = eqm.sendEvents(send_events)
d.addBoth(finished_events)
# Verify the first chunk was sent
self.assertEquals(1, len(results))
self.assertTrue(isinstance(results[0], failure.Failure))
self.assertEquals(1, len(sent_event_chunks))
self.assertEquals(sent_event_chunks[0], perf_events[:5])
# Verify the second chunk (which threw an exception) was added back to
# queues.
self.assertEquals(perf_events[5:], list(eqm.perf_event_queue))
self.assertEquals(events, list(eqm.event_queue))
def testDiscarded(self):
eqm = EventQueueManager(self.createOptions(maxqueuelen=1), log)
evt1 = createTestEvent(device="foo")
eqm.addEvent(evt1)
self.assertEquals(0, eqm.discarded_events)
evt2 = createTestEvent(device="foo2")
eqm.addEvent(evt2)
self.assertEquals(1, eqm.discarded_events)
self.assertEquals(list(eqm.event_queue), [evt2])
def testDiscarded(self):
eqm = EventQueueManager(self.createOptions(maxqueuelen=1), log)
evt1 = createTestEvent(device='foo')
eqm.addEvent(evt1)
self.assertEquals(0, eqm.discarded_events)
evt2 = createTestEvent(device='foo2')
eqm.addEvent(evt2)
self.assertEquals(1, eqm.discarded_events)
self.assertEquals(list(eqm.event_queue), [evt2])
def testDuplicateClears(self):
opts = self.createOptions(allowduplicateclears=True)
eqm = EventQueueManager(opts, log)
sent_events = []
def send_events(evts):
sent_events.extend(evts)
for i in range(5):
eqm.addEvent(createTestEvent(severity=0))
eqm.sendEvents(send_events)
self.assertEquals(5, len(sent_events))
def testNoDuplicateClears(self):
eqm = EventQueueManager(self.createOptions(), log)
for i in range(5):
eqm.addEvent(createTestEvent(severity=0))
self.assertEquals(1, len(eqm.event_queue))
sent_events = []
eqm.sendEvents(lambda evts: sent_events.extend(evts))
for i in range(5):
eqm.addEvent(createTestEvent(severity=0))
self.assertEquals(0, len(eqm.event_queue))
def testNoDuplicateClears(self):
eqm = EventQueueManager(self.createOptions(), log)
for i in range(5):
eqm.addEvent(createTestEvent(severity=0))
self.assertEquals(1, len(eqm.event_queue))
sent_events = []
eqm.sendEvents(lambda evts: sent_events.extend(evts))
for i in range(5):
eqm.addEvent(createTestEvent(severity=0))
self.assertEquals(0, len(eqm.event_queue))
def testDuplicateClears(self):
opts = self.createOptions(allowduplicateclears=True)
eqm = EventQueueManager(opts, log)
sent_events = []
def send_events(evts):
sent_events.extend(evts)
for i in range(5):
eqm.addEvent(createTestEvent(severity=0))
eqm.sendEvents(send_events)
self.assertEquals(5, len(sent_events))
def testAddEventDroppedTransform(self):
class DroppingTransformer(object):
implements(ICollectorEventTransformer)
def __init__(self):
self.num_dropped = 0
def transform(self, event):
self.num_dropped += 1
return TRANSFORM_DROP
transformer = DroppingTransformer()
self.gsm.registerUtility(transformer)
try:
eqm = EventQueueManager(self.createOptions(), log)
for i in range(5):
eqm.addEvent(createTestEvent())
finally:
self.gsm.unregisterUtility(transformer)
self.assertEquals(5, transformer.num_dropped)
def testAddEventDroppedTransform(self):
class DroppingTransformer(object):
implements(ICollectorEventTransformer)
def __init__(self):
self.num_dropped = 0
def transform(self, event):
self.num_dropped += 1
return TRANSFORM_DROP
transformer = DroppingTransformer()
self.gsm.registerUtility(transformer)
try:
eqm = EventQueueManager(self.createOptions(), log)
for i in range(5):
eqm.addEvent(createTestEvent())
finally:
self.gsm.unregisterUtility(transformer)
self.assertEquals(5, transformer.num_dropped)
def testRestoreEventsDiscarded(self):
opts = self.createOptions(eventflushchunksize=5, maxqueuelen=10)
eqm = EventQueueManager(opts, log)
events = []
for i in range(10):
if i == 0:
events.append(createTestEvent(device="dev%d" % i, severity=0))
else:
events.append(createTestEvent(device="dev%d" % i))
eqm.addEvent(events[-1])
def send_events(ignored_events):
# Add 3 new events to the queue (while sending)
for i in range(3):
events.append(createTestEvent(device="dev%d" % (10 + i)))
eqm.addEvent(events[-1])
raise Exception("Failed on first send.")
results = []
def finished_events(result):
results.append(result)
d = eqm.sendEvents(send_events)
d.addBoth(finished_events)
self.assertEquals(1, len(results))
self.assertTrue(isinstance(results[0], failure.Failure))
# 3 earliest events should be discarded
self.assertEquals(3, eqm.discarded_events)
self.assertEquals(10, len(eqm.event_queue))
self.assertEquals(events[3:], list(eqm.event_queue))
# The state of the clear event should be removed, so it should be
# allowed to be sent again.
sent_events = []
eqm.addEvent(events[0])
eqm.sendEvents(lambda evts: sent_events.extend(evts))
self.assertTrue(events[0] in sent_events)
def testRestoreEventsDiscarded(self):
opts = self.createOptions(eventflushchunksize=5, maxqueuelen=10)
eqm = EventQueueManager(opts, log)
events = []
for i in range(10):
if i == 0:
events.append(createTestEvent(device='dev%d' % i, severity=0))
else:
events.append(createTestEvent(device='dev%d' % i))
eqm.addEvent(events[-1])
def send_events(ignored_events):
# Add 3 new events to the queue (while sending)
for i in range(3):
events.append(createTestEvent(device='dev%d' % (10 + i)))
eqm.addEvent(events[-1])
raise Exception("Failed on first send.")
results = []
def finished_events(result):
results.append(result)
d = eqm.sendEvents(send_events)
d.addBoth(finished_events)
self.assertEquals(1, len(results))
self.assertTrue(isinstance(results[0], failure.Failure))
# 3 earliest events should be discarded
self.assertEquals(3, eqm.discarded_events)
self.assertEquals(10, len(eqm.event_queue))
self.assertEquals(events[3:], list(eqm.event_queue))
# The state of the clear event should be removed, so it should be
# allowed to be sent again.
sent_events = []
eqm.addEvent(events[0])
eqm.sendEvents(lambda evts: sent_events.extend(evts))
self.assertTrue(events[0] in sent_events)
def testRestoreEvents(self):
opts = self.createOptions(eventflushchunksize=5)
eqm = EventQueueManager(opts, log)
events = []
for i in range(10):
events.append(createTestEvent(device='dev%d' % i))
eqm.addEvent(events[-1])
perf_events = []
for i in range(7):
perf_events.append(createTestEvent(device='perfdev%d' % i))
eqm.addPerformanceEvent(perf_events[-1])
sent_event_chunks = []
def send_events(events):
# Send an exception on the second batch
if len(sent_event_chunks) == 1:
raise Exception('Test Exception')
sent_event_chunks.append(events)
results = []
def finished_events(result):
results.append(result)
d = eqm.sendEvents(send_events)
d.addBoth(finished_events)
# Verify the first chunk was sent
self.assertEquals(1, len(results))
self.assertTrue(isinstance(results[0], failure.Failure))
self.assertEquals(1, len(sent_event_chunks))
self.assertEquals(sent_event_chunks[0], perf_events[:5])
# Verify the second chunk (which threw an exception) was added back to
# queues.
self.assertEquals(perf_events[5:], list(eqm.perf_event_queue))
self.assertEquals(events, list(eqm.event_queue))
def testDiscardedClearHandling(self):
opts = self.createOptions(maxqueuelen=1)
eqm = EventQueueManager(opts, log)
# Send a clear which is later discarded
eqm.addEvent(createTestEvent(severity=0))
# Send an event that wipes it out
eqm.addEvent(createTestEvent())
self.assertEquals(1, eqm.discarded_events)
# Send events
eqm.sendEvents(lambda evts: None)
# Send the clear again
clearevt = createTestEvent(severity=0)
eqm.addEvent(clearevt)
sent_events = []
def send_events(evts):
sent_events.extend(evts)
eqm.sendEvents(send_events)
self.assertEquals([clearevt], sent_events)
def testDiscardedClearHandling(self):
opts = self.createOptions(maxqueuelen=1)
eqm = EventQueueManager(opts, log)
# Send a clear which is later discarded
eqm.addEvent(createTestEvent(severity=0))
# Send an event that wipes it out
eqm.addEvent(createTestEvent())
self.assertEquals(1, eqm.discarded_events)
# Send events
eqm.sendEvents(lambda evts: None)
# Send the clear again
clearevt = createTestEvent(severity=0)
eqm.addEvent(clearevt)
sent_events = []
def send_events(evts):
sent_events.extend(evts)
eqm.sendEvents(send_events)
self.assertEquals([clearevt], sent_events)