def testPushPopItem(self):
"""Tests the PushItem and PopItem functions."""
test_queue = multi_process.MultiProcessingQueue()
for item in self._ITEMS:
test_queue.PushItem(item)
test_queue.SignalEndOfInput()
test_queue_consumer = test_lib.TestQueueConsumer(test_queue)
test_queue_consumer.ConsumeItems()
self.assertEqual(test_queue_consumer.number_of_items, len(self._ITEMS))
def testPushPopItem(self):
"""Tests the PushItem and PopItem functions."""
test_queue = single_process.SingleProcessQueue()
for item in self._ITEMS:
test_queue.PushItem(item)
test_queue.SignalEndOfInput()
test_queue_consumer = test_lib.TestQueueConsumer(test_queue)
test_queue_consumer.ConsumeItems()
expected_number_of_items = len(self._ITEMS)
self.assertEqual(
test_queue_consumer.number_of_items, expected_number_of_items)
def testPushPopItem(self):
"""Tests the PushItem and PopItem functions."""
test_queue = multi_process.MultiProcessingQueue()
for item in self._ITEMS:
test_queue.PushItem(item)
try:
self.assertEqual(len(test_queue), len(self._ITEMS))
except NotImplementedError:
# On Mac OS X because of broken sem_getvalue()
return
test_queue.SignalEndOfInput()
test_queue_consumer = test_lib.TestQueueConsumer(test_queue)
test_queue_consumer.ConsumeItems()
self.assertEqual(test_queue_consumer.number_of_items, len(self._ITEMS))
def testQueueFull(self):
"""Tests the queue raises the QueueFull exception."""
test_queue = single_process.SingleProcessQueue(
maximum_number_of_queued_items=5)
for item in self._ITEMS:
test_queue.PushItem(item)
with self.assertRaises(errors.QueueFull):
test_queue.PushItem('item5')
with self.assertRaises(errors.QueueFull):
test_queue.PushItem('item6')
test_queue_consumer = test_lib.TestQueueConsumer(test_queue)
test_queue_consumer.ConsumeItems()
expected_number_of_items = len(self._ITEMS)
self.assertEqual(test_queue_consumer.number_of_items,
expected_number_of_items + 1)
def testExtractionWorker(self):
"""Tests the extraction worker functionality."""
collection_queue = single_process.SingleProcessQueue()
storage_queue = single_process.SingleProcessQueue()
parse_error_queue = single_process.SingleProcessQueue()
event_queue_producer = single_process.SingleProcessItemQueueProducer(
storage_queue)
parse_error_queue_producer = single_process.SingleProcessItemQueueProducer(
parse_error_queue)
knowledge_base_object = knowledge_base.KnowledgeBase()
parser_mediator = parsers_mediator.ParserMediator(
event_queue_producer, parse_error_queue_producer,
knowledge_base_object)
resolver_context = context.Context()
extraction_worker = worker.BaseEventExtractionWorker(
0, collection_queue, event_queue_producer, parse_error_queue_producer,
parser_mediator, resolver_context=resolver_context)
self.assertNotEqual(extraction_worker, None)
extraction_worker.InitializeParserObjects()
# Process a file.
source_path = self._GetTestFilePath([u'syslog'])
path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_OS, location=source_path)
collection_queue.PushItem(path_spec)
extraction_worker.Run()
test_queue_consumer = test_lib.TestQueueConsumer(storage_queue)
test_queue_consumer.ConsumeItems()
self.assertEqual(test_queue_consumer.number_of_items, 16)
# Process a compressed file.
source_path = self._GetTestFilePath([u'syslog.gz'])
path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_OS, location=source_path)
collection_queue.PushItem(path_spec)
extraction_worker.Run()
test_queue_consumer = test_lib.TestQueueConsumer(storage_queue)
test_queue_consumer.ConsumeItems()
self.assertEqual(test_queue_consumer.number_of_items, 16)
source_path = self._GetTestFilePath([u'syslog.bz2'])
path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_OS, location=source_path)
collection_queue.PushItem(path_spec)
extraction_worker.Run()
test_queue_consumer = test_lib.TestQueueConsumer(storage_queue)
test_queue_consumer.ConsumeItems()
self.assertEqual(test_queue_consumer.number_of_items, 15)
# Process a file in an archive.
source_path = self._GetTestFilePath([u'syslog.tar'])
path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_OS, location=source_path)
path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_TAR, location=u'/syslog',
parent=path_spec)
collection_queue.PushItem(path_spec)
extraction_worker.Run()
test_queue_consumer = test_lib.TestQueueConsumer(storage_queue)
test_queue_consumer.ConsumeItems()
self.assertEqual(test_queue_consumer.number_of_items, 13)
# Process an archive file without "process archive files" mode.
extraction_worker.SetProcessArchiveFiles(False)
source_path = self._GetTestFilePath([u'syslog.tar'])
path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_OS, location=source_path)
collection_queue.PushItem(path_spec)
extraction_worker.Run()
test_queue_consumer = test_lib.TestQueueConsumer(storage_queue)
test_queue_consumer.ConsumeItems()
self.assertEqual(test_queue_consumer.number_of_items, 3)
# Process an archive file with "process archive files" mode.
extraction_worker.SetProcessArchiveFiles(True)
source_path = self._GetTestFilePath([u'syslog.tar'])
path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_OS, location=source_path)
collection_queue.PushItem(path_spec)
extraction_worker.Run()
test_queue_consumer = test_lib.TestQueueConsumer(storage_queue)
test_queue_consumer.ConsumeItems()
self.assertEqual(test_queue_consumer.number_of_items, 16)
# Process a file in a compressed archive.
source_path = self._GetTestFilePath([u'syslog.tgz'])
path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_OS, location=source_path)
path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_GZIP, parent=path_spec)
path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_TAR, location=u'/syslog',
parent=path_spec)
collection_queue.PushItem(path_spec)
extraction_worker.Run()
test_queue_consumer = test_lib.TestQueueConsumer(storage_queue)
test_queue_consumer.ConsumeItems()
self.assertEqual(test_queue_consumer.number_of_items, 13)
# Process an archive file with "process archive files" mode.
extraction_worker.SetProcessArchiveFiles(True)
source_path = self._GetTestFilePath([u'syslog.tgz'])
path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_OS, location=source_path)
collection_queue.PushItem(path_spec)
extraction_worker.Run()
test_queue_consumer = test_lib.TestQueueConsumer(storage_queue)
test_queue_consumer.ConsumeItems()
self.assertEqual(test_queue_consumer.number_of_items, 17)
