def testGetStatus(self):
"""Tests the _GetStatus function."""
with shared_test_lib.TempDirectory() as temp_directory:
configuration = configurations.ProcessingConfiguration()
configuration.task_storage_path = temp_directory
test_process = extraction_process.ExtractionWorkerProcess(
None, None, None, configuration, name='TestWorker')
status_attributes = test_process._GetStatus()
self.assertIsNotNone(status_attributes)
self.assertEqual(status_attributes['identifier'], 'TestWorker')
self.assertEqual(status_attributes['last_activity_timestamp'], 0.0)
self.assertIsNone(
status_attributes['number_of_produced_extraction_warnings'])
task_storage_writer = self._CreateStorageWriter()
knowledge_base = self._CreateKnowledgeBase()
test_process._parser_mediator = self._CreateParserMediator(
task_storage_writer, knowledge_base)
status_attributes = test_process._GetStatus()
self.assertIsNotNone(status_attributes)
self.assertEqual(status_attributes['identifier'], 'TestWorker')
self.assertEqual(status_attributes['last_activity_timestamp'], 0.0)
self.assertEqual(
status_attributes['number_of_produced_extraction_warnings'], 0)
def testProcessPathSpec(self):
"""Tests the _ProcessPathSpec function."""
with shared_test_lib.TempDirectory() as temp_directory:
configuration = configurations.ProcessingConfiguration()
configuration.task_storage_path = temp_directory
test_process = extraction_process.ExtractionWorkerProcess(
None, None, None, configuration, name='TestWorker')
task_storage_writer = self._CreateStorageWriter()
knowledge_base = self._CreateKnowledgeBase()
parser_mediator = self._CreateParserMediator(
task_storage_writer, knowledge_base)
path_spec = fake_path_spec.FakePathSpec(location='/test/file')
extraction_worker = TestEventExtractionWorker()
test_process._ProcessPathSpec(extraction_worker, parser_mediator,
path_spec)
self.assertEqual(parser_mediator._number_of_extraction_warnings, 0)
extraction_worker = TestFailureEventExtractionWorker()
test_process._ProcessPathSpec(extraction_worker, parser_mediator,
path_spec)
self.assertEqual(parser_mediator._number_of_extraction_warnings, 0)
self.assertTrue(test_process._abort)
test_process._ProcessPathSpec(None, parser_mediator, path_spec)
self.assertEqual(parser_mediator._number_of_extraction_warnings, 1)
def testMain(self):
"""Tests the _Main function."""
output_task_queue = zeromq_queue.ZeroMQBufferedReplyBindQueue(
delay_open=True,
linger_seconds=0,
maximum_items=1,
name='test output task queue',
timeout_seconds=self._QUEUE_TIMEOUT)
output_task_queue.Open()
input_task_queue = zeromq_queue.ZeroMQRequestConnectQueue(
delay_open=True,
linger_seconds=0,
name='test input task queue',
port=output_task_queue.port,
timeout_seconds=self._QUEUE_TIMEOUT)
with shared_test_lib.TempDirectory() as temp_directory:
configuration = configurations.ProcessingConfiguration()
configuration.task_storage_path = temp_directory
test_process = extraction_process.ExtractionWorkerProcess(
input_task_queue, None, None, configuration, name='TestWorker')
test_process.start()
output_task_queue.PushItem(plaso_queue.QueueAbort(), block=False)
output_task_queue.Close(abort=True)
def _StartWorkerProcess(self, process_name):
"""Creates, starts, monitors and registers a worker process.
Args:
process_name (str): process name.
Returns:
MultiProcessWorkerProcess: extraction worker process or None if the
process could not be started.
"""
logger.debug('Starting worker process {0:s}'.format(process_name))
queue_name = '{0:s} task queue'.format(process_name)
task_queue = zeromq_queue.ZeroMQRequestConnectQueue(
delay_open=True,
linger_seconds=0,
name=queue_name,
port=self._task_queue_port,
timeout_seconds=self._TASK_QUEUE_TIMEOUT_SECONDS)
process = extraction_process.ExtractionWorkerProcess(
task_queue,
self.collection_filters_helper,
self.knowledge_base,
self._processing_configuration,
enable_sigsegv_handler=self._enable_sigsegv_handler,
name=process_name)
# Remove all possible log handlers to prevent a child process from logging
# to the main process log file and garbling the log. The log handlers are
# recreated after the worker process has been started.
for handler in logging.root.handlers:
logging.root.removeHandler(handler)
handler.close()
process.start()
loggers.ConfigureLogging(debug_output=self._debug_output,
filename=self._log_filename,
mode='a',
quiet_mode=self._quiet_mode)
try:
self._StartMonitoringProcess(process)
except (IOError, KeyError) as exception:
pid = process.pid
logger.error(
('Unable to monitor replacement worker process: {0:s} '
'(PID: {1:d}) with error: {2!s}').format(
process_name, pid, exception))
self._TerminateProcess(process)
return None
self._RegisterProcess(process)
self._last_worker_number += 1
return process
def testInitialization(self):
"""Tests the initialization."""
with shared_test_lib.TempDirectory() as temp_directory:
configuration = configurations.ProcessingConfiguration()
configuration.task_storage_path = temp_directory
test_process = extraction_process.ExtractionWorkerProcess(
None, None, None, configuration, name='TestWorker')
self.assertIsNotNone(test_process)
def testSignalAbort(self):
"""Tests the SignalAbort function."""
with shared_test_lib.TempDirectory() as temp_directory:
configuration = configurations.ProcessingConfiguration()
configuration.task_storage_path = temp_directory
test_process = extraction_process.ExtractionWorkerProcess(
None, None, None, configuration, name='TestWorker')
test_process.SignalAbort()
def testStartAndStopProfiling(self):
"""Tests the _StartProfiling and _StopProfiling functions."""
with shared_test_lib.TempDirectory() as temp_directory:
configuration = configurations.ProcessingConfiguration()
configuration.profiling.directory = temp_directory
configuration.profiling.profilers = set([
'memory', 'parsers', 'processing', 'serializers', 'storage',
'task_queue'])
configuration.task_storage_path = temp_directory
test_process = extraction_process.ExtractionWorkerProcess(
None, None, None, None, configuration, name='TestWorker')
test_process._extraction_worker = TestEventExtractionWorker()
test_process._StartProfiling(None)
test_process._StartProfiling(configuration.profiling)
test_process._StopProfiling()
def testProcessTask(self):
"""Tests the _ProcessTask function."""
session = sessions.Session()
knowledge_base = self._CreateKnowledgeBase()
with shared_test_lib.TempDirectory() as temp_directory:
configuration = configurations.ProcessingConfiguration()
configuration.task_storage_path = temp_directory
configuration.task_storage_format = definitions.STORAGE_FORMAT_SQLITE
test_process = extraction_process.ExtractionWorkerProcess(
None, None, knowledge_base, configuration, name='TestWorker')
test_process._extraction_worker = TestEventExtractionWorker()
task_storage_writer = self._CreateStorageWriter()
test_process._parser_mediator = self._CreateParserMediator(
task_storage_writer, knowledge_base)
task = tasks.Task(session_identifier=session.identifier)
test_process._ProcessTask(task)