def testGetStatus(self):
"""Tests the _GetStatus function."""
test_process = worker_process.WorkerProcess(None,
None,
None,
None,
None,
name='TestWorker')
status_attributes = test_process._GetStatus()
self.assertIsNotNone(status_attributes)
self.assertEqual(status_attributes['identifier'], 'TestWorker')
self.assertEqual(status_attributes['last_activity_timestamp'], 0.0)
self.assertIsNone(status_attributes['number_of_produced_errors'])
session = sessions.Session()
storage_writer = self._CreateStorageWriter(session)
knowledge_base = self._CreateKnowledgeBase()
test_process._parser_mediator = self._CreateParserMediator(
storage_writer, knowledge_base)
status_attributes = test_process._GetStatus()
self.assertIsNotNone(status_attributes)
self.assertEqual(status_attributes['identifier'], 'TestWorker')
self.assertEqual(status_attributes['last_activity_timestamp'], 0.0)
self.assertEqual(status_attributes['number_of_produced_errors'], 0)
def testProcessPathSpec(self):
"""Tests the _ProcessPathSpec function."""
configuration = configurations.ProcessingConfiguration()
test_process = worker_process.WorkerProcess(
None, None, None, None, None, configuration, name='TestWorker')
session = sessions.Session()
storage_writer = self._CreateStorageWriter(session)
knowledge_base = self._CreateKnowledgeBase()
parser_mediator = self._CreateParserMediator(storage_writer, knowledge_base)
path_spec = fake_path_spec.FakePathSpec(location='/test/file')
extraction_worker = TestEventExtractionWorker()
test_process._ProcessPathSpec(extraction_worker, parser_mediator, path_spec)
self.assertEqual(parser_mediator._number_of_warnings, 0)
extraction_worker = TestFailureEventExtractionWorker()
test_process._ProcessPathSpec(extraction_worker, parser_mediator, path_spec)
self.assertEqual(parser_mediator._number_of_warnings, 0)
self.assertTrue(test_process._abort)
test_process._ProcessPathSpec(None, parser_mediator, path_spec)
self.assertEqual(parser_mediator._number_of_warnings, 1)
def testMain(self):
"""Tests the _Main function."""
output_task_queue = zeromq_queue.ZeroMQBufferedReplyBindQueue(
delay_open=True,
linger_seconds=0,
maximum_items=1,
name='test output task queue',
timeout_seconds=self._QUEUE_TIMEOUT)
output_task_queue.Open()
input_task_queue = zeromq_queue.ZeroMQRequestConnectQueue(
delay_open=True,
linger_seconds=0,
name='test input task queue',
port=output_task_queue.port,
timeout_seconds=self._QUEUE_TIMEOUT)
configuration = configurations.ProcessingConfiguration()
test_process = worker_process.WorkerProcess(input_task_queue,
None,
None,
None,
None,
configuration,
name='TestWorker')
test_process.start()
output_task_queue.PushItem(plaso_queue.QueueAbort(), block=False)
output_task_queue.Close(abort=True)
def _StartWorkerProcess(self, process_name, storage_writer):
"""Creates, starts, monitors and registers a worker process.
Args:
process_name (str): process name.
storage_writer (StorageWriter): storage writer for a session storage used
to create task storage.
Returns:
MultiProcessWorkerProcess: extraction worker process or None if the
process could not be started.
"""
process_name = 'Worker_{0:02d}'.format(self._last_worker_number)
logger.debug('Starting worker process {0:s}'.format(process_name))
if self._use_zeromq:
queue_name = '{0:s} task queue'.format(process_name)
task_queue = zeromq_queue.ZeroMQRequestConnectQueue(
delay_open=True, linger_seconds=0, name=queue_name,
port=self._task_queue_port,
timeout_seconds=self._TASK_QUEUE_TIMEOUT_SECONDS)
else:
task_queue = self._task_queue
process = worker_process.WorkerProcess(
task_queue, storage_writer, self.knowledge_base,
self._session_identifier, self._processing_configuration,
enable_sigsegv_handler=self._enable_sigsegv_handler, name=process_name)
# Remove all possible log handlers to prevent a child process from logging
# to the main process log file and garbling the log. The log handlers are
# recreated after the worker process has been started.
for handler in logging.root.handlers:
logging.root.removeHandler(handler)
handler.close()
process.start()
loggers.ConfigureLogging(
debug_output=self._debug_output, filename=self._log_filename,
mode='a', quiet_mode=self._quiet_mode)
try:
self._StartMonitoringProcess(process)
except (IOError, KeyError) as exception:
pid = process.pid
logger.error((
'Unable to monitor replacement worker process: {0:s} '
'(PID: {1:d}) with error: {2!s}').format(
process_name, pid, exception))
self._TerminateProcess(process)
return None
self._RegisterProcess(process)
self._last_worker_number += 1
return process
def testSignalAbort(self):
"""Tests the SignalAbort function."""
test_process = worker_process.WorkerProcess(None,
None,
None,
None,
name=u'TestWorker')
test_process.SignalAbort()
def testInitialization(self):
"""Tests the initialization."""
test_process = worker_process.WorkerProcess(None,
None,
None,
None,
name=u'TestWorker')
self.assertIsNotNone(test_process)
def testGetStatus(self):
"""Tests the _GetStatus function."""
test_process = worker_process.WorkerProcess(None,
None,
None,
None,
name=u'TestWorker')
status_attributes = test_process._GetStatus()
self.assertIsNotNone(status_attributes)
self.assertEqual(status_attributes[u'identifier'], u'TestWorker')
def testMain(self):
"""Tests the _Main function."""
task_queue = multi_process_queue.MultiProcessingQueue(timeout=1)
configuration = configurations.ProcessingConfiguration()
test_process = worker_process.WorkerProcess(
task_queue, None, None, None, None, configuration, name='TestWorker')
test_process._abort = True
test_process._pid = 0
test_process._Main()
def _StartWorkerProcess(self, process_name, storage_writer):
"""Creates, starts, monitors and registers a worker process.
Args:
process_name (str): process name.
storage_writer (StorageWriter): storage writer for a session storage used
to create task storage.
Returns:
MultiProcessWorkerProcess: extraction worker process or None if the
process could not be started.
"""
process_name = 'Worker_{0:02d}'.format(self._last_worker_number)
logging.debug('Starting worker process {0:s}'.format(process_name))
if self._use_zeromq:
queue_name = '{0:s} task queue'.format(process_name)
task_queue = zeromq_queue.ZeroMQRequestConnectQueue(
delay_open=True, linger_seconds=0, name=queue_name,
port=self._task_queue_port,
timeout_seconds=self._TASK_QUEUE_TIMEOUT_SECONDS)
else:
task_queue = self._task_queue
process = worker_process.WorkerProcess(
task_queue, storage_writer, self.knowledge_base,
self._session_identifier, self._processing_configuration,
enable_sigsegv_handler=self._enable_sigsegv_handler, name=process_name)
process.start()
try:
self._StartMonitoringProcess(process)
except (IOError, KeyError) as exception:
pid = process.pid
logging.error((
'Unable to monitor replacement worker process: {0:s} '
'(PID: {1:d}) with error: {2!s}').format(
process_name, pid, exception))
self._TerminateProcess(process)
return
self._RegisterProcess(process)
self._last_worker_number += 1
return process
def testProcessTask(self):
"""Tests the _ProcessTask function."""
session = sessions.Session()
storage_writer = self._CreateStorageWriter(session)
knowledge_base = self._CreateKnowledgeBase()
configuration = configurations.ProcessingConfiguration()
test_process = worker_process.WorkerProcess(
None, storage_writer, knowledge_base, session.identifier, configuration,
name='TestWorker')
test_process._parser_mediator = self._CreateParserMediator(
storage_writer, knowledge_base)
task = tasks.Task(session_identifier=session.identifier)
test_process._ProcessTask(task)
def testStartAndStopProfiling(self):
"""Tests the _StartProfiling and _StopProfiling functions."""
with shared_test_lib.TempDirectory() as temp_directory:
configuration = configurations.ProcessingConfiguration()
configuration.profiling.directory = temp_directory
configuration.profiling.profilers = set([
'memory', 'parsers', 'processing', 'serializers', 'storage',
'task_queue'])
test_process = worker_process.WorkerProcess(
None, None, None, None, None, configuration, name='TestWorker')
test_process._extraction_worker = TestEventExtractionWorker()
test_process._StartProfiling(None)
test_process._StartProfiling(configuration.profiling)
test_process._StopProfiling()
def _StartExtractionWorkerProcess(self, storage_writer):
"""Creates, starts and registers an extraction worker process.
Args:
storage_writer (StorageWriter): storage writer for a session storage used
to create task storage.
Returns:
MultiProcessWorkerProcess: extraction worker process.
"""
process_name = u'Worker_{0:02d}'.format(self._last_worker_number)
logging.debug(u'Starting worker process {0:s}'.format(process_name))
if self._use_zeromq:
task_queue = zeromq_queue.ZeroMQRequestConnectQueue(
delay_open=True, name=u'{0:s} task queue'.format(process_name),
linger_seconds=0, port=self._task_queue_port,
timeout_seconds=self._TASK_QUEUE_TIMEOUT_SECONDS)
else:
task_queue = self._task_queue
process = worker_process.WorkerProcess(
task_queue, storage_writer, self.knowledge_base,
self._session_identifier, debug_output=self._debug_output,
enable_profiling=self._enable_profiling,
enable_sigsegv_handler=self._enable_sigsegv_handler,
filter_object=self._filter_object,
hasher_names_string=self._hasher_names_string,
mount_path=self._mount_path, name=process_name,
parser_filter_expression=self._parser_filter_expression,
preferred_year=self._preferred_year,
process_archives=self._process_archives,
process_compressed_streams=self._process_compressed_streams,
profiling_directory=self._profiling_directory,
profiling_sample_rate=self._profiling_sample_rate,
profiling_type=self._profiling_type,
temporary_directory=self._temporary_directory,
text_prepend=self._text_prepend,
yara_rules_string=self._yara_rules_string)
process.start()
self._last_worker_number += 1
self._RegisterProcess(process)
return process
def _StartWorkerProcess(self, storage_writer):
"""Creates, starts and registers a worker process.
Args:
storage_writer (StorageWriter): storage writer for a session storage used
to create task storage.
Returns:
MultiProcessWorkerProcess: extraction worker process.
"""
process_name = u'Worker_{0:02d}'.format(self._last_worker_number)
logging.debug(u'Starting worker process {0:s}'.format(process_name))
if self._use_zeromq:
task_queue = zeromq_queue.ZeroMQRequestConnectQueue(
delay_open=True,
name=u'{0:s} task queue'.format(process_name),
linger_seconds=0,
port=self._task_queue_port,
timeout_seconds=self._TASK_QUEUE_TIMEOUT_SECONDS)
else:
task_queue = self._task_queue
process = worker_process.WorkerProcess(
task_queue,
storage_writer,
self.knowledge_base,
self._session_identifier,
self._processing_configuration,
enable_sigsegv_handler=self._enable_sigsegv_handler,
name=process_name)
process.start()
self._last_worker_number += 1
self._RegisterProcess(process)
return process