def testMain(self):
"""Tests the _Main function."""
output_task_queue = zeromq_queue.ZeroMQBufferedReplyBindQueue(
delay_open=True,
linger_seconds=0,
maximum_items=1,
name='test output task queue',
timeout_seconds=self._QUEUE_TIMEOUT)
output_task_queue.Open()
input_task_queue = zeromq_queue.ZeroMQRequestConnectQueue(
delay_open=True,
linger_seconds=0,
name='test input task queue',
port=output_task_queue.port,
timeout_seconds=self._QUEUE_TIMEOUT)
configuration = configurations.ProcessingConfiguration()
test_process = worker_process.WorkerProcess(input_task_queue,
None,
None,
None,
None,
configuration,
name='TestWorker')
test_process.start()
output_task_queue.PushItem(plaso_queue.QueueAbort(), block=False)
output_task_queue.Close(abort=True)
def testRequestAndBufferedReplyQueues(self):
"""Tests REQ and buffered REP queue pairs."""
reply_queue = zeromq_queue.ZeroMQBufferedReplyBindQueue(
name='requestbufferedreply_replybind',
delay_open=False,
linger_seconds=1)
request_queue = zeromq_queue.ZeroMQRequestConnectQueue(
name='requestbufferedreply_requestconnect',
delay_open=False,
port=reply_queue.port,
linger_seconds=1)
self._testItemTransferred(reply_queue, request_queue)
reply_queue.Close()
request_queue.Close()
request_queue = ZeroMQRequestBindQueue(
name='requestbufferedreply_requestbind',
delay_open=False,
linger_seconds=1)
reply_queue = ZeroMQBufferedReplyConnectQueue(
name='requestbufferedreply_replyconnect',
delay_open=False,
port=request_queue.port,
linger_seconds=0)
self._testItemTransferred(reply_queue, request_queue)
reply_queue.Close()
request_queue.Close()
def _StartWorkerProcess(self, process_name, storage_writer):
"""Creates, starts, monitors and registers a worker process.
Args:
process_name (str): process name.
storage_writer (StorageWriter): storage writer for a session storage used
to create task storage.
Returns:
MultiProcessWorkerProcess: extraction worker process or None if the
process could not be started.
"""
process_name = 'Worker_{0:02d}'.format(self._last_worker_number)
logger.debug('Starting worker process {0:s}'.format(process_name))
if self._use_zeromq:
queue_name = '{0:s} task queue'.format(process_name)
task_queue = zeromq_queue.ZeroMQRequestConnectQueue(
delay_open=True, linger_seconds=0, name=queue_name,
port=self._task_queue_port,
timeout_seconds=self._TASK_QUEUE_TIMEOUT_SECONDS)
else:
task_queue = self._task_queue
process = worker_process.WorkerProcess(
task_queue, storage_writer, self.knowledge_base,
self._session_identifier, self._processing_configuration,
enable_sigsegv_handler=self._enable_sigsegv_handler, name=process_name)
# Remove all possible log handlers to prevent a child process from logging
# to the main process log file and garbling the log. The log handlers are
# recreated after the worker process has been started.
for handler in logging.root.handlers:
logging.root.removeHandler(handler)
handler.close()
process.start()
loggers.ConfigureLogging(
debug_output=self._debug_output, filename=self._log_filename,
mode='a', quiet_mode=self._quiet_mode)
try:
self._StartMonitoringProcess(process)
except (IOError, KeyError) as exception:
pid = process.pid
logger.error((
'Unable to monitor replacement worker process: {0:s} '
'(PID: {1:d}) with error: {2!s}').format(
process_name, pid, exception))
self._TerminateProcess(process)
return None
self._RegisterProcess(process)
self._last_worker_number += 1
return process
def testRequestAndBufferedReplyQueues(self):
"""Tests REQ and buffered REP queue pairs."""
reply_queue = zeromq_queue.ZeroMQBufferedReplyBindQueue(
delay_open=False)
request_queue = zeromq_queue.ZeroMQRequestConnectQueue(
delay_open=False, port=reply_queue.port)
self._testItemTransferred(reply_queue, request_queue)
request_queue = zeromq_queue.ZeroMQRequestBindQueue(delay_open=False)
reply_queue = zeromq_queue.ZeroMQBufferedReplyConnectQueue(
delay_open=False, port=request_queue.port)
self._testItemTransferred(reply_queue, request_queue)
def _StartWorkerProcess(self, process_name, storage_writer):
"""Creates, starts, monitors and registers a worker process.
Args:
process_name (str): process name.
storage_writer (StorageWriter): storage writer for a session storage used
to create task storage.
Returns:
MultiProcessWorkerProcess: extraction worker process or None if the
process could not be started.
"""
process_name = 'Worker_{0:02d}'.format(self._last_worker_number)
logging.debug('Starting worker process {0:s}'.format(process_name))
if self._use_zeromq:
queue_name = '{0:s} task queue'.format(process_name)
task_queue = zeromq_queue.ZeroMQRequestConnectQueue(
delay_open=True, linger_seconds=0, name=queue_name,
port=self._task_queue_port,
timeout_seconds=self._TASK_QUEUE_TIMEOUT_SECONDS)
else:
task_queue = self._task_queue
process = worker_process.WorkerProcess(
task_queue, storage_writer, self.knowledge_base,
self._session_identifier, self._processing_configuration,
enable_sigsegv_handler=self._enable_sigsegv_handler, name=process_name)
process.start()
try:
self._StartMonitoringProcess(process)
except (IOError, KeyError) as exception:
pid = process.pid
logging.error((
'Unable to monitor replacement worker process: {0:s} '
'(PID: {1:d}) with error: {2!s}').format(
process_name, pid, exception))
self._TerminateProcess(process)
return
self._RegisterProcess(process)
self._last_worker_number += 1
return process
def _StartExtractionWorkerProcess(self, storage_writer):
"""Creates, starts and registers an extraction worker process.
Args:
storage_writer (StorageWriter): storage writer for a session storage used
to create task storage.
Returns:
MultiProcessWorkerProcess: extraction worker process.
"""
process_name = u'Worker_{0:02d}'.format(self._last_worker_number)
logging.debug(u'Starting worker process {0:s}'.format(process_name))
if self._use_zeromq:
task_queue = zeromq_queue.ZeroMQRequestConnectQueue(
delay_open=True, name=u'{0:s} task queue'.format(process_name),
linger_seconds=0, port=self._task_queue_port,
timeout_seconds=self._TASK_QUEUE_TIMEOUT_SECONDS)
else:
task_queue = self._task_queue
process = worker_process.WorkerProcess(
task_queue, storage_writer, self.knowledge_base,
self._session_identifier, debug_output=self._debug_output,
enable_profiling=self._enable_profiling,
enable_sigsegv_handler=self._enable_sigsegv_handler,
filter_object=self._filter_object,
hasher_names_string=self._hasher_names_string,
mount_path=self._mount_path, name=process_name,
parser_filter_expression=self._parser_filter_expression,
preferred_year=self._preferred_year,
process_archives=self._process_archives,
process_compressed_streams=self._process_compressed_streams,
profiling_directory=self._profiling_directory,
profiling_sample_rate=self._profiling_sample_rate,
profiling_type=self._profiling_type,
temporary_directory=self._temporary_directory,
text_prepend=self._text_prepend,
yara_rules_string=self._yara_rules_string)
process.start()
self._last_worker_number += 1
self._RegisterProcess(process)
return process
def _StartWorkerProcess(self, storage_writer):
"""Creates, starts and registers a worker process.
Args:
storage_writer (StorageWriter): storage writer for a session storage used
to create task storage.
Returns:
MultiProcessWorkerProcess: extraction worker process.
"""
process_name = u'Worker_{0:02d}'.format(self._last_worker_number)
logging.debug(u'Starting worker process {0:s}'.format(process_name))
if self._use_zeromq:
task_queue = zeromq_queue.ZeroMQRequestConnectQueue(
delay_open=True,
name=u'{0:s} task queue'.format(process_name),
linger_seconds=0,
port=self._task_queue_port,
timeout_seconds=self._TASK_QUEUE_TIMEOUT_SECONDS)
else:
task_queue = self._task_queue
process = worker_process.WorkerProcess(
task_queue,
storage_writer,
self.knowledge_base,
self._session_identifier,
self._processing_configuration,
enable_sigsegv_handler=self._enable_sigsegv_handler,
name=process_name)
process.start()
self._last_worker_number += 1
self._RegisterProcess(process)
return process
