def _StartWorkerProcess(self, process_name, storage_writer):
"""Creates, starts, monitors and registers a worker process.
Args:
process_name (str): process name.
storage_writer (StorageWriter): storage writer for a session storage used
to create task storage.
Returns:
MultiProcessWorkerProcess: extraction worker process or None on error.
"""
analysis_plugin = self._analysis_plugins.get(process_name, None)
if not analysis_plugin:
logger.error('Missing analysis plugin: {0:s}'.format(process_name))
return None
queue_name = '{0:s} output event queue'.format(process_name)
output_event_queue = zeromq_queue.ZeroMQPushBindQueue(
name=queue_name, timeout_seconds=self._QUEUE_TIMEOUT)
# Open the queue so it can bind to a random port, and we can get the
# port number to use in the input queue.
output_event_queue.Open()
self._event_queues[process_name] = output_event_queue
queue_name = '{0:s} input event queue'.format(process_name)
input_event_queue = zeromq_queue.ZeroMQPullConnectQueue(
name=queue_name,
delay_open=True,
port=output_event_queue.port,
timeout_seconds=self._QUEUE_TIMEOUT)
process = analysis_process.AnalysisProcess(
input_event_queue,
storage_writer,
self._knowledge_base,
analysis_plugin,
self._processing_configuration,
data_location=self._data_location,
event_filter_expression=self._event_filter_expression,
name=process_name)
process.start()
logger.info('Started analysis plugin: {0:s} (PID: {1:d}).'.format(
process_name, process.pid))
try:
self._StartMonitoringProcess(process)
except (IOError, KeyError) as exception:
logger.error(
('Unable to monitor analysis plugin: {0:s} (PID: {1:d}) '
'with error: {2!s}').format(process_name, process.pid,
exception))
process.terminate()
return None
self._RegisterProcess(process)
return process
def testInitialization(self):
"""Tests the initialization."""
configuration = configurations.ProcessingConfiguration()
test_process = analysis_process.AnalysisProcess(
None, None, None, None, configuration, name='TestAnalysis')
self.assertIsNotNone(test_process)
def testSignalAbort(self):
"""Tests the SignalAbort function."""
configuration = configurations.ProcessingConfiguration()
test_process = analysis_process.AnalysisProcess(
None, None, None, None, configuration, name='TestAnalysis')
test_process.SignalAbort()
def testMain(self):
"""Tests the _Main function."""
output_event_queue = zeromq_queue.ZeroMQPushBindQueue(
name='test output event queue', timeout_seconds=self._QUEUE_TIMEOUT)
output_event_queue.Open()
input_event_queue = zeromq_queue.ZeroMQPullConnectQueue(
name='test input event queue', delay_open=True,
port=output_event_queue.port,
timeout_seconds=self._QUEUE_TIMEOUT)
session = sessions.Session()
storage_writer = self._CreateStorageWriter(session)
analysis_plugin = TestAnalysisPlugin()
configuration = configurations.ProcessingConfiguration()
test_process = analysis_process.AnalysisProcess(
input_event_queue, storage_writer, None, analysis_plugin, configuration,
name='TestAnalysis')
test_process._FOREMAN_STATUS_WAIT = 1
test_process.start()
output_event_queue.PushItem(plaso_queue.QueueAbort(), block=False)
output_event_queue.Close(abort=True)
def testSignalAbort(self):
"""Tests the SignalAbort function."""
test_process = analysis_process.AnalysisProcess(None,
None,
None,
None,
name=u'TestAnalysis')
test_process.SignalAbort()
def testInitialization(self):
"""Tests the initialization."""
test_process = analysis_process.AnalysisProcess(None,
None,
None,
None,
name=u'TestAnalysis')
self.assertIsNotNone(test_process)
def _StartAnalysisProcesses(
self, knowledge_base_object, storage_writer, analysis_plugins,
data_location, event_filter_expression=None):
"""Starts the analysis processes.
Args:
knowledge_base_object (KnowledgeBase): contains information from
the source data needed for processing.
storage_writer (StorageWriter): storage writer.
analysis_plugins (list[AnalysisPlugin]): analysis plugins that should
be run.
data_location (str): path to the location that data files should
be loaded from.
event_filter_expression (Optional[str]): event filter expression.
"""
logging.info(u'Starting analysis plugins.')
for analysis_plugin in analysis_plugins:
if self._use_zeromq:
queue_name = u'{0:s} output event queue'.format(analysis_plugin.NAME)
output_event_queue = zeromq_queue.ZeroMQPushBindQueue(
name=queue_name, timeout_seconds=self._QUEUE_TIMEOUT)
# Open the queue so it can bind to a random port, and we can get the
# port number to use in the input queue.
output_event_queue.Open()
else:
output_event_queue = multi_process_queue.MultiProcessingQueue(
timeout=self._QUEUE_TIMEOUT)
self._event_queues[analysis_plugin.NAME] = output_event_queue
if self._use_zeromq:
queue_name = u'{0:s} input event queue'.format(analysis_plugin.NAME)
input_event_queue = zeromq_queue.ZeroMQPullConnectQueue(
name=queue_name, delay_open=True, port=output_event_queue.port,
timeout_seconds=self._QUEUE_TIMEOUT)
else:
input_event_queue = output_event_queue
process = analysis_process.AnalysisProcess(
input_event_queue, storage_writer, knowledge_base_object,
analysis_plugin, data_location=data_location,
event_filter_expression=event_filter_expression,
name=analysis_plugin.plugin_name)
process.start()
logging.info(u'Started analysis plugin: {0:s} (PID: {1:d}).'.format(
analysis_plugin.plugin_name, process.pid))
self._RegisterProcess(process)
self._StartMonitoringProcess(process.pid)
logging.info(u'Analysis plugins running')
def testGetStatus(self):
"""Tests the _GetStatus function."""
configuration = configurations.ProcessingConfiguration()
test_process = analysis_process.AnalysisProcess(
None, None, None, None, configuration, name='TestAnalysis')
status_attributes = test_process._GetStatus()
self.assertIsNotNone(status_attributes)
self.assertEqual(status_attributes['identifier'], 'TestAnalysis')
self.assertIsNone(status_attributes['number_of_produced_reports'])
def testGetStatus(self):
"""Tests the _GetStatus function."""
test_process = analysis_process.AnalysisProcess(None,
None,
None,
None,
name=u'TestAnalysis')
status_attributes = test_process._GetStatus()
self.assertIsNotNone(status_attributes)
self.assertEqual(status_attributes[u'identifier'], u'TestAnalysis')
self.assertIsNone(status_attributes['number_of_produced_reports'])
def testMain(self):
"""Tests the _Main function."""
event_queue = multi_process_queue.MultiProcessingQueue(timeout=1)
session = sessions.Session()
storage_writer = self._CreateStorageWriter(session)
analysis_plugin = TestAnalysisPlugin()
configuration = configurations.ProcessingConfiguration()
test_process = analysis_process.AnalysisProcess(
event_queue, storage_writer, None, analysis_plugin, configuration,
name='TestAnalysis')
test_process._abort = True
test_process._FOREMAN_STATUS_WAIT = 1
test_process._pid = 0
test_process._Main()
