def testMain(self):
"""Tests the _Main function."""
output_event_queue = zeromq_queue.ZeroMQPushBindQueue(
name='test output event queue', timeout_seconds=self._QUEUE_TIMEOUT)
output_event_queue.Open()
input_event_queue = zeromq_queue.ZeroMQPullConnectQueue(
name='test input event queue', delay_open=True,
port=output_event_queue.port,
timeout_seconds=self._QUEUE_TIMEOUT)
session = sessions.Session()
storage_writer = self._CreateStorageWriter(session)
analysis_plugin = TestAnalysisPlugin()
configuration = configurations.ProcessingConfiguration()
test_process = analysis_process.AnalysisProcess(
input_event_queue, storage_writer, None, analysis_plugin, configuration,
name='TestAnalysis')
test_process._FOREMAN_STATUS_WAIT = 1
test_process.start()
output_event_queue.PushItem(plaso_queue.QueueAbort(), block=False)
output_event_queue.Close(abort=True)
def _StartWorkerProcess(self, process_name, storage_writer):
"""Creates, starts, monitors and registers a worker process.
Args:
process_name (str): process name.
storage_writer (StorageWriter): storage writer for a session storage used
to create task storage.
Returns:
MultiProcessWorkerProcess: extraction worker process or None on error.
"""
analysis_plugin = self._analysis_plugins.get(process_name, None)
if not analysis_plugin:
logger.error('Missing analysis plugin: {0:s}'.format(process_name))
return None
queue_name = '{0:s} output event queue'.format(process_name)
output_event_queue = zeromq_queue.ZeroMQPushBindQueue(
name=queue_name, timeout_seconds=self._QUEUE_TIMEOUT)
# Open the queue so it can bind to a random port, and we can get the
# port number to use in the input queue.
output_event_queue.Open()
self._event_queues[process_name] = output_event_queue
queue_name = '{0:s} input event queue'.format(process_name)
input_event_queue = zeromq_queue.ZeroMQPullConnectQueue(
name=queue_name,
delay_open=True,
port=output_event_queue.port,
timeout_seconds=self._QUEUE_TIMEOUT)
process = analysis_process.AnalysisProcess(
input_event_queue,
storage_writer,
self._knowledge_base,
analysis_plugin,
self._processing_configuration,
data_location=self._data_location,
event_filter_expression=self._event_filter_expression,
name=process_name)
process.start()
logger.info('Started analysis plugin: {0:s} (PID: {1:d}).'.format(
process_name, process.pid))
try:
self._StartMonitoringProcess(process)
except (IOError, KeyError) as exception:
logger.error(
('Unable to monitor analysis plugin: {0:s} (PID: {1:d}) '
'with error: {2!s}').format(process_name, process.pid,
exception))
process.terminate()
return None
self._RegisterProcess(process)
return process
def testItemCanBeQueuedAndDequeued(self):
"""Tests than an item can be transferred between push and pull queues."""
push_queue = zeromq_queue.ZeroMQPushBindQueue(delay_start=False)
pull_queue = zeromq_queue.ZeroMQPullConnectQueue(delay_start=False,
port=push_queue.port)
item = u'This is an item.'
push_queue.PushItem(item)
popped_item = pull_queue.PopItem()
self.assertEqual(item, popped_item)
def _StartAnalysisProcesses(
self, knowledge_base_object, storage_writer, analysis_plugins,
data_location, event_filter_expression=None):
"""Starts the analysis processes.
Args:
knowledge_base_object (KnowledgeBase): contains information from
the source data needed for processing.
storage_writer (StorageWriter): storage writer.
analysis_plugins (list[AnalysisPlugin]): analysis plugins that should
be run.
data_location (str): path to the location that data files should
be loaded from.
event_filter_expression (Optional[str]): event filter expression.
"""
logging.info(u'Starting analysis plugins.')
for analysis_plugin in analysis_plugins:
if self._use_zeromq:
queue_name = u'{0:s} output event queue'.format(analysis_plugin.NAME)
output_event_queue = zeromq_queue.ZeroMQPushBindQueue(
name=queue_name, timeout_seconds=self._QUEUE_TIMEOUT)
# Open the queue so it can bind to a random port, and we can get the
# port number to use in the input queue.
output_event_queue.Open()
else:
output_event_queue = multi_process_queue.MultiProcessingQueue(
timeout=self._QUEUE_TIMEOUT)
self._event_queues[analysis_plugin.NAME] = output_event_queue
if self._use_zeromq:
queue_name = u'{0:s} input event queue'.format(analysis_plugin.NAME)
input_event_queue = zeromq_queue.ZeroMQPullConnectQueue(
name=queue_name, delay_open=True, port=output_event_queue.port,
timeout_seconds=self._QUEUE_TIMEOUT)
else:
input_event_queue = output_event_queue
process = analysis_process.AnalysisProcess(
input_event_queue, storage_writer, knowledge_base_object,
analysis_plugin, data_location=data_location,
event_filter_expression=event_filter_expression,
name=analysis_plugin.plugin_name)
process.start()
logging.info(u'Started analysis plugin: {0:s} (PID: {1:d}).'.format(
analysis_plugin.plugin_name, process.pid))
self._RegisterProcess(process)
self._StartMonitoringProcess(process.pid)
logging.info(u'Analysis plugins running')
def testPushPullQueues(self):
"""Tests than an item can be transferred between push and pull queues."""
push_queue = zeromq_queue.ZeroMQPushBindQueue(delay_open=False)
pull_queue = zeromq_queue.ZeroMQPullConnectQueue(delay_open=False,
port=push_queue.port)
self._testItemTransferred(push_queue, pull_queue)
pull_queue = zeromq_queue.ZeroMQPullBindQueue(delay_open=False)
push_queue = zeromq_queue.ZeroMQPushConnectQueue(delay_open=False,
port=pull_queue.port)
self._testItemTransferred(push_queue, pull_queue)
def testPushPullQueues(self):
"""Tests than an item can be transferred between push and pull queues."""
push_queue = zeromq_queue.ZeroMQPushBindQueue(
name=u'pushpull_pushbind', delay_open=False, linger_seconds=1)
pull_queue = zeromq_queue.ZeroMQPullConnectQueue(
name=u'pushpull_pullconnect', delay_open=False, port=push_queue.port,
linger_seconds=1)
self._testItemTransferred(push_queue, pull_queue)
push_queue.Close()
pull_queue.Close()
pull_queue = ZeroMQPullBindQueue(
name=u'pushpull_pullbind', delay_open=False, linger_seconds=1)
push_queue = ZeroMQPushConnectQueue(
name=u'pushpull_pushconnect', delay_open=False, port=pull_queue.port,
linger_seconds=1)
self._testItemTransferred(push_queue, pull_queue)
push_queue.Close()
pull_queue.Close()
def GetAnalysisPluginsAndEventQueues(self, analysis_plugins_string):
"""Return a list of analysis plugins and event queues.
Args:
analysis_plugins_string: comma separated string with names of analysis
plugins to load.
Returns:
A tuple of two lists, one containing list of analysis plugins
and the other a list of event queues.
"""
if not analysis_plugins_string:
return [], []
event_producers = []
# These are the queues analysis plugins will read from.
analysis_plugin_input_queues = []
analysis_plugins_list = [
name.strip() for name in analysis_plugins_string.split(u',')
]
for _ in range(0, len(analysis_plugins_list)):
if self._use_zeromq:
output_queue = zeromq_queue.ZeroMQPushBindQueue()
# Open the queue so it can bind to a random port, and we can get the
# port number to use in the input queue.
output_queue.Open()
queue_port = output_queue.port
input_queue = zeromq_queue.ZeroMQPullConnectQueue(
port=queue_port, delay_open=True)
analysis_plugin_input_queues.append(input_queue)
else:
input_queue = multi_process.MultiProcessingQueue(timeout=5)
analysis_plugin_input_queues.append(input_queue)
output_queue = input_queue
event_producers.append(queue.ItemQueueProducer(output_queue))
analysis_plugins = analysis_manager.AnalysisPluginManager.LoadPlugins(
analysis_plugins_list, analysis_plugin_input_queues)
analysis_plugins = list(analysis_plugins)
return analysis_plugins, event_producers
def testMain(self):
"""Tests the _Main function."""
output_event_queue = zeromq_queue.ZeroMQPushBindQueue(
name='test output event queue',
timeout_seconds=self._QUEUE_TIMEOUT)
output_event_queue.Open()
input_event_queue = zeromq_queue.ZeroMQPullConnectQueue(
name='test input event queue',
delay_open=True,
port=output_event_queue.port,
timeout_seconds=self._QUEUE_TIMEOUT)
session = sessions.Session()
analysis_plugin = TestAnalysisPlugin()
with shared_test_lib.TempDirectory() as temp_directory:
# Set up the processed for the task storage file generated by the
# analysis plugin.
os.mkdir(os.path.join(temp_directory, 'processed'))
configuration = configurations.ProcessingConfiguration()
configuration.task_storage_path = temp_directory
test_process = analysis_process.AnalysisProcess(
input_event_queue,
None,
session,
analysis_plugin,
configuration,
name='TestAnalysis')
test_process._FOREMAN_STATUS_WAIT = 1
test_process.start()
output_event_queue.PushItem(plaso_queue.QueueAbort(), block=False)
output_event_queue.Close(abort=True)
# Sleep for 1 second to allow the analysis process to terminate.
# Before the temporary directory is removed.
time.sleep(1)
