def testMain(self): """Tests the _Main function.""" output_event_queue = zeromq_queue.ZeroMQPushBindQueue( name='test output event queue', timeout_seconds=self._QUEUE_TIMEOUT) output_event_queue.Open() input_event_queue = zeromq_queue.ZeroMQPullConnectQueue( name='test input event queue', delay_open=True, port=output_event_queue.port, timeout_seconds=self._QUEUE_TIMEOUT) session = sessions.Session() storage_writer = self._CreateStorageWriter(session) analysis_plugin = TestAnalysisPlugin() configuration = configurations.ProcessingConfiguration() test_process = analysis_process.AnalysisProcess( input_event_queue, storage_writer, None, analysis_plugin, configuration, name='TestAnalysis') test_process._FOREMAN_STATUS_WAIT = 1 test_process.start() output_event_queue.PushItem(plaso_queue.QueueAbort(), block=False) output_event_queue.Close(abort=True)
def _StartWorkerProcess(self, process_name, storage_writer): """Creates, starts, monitors and registers a worker process. Args: process_name (str): process name. storage_writer (StorageWriter): storage writer for a session storage used to create task storage. Returns: MultiProcessWorkerProcess: extraction worker process or None on error. """ analysis_plugin = self._analysis_plugins.get(process_name, None) if not analysis_plugin: logger.error('Missing analysis plugin: {0:s}'.format(process_name)) return None queue_name = '{0:s} output event queue'.format(process_name) output_event_queue = zeromq_queue.ZeroMQPushBindQueue( name=queue_name, timeout_seconds=self._QUEUE_TIMEOUT) # Open the queue so it can bind to a random port, and we can get the # port number to use in the input queue. output_event_queue.Open() self._event_queues[process_name] = output_event_queue queue_name = '{0:s} input event queue'.format(process_name) input_event_queue = zeromq_queue.ZeroMQPullConnectQueue( name=queue_name, delay_open=True, port=output_event_queue.port, timeout_seconds=self._QUEUE_TIMEOUT) process = analysis_process.AnalysisProcess( input_event_queue, storage_writer, self._knowledge_base, analysis_plugin, self._processing_configuration, data_location=self._data_location, event_filter_expression=self._event_filter_expression, name=process_name) process.start() logger.info('Started analysis plugin: {0:s} (PID: {1:d}).'.format( process_name, process.pid)) try: self._StartMonitoringProcess(process) except (IOError, KeyError) as exception: logger.error( ('Unable to monitor analysis plugin: {0:s} (PID: {1:d}) ' 'with error: {2!s}').format(process_name, process.pid, exception)) process.terminate() return None self._RegisterProcess(process) return process
def testItemCanBeQueuedAndDequeued(self): """Tests than an item can be transferred between push and pull queues.""" push_queue = zeromq_queue.ZeroMQPushBindQueue(delay_start=False) pull_queue = zeromq_queue.ZeroMQPullConnectQueue(delay_start=False, port=push_queue.port) item = u'This is an item.' push_queue.PushItem(item) popped_item = pull_queue.PopItem() self.assertEqual(item, popped_item)
def _StartAnalysisProcesses( self, knowledge_base_object, storage_writer, analysis_plugins, data_location, event_filter_expression=None): """Starts the analysis processes. Args: knowledge_base_object (KnowledgeBase): contains information from the source data needed for processing. storage_writer (StorageWriter): storage writer. analysis_plugins (list[AnalysisPlugin]): analysis plugins that should be run. data_location (str): path to the location that data files should be loaded from. event_filter_expression (Optional[str]): event filter expression. """ logging.info(u'Starting analysis plugins.') for analysis_plugin in analysis_plugins: if self._use_zeromq: queue_name = u'{0:s} output event queue'.format(analysis_plugin.NAME) output_event_queue = zeromq_queue.ZeroMQPushBindQueue( name=queue_name, timeout_seconds=self._QUEUE_TIMEOUT) # Open the queue so it can bind to a random port, and we can get the # port number to use in the input queue. output_event_queue.Open() else: output_event_queue = multi_process_queue.MultiProcessingQueue( timeout=self._QUEUE_TIMEOUT) self._event_queues[analysis_plugin.NAME] = output_event_queue if self._use_zeromq: queue_name = u'{0:s} input event queue'.format(analysis_plugin.NAME) input_event_queue = zeromq_queue.ZeroMQPullConnectQueue( name=queue_name, delay_open=True, port=output_event_queue.port, timeout_seconds=self._QUEUE_TIMEOUT) else: input_event_queue = output_event_queue process = analysis_process.AnalysisProcess( input_event_queue, storage_writer, knowledge_base_object, analysis_plugin, data_location=data_location, event_filter_expression=event_filter_expression, name=analysis_plugin.plugin_name) process.start() logging.info(u'Started analysis plugin: {0:s} (PID: {1:d}).'.format( analysis_plugin.plugin_name, process.pid)) self._RegisterProcess(process) self._StartMonitoringProcess(process.pid) logging.info(u'Analysis plugins running')
def testQueueStart(self): """Tests that delayed creation of ZeroMQ sockets occurs correctly.""" push_queue = zeromq_queue.ZeroMQPushBindQueue(delay_start=True) pull_queue = zeromq_queue.ZeroMQPullBindQueue(delay_start=True) self.assertIsNone(push_queue._zmq_socket) self.assertIsNone(pull_queue._zmq_socket) push_queue.Start() self.assertIsNotNone(push_queue._zmq_socket) pull_queue.Start() self.assertIsNotNone(pull_queue._zmq_socket)
def testPushPullQueues(self): """Tests than an item can be transferred between push and pull queues.""" push_queue = zeromq_queue.ZeroMQPushBindQueue(delay_open=False) pull_queue = zeromq_queue.ZeroMQPullConnectQueue(delay_open=False, port=push_queue.port) self._testItemTransferred(push_queue, pull_queue) pull_queue = zeromq_queue.ZeroMQPullBindQueue(delay_open=False) push_queue = zeromq_queue.ZeroMQPushConnectQueue(delay_open=False, port=pull_queue.port) self._testItemTransferred(push_queue, pull_queue)
def testPushPullQueues(self): """Tests than an item can be transferred between push and pull queues.""" push_queue = zeromq_queue.ZeroMQPushBindQueue( name=u'pushpull_pushbind', delay_open=False, linger_seconds=1) pull_queue = zeromq_queue.ZeroMQPullConnectQueue( name=u'pushpull_pullconnect', delay_open=False, port=push_queue.port, linger_seconds=1) self._testItemTransferred(push_queue, pull_queue) push_queue.Close() pull_queue.Close() pull_queue = ZeroMQPullBindQueue( name=u'pushpull_pullbind', delay_open=False, linger_seconds=1) push_queue = ZeroMQPushConnectQueue( name=u'pushpull_pushconnect', delay_open=False, port=pull_queue.port, linger_seconds=1) self._testItemTransferred(push_queue, pull_queue) push_queue.Close() pull_queue.Close()
def GetAnalysisPluginsAndEventQueues(self, analysis_plugins_string): """Return a list of analysis plugins and event queues. Args: analysis_plugins_string: comma separated string with names of analysis plugins to load. Returns: A tuple of two lists, one containing list of analysis plugins and the other a list of event queues. """ if not analysis_plugins_string: return [], [] event_producers = [] # These are the queues analysis plugins will read from. analysis_plugin_input_queues = [] analysis_plugins_list = [ name.strip() for name in analysis_plugins_string.split(u',') ] for _ in range(0, len(analysis_plugins_list)): if self._use_zeromq: output_queue = zeromq_queue.ZeroMQPushBindQueue() # Open the queue so it can bind to a random port, and we can get the # port number to use in the input queue. output_queue.Open() queue_port = output_queue.port input_queue = zeromq_queue.ZeroMQPullConnectQueue( port=queue_port, delay_open=True) analysis_plugin_input_queues.append(input_queue) else: input_queue = multi_process.MultiProcessingQueue(timeout=5) analysis_plugin_input_queues.append(input_queue) output_queue = input_queue event_producers.append(queue.ItemQueueProducer(output_queue)) analysis_plugins = analysis_manager.AnalysisPluginManager.LoadPlugins( analysis_plugins_list, analysis_plugin_input_queues) analysis_plugins = list(analysis_plugins) return analysis_plugins, event_producers
def testMain(self): """Tests the _Main function.""" output_event_queue = zeromq_queue.ZeroMQPushBindQueue( name='test output event queue', timeout_seconds=self._QUEUE_TIMEOUT) output_event_queue.Open() input_event_queue = zeromq_queue.ZeroMQPullConnectQueue( name='test input event queue', delay_open=True, port=output_event_queue.port, timeout_seconds=self._QUEUE_TIMEOUT) session = sessions.Session() analysis_plugin = TestAnalysisPlugin() with shared_test_lib.TempDirectory() as temp_directory: # Set up the processed for the task storage file generated by the # analysis plugin. os.mkdir(os.path.join(temp_directory, 'processed')) configuration = configurations.ProcessingConfiguration() configuration.task_storage_path = temp_directory test_process = analysis_process.AnalysisProcess( input_event_queue, None, session, analysis_plugin, configuration, name='TestAnalysis') test_process._FOREMAN_STATUS_WAIT = 1 test_process.start() output_event_queue.PushItem(plaso_queue.QueueAbort(), block=False) output_event_queue.Close(abort=True) # Sleep for 1 second to allow the analysis process to terminate. # Before the temporary directory is removed. time.sleep(1)
def testSocketCreation(self): """Tests that ZeroMQ sockets are created when a new queue is created.""" push_queue = zeromq_queue.ZeroMQPushBindQueue(delay_start=False) pull_queue = zeromq_queue.ZeroMQPullBindQueue(delay_start=False) self.assertIsNotNone(push_queue._zmq_socket) self.assertIsNotNone(pull_queue._zmq_socket)