def _StartWorkerProcess(self, process_name, storage_writer): """Creates, starts, monitors and registers a worker process. Args: process_name (str): process name. storage_writer (StorageWriter): storage writer for a session storage used to create task storage. Returns: MultiProcessWorkerProcess: extraction worker process or None on error. """ analysis_plugin = self._analysis_plugins.get(process_name, None) if not analysis_plugin: logger.error('Missing analysis plugin: {0:s}'.format(process_name)) return None queue_name = '{0:s} output event queue'.format(process_name) output_event_queue = zeromq_queue.ZeroMQPushBindQueue( name=queue_name, timeout_seconds=self._QUEUE_TIMEOUT) # Open the queue so it can bind to a random port, and we can get the # port number to use in the input queue. output_event_queue.Open() self._event_queues[process_name] = output_event_queue queue_name = '{0:s} input event queue'.format(process_name) input_event_queue = zeromq_queue.ZeroMQPullConnectQueue( name=queue_name, delay_open=True, port=output_event_queue.port, timeout_seconds=self._QUEUE_TIMEOUT) process = analysis_process.AnalysisProcess( input_event_queue, storage_writer, self._knowledge_base, analysis_plugin, self._processing_configuration, data_location=self._data_location, event_filter_expression=self._event_filter_expression, name=process_name) process.start() logger.info('Started analysis plugin: {0:s} (PID: {1:d}).'.format( process_name, process.pid)) try: self._StartMonitoringProcess(process) except (IOError, KeyError) as exception: logger.error( ('Unable to monitor analysis plugin: {0:s} (PID: {1:d}) ' 'with error: {2!s}').format(process_name, process.pid, exception)) process.terminate() return None self._RegisterProcess(process) return process
def testInitialization(self): """Tests the initialization.""" configuration = configurations.ProcessingConfiguration() test_process = analysis_process.AnalysisProcess( None, None, None, None, configuration, name='TestAnalysis') self.assertIsNotNone(test_process)
def testSignalAbort(self): """Tests the SignalAbort function.""" configuration = configurations.ProcessingConfiguration() test_process = analysis_process.AnalysisProcess( None, None, None, None, configuration, name='TestAnalysis') test_process.SignalAbort()
def testMain(self): """Tests the _Main function.""" output_event_queue = zeromq_queue.ZeroMQPushBindQueue( name='test output event queue', timeout_seconds=self._QUEUE_TIMEOUT) output_event_queue.Open() input_event_queue = zeromq_queue.ZeroMQPullConnectQueue( name='test input event queue', delay_open=True, port=output_event_queue.port, timeout_seconds=self._QUEUE_TIMEOUT) session = sessions.Session() storage_writer = self._CreateStorageWriter(session) analysis_plugin = TestAnalysisPlugin() configuration = configurations.ProcessingConfiguration() test_process = analysis_process.AnalysisProcess( input_event_queue, storage_writer, None, analysis_plugin, configuration, name='TestAnalysis') test_process._FOREMAN_STATUS_WAIT = 1 test_process.start() output_event_queue.PushItem(plaso_queue.QueueAbort(), block=False) output_event_queue.Close(abort=True)
def testSignalAbort(self): """Tests the SignalAbort function.""" test_process = analysis_process.AnalysisProcess(None, None, None, None, name=u'TestAnalysis') test_process.SignalAbort()
def testInitialization(self): """Tests the initialization.""" test_process = analysis_process.AnalysisProcess(None, None, None, None, name=u'TestAnalysis') self.assertIsNotNone(test_process)
def _StartAnalysisProcesses( self, knowledge_base_object, storage_writer, analysis_plugins, data_location, event_filter_expression=None): """Starts the analysis processes. Args: knowledge_base_object (KnowledgeBase): contains information from the source data needed for processing. storage_writer (StorageWriter): storage writer. analysis_plugins (list[AnalysisPlugin]): analysis plugins that should be run. data_location (str): path to the location that data files should be loaded from. event_filter_expression (Optional[str]): event filter expression. """ logging.info(u'Starting analysis plugins.') for analysis_plugin in analysis_plugins: if self._use_zeromq: queue_name = u'{0:s} output event queue'.format(analysis_plugin.NAME) output_event_queue = zeromq_queue.ZeroMQPushBindQueue( name=queue_name, timeout_seconds=self._QUEUE_TIMEOUT) # Open the queue so it can bind to a random port, and we can get the # port number to use in the input queue. output_event_queue.Open() else: output_event_queue = multi_process_queue.MultiProcessingQueue( timeout=self._QUEUE_TIMEOUT) self._event_queues[analysis_plugin.NAME] = output_event_queue if self._use_zeromq: queue_name = u'{0:s} input event queue'.format(analysis_plugin.NAME) input_event_queue = zeromq_queue.ZeroMQPullConnectQueue( name=queue_name, delay_open=True, port=output_event_queue.port, timeout_seconds=self._QUEUE_TIMEOUT) else: input_event_queue = output_event_queue process = analysis_process.AnalysisProcess( input_event_queue, storage_writer, knowledge_base_object, analysis_plugin, data_location=data_location, event_filter_expression=event_filter_expression, name=analysis_plugin.plugin_name) process.start() logging.info(u'Started analysis plugin: {0:s} (PID: {1:d}).'.format( analysis_plugin.plugin_name, process.pid)) self._RegisterProcess(process) self._StartMonitoringProcess(process.pid) logging.info(u'Analysis plugins running')
def testGetStatus(self): """Tests the _GetStatus function.""" configuration = configurations.ProcessingConfiguration() test_process = analysis_process.AnalysisProcess( None, None, None, None, configuration, name='TestAnalysis') status_attributes = test_process._GetStatus() self.assertIsNotNone(status_attributes) self.assertEqual(status_attributes['identifier'], 'TestAnalysis') self.assertIsNone(status_attributes['number_of_produced_reports'])
def testGetStatus(self): """Tests the _GetStatus function.""" test_process = analysis_process.AnalysisProcess(None, None, None, None, name=u'TestAnalysis') status_attributes = test_process._GetStatus() self.assertIsNotNone(status_attributes) self.assertEqual(status_attributes[u'identifier'], u'TestAnalysis') self.assertIsNone(status_attributes['number_of_produced_reports'])
def testMain(self): """Tests the _Main function.""" event_queue = multi_process_queue.MultiProcessingQueue(timeout=1) session = sessions.Session() storage_writer = self._CreateStorageWriter(session) analysis_plugin = TestAnalysisPlugin() configuration = configurations.ProcessingConfiguration() test_process = analysis_process.AnalysisProcess( event_queue, storage_writer, None, analysis_plugin, configuration, name='TestAnalysis') test_process._abort = True test_process._FOREMAN_STATUS_WAIT = 1 test_process._pid = 0 test_process._Main()