def testGetStatus(self): """Tests the _GetStatus function.""" test_process = worker_process.WorkerProcess(None, None, None, None, None, name='TestWorker') status_attributes = test_process._GetStatus() self.assertIsNotNone(status_attributes) self.assertEqual(status_attributes['identifier'], 'TestWorker') self.assertEqual(status_attributes['last_activity_timestamp'], 0.0) self.assertIsNone(status_attributes['number_of_produced_errors']) session = sessions.Session() storage_writer = self._CreateStorageWriter(session) knowledge_base = self._CreateKnowledgeBase() test_process._parser_mediator = self._CreateParserMediator( storage_writer, knowledge_base) status_attributes = test_process._GetStatus() self.assertIsNotNone(status_attributes) self.assertEqual(status_attributes['identifier'], 'TestWorker') self.assertEqual(status_attributes['last_activity_timestamp'], 0.0) self.assertEqual(status_attributes['number_of_produced_errors'], 0)
def testProcessPathSpec(self): """Tests the _ProcessPathSpec function.""" configuration = configurations.ProcessingConfiguration() test_process = worker_process.WorkerProcess( None, None, None, None, None, configuration, name='TestWorker') session = sessions.Session() storage_writer = self._CreateStorageWriter(session) knowledge_base = self._CreateKnowledgeBase() parser_mediator = self._CreateParserMediator(storage_writer, knowledge_base) path_spec = fake_path_spec.FakePathSpec(location='/test/file') extraction_worker = TestEventExtractionWorker() test_process._ProcessPathSpec(extraction_worker, parser_mediator, path_spec) self.assertEqual(parser_mediator._number_of_warnings, 0) extraction_worker = TestFailureEventExtractionWorker() test_process._ProcessPathSpec(extraction_worker, parser_mediator, path_spec) self.assertEqual(parser_mediator._number_of_warnings, 0) self.assertTrue(test_process._abort) test_process._ProcessPathSpec(None, parser_mediator, path_spec) self.assertEqual(parser_mediator._number_of_warnings, 1)
def testMain(self): """Tests the _Main function.""" output_task_queue = zeromq_queue.ZeroMQBufferedReplyBindQueue( delay_open=True, linger_seconds=0, maximum_items=1, name='test output task queue', timeout_seconds=self._QUEUE_TIMEOUT) output_task_queue.Open() input_task_queue = zeromq_queue.ZeroMQRequestConnectQueue( delay_open=True, linger_seconds=0, name='test input task queue', port=output_task_queue.port, timeout_seconds=self._QUEUE_TIMEOUT) configuration = configurations.ProcessingConfiguration() test_process = worker_process.WorkerProcess(input_task_queue, None, None, None, None, configuration, name='TestWorker') test_process.start() output_task_queue.PushItem(plaso_queue.QueueAbort(), block=False) output_task_queue.Close(abort=True)
def _StartWorkerProcess(self, process_name, storage_writer): """Creates, starts, monitors and registers a worker process. Args: process_name (str): process name. storage_writer (StorageWriter): storage writer for a session storage used to create task storage. Returns: MultiProcessWorkerProcess: extraction worker process or None if the process could not be started. """ process_name = 'Worker_{0:02d}'.format(self._last_worker_number) logger.debug('Starting worker process {0:s}'.format(process_name)) if self._use_zeromq: queue_name = '{0:s} task queue'.format(process_name) task_queue = zeromq_queue.ZeroMQRequestConnectQueue( delay_open=True, linger_seconds=0, name=queue_name, port=self._task_queue_port, timeout_seconds=self._TASK_QUEUE_TIMEOUT_SECONDS) else: task_queue = self._task_queue process = worker_process.WorkerProcess( task_queue, storage_writer, self.knowledge_base, self._session_identifier, self._processing_configuration, enable_sigsegv_handler=self._enable_sigsegv_handler, name=process_name) # Remove all possible log handlers to prevent a child process from logging # to the main process log file and garbling the log. The log handlers are # recreated after the worker process has been started. for handler in logging.root.handlers: logging.root.removeHandler(handler) handler.close() process.start() loggers.ConfigureLogging( debug_output=self._debug_output, filename=self._log_filename, mode='a', quiet_mode=self._quiet_mode) try: self._StartMonitoringProcess(process) except (IOError, KeyError) as exception: pid = process.pid logger.error(( 'Unable to monitor replacement worker process: {0:s} ' '(PID: {1:d}) with error: {2!s}').format( process_name, pid, exception)) self._TerminateProcess(process) return None self._RegisterProcess(process) self._last_worker_number += 1 return process
def testSignalAbort(self): """Tests the SignalAbort function.""" test_process = worker_process.WorkerProcess(None, None, None, None, name=u'TestWorker') test_process.SignalAbort()
def testInitialization(self): """Tests the initialization.""" test_process = worker_process.WorkerProcess(None, None, None, None, name=u'TestWorker') self.assertIsNotNone(test_process)
def testGetStatus(self): """Tests the _GetStatus function.""" test_process = worker_process.WorkerProcess(None, None, None, None, name=u'TestWorker') status_attributes = test_process._GetStatus() self.assertIsNotNone(status_attributes) self.assertEqual(status_attributes[u'identifier'], u'TestWorker')
def testMain(self): """Tests the _Main function.""" task_queue = multi_process_queue.MultiProcessingQueue(timeout=1) configuration = configurations.ProcessingConfiguration() test_process = worker_process.WorkerProcess( task_queue, None, None, None, None, configuration, name='TestWorker') test_process._abort = True test_process._pid = 0 test_process._Main()
def _StartWorkerProcess(self, process_name, storage_writer): """Creates, starts, monitors and registers a worker process. Args: process_name (str): process name. storage_writer (StorageWriter): storage writer for a session storage used to create task storage. Returns: MultiProcessWorkerProcess: extraction worker process or None if the process could not be started. """ process_name = 'Worker_{0:02d}'.format(self._last_worker_number) logging.debug('Starting worker process {0:s}'.format(process_name)) if self._use_zeromq: queue_name = '{0:s} task queue'.format(process_name) task_queue = zeromq_queue.ZeroMQRequestConnectQueue( delay_open=True, linger_seconds=0, name=queue_name, port=self._task_queue_port, timeout_seconds=self._TASK_QUEUE_TIMEOUT_SECONDS) else: task_queue = self._task_queue process = worker_process.WorkerProcess( task_queue, storage_writer, self.knowledge_base, self._session_identifier, self._processing_configuration, enable_sigsegv_handler=self._enable_sigsegv_handler, name=process_name) process.start() try: self._StartMonitoringProcess(process) except (IOError, KeyError) as exception: pid = process.pid logging.error(( 'Unable to monitor replacement worker process: {0:s} ' '(PID: {1:d}) with error: {2!s}').format( process_name, pid, exception)) self._TerminateProcess(process) return self._RegisterProcess(process) self._last_worker_number += 1 return process
def testProcessTask(self): """Tests the _ProcessTask function.""" session = sessions.Session() storage_writer = self._CreateStorageWriter(session) knowledge_base = self._CreateKnowledgeBase() configuration = configurations.ProcessingConfiguration() test_process = worker_process.WorkerProcess( None, storage_writer, knowledge_base, session.identifier, configuration, name='TestWorker') test_process._parser_mediator = self._CreateParserMediator( storage_writer, knowledge_base) task = tasks.Task(session_identifier=session.identifier) test_process._ProcessTask(task)
def testStartAndStopProfiling(self): """Tests the _StartProfiling and _StopProfiling functions.""" with shared_test_lib.TempDirectory() as temp_directory: configuration = configurations.ProcessingConfiguration() configuration.profiling.directory = temp_directory configuration.profiling.profilers = set([ 'memory', 'parsers', 'processing', 'serializers', 'storage', 'task_queue']) test_process = worker_process.WorkerProcess( None, None, None, None, None, configuration, name='TestWorker') test_process._extraction_worker = TestEventExtractionWorker() test_process._StartProfiling(None) test_process._StartProfiling(configuration.profiling) test_process._StopProfiling()
def _StartExtractionWorkerProcess(self, storage_writer): """Creates, starts and registers an extraction worker process. Args: storage_writer (StorageWriter): storage writer for a session storage used to create task storage. Returns: MultiProcessWorkerProcess: extraction worker process. """ process_name = u'Worker_{0:02d}'.format(self._last_worker_number) logging.debug(u'Starting worker process {0:s}'.format(process_name)) if self._use_zeromq: task_queue = zeromq_queue.ZeroMQRequestConnectQueue( delay_open=True, name=u'{0:s} task queue'.format(process_name), linger_seconds=0, port=self._task_queue_port, timeout_seconds=self._TASK_QUEUE_TIMEOUT_SECONDS) else: task_queue = self._task_queue process = worker_process.WorkerProcess( task_queue, storage_writer, self.knowledge_base, self._session_identifier, debug_output=self._debug_output, enable_profiling=self._enable_profiling, enable_sigsegv_handler=self._enable_sigsegv_handler, filter_object=self._filter_object, hasher_names_string=self._hasher_names_string, mount_path=self._mount_path, name=process_name, parser_filter_expression=self._parser_filter_expression, preferred_year=self._preferred_year, process_archives=self._process_archives, process_compressed_streams=self._process_compressed_streams, profiling_directory=self._profiling_directory, profiling_sample_rate=self._profiling_sample_rate, profiling_type=self._profiling_type, temporary_directory=self._temporary_directory, text_prepend=self._text_prepend, yara_rules_string=self._yara_rules_string) process.start() self._last_worker_number += 1 self._RegisterProcess(process) return process
def _StartWorkerProcess(self, storage_writer): """Creates, starts and registers a worker process. Args: storage_writer (StorageWriter): storage writer for a session storage used to create task storage. Returns: MultiProcessWorkerProcess: extraction worker process. """ process_name = u'Worker_{0:02d}'.format(self._last_worker_number) logging.debug(u'Starting worker process {0:s}'.format(process_name)) if self._use_zeromq: task_queue = zeromq_queue.ZeroMQRequestConnectQueue( delay_open=True, name=u'{0:s} task queue'.format(process_name), linger_seconds=0, port=self._task_queue_port, timeout_seconds=self._TASK_QUEUE_TIMEOUT_SECONDS) else: task_queue = self._task_queue process = worker_process.WorkerProcess( task_queue, storage_writer, self.knowledge_base, self._session_identifier, self._processing_configuration, enable_sigsegv_handler=self._enable_sigsegv_handler, name=process_name) process.start() self._last_worker_number += 1 self._RegisterProcess(process) return process