def testMain(self): """Tests the _Main function.""" output_task_queue = zeromq_queue.ZeroMQBufferedReplyBindQueue( delay_open=True, linger_seconds=0, maximum_items=1, name='test output task queue', timeout_seconds=self._QUEUE_TIMEOUT) output_task_queue.Open() input_task_queue = zeromq_queue.ZeroMQRequestConnectQueue( delay_open=True, linger_seconds=0, name='test input task queue', port=output_task_queue.port, timeout_seconds=self._QUEUE_TIMEOUT) configuration = configurations.ProcessingConfiguration() test_process = worker_process.WorkerProcess(input_task_queue, None, None, None, None, configuration, name='TestWorker') test_process.start() output_task_queue.PushItem(plaso_queue.QueueAbort(), block=False) output_task_queue.Close(abort=True)
def testRequestAndBufferedReplyQueues(self): """Tests REQ and buffered REP queue pairs.""" reply_queue = zeromq_queue.ZeroMQBufferedReplyBindQueue( name='requestbufferedreply_replybind', delay_open=False, linger_seconds=1) request_queue = zeromq_queue.ZeroMQRequestConnectQueue( name='requestbufferedreply_requestconnect', delay_open=False, port=reply_queue.port, linger_seconds=1) self._testItemTransferred(reply_queue, request_queue) reply_queue.Close() request_queue.Close() request_queue = ZeroMQRequestBindQueue( name='requestbufferedreply_requestbind', delay_open=False, linger_seconds=1) reply_queue = ZeroMQBufferedReplyConnectQueue( name='requestbufferedreply_replyconnect', delay_open=False, port=request_queue.port, linger_seconds=0) self._testItemTransferred(reply_queue, request_queue) reply_queue.Close() request_queue.Close()
def _StartWorkerProcess(self, process_name, storage_writer): """Creates, starts, monitors and registers a worker process. Args: process_name (str): process name. storage_writer (StorageWriter): storage writer for a session storage used to create task storage. Returns: MultiProcessWorkerProcess: extraction worker process or None if the process could not be started. """ process_name = 'Worker_{0:02d}'.format(self._last_worker_number) logger.debug('Starting worker process {0:s}'.format(process_name)) if self._use_zeromq: queue_name = '{0:s} task queue'.format(process_name) task_queue = zeromq_queue.ZeroMQRequestConnectQueue( delay_open=True, linger_seconds=0, name=queue_name, port=self._task_queue_port, timeout_seconds=self._TASK_QUEUE_TIMEOUT_SECONDS) else: task_queue = self._task_queue process = worker_process.WorkerProcess( task_queue, storage_writer, self.knowledge_base, self._session_identifier, self._processing_configuration, enable_sigsegv_handler=self._enable_sigsegv_handler, name=process_name) # Remove all possible log handlers to prevent a child process from logging # to the main process log file and garbling the log. The log handlers are # recreated after the worker process has been started. for handler in logging.root.handlers: logging.root.removeHandler(handler) handler.close() process.start() loggers.ConfigureLogging( debug_output=self._debug_output, filename=self._log_filename, mode='a', quiet_mode=self._quiet_mode) try: self._StartMonitoringProcess(process) except (IOError, KeyError) as exception: pid = process.pid logger.error(( 'Unable to monitor replacement worker process: {0:s} ' '(PID: {1:d}) with error: {2!s}').format( process_name, pid, exception)) self._TerminateProcess(process) return None self._RegisterProcess(process) self._last_worker_number += 1 return process
def testRequestAndBufferedReplyQueues(self): """Tests REQ and buffered REP queue pairs.""" reply_queue = zeromq_queue.ZeroMQBufferedReplyBindQueue( delay_open=False) request_queue = zeromq_queue.ZeroMQRequestConnectQueue( delay_open=False, port=reply_queue.port) self._testItemTransferred(reply_queue, request_queue) request_queue = zeromq_queue.ZeroMQRequestBindQueue(delay_open=False) reply_queue = zeromq_queue.ZeroMQBufferedReplyConnectQueue( delay_open=False, port=request_queue.port) self._testItemTransferred(reply_queue, request_queue)
def _StartWorkerProcess(self, process_name, storage_writer): """Creates, starts, monitors and registers a worker process. Args: process_name (str): process name. storage_writer (StorageWriter): storage writer for a session storage used to create task storage. Returns: MultiProcessWorkerProcess: extraction worker process or None if the process could not be started. """ process_name = 'Worker_{0:02d}'.format(self._last_worker_number) logging.debug('Starting worker process {0:s}'.format(process_name)) if self._use_zeromq: queue_name = '{0:s} task queue'.format(process_name) task_queue = zeromq_queue.ZeroMQRequestConnectQueue( delay_open=True, linger_seconds=0, name=queue_name, port=self._task_queue_port, timeout_seconds=self._TASK_QUEUE_TIMEOUT_SECONDS) else: task_queue = self._task_queue process = worker_process.WorkerProcess( task_queue, storage_writer, self.knowledge_base, self._session_identifier, self._processing_configuration, enable_sigsegv_handler=self._enable_sigsegv_handler, name=process_name) process.start() try: self._StartMonitoringProcess(process) except (IOError, KeyError) as exception: pid = process.pid logging.error(( 'Unable to monitor replacement worker process: {0:s} ' '(PID: {1:d}) with error: {2!s}').format( process_name, pid, exception)) self._TerminateProcess(process) return self._RegisterProcess(process) self._last_worker_number += 1 return process
def _StartExtractionWorkerProcess(self, storage_writer): """Creates, starts and registers an extraction worker process. Args: storage_writer (StorageWriter): storage writer for a session storage used to create task storage. Returns: MultiProcessWorkerProcess: extraction worker process. """ process_name = u'Worker_{0:02d}'.format(self._last_worker_number) logging.debug(u'Starting worker process {0:s}'.format(process_name)) if self._use_zeromq: task_queue = zeromq_queue.ZeroMQRequestConnectQueue( delay_open=True, name=u'{0:s} task queue'.format(process_name), linger_seconds=0, port=self._task_queue_port, timeout_seconds=self._TASK_QUEUE_TIMEOUT_SECONDS) else: task_queue = self._task_queue process = worker_process.WorkerProcess( task_queue, storage_writer, self.knowledge_base, self._session_identifier, debug_output=self._debug_output, enable_profiling=self._enable_profiling, enable_sigsegv_handler=self._enable_sigsegv_handler, filter_object=self._filter_object, hasher_names_string=self._hasher_names_string, mount_path=self._mount_path, name=process_name, parser_filter_expression=self._parser_filter_expression, preferred_year=self._preferred_year, process_archives=self._process_archives, process_compressed_streams=self._process_compressed_streams, profiling_directory=self._profiling_directory, profiling_sample_rate=self._profiling_sample_rate, profiling_type=self._profiling_type, temporary_directory=self._temporary_directory, text_prepend=self._text_prepend, yara_rules_string=self._yara_rules_string) process.start() self._last_worker_number += 1 self._RegisterProcess(process) return process
def _StartWorkerProcess(self, storage_writer): """Creates, starts and registers a worker process. Args: storage_writer (StorageWriter): storage writer for a session storage used to create task storage. Returns: MultiProcessWorkerProcess: extraction worker process. """ process_name = u'Worker_{0:02d}'.format(self._last_worker_number) logging.debug(u'Starting worker process {0:s}'.format(process_name)) if self._use_zeromq: task_queue = zeromq_queue.ZeroMQRequestConnectQueue( delay_open=True, name=u'{0:s} task queue'.format(process_name), linger_seconds=0, port=self._task_queue_port, timeout_seconds=self._TASK_QUEUE_TIMEOUT_SECONDS) else: task_queue = self._task_queue process = worker_process.WorkerProcess( task_queue, storage_writer, self.knowledge_base, self._session_identifier, self._processing_configuration, enable_sigsegv_handler=self._enable_sigsegv_handler, name=process_name) process.start() self._last_worker_number += 1 self._RegisterProcess(process) return process