def __init__(self): """Initializes a psort multi-processing engine.""" super(PsortMultiProcessEngine, self).__init__() self._analysis_plugins = {} self._completed_analysis_processes = set() self._data_location = None self._event_filter_expression = None self._event_queues = {} self._event_tag_index = event_tag_index.EventTagIndex() self._events_status = processing_status.EventsStatus() # The export event heap is used to make sure the events are sorted in # a deterministic way. self._export_event_heap = PsortEventHeap() self._export_event_timestamp = 0 self._knowledge_base = None self._memory_profiler = None self._merge_task = None self._number_of_consumed_event_tags = 0 self._number_of_consumed_events = 0 self._number_of_consumed_reports = 0 self._number_of_consumed_sources = 0 self._number_of_consumed_warnings = 0 self._number_of_produced_event_tags = 0 self._number_of_produced_events = 0 self._number_of_produced_reports = 0 self._number_of_produced_sources = 0 self._number_of_produced_warnings = 0 self._processing_configuration = None self._processing_profiler = None self._serializers_profiler = None self._status = definitions.STATUS_INDICATOR_IDLE self._status_update_callback = None self._worker_memory_limit = definitions.DEFAULT_WORKER_MEMORY_LIMIT
def testBuild(self): """Tests the _Build function.""" test_index = event_tag_index.EventTagIndex() self.assertIsNone(test_index._index) test_file = self._GetTestFilePath(['psort_test.plaso']) storage_file = sqlite_file.SQLiteStorageFile() storage_file.Open(path=test_file) test_index._Build(storage_file) storage_file.Close() self.assertIsNotNone(test_index._index)
def __init__(self): """Initializes an output and formatting multi-processing engine.""" super(OutputAndFormattingMultiProcessEngine, self).__init__() # The export event heap is used to make sure the events are sorted in # a deterministic way. self._event_tag_index = event_tag_index.EventTagIndex() self._events_status = processing_status.EventsStatus() self._export_event_heap = PsortEventHeap() self._export_event_timestamp = 0 self._knowledge_base = None self._number_of_consumed_events = 0 self._processing_configuration = None self._status = definitions.STATUS_INDICATOR_IDLE self._status_update_callback = None
def __init__(self, worker_memory_limit=None, worker_timeout=None): """Initializes a psort multi-processing engine. Args: worker_memory_limit (Optional[int]): maximum amount of memory a worker is allowed to consume, where None represents the default memory limit and 0 represents no limit. worker_timeout (Optional[float]): number of minutes before a worker process that is not providing status updates is considered inactive, where None or 0.0 represents the default timeout. """ if worker_memory_limit is None: worker_memory_limit = definitions.DEFAULT_WORKER_MEMORY_LIMIT if not worker_timeout: worker_timeout = definitions.DEFAULT_WORKER_TIMEOUT super(PsortMultiProcessEngine, self).__init__() self._analysis_plugins = {} self._completed_analysis_processes = set() self._data_location = None self._event_filter_expression = None self._event_queues = {} self._event_tag_index = event_tag_index.EventTagIndex() self._events_status = processing_status.EventsStatus() # The export event heap is used to make sure the events are sorted in # a deterministic way. self._export_event_heap = PsortEventHeap() self._export_event_timestamp = 0 self._knowledge_base = None self._memory_profiler = None self._merge_task = None self._number_of_consumed_event_tags = 0 self._number_of_consumed_events = 0 self._number_of_consumed_reports = 0 self._number_of_consumed_sources = 0 self._number_of_consumed_warnings = 0 self._number_of_produced_event_tags = 0 self._number_of_produced_events = 0 self._number_of_produced_reports = 0 self._number_of_produced_sources = 0 self._number_of_produced_warnings = 0 self._processing_configuration = None self._processing_profiler = None self._serializers_profiler = None self._status = definitions.STATUS_INDICATOR_IDLE self._status_update_callback = None self._worker_memory_limit = worker_memory_limit self._worker_timeout = worker_timeout
def testBuild(self): """Tests the _Build function.""" test_index = event_tag_index.EventTagIndex() self.assertIsNone(test_index._index) test_file_path = self._GetTestFilePath(['psort_test.plaso']) self._SkipIfPathNotExists(test_file_path) storage_reader = sqlite_file_reader.SQLiteStorageFileReader( test_file_path) test_index._Build(storage_reader) storage_reader.Close() self.assertIsNotNone(test_index._index)
def __init__(self, use_zeromq=True): """Initializes an engine object. Args: use_zeromq (Optional[bool]): True if ZeroMQ should be used for queuing instead of Python's multiprocessing queue. """ super(PsortMultiProcessEngine, self).__init__() self._analysis_plugins = {} self._completed_analysis_processes = set() self._data_location = None self._event_filter_expression = None self._event_queues = {} self._event_tag_index = event_tag_index.EventTagIndex() self._events_status = processing_status.EventsStatus() # The export event heap is used to make sure the events are sorted in # a deterministic way. self._export_event_heap = PsortEventHeap() self._export_event_timestamp = 0 self._guppy_memory_profiler = None self._knowledge_base = None self._memory_profiler = None self._merge_task = None self._number_of_consumed_event_tags = 0 self._number_of_consumed_events = 0 self._number_of_consumed_reports = 0 self._number_of_consumed_sources = 0 self._number_of_consumed_warnings = 0 self._number_of_duplicate_events = 0 self._number_of_macb_grouped_events = 0 self._number_of_produced_event_tags = 0 self._number_of_produced_events = 0 self._number_of_produced_reports = 0 self._number_of_produced_sources = 0 self._number_of_produced_warnings = 0 self._processing_configuration = None self._processing_profiler = None self._serializers_profiler = None self._status = definitions.PROCESSING_STATUS_IDLE self._status_update_callback = None self._use_zeromq = use_zeromq self._worker_memory_limit = definitions.DEFAULT_WORKER_MEMORY_LIMIT
def testGetEventTagByIdentifier(self): """Tests the GetEventTagByIdentifier function.""" test_index = event_tag_index.EventTagIndex() with shared_test_lib.TempDirectory() as temp_directory: temp_file = os.path.join(temp_directory, 'storage.plaso') self._CreateTestStorageFileWithTags(temp_file) storage_reader = sqlite_file_reader.SQLiteStorageFileReader( temp_file) event_identifier = identifiers.SQLTableIdentifier('event', 1) event_tag = test_index.GetEventTagByIdentifier( storage_reader, event_identifier) self.assertIsNotNone(event_tag) event_identifier = identifiers.SQLTableIdentifier('event', 99) event_tag = test_index.GetEventTagByIdentifier( storage_reader, event_identifier) self.assertIsNone(event_tag) storage_reader.Close()