def __init__(self):
"""Initializes a psort multi-processing engine."""
super(PsortMultiProcessEngine, self).__init__()
self._analysis_plugins = {}
self._completed_analysis_processes = set()
self._data_location = None
self._event_filter_expression = None
self._event_queues = {}
self._event_tag_index = event_tag_index.EventTagIndex()
self._events_status = processing_status.EventsStatus()
# The export event heap is used to make sure the events are sorted in
# a deterministic way.
self._export_event_heap = PsortEventHeap()
self._export_event_timestamp = 0
self._knowledge_base = None
self._memory_profiler = None
self._merge_task = None
self._number_of_consumed_event_tags = 0
self._number_of_consumed_events = 0
self._number_of_consumed_reports = 0
self._number_of_consumed_sources = 0
self._number_of_consumed_warnings = 0
self._number_of_produced_event_tags = 0
self._number_of_produced_events = 0
self._number_of_produced_reports = 0
self._number_of_produced_sources = 0
self._number_of_produced_warnings = 0
self._processing_configuration = None
self._processing_profiler = None
self._serializers_profiler = None
self._status = definitions.STATUS_INDICATOR_IDLE
self._status_update_callback = None
self._worker_memory_limit = definitions.DEFAULT_WORKER_MEMORY_LIMIT
def testBuild(self):
"""Tests the _Build function."""
test_index = event_tag_index.EventTagIndex()
self.assertIsNone(test_index._index)
test_file = self._GetTestFilePath(['psort_test.plaso'])
storage_file = sqlite_file.SQLiteStorageFile()
storage_file.Open(path=test_file)
test_index._Build(storage_file)
storage_file.Close()
self.assertIsNotNone(test_index._index)
def __init__(self):
"""Initializes an output and formatting multi-processing engine."""
super(OutputAndFormattingMultiProcessEngine, self).__init__()
# The export event heap is used to make sure the events are sorted in
# a deterministic way.
self._event_tag_index = event_tag_index.EventTagIndex()
self._events_status = processing_status.EventsStatus()
self._export_event_heap = PsortEventHeap()
self._export_event_timestamp = 0
self._knowledge_base = None
self._number_of_consumed_events = 0
self._processing_configuration = None
self._status = definitions.STATUS_INDICATOR_IDLE
self._status_update_callback = None
def __init__(self, worker_memory_limit=None, worker_timeout=None):
"""Initializes a psort multi-processing engine.
Args:
worker_memory_limit (Optional[int]): maximum amount of memory a worker is
allowed to consume, where None represents the default memory limit
and 0 represents no limit.
worker_timeout (Optional[float]): number of minutes before a worker
process that is not providing status updates is considered inactive,
where None or 0.0 represents the default timeout.
"""
if worker_memory_limit is None:
worker_memory_limit = definitions.DEFAULT_WORKER_MEMORY_LIMIT
if not worker_timeout:
worker_timeout = definitions.DEFAULT_WORKER_TIMEOUT
super(PsortMultiProcessEngine, self).__init__()
self._analysis_plugins = {}
self._completed_analysis_processes = set()
self._data_location = None
self._event_filter_expression = None
self._event_queues = {}
self._event_tag_index = event_tag_index.EventTagIndex()
self._events_status = processing_status.EventsStatus()
# The export event heap is used to make sure the events are sorted in
# a deterministic way.
self._export_event_heap = PsortEventHeap()
self._export_event_timestamp = 0
self._knowledge_base = None
self._memory_profiler = None
self._merge_task = None
self._number_of_consumed_event_tags = 0
self._number_of_consumed_events = 0
self._number_of_consumed_reports = 0
self._number_of_consumed_sources = 0
self._number_of_consumed_warnings = 0
self._number_of_produced_event_tags = 0
self._number_of_produced_events = 0
self._number_of_produced_reports = 0
self._number_of_produced_sources = 0
self._number_of_produced_warnings = 0
self._processing_configuration = None
self._processing_profiler = None
self._serializers_profiler = None
self._status = definitions.STATUS_INDICATOR_IDLE
self._status_update_callback = None
self._worker_memory_limit = worker_memory_limit
self._worker_timeout = worker_timeout
def testBuild(self):
"""Tests the _Build function."""
test_index = event_tag_index.EventTagIndex()
self.assertIsNone(test_index._index)
test_file_path = self._GetTestFilePath(['psort_test.plaso'])
self._SkipIfPathNotExists(test_file_path)
storage_reader = sqlite_file_reader.SQLiteStorageFileReader(
test_file_path)
test_index._Build(storage_reader)
storage_reader.Close()
self.assertIsNotNone(test_index._index)
def __init__(self, use_zeromq=True):
"""Initializes an engine object.
Args:
use_zeromq (Optional[bool]): True if ZeroMQ should be used for queuing
instead of Python's multiprocessing queue.
"""
super(PsortMultiProcessEngine, self).__init__()
self._analysis_plugins = {}
self._completed_analysis_processes = set()
self._data_location = None
self._event_filter_expression = None
self._event_queues = {}
self._event_tag_index = event_tag_index.EventTagIndex()
self._events_status = processing_status.EventsStatus()
# The export event heap is used to make sure the events are sorted in
# a deterministic way.
self._export_event_heap = PsortEventHeap()
self._export_event_timestamp = 0
self._guppy_memory_profiler = None
self._knowledge_base = None
self._memory_profiler = None
self._merge_task = None
self._number_of_consumed_event_tags = 0
self._number_of_consumed_events = 0
self._number_of_consumed_reports = 0
self._number_of_consumed_sources = 0
self._number_of_consumed_warnings = 0
self._number_of_duplicate_events = 0
self._number_of_macb_grouped_events = 0
self._number_of_produced_event_tags = 0
self._number_of_produced_events = 0
self._number_of_produced_reports = 0
self._number_of_produced_sources = 0
self._number_of_produced_warnings = 0
self._processing_configuration = None
self._processing_profiler = None
self._serializers_profiler = None
self._status = definitions.PROCESSING_STATUS_IDLE
self._status_update_callback = None
self._use_zeromq = use_zeromq
self._worker_memory_limit = definitions.DEFAULT_WORKER_MEMORY_LIMIT
def testGetEventTagByIdentifier(self):
"""Tests the GetEventTagByIdentifier function."""
test_index = event_tag_index.EventTagIndex()
with shared_test_lib.TempDirectory() as temp_directory:
temp_file = os.path.join(temp_directory, 'storage.plaso')
self._CreateTestStorageFileWithTags(temp_file)
storage_reader = sqlite_file_reader.SQLiteStorageFileReader(
temp_file)
event_identifier = identifiers.SQLTableIdentifier('event', 1)
event_tag = test_index.GetEventTagByIdentifier(
storage_reader, event_identifier)
self.assertIsNotNone(event_tag)
event_identifier = identifiers.SQLTableIdentifier('event', 99)
event_tag = test_index.GetEventTagByIdentifier(
storage_reader, event_identifier)
self.assertIsNone(event_tag)
storage_reader.Close()
