def testGetSortedEntry(self):
"""Tests the GetSortedEntry function."""
test_file = self._GetTestFilePath([u'psort_test.json.plaso'])
storage_file = zip_file.StorageFile(test_file, read_only=True)
expected_timestamp = 1453449153000000
event_object = storage_file.GetSortedEntry()
self.assertEqual(event_object.timestamp, expected_timestamp)
# Test lower bound time range filter.
test_time_range = time_range.TimeRange(
timelib.Timestamp.CopyFromString(u'2016-04-30 06:41:49'),
timelib.Timestamp.CopyFromString(u'2030-12-31 23:59:59'))
storage_file.Close()
storage_file = zip_file.StorageFile(test_file, read_only=True)
expected_timestamp = 1462105168000000
event_object = storage_file.GetSortedEntry(time_range=test_time_range)
self.assertEqual(event_object.timestamp, expected_timestamp)
# Test upper bound time range filter.
test_time_range = time_range.TimeRange(
timelib.Timestamp.CopyFromString(u'2000-01-01 00:00:00'),
timelib.Timestamp.CopyFromString(u'2016-04-30 06:41:49'))
storage_file.Close()
storage_file = zip_file.StorageFile(test_file, read_only=True)
expected_timestamp = 1453449153000000
event_object = storage_file.GetSortedEntry(time_range=test_time_range)
self.assertEqual(event_object.timestamp, expected_timestamp)
storage_file.Close()
def testReadEntries(self):
"""Ensure returned EventObjects from the storage are within time bounds."""
storage_file_path = self._GetTestFilePath([u'psort_test.proto.plaso'])
storage_file = storage_zip_file.StorageFile(storage_file_path,
read_only=True)
time_range = storage_time_range.TimeRange(self._start_timestamp,
self._end_timestamp)
timestamp_list = []
with storage_zip_file.ZIPStorageFileReader(
storage_file) as storage_reader:
for event_object in storage_reader.GetEvents(
time_range=time_range):
timestamp_list.append(event_object.timestamp)
self.assertEqual(len(timestamp_list), 15)
self.assertEqual(timestamp_list[0], self._start_timestamp)
self.assertEqual(timestamp_list[-1], self._end_timestamp)
def _ExportEvents(
self, storage_reader, output_module, deduplicate_events=True,
event_filter=None, time_slice=None, use_time_slicer=False):
"""Exports events using an output module.
Args:
storage_reader (StorageReader): storage reader.
output_module (OutputModule): output module.
deduplicate_events (Optional[bool]): True if events should be
deduplicated.
event_filter (Optional[EventObjectFilter]): event filter.
time_slice (Optional[TimeRange]): time range that defines a time slice
to filter events.
use_time_slicer (Optional[bool]): True if the 'time slicer' should be
used. The 'time slicer' will provide a context of events around
an event of interest.
"""
self._status = definitions.STATUS_INDICATOR_EXPORTING
time_slice_buffer = None
time_slice_range = None
if time_slice:
if time_slice.event_timestamp is not None:
time_slice_range = storage_time_range.TimeRange(
time_slice.start_timestamp, time_slice.end_timestamp)
if use_time_slicer:
time_slice_buffer = bufferlib.CircularBuffer(time_slice.duration)
filter_limit = getattr(event_filter, 'limit', None)
forward_entries = 0
self._events_status.number_of_filtered_events = 0
self._events_status.number_of_events_from_time_slice = 0
for event in storage_reader.GetSortedEvents(time_range=time_slice_range):
event_data_identifier = event.GetEventDataIdentifier()
event_data = storage_reader.GetEventDataByIdentifier(
event_data_identifier)
event_data_stream_identifier = event_data.GetEventDataStreamIdentifier()
if event_data_stream_identifier:
event_data_stream = storage_reader.GetEventDataStreamByIdentifier(
event_data_stream_identifier)
else:
event_data_stream = None
event_identifier = event.GetIdentifier()
event_tag = self._event_tag_index.GetEventTagByIdentifier(
storage_reader, event_identifier)
if time_slice_range and event.timestamp != time_slice.event_timestamp:
self._events_status.number_of_events_from_time_slice += 1
if event_filter:
filter_match = event_filter.Match(
event, event_data, event_data_stream, event_tag)
else:
filter_match = None
# pylint: disable=singleton-comparison
if filter_match == False:
if not time_slice_buffer:
self._events_status.number_of_filtered_events += 1
elif forward_entries == 0:
time_slice_buffer.Append((event, event_data))
self._events_status.number_of_filtered_events += 1
elif forward_entries <= time_slice_buffer.size:
self._ExportEvent(
storage_reader, output_module, event, event_data,
event_data_stream, deduplicate_events=deduplicate_events)
self._number_of_consumed_events += 1
self._events_status.number_of_events_from_time_slice += 1
forward_entries += 1
else:
# We reached the maximum size of the time slice and don't need to
# include other entries.
self._events_status.number_of_filtered_events += 1
forward_entries = 0
else:
# pylint: disable=singleton-comparison
if filter_match == True and time_slice_buffer:
# Empty the time slice buffer.
for event_in_buffer, event_data_in_buffer in (
time_slice_buffer.Flush()):
self._ExportEvent(
storage_reader, output_module, event_in_buffer,
event_data_in_buffer, event_data_stream,
deduplicate_events=deduplicate_events)
self._number_of_consumed_events += 1
self._events_status.number_of_filtered_events += 1
self._events_status.number_of_events_from_time_slice += 1
forward_entries = 1
self._ExportEvent(
storage_reader, output_module, event, event_data, event_data_stream,
deduplicate_events=deduplicate_events)
self._number_of_consumed_events += 1
# pylint: disable=singleton-comparison
if (filter_match == True and filter_limit and
filter_limit == self._number_of_consumed_events):
break
self._FlushExportBuffer(storage_reader, output_module)
def _ExportEvents(self,
storage_reader,
output_module,
deduplicate_events=True,
event_filter=None,
time_slice=None,
use_time_slicer=False):
"""Exports events using an output module.
Args:
storage_reader (StorageReader): storage reader.
output_module (OutputModule): output module.
deduplicate_events (Optional[bool]): True if events should be
deduplicated.
event_filter (Optional[FilterObject]): event filter.
time_slice (Optional[TimeRange]): time range that defines a time slice
to filter events.
use_time_slicer (Optional[bool]): True if the 'time slicer' should be
used. The 'time slicer' will provide a context of events around
an event of interest.
Returns:
collections.Counter: counter that tracks the number of unique events
read from storage.
"""
self._status = definitions.PROCESSING_STATUS_EXPORTING
time_slice_buffer = None
time_slice_range = None
if time_slice:
if time_slice.event_timestamp is not None:
time_slice_range = storage_time_range.TimeRange(
time_slice.start_timestamp, time_slice.end_timestamp)
if use_time_slicer:
time_slice_buffer = bufferlib.CircularBuffer(
time_slice.duration)
filter_limit = getattr(event_filter, 'limit', None)
forward_entries = 0
number_of_filtered_events = 0
number_of_events_from_time_slice = 0
for event in storage_reader.GetSortedEvents(
time_range=time_slice_range):
event_data_identifier = event.GetEventDataIdentifier()
if event_data_identifier:
event_data = storage_reader.GetEventDataByIdentifier(
event_data_identifier)
if event_data:
for attribute_name, attribute_value in event_data.GetAttributes(
):
setattr(event, attribute_name, attribute_value)
event_identifier = event.GetIdentifier()
event.tag = self._event_tag_index.GetEventTagByIdentifier(
storage_reader, event_identifier)
if time_slice_range and event.timestamp != time_slice.event_timestamp:
number_of_events_from_time_slice += 1
if event_filter:
filter_match = event_filter.Match(event)
else:
filter_match = None
# pylint: disable=singleton-comparison
if filter_match == False:
if not time_slice_buffer:
number_of_filtered_events += 1
elif forward_entries == 0:
time_slice_buffer.Append(event)
number_of_filtered_events += 1
elif forward_entries <= time_slice_buffer.size:
self._ExportEvent(output_module,
event,
deduplicate_events=deduplicate_events)
self._number_of_consumed_events += 1
number_of_events_from_time_slice += 1
forward_entries += 1
else:
# We reached the maximum size of the time slice and don't need to
# include other entries.
number_of_filtered_events += 1
forward_entries = 0
else:
# pylint: disable=singleton-comparison
if filter_match == True and time_slice_buffer:
# Empty the time slice buffer.
for event_in_buffer in time_slice_buffer.Flush():
self._ExportEvent(
output_module,
event_in_buffer,
deduplicate_events=deduplicate_events)
self._number_of_consumed_events += 1
number_of_filtered_events += 1
number_of_events_from_time_slice += 1
forward_entries = 1
self._ExportEvent(output_module,
event,
deduplicate_events=deduplicate_events)
self._number_of_consumed_events += 1
# pylint: disable=singleton-comparison
if (filter_match == True and filter_limit
and filter_limit == self._number_of_consumed_events):
break
self._FlushExportBuffer(output_module)
events_counter = collections.Counter()
events_counter['Events filtered'] = number_of_filtered_events
events_counter[
'Events from time slice'] = number_of_events_from_time_slice
events_counter['Events processed'] = self._number_of_consumed_events
if self._number_of_duplicate_events:
events_counter['Duplicate events removed'] = (
self._number_of_duplicate_events)
if self._number_of_macb_grouped_events:
events_counter['Events MACB grouped'] = (
self._number_of_macb_grouped_events)
if filter_limit:
events_counter['Limited By'] = filter_limit
return events_counter
def ProcessStorage(self,
output_module,
storage_file,
storage_file_path,
analysis_plugins,
event_queue_producers,
command_line_arguments=None,
deduplicate_events=True,
preferred_encoding=u'utf-8',
time_slice=None,
use_time_slicer=False):
"""Processes a plaso storage file.
Args:
output_module: an output module (instance of OutputModule).
storage_file: the storage file object (instance of StorageFile).
storage_file_path: string containing the path of the storage file.
analysis_plugins: list of analysis plugin objects (instance of
AnalysisPlugin).
event_queue_producers: list of event queue producer objects (instance
of ItemQueueProducer).
command_line_arguments: optional string of the command line arguments or
None if not set.
deduplicate_events: optional boolean value to indicate if the event
objects should be deduplicated.
preferred_encoding: optional preferred encoding.
time_slice: optional time slice object (instance of TimeSlice).
use_time_slicer: optional boolean value to indicate the 'time slicer'
should be used. The 'time slicer' will provide a
context of events around an event of interest.
Returns:
A counter (an instance of collections.Counter) that tracks the number of
events extracted from storage, and the analysis plugin results.
Raises:
RuntimeError: if a non-recoverable situation is encountered.
"""
time_slice = None
if time_slice:
if time_slice.event_timestamp is not None:
time_slice = storage_time_range.TimeRange(
time_slice.start_timestamp, time_slice.end_timestamp)
elif use_time_slicer:
self._filter_buffer = bufferlib.CircularBuffer(
time_slice.duration)
with storage_file:
# TODO: allow for single processing.
# TODO: add upper queue limit.
analysis_queue_port = None
if self._use_zeromq:
analysis_report_incoming_queue = zeromq_queue.ZeroMQPullBindQueue(
delay_open=False, port=None, linger_seconds=5)
analysis_queue_port = analysis_report_incoming_queue.port
else:
analysis_report_incoming_queue = multi_process.MultiProcessingQueue(
timeout=5)
pre_obj = self._GetLastGoodPreprocess(storage_file)
if pre_obj is None:
pre_obj = event.PreprocessObject()
if analysis_plugins:
self._StartAnalysisPlugins(
storage_file_path,
analysis_plugins,
pre_obj,
analysis_queue_port=analysis_queue_port,
analysis_report_incoming_queue=
analysis_report_incoming_queue,
command_line_arguments=command_line_arguments)
# Assign the preprocessing object to the storage.
# This is normally done in the construction of the storage object,
# however we cannot do that here since the preprocessing object is
# stored inside the storage file, so we need to open it first to
# be able to read it in, before we make changes to it. Thus we need
# to access this protected member of the class.
# pylint: disable=protected-access
storage_file._pre_obj = pre_obj
else:
event_queue_producers = []
output_buffer = output_event_buffer.EventBuffer(
output_module, deduplicate_events)
with output_buffer:
counter = self.ProcessEventsFromStorage(
storage_file,
output_buffer,
analysis_queues=event_queue_producers,
filter_buffer=self._filter_buffer,
my_filter=self._filter_object,
time_slice=time_slice)
for information in storage_file.GetStorageInformation():
if hasattr(information, u'counter'):
counter[u'Stored Events'] += information.counter[u'total']
if not self._quiet_mode:
logging.info(u'Output processing is done.')
# Get all reports and tags from analysis plugins.
self._ProcessAnalysisPlugins(analysis_plugins,
analysis_report_incoming_queue,
storage_file,
counter,
preferred_encoding=preferred_encoding)
if self._filter_object and not counter[u'Limited By']:
counter[u'Filter By Date'] = (counter[u'Stored Events'] -
counter[u'Events Included'] -
counter[u'Events Filtered Out'])
return counter
def testGetEvents(self):
"""Tests the GetEvents function."""
test_file = self._GetTestFilePath([u'psort_test.json.plaso'])
storage_file = zip_file.StorageFile(test_file, read_only=True)
timestamps = []
with zip_file.ZIPStorageFileReader(storage_file) as storage_reader:
for event_object in storage_reader.GetEvents():
timestamps.append(event_object.timestamp)
expected_timestamps = [
1453449153000000, 1453449153000000, 1453449153000000, 1453449153000000,
1453449181000000, 1453449181000000, 1453449241000000, 1453449241000000,
1453449241000000, 1453449241000000, 1453449272000000, 1453449272000000,
1456708543000000, 1456708543000000, 1462105168000000, 1462105168000000,
1462105168000000, 1462105168000000, 1462105169000000, 1462105170000000,
1482083672000000, 1482083672000000, 1490310078000000, 1490310078000000,
1490310078000123, 1490310078000123, 1514742872000000, 1514742872000000,
1542503720000000, 1542503720000000, 1542503743000000, 1542503743000000]
self.assertEqual(len(timestamps), 32)
self.assertEqual(sorted(timestamps), expected_timestamps)
# Test lower bound time range filter.
test_time_range = time_range.TimeRange(
timelib.Timestamp.CopyFromString(u'2016-04-30 06:41:49'),
timelib.Timestamp.CopyFromString(u'2030-12-31 23:59:59'))
storage_file = zip_file.StorageFile(test_file, read_only=True)
timestamps = []
with zip_file.ZIPStorageFileReader(storage_file) as storage_reader:
for event_object in storage_reader.GetEvents(
time_range=test_time_range):
timestamps.append(event_object.timestamp)
expected_timestamps = [
1462105168000000, 1462105168000000, 1462105168000000, 1462105168000000,
1462105169000000, 1462105170000000, 1482083672000000, 1482083672000000,
1490310078000000, 1490310078000000, 1490310078000123, 1490310078000123,
1514742872000000, 1514742872000000, 1542503720000000, 1542503720000000,
1542503743000000, 1542503743000000]
self.assertEqual(sorted(timestamps), expected_timestamps)
# Test upper bound time range filter.
test_time_range = time_range.TimeRange(
timelib.Timestamp.CopyFromString(u'2000-01-01 00:00:00'),
timelib.Timestamp.CopyFromString(u'2016-04-30 06:41:49'))
storage_file = zip_file.StorageFile(test_file, read_only=True)
timestamps = []
with zip_file.ZIPStorageFileReader(storage_file) as storage_reader:
for event_object in storage_reader.GetEvents(
time_range=test_time_range):
timestamps.append(event_object.timestamp)
expected_timestamps = [
1453449153000000, 1453449153000000, 1453449153000000, 1453449153000000,
1453449181000000, 1453449181000000, 1453449241000000, 1453449241000000,
1453449241000000, 1453449241000000, 1453449272000000, 1453449272000000,
1456708543000000, 1456708543000000]
self.assertEqual(sorted(timestamps), expected_timestamps)
def _ExportEvents(
self, storage_reader, event_buffer, event_filter=None, time_slice=None,
use_time_slicer=False):
"""Exports events using an output module.
Args:
storage_reader (StorageReader): storage reader.
event_buffer (EventBuffer): event buffer.
event_filter (Optional[FilterObject]): event filter.
time_slice (Optional[TimeRange]): time range that defines a time slice
to filter events.
use_time_slicer (Optional[bool]): True if the 'time slicer' should be
used. The 'time slicer' will provide a context of events around
an event of interest.
Returns:
collections.Counter: counter that tracks the number of unique events
read from storage.
"""
self._status = definitions.PROCESSING_STATUS_EXPORTING
time_slice_buffer = None
if time_slice:
if time_slice.event_timestamp is not None:
time_slice = storage_time_range.TimeRange(
time_slice.start_timestamp, time_slice.end_timestamp)
if use_time_slicer:
time_slice_buffer = bufferlib.CircularBuffer(time_slice.duration)
filter_limit = getattr(event_filter, u'limit', None)
forward_entries = 0
number_of_filtered_events = 0
number_of_events_from_time_slice = 0
for event in storage_reader.GetEvents(time_range=time_slice):
if event_filter:
filter_match = event_filter.Match(event)
else:
filter_match = None
# pylint: disable=singleton-comparison
if filter_match == False:
if not time_slice_buffer:
number_of_filtered_events += 1
elif forward_entries == 0:
time_slice_buffer.Append(event)
number_of_filtered_events += 1
elif forward_entries <= time_slice_buffer.size:
event_buffer.Append(event)
self._number_of_consumed_events += 1
number_of_events_from_time_slice += 1
forward_entries += 1
else:
# We reached the maximum size of the time slice and don't need to
# include other entries.
number_of_filtered_events += 1
forward_entries = 0
else:
# pylint: disable=singleton-comparison
if filter_match == True and time_slice_buffer:
# Empty the time slice buffer.
for event_in_buffer in time_slice_buffer.Flush():
event_buffer.Append(event_in_buffer)
self._number_of_consumed_events += 1
number_of_filtered_events += 1
number_of_events_from_time_slice += 1
forward_entries = 1
event_buffer.Append(event)
self._number_of_consumed_events += 1
# pylint: disable=singleton-comparison
if (filter_match == True and filter_limit and
filter_limit == self._number_of_consumed_events):
break
events_counter = collections.Counter()
events_counter[u'Events filtered'] = number_of_filtered_events
events_counter[u'Events from time slice'] = number_of_events_from_time_slice
events_counter[u'Events processed'] = self._number_of_consumed_events
if event_buffer.duplicate_counter:
events_counter[u'Duplicate events removed'] = (
event_buffer.duplicate_counter)
if filter_limit:
events_counter[u'Limited By'] = filter_limit
return events_counter
def testGetSortedEntries(self):
"""Tests the GetSortedEntries function."""
test_file = self._GetTestFilePath([u'psort_test.proto.plaso'])
storage_file = zip_file.StorageFile(test_file, read_only=True)
timestamps = []
for event_object in storage_file.GetSortedEntries():
timestamps.append(event_object.timestamp)
expected_timestamps = [
1343166324000000, 1344270407000000, 1390377153000000,
1390377153000000, 1390377181000000, 1390377241000000,
1390377241000000, 1390377272000000, 1392438730000000,
1418925272000000, 1427151678000000, 1427151678000123,
1451584472000000, 1479431720000000, 1479431743000000
]
self.assertEqual(sorted(timestamps), expected_timestamps)
# Test lower bound time range filter.
test_time_range = time_range.TimeRange(
timelib.Timestamp.CopyFromString(u'2014-02-16 00:00:00'),
timelib.Timestamp.CopyFromString(u'2030-12-31 23:59:59'))
storage_file.Close()
storage_file = zip_file.StorageFile(test_file, read_only=True)
timestamps = []
for event_object in storage_file.GetSortedEntries(
time_range=test_time_range):
timestamps.append(event_object.timestamp)
expected_timestamps = [
1418925272000000, 1427151678000000, 1427151678000123,
1451584472000000, 1479431720000000, 1479431743000000
]
self.assertEqual(sorted(timestamps), expected_timestamps)
# Test upper bound time range filter.
test_time_range = time_range.TimeRange(
timelib.Timestamp.CopyFromString(u'2000-01-01 00:00:00'),
timelib.Timestamp.CopyFromString(u'2014-02-16 00:00:00'))
storage_file.Close()
storage_file = zip_file.StorageFile(test_file, read_only=True)
timestamps = []
for event_object in storage_file.GetSortedEntries(
time_range=test_time_range):
timestamps.append(event_object.timestamp)
expected_timestamps = [
1343166324000000, 1344270407000000, 1390377153000000,
1390377153000000, 1390377181000000, 1390377241000000,
1390377241000000, 1390377272000000, 1392438730000000
]
self.assertEqual(sorted(timestamps), expected_timestamps)
storage_file.Close()
