def testGetSortedEntry(self): """Tests the GetSortedEntry function.""" test_file = self._GetTestFilePath([u'psort_test.json.plaso']) storage_file = zip_file.StorageFile(test_file, read_only=True) expected_timestamp = 1453449153000000 event_object = storage_file.GetSortedEntry() self.assertEqual(event_object.timestamp, expected_timestamp) # Test lower bound time range filter. test_time_range = time_range.TimeRange( timelib.Timestamp.CopyFromString(u'2016-04-30 06:41:49'), timelib.Timestamp.CopyFromString(u'2030-12-31 23:59:59')) storage_file.Close() storage_file = zip_file.StorageFile(test_file, read_only=True) expected_timestamp = 1462105168000000 event_object = storage_file.GetSortedEntry(time_range=test_time_range) self.assertEqual(event_object.timestamp, expected_timestamp) # Test upper bound time range filter. test_time_range = time_range.TimeRange( timelib.Timestamp.CopyFromString(u'2000-01-01 00:00:00'), timelib.Timestamp.CopyFromString(u'2016-04-30 06:41:49')) storage_file.Close() storage_file = zip_file.StorageFile(test_file, read_only=True) expected_timestamp = 1453449153000000 event_object = storage_file.GetSortedEntry(time_range=test_time_range) self.assertEqual(event_object.timestamp, expected_timestamp) storage_file.Close()
def testReadEntries(self): """Ensure returned EventObjects from the storage are within time bounds.""" storage_file_path = self._GetTestFilePath([u'psort_test.proto.plaso']) storage_file = storage_zip_file.StorageFile(storage_file_path, read_only=True) time_range = storage_time_range.TimeRange(self._start_timestamp, self._end_timestamp) timestamp_list = [] with storage_zip_file.ZIPStorageFileReader( storage_file) as storage_reader: for event_object in storage_reader.GetEvents( time_range=time_range): timestamp_list.append(event_object.timestamp) self.assertEqual(len(timestamp_list), 15) self.assertEqual(timestamp_list[0], self._start_timestamp) self.assertEqual(timestamp_list[-1], self._end_timestamp)
def _ExportEvents( self, storage_reader, output_module, deduplicate_events=True, event_filter=None, time_slice=None, use_time_slicer=False): """Exports events using an output module. Args: storage_reader (StorageReader): storage reader. output_module (OutputModule): output module. deduplicate_events (Optional[bool]): True if events should be deduplicated. event_filter (Optional[EventObjectFilter]): event filter. time_slice (Optional[TimeRange]): time range that defines a time slice to filter events. use_time_slicer (Optional[bool]): True if the 'time slicer' should be used. The 'time slicer' will provide a context of events around an event of interest. """ self._status = definitions.STATUS_INDICATOR_EXPORTING time_slice_buffer = None time_slice_range = None if time_slice: if time_slice.event_timestamp is not None: time_slice_range = storage_time_range.TimeRange( time_slice.start_timestamp, time_slice.end_timestamp) if use_time_slicer: time_slice_buffer = bufferlib.CircularBuffer(time_slice.duration) filter_limit = getattr(event_filter, 'limit', None) forward_entries = 0 self._events_status.number_of_filtered_events = 0 self._events_status.number_of_events_from_time_slice = 0 for event in storage_reader.GetSortedEvents(time_range=time_slice_range): event_data_identifier = event.GetEventDataIdentifier() event_data = storage_reader.GetEventDataByIdentifier( event_data_identifier) event_data_stream_identifier = event_data.GetEventDataStreamIdentifier() if event_data_stream_identifier: event_data_stream = storage_reader.GetEventDataStreamByIdentifier( event_data_stream_identifier) else: event_data_stream = None event_identifier = event.GetIdentifier() event_tag = self._event_tag_index.GetEventTagByIdentifier( storage_reader, event_identifier) if time_slice_range and event.timestamp != time_slice.event_timestamp: self._events_status.number_of_events_from_time_slice += 1 if event_filter: filter_match = event_filter.Match( event, event_data, event_data_stream, event_tag) else: filter_match = None # pylint: disable=singleton-comparison if filter_match == False: if not time_slice_buffer: self._events_status.number_of_filtered_events += 1 elif forward_entries == 0: time_slice_buffer.Append((event, event_data)) self._events_status.number_of_filtered_events += 1 elif forward_entries <= time_slice_buffer.size: self._ExportEvent( storage_reader, output_module, event, event_data, event_data_stream, deduplicate_events=deduplicate_events) self._number_of_consumed_events += 1 self._events_status.number_of_events_from_time_slice += 1 forward_entries += 1 else: # We reached the maximum size of the time slice and don't need to # include other entries. self._events_status.number_of_filtered_events += 1 forward_entries = 0 else: # pylint: disable=singleton-comparison if filter_match == True and time_slice_buffer: # Empty the time slice buffer. for event_in_buffer, event_data_in_buffer in ( time_slice_buffer.Flush()): self._ExportEvent( storage_reader, output_module, event_in_buffer, event_data_in_buffer, event_data_stream, deduplicate_events=deduplicate_events) self._number_of_consumed_events += 1 self._events_status.number_of_filtered_events += 1 self._events_status.number_of_events_from_time_slice += 1 forward_entries = 1 self._ExportEvent( storage_reader, output_module, event, event_data, event_data_stream, deduplicate_events=deduplicate_events) self._number_of_consumed_events += 1 # pylint: disable=singleton-comparison if (filter_match == True and filter_limit and filter_limit == self._number_of_consumed_events): break self._FlushExportBuffer(storage_reader, output_module)
def _ExportEvents(self, storage_reader, output_module, deduplicate_events=True, event_filter=None, time_slice=None, use_time_slicer=False): """Exports events using an output module. Args: storage_reader (StorageReader): storage reader. output_module (OutputModule): output module. deduplicate_events (Optional[bool]): True if events should be deduplicated. event_filter (Optional[FilterObject]): event filter. time_slice (Optional[TimeRange]): time range that defines a time slice to filter events. use_time_slicer (Optional[bool]): True if the 'time slicer' should be used. The 'time slicer' will provide a context of events around an event of interest. Returns: collections.Counter: counter that tracks the number of unique events read from storage. """ self._status = definitions.PROCESSING_STATUS_EXPORTING time_slice_buffer = None time_slice_range = None if time_slice: if time_slice.event_timestamp is not None: time_slice_range = storage_time_range.TimeRange( time_slice.start_timestamp, time_slice.end_timestamp) if use_time_slicer: time_slice_buffer = bufferlib.CircularBuffer( time_slice.duration) filter_limit = getattr(event_filter, 'limit', None) forward_entries = 0 number_of_filtered_events = 0 number_of_events_from_time_slice = 0 for event in storage_reader.GetSortedEvents( time_range=time_slice_range): event_data_identifier = event.GetEventDataIdentifier() if event_data_identifier: event_data = storage_reader.GetEventDataByIdentifier( event_data_identifier) if event_data: for attribute_name, attribute_value in event_data.GetAttributes( ): setattr(event, attribute_name, attribute_value) event_identifier = event.GetIdentifier() event.tag = self._event_tag_index.GetEventTagByIdentifier( storage_reader, event_identifier) if time_slice_range and event.timestamp != time_slice.event_timestamp: number_of_events_from_time_slice += 1 if event_filter: filter_match = event_filter.Match(event) else: filter_match = None # pylint: disable=singleton-comparison if filter_match == False: if not time_slice_buffer: number_of_filtered_events += 1 elif forward_entries == 0: time_slice_buffer.Append(event) number_of_filtered_events += 1 elif forward_entries <= time_slice_buffer.size: self._ExportEvent(output_module, event, deduplicate_events=deduplicate_events) self._number_of_consumed_events += 1 number_of_events_from_time_slice += 1 forward_entries += 1 else: # We reached the maximum size of the time slice and don't need to # include other entries. number_of_filtered_events += 1 forward_entries = 0 else: # pylint: disable=singleton-comparison if filter_match == True and time_slice_buffer: # Empty the time slice buffer. for event_in_buffer in time_slice_buffer.Flush(): self._ExportEvent( output_module, event_in_buffer, deduplicate_events=deduplicate_events) self._number_of_consumed_events += 1 number_of_filtered_events += 1 number_of_events_from_time_slice += 1 forward_entries = 1 self._ExportEvent(output_module, event, deduplicate_events=deduplicate_events) self._number_of_consumed_events += 1 # pylint: disable=singleton-comparison if (filter_match == True and filter_limit and filter_limit == self._number_of_consumed_events): break self._FlushExportBuffer(output_module) events_counter = collections.Counter() events_counter['Events filtered'] = number_of_filtered_events events_counter[ 'Events from time slice'] = number_of_events_from_time_slice events_counter['Events processed'] = self._number_of_consumed_events if self._number_of_duplicate_events: events_counter['Duplicate events removed'] = ( self._number_of_duplicate_events) if self._number_of_macb_grouped_events: events_counter['Events MACB grouped'] = ( self._number_of_macb_grouped_events) if filter_limit: events_counter['Limited By'] = filter_limit return events_counter
def ProcessStorage(self, output_module, storage_file, storage_file_path, analysis_plugins, event_queue_producers, command_line_arguments=None, deduplicate_events=True, preferred_encoding=u'utf-8', time_slice=None, use_time_slicer=False): """Processes a plaso storage file. Args: output_module: an output module (instance of OutputModule). storage_file: the storage file object (instance of StorageFile). storage_file_path: string containing the path of the storage file. analysis_plugins: list of analysis plugin objects (instance of AnalysisPlugin). event_queue_producers: list of event queue producer objects (instance of ItemQueueProducer). command_line_arguments: optional string of the command line arguments or None if not set. deduplicate_events: optional boolean value to indicate if the event objects should be deduplicated. preferred_encoding: optional preferred encoding. time_slice: optional time slice object (instance of TimeSlice). use_time_slicer: optional boolean value to indicate the 'time slicer' should be used. The 'time slicer' will provide a context of events around an event of interest. Returns: A counter (an instance of collections.Counter) that tracks the number of events extracted from storage, and the analysis plugin results. Raises: RuntimeError: if a non-recoverable situation is encountered. """ time_slice = None if time_slice: if time_slice.event_timestamp is not None: time_slice = storage_time_range.TimeRange( time_slice.start_timestamp, time_slice.end_timestamp) elif use_time_slicer: self._filter_buffer = bufferlib.CircularBuffer( time_slice.duration) with storage_file: # TODO: allow for single processing. # TODO: add upper queue limit. analysis_queue_port = None if self._use_zeromq: analysis_report_incoming_queue = zeromq_queue.ZeroMQPullBindQueue( delay_open=False, port=None, linger_seconds=5) analysis_queue_port = analysis_report_incoming_queue.port else: analysis_report_incoming_queue = multi_process.MultiProcessingQueue( timeout=5) pre_obj = self._GetLastGoodPreprocess(storage_file) if pre_obj is None: pre_obj = event.PreprocessObject() if analysis_plugins: self._StartAnalysisPlugins( storage_file_path, analysis_plugins, pre_obj, analysis_queue_port=analysis_queue_port, analysis_report_incoming_queue= analysis_report_incoming_queue, command_line_arguments=command_line_arguments) # Assign the preprocessing object to the storage. # This is normally done in the construction of the storage object, # however we cannot do that here since the preprocessing object is # stored inside the storage file, so we need to open it first to # be able to read it in, before we make changes to it. Thus we need # to access this protected member of the class. # pylint: disable=protected-access storage_file._pre_obj = pre_obj else: event_queue_producers = [] output_buffer = output_event_buffer.EventBuffer( output_module, deduplicate_events) with output_buffer: counter = self.ProcessEventsFromStorage( storage_file, output_buffer, analysis_queues=event_queue_producers, filter_buffer=self._filter_buffer, my_filter=self._filter_object, time_slice=time_slice) for information in storage_file.GetStorageInformation(): if hasattr(information, u'counter'): counter[u'Stored Events'] += information.counter[u'total'] if not self._quiet_mode: logging.info(u'Output processing is done.') # Get all reports and tags from analysis plugins. self._ProcessAnalysisPlugins(analysis_plugins, analysis_report_incoming_queue, storage_file, counter, preferred_encoding=preferred_encoding) if self._filter_object and not counter[u'Limited By']: counter[u'Filter By Date'] = (counter[u'Stored Events'] - counter[u'Events Included'] - counter[u'Events Filtered Out']) return counter
def testGetEvents(self): """Tests the GetEvents function.""" test_file = self._GetTestFilePath([u'psort_test.json.plaso']) storage_file = zip_file.StorageFile(test_file, read_only=True) timestamps = [] with zip_file.ZIPStorageFileReader(storage_file) as storage_reader: for event_object in storage_reader.GetEvents(): timestamps.append(event_object.timestamp) expected_timestamps = [ 1453449153000000, 1453449153000000, 1453449153000000, 1453449153000000, 1453449181000000, 1453449181000000, 1453449241000000, 1453449241000000, 1453449241000000, 1453449241000000, 1453449272000000, 1453449272000000, 1456708543000000, 1456708543000000, 1462105168000000, 1462105168000000, 1462105168000000, 1462105168000000, 1462105169000000, 1462105170000000, 1482083672000000, 1482083672000000, 1490310078000000, 1490310078000000, 1490310078000123, 1490310078000123, 1514742872000000, 1514742872000000, 1542503720000000, 1542503720000000, 1542503743000000, 1542503743000000] self.assertEqual(len(timestamps), 32) self.assertEqual(sorted(timestamps), expected_timestamps) # Test lower bound time range filter. test_time_range = time_range.TimeRange( timelib.Timestamp.CopyFromString(u'2016-04-30 06:41:49'), timelib.Timestamp.CopyFromString(u'2030-12-31 23:59:59')) storage_file = zip_file.StorageFile(test_file, read_only=True) timestamps = [] with zip_file.ZIPStorageFileReader(storage_file) as storage_reader: for event_object in storage_reader.GetEvents( time_range=test_time_range): timestamps.append(event_object.timestamp) expected_timestamps = [ 1462105168000000, 1462105168000000, 1462105168000000, 1462105168000000, 1462105169000000, 1462105170000000, 1482083672000000, 1482083672000000, 1490310078000000, 1490310078000000, 1490310078000123, 1490310078000123, 1514742872000000, 1514742872000000, 1542503720000000, 1542503720000000, 1542503743000000, 1542503743000000] self.assertEqual(sorted(timestamps), expected_timestamps) # Test upper bound time range filter. test_time_range = time_range.TimeRange( timelib.Timestamp.CopyFromString(u'2000-01-01 00:00:00'), timelib.Timestamp.CopyFromString(u'2016-04-30 06:41:49')) storage_file = zip_file.StorageFile(test_file, read_only=True) timestamps = [] with zip_file.ZIPStorageFileReader(storage_file) as storage_reader: for event_object in storage_reader.GetEvents( time_range=test_time_range): timestamps.append(event_object.timestamp) expected_timestamps = [ 1453449153000000, 1453449153000000, 1453449153000000, 1453449153000000, 1453449181000000, 1453449181000000, 1453449241000000, 1453449241000000, 1453449241000000, 1453449241000000, 1453449272000000, 1453449272000000, 1456708543000000, 1456708543000000] self.assertEqual(sorted(timestamps), expected_timestamps)
def _ExportEvents( self, storage_reader, event_buffer, event_filter=None, time_slice=None, use_time_slicer=False): """Exports events using an output module. Args: storage_reader (StorageReader): storage reader. event_buffer (EventBuffer): event buffer. event_filter (Optional[FilterObject]): event filter. time_slice (Optional[TimeRange]): time range that defines a time slice to filter events. use_time_slicer (Optional[bool]): True if the 'time slicer' should be used. The 'time slicer' will provide a context of events around an event of interest. Returns: collections.Counter: counter that tracks the number of unique events read from storage. """ self._status = definitions.PROCESSING_STATUS_EXPORTING time_slice_buffer = None if time_slice: if time_slice.event_timestamp is not None: time_slice = storage_time_range.TimeRange( time_slice.start_timestamp, time_slice.end_timestamp) if use_time_slicer: time_slice_buffer = bufferlib.CircularBuffer(time_slice.duration) filter_limit = getattr(event_filter, u'limit', None) forward_entries = 0 number_of_filtered_events = 0 number_of_events_from_time_slice = 0 for event in storage_reader.GetEvents(time_range=time_slice): if event_filter: filter_match = event_filter.Match(event) else: filter_match = None # pylint: disable=singleton-comparison if filter_match == False: if not time_slice_buffer: number_of_filtered_events += 1 elif forward_entries == 0: time_slice_buffer.Append(event) number_of_filtered_events += 1 elif forward_entries <= time_slice_buffer.size: event_buffer.Append(event) self._number_of_consumed_events += 1 number_of_events_from_time_slice += 1 forward_entries += 1 else: # We reached the maximum size of the time slice and don't need to # include other entries. number_of_filtered_events += 1 forward_entries = 0 else: # pylint: disable=singleton-comparison if filter_match == True and time_slice_buffer: # Empty the time slice buffer. for event_in_buffer in time_slice_buffer.Flush(): event_buffer.Append(event_in_buffer) self._number_of_consumed_events += 1 number_of_filtered_events += 1 number_of_events_from_time_slice += 1 forward_entries = 1 event_buffer.Append(event) self._number_of_consumed_events += 1 # pylint: disable=singleton-comparison if (filter_match == True and filter_limit and filter_limit == self._number_of_consumed_events): break events_counter = collections.Counter() events_counter[u'Events filtered'] = number_of_filtered_events events_counter[u'Events from time slice'] = number_of_events_from_time_slice events_counter[u'Events processed'] = self._number_of_consumed_events if event_buffer.duplicate_counter: events_counter[u'Duplicate events removed'] = ( event_buffer.duplicate_counter) if filter_limit: events_counter[u'Limited By'] = filter_limit return events_counter
def testGetSortedEntries(self): """Tests the GetSortedEntries function.""" test_file = self._GetTestFilePath([u'psort_test.proto.plaso']) storage_file = zip_file.StorageFile(test_file, read_only=True) timestamps = [] for event_object in storage_file.GetSortedEntries(): timestamps.append(event_object.timestamp) expected_timestamps = [ 1343166324000000, 1344270407000000, 1390377153000000, 1390377153000000, 1390377181000000, 1390377241000000, 1390377241000000, 1390377272000000, 1392438730000000, 1418925272000000, 1427151678000000, 1427151678000123, 1451584472000000, 1479431720000000, 1479431743000000 ] self.assertEqual(sorted(timestamps), expected_timestamps) # Test lower bound time range filter. test_time_range = time_range.TimeRange( timelib.Timestamp.CopyFromString(u'2014-02-16 00:00:00'), timelib.Timestamp.CopyFromString(u'2030-12-31 23:59:59')) storage_file.Close() storage_file = zip_file.StorageFile(test_file, read_only=True) timestamps = [] for event_object in storage_file.GetSortedEntries( time_range=test_time_range): timestamps.append(event_object.timestamp) expected_timestamps = [ 1418925272000000, 1427151678000000, 1427151678000123, 1451584472000000, 1479431720000000, 1479431743000000 ] self.assertEqual(sorted(timestamps), expected_timestamps) # Test upper bound time range filter. test_time_range = time_range.TimeRange( timelib.Timestamp.CopyFromString(u'2000-01-01 00:00:00'), timelib.Timestamp.CopyFromString(u'2014-02-16 00:00:00')) storage_file.Close() storage_file = zip_file.StorageFile(test_file, read_only=True) timestamps = [] for event_object in storage_file.GetSortedEntries( time_range=test_time_range): timestamps.append(event_object.timestamp) expected_timestamps = [ 1343166324000000, 1344270407000000, 1390377153000000, 1390377153000000, 1390377181000000, 1390377241000000, 1390377241000000, 1390377272000000, 1392438730000000 ] self.assertEqual(sorted(timestamps), expected_timestamps) storage_file.Close()