def testOutput(self):
with TempDirectory() as dirname:
dump_file = os.path.join(dirname, u'plaso.db')
# Copy events to pstorage dump.
with storage.StorageFile(self.test_filename,
read_only=True) as store:
formatter = pstorage.PlasoStorageOutputFormatter(
store, self._formatter_mediator, filehandle=dump_file)
with interface.EventBuffer(
formatter, check_dedups=False) as output_buffer:
event_object = store.GetSortedEntry()
while event_object:
output_buffer.Append(event_object)
event_object = store.GetSortedEntry()
# Make sure original and dump have the same events.
original = storage.StorageFile(self.test_filename, read_only=True)
dump = storage.StorageFile(dump_file, read_only=True)
event_object_original = original.GetSortedEntry()
event_object_dump = dump.GetSortedEntry()
original_list = []
dump_list = []
while event_object_original:
original_list.append(event_object_original.EqualityString())
dump_list.append(event_object_dump.EqualityString())
event_object_original = original.GetSortedEntry()
event_object_dump = dump.GetSortedEntry()
self.assertFalse(event_object_dump)
for original_str, dump_str in zip(sorted(original_list),
sorted(dump_list)):
self.assertEqual(original_str, dump_str)
def testGroupingEngine(self):
"""Tests the Grouping engine's functionality."""
pfilter.TimeRangeCache.ResetTimeConstraints()
tagging_engine = plasm.TaggingEngine(self._storage_filename,
self._tag_input_filename,
quiet=True)
tagging_engine.Run()
storage_file = storage.StorageFile(self._storage_filename,
read_only=False)
grouping_engine = plasm.GroupingEngine()
grouping_engine.Run(storage_file, quiet=True)
storage_file.Close()
storage_file = storage.StorageFile(self._storage_filename,
read_only=True)
storage_file.SetStoreLimit()
self.assertTrue(storage_file.HasGrouping())
groups = storage_file.GetGrouping()
count = 0
for group_event in groups:
count += 1
self.assertEquals(group_event.category, 'Test Tag')
self.assertEquals(count, 2)
storage_file.Close()
def testOutput(self):
"""Testing if psort can output data."""
formatters_manager.FormattersManager.RegisterFormatter(
PsortTestEventFormatter)
events = []
events.append(PsortTestEvent(5134324321))
events.append(PsortTestEvent(2134324321))
events.append(PsortTestEvent(9134324321))
events.append(PsortTestEvent(15134324321))
events.append(PsortTestEvent(5134324322))
events.append(PsortTestEvent(5134024321))
output_writer = cli_test_lib.TestOutputWriter()
with shared_test_lib.TempDirectory() as dirname:
temp_file = os.path.join(dirname, u'plaso.db')
storage_file = storage.StorageFile(temp_file, read_only=False)
pfilter.TimeRangeCache.ResetTimeConstraints()
storage_file.SetStoreLimit()
storage_file.AddEventObjects(events)
storage_file.Close()
with storage.StorageFile(temp_file) as storage_file:
storage_file.store_range = [1]
output_mediator_object = output_mediator.OutputMediator(
self._formatter_mediator, storage_file)
output_module = TestOutputModule(output_mediator_object)
output_module.SetOutputWriter(output_writer)
event_buffer = TestEventBuffer(output_module,
check_dedups=False,
store=storage_file)
self._front_end.ProcessEventsFromStorage(
storage_file, event_buffer)
event_buffer.Flush()
lines = []
output = output_writer.ReadOutput()
for line in output.split(b'\n'):
if line == b'.':
continue
if line:
lines.append(line)
# One more line than events (header row).
self.assertEqual(len(lines), 7)
self.assertTrue(b'My text goes along: My text dude. lines' in lines[2])
self.assertTrue(b'LOG/' in lines[2])
self.assertTrue(b'None in Particular' in lines[2])
self.assertEqual(lines[0], (
b'date,time,timezone,MACB,source,sourcetype,type,user,host,short,desc,'
b'version,filename,inode,notes,format,extra'))
formatters_manager.FormattersManager.DeregisterFormatter(
PsortTestEventFormatter)
def testGroupingEngineUntagged(self):
"""Grouping engine should do nothing if dealing with untagged storage."""
storage_file = storage.StorageFile(self._storage_filename, read_only=False)
grouping_engine = plasm.GroupingEngine()
grouping_engine.Run(storage_file, quiet=True)
storage_file.Close()
storage_file = storage.StorageFile(self._storage_filename, read_only=True)
self.assertFalse(storage_file.HasGrouping())
storage_file.Close()
def testOutput(self):
"""Testing if psort can output data."""
events = []
events.append(PsortTestEvent(5134324321))
events.append(PsortTestEvent(2134324321))
events.append(PsortTestEvent(9134324321))
events.append(PsortTestEvent(15134324321))
events.append(PsortTestEvent(5134324322))
events.append(PsortTestEvent(5134024321))
output_fd = io.StringIO()
with test_lib.TempDirectory() as dirname:
temp_file = os.path.join(dirname, u'plaso.db')
storage_file = storage.StorageFile(temp_file, read_only=False)
pfilter.TimeRangeCache.ResetTimeConstraints()
storage_file.SetStoreLimit()
storage_file.AddEventObjects(events)
storage_file.Close()
with storage.StorageFile(temp_file) as storage_file:
storage_file.store_range = [1]
formatter = TestFormatter(storage_file,
self._formatter_mediator,
filehandle=output_fd)
event_buffer = TestEventBuffer(formatter,
check_dedups=False,
store=storage_file)
self._front_end.ProcessOutput(storage_file, event_buffer)
event_buffer.Flush()
lines = []
for line in output_fd.getvalue().split(u'\n'):
if line == u'.':
continue
if line:
lines.append(line)
# One more line than events (header row).
self.assertEqual(len(lines), 7)
self.assertTrue(u'My text goes along: My text dude. lines' in lines[2])
self.assertTrue(u'LOG/' in lines[2])
self.assertTrue(u'None in Particular' in lines[2])
self.assertEqual(lines[0], (
u'date,time,timezone,MACB,source,sourcetype,type,user,host,short,desc,'
u'version,filename,inode,notes,format,extra'))
def testGetStorageInformation(self):
"""Tests the get storage information function."""
test_front_end = log2timeline.Log2TimelineFrontend()
options = test_lib.Options()
options.source = self._GetTestFilePath(['image.dd'])
storage_file_path = os.path.join(self._temp_directory, 'plaso.db')
test_front_end.ParseOptions(options)
test_front_end.SetStorageFile(storage_file_path=storage_file_path)
test_front_end.SetRunForeman(run_foreman=False)
test_front_end.ProcessSource(options)
try:
storage_file = storage.StorageFile(storage_file_path,
read_only=True)
except IOError:
# This is not a storage file, we should fail.
self.assertTrue(False)
# Make sure we can read an event out of the storage.
event_object = storage_file.GetSortedEntry()
self.assertIsNotNone(event_object)
def testStorageSort(self):
"""This test ensures that items read and output are in the expected order.
This method by design outputs data as it runs. In order to test this a
a modified output renderer is used for which the flush functionality has
been removed.
The test will be to read the TestEventBuffer storage and check to see
if it matches the known good sort order.
"""
pfilter.TimeRangeCache.ResetTimeConstraints()
pfilter.TimeRangeCache.SetUpperTimestamp(self.last)
pfilter.TimeRangeCache.SetLowerTimestamp(self.first)
store = storage.StorageFile(self.test_file, read_only=True)
store.store_range = [1, 5, 6]
read_list = []
event_object = store.GetSortedEntry()
while event_object:
read_list.append(event_object.timestamp)
event_object = store.GetSortedEntry()
expected_timestamps = [
1344270407000000L, 1392438730000000L, 1427151678000000L,
1451584472000000L]
self.assertEquals(read_list, expected_timestamps)
def setUp(self):
"""Sets up the objects used throughout the test."""
self._temp_directory = tempfile.mkdtemp()
self._storage_filename = os.path.join(self._temp_directory, 'plaso.db')
self._tag_input_filename = os.path.join(self._temp_directory, 'input1.tag')
tag_input_file = open(self._tag_input_filename, 'wb')
tag_input_file.write('\n'.join([
'Test Tag',
' filename contains \'/tmp/whoaaaa\'',
' parser is \'TestEvent\' and stuff is \'dude\'']))
tag_input_file.close()
pfilter.TimeRangeCache.ResetTimeConstraints()
# TODO: add upper queue limit.
test_queue = multi_process.MultiProcessingQueue()
test_queue_producer = queue.ItemQueueProducer(test_queue)
test_queue_producer.ProduceItems([
TestEvent(0),
TestEvent(1000),
TestEvent(2000000, '/tmp/whoaaaaa'),
TestEvent(2500000, '/tmp/whoaaaaa'),
TestEvent(5000000, '/tmp/whoaaaaa', 'dude')])
test_queue_producer.SignalEndOfInput()
storage_writer = storage.StorageFileWriter(
test_queue, self._storage_filename)
storage_writer.WriteEventObjects()
self._storage_file = storage.StorageFile(self._storage_filename)
self._storage_file.SetStoreLimit()
def testProcessSources(self):
"""Tests the ProcessSources function."""
test_front_end = extraction_frontend.ExtractionFrontend()
storage_file_path = os.path.join(self._temp_directory, u'plaso.db')
test_front_end.SetStorageFile(storage_file_path=storage_file_path)
test_file = self._GetTestFilePath([u'ímynd.dd'])
volume_path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_OS, location=test_file)
path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_TSK,
location=u'/',
parent=volume_path_spec)
# TODO: move source_scanner.SourceScannerContext.SOURCE_TYPE_
# to definitions.SOURCE_TYPE_.
source_type = (source_scanner.SourceScannerContext.
SOURCE_TYPE_STORAGE_MEDIA_IMAGE)
test_front_end.ProcessSources([path_spec], source_type)
try:
storage_file = storage.StorageFile(storage_file_path,
read_only=True)
except IOError:
self.fail(u'Not a storage file.')
# Make sure we can read an event out of the storage.
event_object = storage_file.GetSortedEntry()
self.assertIsNotNone(event_object)
def _Open(self):
"""Opens the storage writer."""
self._storage_file = storage.StorageFile(
self._output_file, buffer_size=self._buffer_size, pre_obj=self._pre_obj,
serializer_format=self._serializer_format)
self._storage_file.SetEnableProfiling(
self._enable_profiling, profiling_type=self._profiling_type)
def _PreprocessSource(self, source_path_specs, source_type):
"""Preprocesses the source.
Args:
source_path_specs: list of path specifications (instances of
dfvfs.PathSpec) to process.
source_type: the dfVFS source type definition.
Returns:
The preprocessing object (instance of PreprocessObject).
"""
pre_obj = None
if self._use_old_preprocess and os.path.isfile(self._storage_file_path):
# Check if the storage file contains a preprocessing object.
try:
with storage.StorageFile(
self._storage_file_path, read_only=True) as storage_file:
storage_information = storage_file.GetStorageInformation()
if storage_information:
logging.info(u'Using preprocessing information from a prior run.')
pre_obj = storage_information[-1]
self._enable_preprocessing = False
except IOError:
logging.warning(u'Storage file does not exist, running preprocess.')
logging.debug(u'Starting preprocessing.')
# TODO: move source_scanner.SourceScannerContext.SOURCE_TYPE_
# to definitions.SOURCE_TYPE_.
if (self._enable_preprocessing and source_type in [
source_scanner.SourceScannerContext.SOURCE_TYPE_DIRECTORY,
source_scanner.SourceScannerContext.SOURCE_TYPE_STORAGE_MEDIA_DEVICE,
source_scanner.SourceScannerContext.SOURCE_TYPE_STORAGE_MEDIA_IMAGE]):
try:
self._engine.PreprocessSource(
source_path_specs, self._operating_system,
resolver_context=self._resolver_context)
except IOError as exception:
logging.error(u'Unable to preprocess with error: {0:s}'.format(
exception))
return event.PreprocessObject()
logging.debug(u'Preprocessing done.')
# TODO: Remove the need for direct access to the pre_obj in favor
# of the knowledge base.
pre_obj = getattr(self._engine.knowledge_base, u'_pre_obj', None)
if not pre_obj:
pre_obj = event.PreprocessObject()
return pre_obj
def Start(self):
"""Sets up the output storage file."""
pre_obj = event.PreprocessObject()
pre_obj.collection_information = {}
pre_obj.collection_information['time_of_run'] = timelib.Timestamp.GetNow()
if hasattr(self._config, 'filter') and self._config.filter:
pre_obj.collection_information['filter'] = self._config.filter
if hasattr(self._config, 'storagefile') and self._config.storagefile:
pre_obj.collection_information[
'file_processed'] = self._config.storagefile
self._storage = storage.StorageFile(self.filehandle, pre_obj=pre_obj)
def OpenStorageFile(storage_path):
"""Opens a storage file and returns the storage file object."""
if not os.path.isfile(storage_path):
return
try:
store = storage.StorageFile(storage_path, read_only=True)
except IOError:
print 'Unable to load storage file, not a storage file?'
return store
def Open(self):
"""Opens the plaso storage file."""
pre_obj = event.PreprocessObject()
pre_obj.collection_information = {
'time_of_run': timelib.Timestamp.GetNow()
}
if hasattr(self._config, 'filter') and self._config.filter:
pre_obj.collection_information['filter'] = self._config.filter
if hasattr(self._config, 'storagefile') and self._config.storagefile:
pre_obj.collection_information[
'file_processed'] = self._config.storagefile
self._storage = storage.StorageFile(self.filehandle, pre_obj=pre_obj)
def OpenStorageFile(self, read_only=True):
"""Opens the storage file.
Args:
read_only: optional boolean value to indicate the storage file should
be opened in read-only mode. The default is True.
Returns:
The storage file object (instance of StorageFile).
"""
return storage.StorageFile(self._storage_file_path,
read_only=read_only)
def testOutput(self):
with shared_test_lib.TempDirectory() as dirname:
storage_file = os.path.join(dirname, u'plaso.plaso')
# Copy events to pstorage dump.
with storage.StorageFile(self.test_filename,
read_only=True) as store:
output_mediator = self._CreateOutputMediator(
storage_object=store)
output_module = pstorage.PlasoStorageOutputModule(
output_mediator)
output_module.SetFilePath(storage_file)
with interface.EventBuffer(
output_module, check_dedups=False) as output_buffer:
event_object = store.GetSortedEntry()
while event_object:
output_buffer.Append(event_object)
event_object = store.GetSortedEntry()
# Make sure original and dump have the same events.
original = storage.StorageFile(self.test_filename, read_only=True)
dump = storage.StorageFile(storage_file, read_only=True)
event_object_original = original.GetSortedEntry()
event_object_dump = dump.GetSortedEntry()
original_list = []
dump_list = []
while event_object_original:
original_list.append(event_object_original.EqualityString())
dump_list.append(event_object_dump.EqualityString())
event_object_original = original.GetSortedEntry()
event_object_dump = dump.GetSortedEntry()
self.assertFalse(event_object_dump)
for original_str, dump_str in zip(sorted(original_list),
sorted(dump_list)):
self.assertEqual(original_str, dump_str)
def testTaggingEngine(self):
"""Tests the Tagging engine's functionality."""
self.assertFalse(self._storage_file.HasTagging())
tagging_engine = plasm.TaggingEngine(
self._storage_filename, self._tag_input_filename, quiet=True)
tagging_engine.Run()
test = storage.StorageFile(self._storage_filename)
self.assertTrue(test.HasTagging())
tagging = test.GetTagging()
count = 0
for tag_event in tagging:
count += 1
self.assertEqual(tag_event.tags, ['Test Tag'])
self.assertEqual(count, 3)
def Open(self):
"""Opens the plaso storage file."""
pre_obj = event.PreprocessObject()
pre_obj.collection_information = {
u'time_of_run': timelib.Timestamp.GetNow()}
filter_expression = self._output_mediator.filter_expression
if filter_expression:
pre_obj.collection_information[u'filter'] = filter_expression
storage_file_path = self._output_mediator.storage_file_path
if storage_file_path:
pre_obj.collection_information[u'file_processed'] = storage_file_path
self._storage = storage.StorageFile(self._file_object, pre_obj=pre_obj)
def PreprocessSource(self, options):
"""Preprocesses the source.
Args:
options: the command line arguments (instance of argparse.Namespace).
Returns:
The preprocessing object (instance of PreprocessObject).
"""
pre_obj = None
if self._old_preprocess and os.path.isfile(self._storage_file_path):
# Check if the storage file contains a preprocessing object.
try:
with storage.StorageFile(
self._storage_file_path, read_only=True) as store:
storage_information = store.GetStorageInformation()
if storage_information:
logging.info(u'Using preprocessing information from a prior run.')
pre_obj = storage_information[-1]
self._preprocess = False
except IOError:
logging.warning(u'Storage file does not exist, running preprocess.')
if self._preprocess and self._scan_context.source_type in [
self._scan_context.SOURCE_TYPE_DIRECTORY,
self._scan_context.SOURCE_TYPE_STORAGE_MEDIA_DEVICE,
self._scan_context.SOURCE_TYPE_STORAGE_MEDIA_IMAGE]:
try:
self._engine.PreprocessSource(
self._operating_system, resolver_context=self._resolver_context)
except IOError as exception:
logging.error(u'Unable to preprocess with error: {0:s}'.format(
exception))
return
# TODO: Remove the need for direct access to the pre_obj in favor
# of the knowledge base.
pre_obj = getattr(self._engine.knowledge_base, '_pre_obj', None)
if not pre_obj:
pre_obj = event.PreprocessObject()
self._PreprocessSetTimezone(options, pre_obj)
self._PreprocessSetParserFilter(options, pre_obj)
return pre_obj
def testIteratingOverPlasoStore(self):
"""Tests the plaso storage iterator"""
counter = 0
for _ in plasm.EventObjectGenerator(self._storage_file, quiet=True):
counter += 1
self.assertEqual(counter, 5)
self._storage_file.Close()
pfilter.TimeRangeCache.ResetTimeConstraints()
self._storage_file = storage.StorageFile(self._storage_filename)
self._storage_file.SetStoreLimit()
counter = 0
for _ in plasm.EventObjectGenerator(self._storage_file, quiet=False):
counter += 1
self.assertEqual(counter, 5)
def _PreprocessSource(self):
"""Preprocesses the source.
Returns:
The preprocessing object (instance of PreprocessObject).
"""
pre_obj = None
if self._old_preprocess and os.path.isfile(self._storage_file_path):
# Check if the storage file contains a preprocessing object.
try:
with storage.StorageFile(
self._storage_file_path, read_only=True) as store:
storage_information = store.GetStorageInformation()
if storage_information:
logging.info(u'Using preprocessing information from a prior run.')
pre_obj = storage_information[-1]
self._preprocess = False
except IOError:
logging.warning(u'Storage file does not exist, running preprocess.')
logging.debug(u'Starting preprocessing.')
if (self._preprocess and
(self.SourceIsDirectory() or self.SourceIsStorageMediaImage())):
try:
self._engine.PreprocessSource(
self._operating_system, resolver_context=self._resolver_context)
except IOError as exception:
logging.error(u'Unable to preprocess with error: {0:s}'.format(
exception))
return
logging.debug(u'Preprocessing done.')
# TODO: Remove the need for direct access to the pre_obj in favor
# of the knowledge base.
pre_obj = getattr(self._engine.knowledge_base, u'_pre_obj', None)
if not pre_obj:
pre_obj = event.PreprocessObject()
return pre_obj
def Open(self):
"""Opens the plaso storage file."""
pre_obj = event.PreprocessObject()
pre_obj.collection_information = {
u'time_of_run': timelib.Timestamp.GetNow()
}
configuration_value = self._output_mediator.GetConfigurationValue(
u'filter')
if configuration_value:
pre_obj.collection_information[u'filter'] = configuration_value
configuration_value = self._output_mediator.GetConfigurationValue(
u'storagefile')
if configuration_value:
pre_obj.collection_information[
u'file_processed'] = configuration_value
self._storage = storage.StorageFile(self._file_object, pre_obj=pre_obj)
def testProcessSource(self):
"""Tests the ProcessSource function."""
test_front_end = extraction_frontend.ExtractionFrontend()
source_file = self._GetTestFilePath([u'ímynd.dd'])
storage_file_path = os.path.join(self._temp_directory, u'plaso.db')
test_front_end.SetStorageFile(storage_file_path=storage_file_path)
test_front_end.ScanSource(source_file)
test_front_end.ProcessSource()
try:
storage_file = storage.StorageFile(storage_file_path, read_only=True)
except IOError:
self.fail(u'Not a storage file.')
# Make sure we can read an event out of the storage.
event_object = storage_file.GetSortedEntry()
self.assertIsNotNone(event_object)
def testReadEntries(self):
"""Ensure returned EventObjects from the storage are within timebounds."""
timestamp_list = []
pfilter.TimeRangeCache.ResetTimeConstraints()
pfilter.TimeRangeCache.SetUpperTimestamp(self.last)
pfilter.TimeRangeCache.SetLowerTimestamp(self.first)
storage_file = storage.StorageFile(self._test_file, read_only=True)
storage_file.SetStoreLimit()
event_object = storage_file.GetSortedEntry()
while event_object:
timestamp_list.append(event_object.timestamp)
event_object = storage_file.GetSortedEntry()
self.assertEquals(len(timestamp_list), 8)
self.assertTrue(timestamp_list[0] >= self.first
and timestamp_list[-1] <= self.last)
storage_file.Close()
def SetupStorage(input_file_path, pre_obj=None):
"""Sets up the storage object.
Attempts to initialize a storage file. If we fail on a IOError, for which
a common cause are typos, log a warning and gracefully exit.
Args:
input_file_path: Filesystem path to the plaso storage container.
pre_obj: A plaso preprocessing object.
Returns:
A storage.StorageFile object.
"""
try:
return storage.StorageFile(input_file_path,
pre_obj=pre_obj,
read_only=False)
except IOError as exception:
logging.error(u'IO ERROR: {0:s}'.format(exception))
else:
logging.error(u'Other Critical Failure Reading Files')
sys.exit(1)
def testStorageSort(self):
"""This test ensures that items read and output are in the expected order.
This method by design outputs data as it runs. In order to test this a
a modified output renderer is used for which the flush functionality has
been removed.
The test will be to read the TestEventBuffer storage and check to see
if it matches the known good sort order.
"""
# TODO: have sample output generated from the test.
# TODO: Use input data with a defined year. syslog parser chooses a
# year based on system clock; forcing updates to test file if regenerated.
test_file = os.path.join(u'test_data', u'psort_test.proto.plaso')
first = timelib.Timestamp.CopyFromString(u'2012-07-20 15:44:14')
last = timelib.Timestamp.CopyFromString(u'2016-11-18 01:15:43')
pfilter.TimeRangeCache.ResetTimeConstraints()
pfilter.TimeRangeCache.SetUpperTimestamp(last)
pfilter.TimeRangeCache.SetLowerTimestamp(first)
store = storage.StorageFile(test_file, read_only=True)
store.store_range = [1, 5, 6]
read_list = []
event_object = store.GetSortedEntry()
while event_object:
read_list.append(event_object.timestamp)
event_object = store.GetSortedEntry()
expected_timestamps = [
1344270407000000, 1392438730000000, 1427151678000000,
1451584472000000
]
self.assertEqual(read_list, expected_timestamps)
def Main():
"""Run the tool."""
arg_parser = argparse.ArgumentParser(description=(
'plaso_extract_search_history is a simple script that reads the '
'content of a plaso storage file and tries to extract known search '
'engine history from it'))
arg_parser.add_argument('-w',
'--write',
metavar='FILENAME',
action='store',
dest='output_file',
default='',
help='Write results to a file.')
arg_parser.add_argument('filename',
action='store',
metavar='STORAGE_FILE',
help=('The path to the plaso storage file.'))
options = arg_parser.parse_args()
preferred_encoding = locale.getpreferredencoding()
if preferred_encoding.lower() == 'ascii':
preferred_encoding = 'utf-8'
if not os.path.isfile(options.filename):
raise RuntimeError(u'File {} does not exist'.format(options.filename))
results = {}
result_count = {}
output_filehandle = output.OutputFilehandle(preferred_encoding)
if options.output_file:
output_filehandle.Open(path=options.output_file)
else:
output_filehandle.Open(sys.stdout)
# Build filters.
filter_dict = {}
for filter_str, call_back in FILTERS:
filter_obj = filters.GetFilter(filter_str)
call_back_obj = getattr(FilterClass, call_back, None)
results[call_back] = []
if filter_obj and call_back_obj:
filter_dict[filter_obj] = (call_back, call_back_obj)
with storage.StorageFile(options.filename, read_only=True) as store:
event_object = store.GetSortedEntry()
while event_object:
for filter_obj, call_backs in filter_dict.items():
call_back_name, call_back_object = call_backs
if filter_obj.Match(event_object):
url_attribute = getattr(event_object, 'url', None)
if not url_attribute:
continue
ret_line = ScrubLine(call_back_object(url_attribute))
if not ret_line:
continue
if ret_line in results[call_back_name]:
result_count[u'{}:{}'.format(call_back_name,
ret_line)] += 1
else:
results[call_back_name].append(ret_line)
result_count[u'{}:{}'.format(call_back_name,
ret_line)] = 1
event_object = store.GetSortedEntry()
for engine_name, result_list in results.items():
results_with_count = []
for result in result_list:
results_with_count.append(
(result_count[u'{}:{}'.format(engine_name, result)], result))
header = u' == ENGINE: {0:s} ==\n'.format(engine_name)
output_filehandle.WriteLine(header)
for count, result in sorted(results_with_count, reverse=True):
line = u'{} {}\n'.format(count, result)
output_filehandle.WriteLine(line)
output_filehandle.WriteLine('\n')
def testStorage(self):
"""Test the storage object."""
event_objects = []
timestamps = []
group_mock = GroupMock()
tags = []
tags_mock = []
groups = []
group_events = []
same_events = []
serializer = protobuf_serializer.ProtobufEventObjectSerializer
with TempDirectory() as dirname:
temp_file = os.path.join(dirname, 'plaso.db')
store = storage.StorageFile(temp_file)
store.AddEventObjects(self._event_objects)
# Add tagging.
tag_1 = event.EventTag()
tag_1.store_index = 0
tag_1.store_number = 1
tag_1.comment = 'My comment'
tag_1.color = 'blue'
tags_mock.append(tag_1)
tag_2 = event.EventTag()
tag_2.store_index = 1
tag_2.store_number = 1
tag_2.tags = ['Malware']
tag_2.color = 'red'
tags_mock.append(tag_2)
tag_3 = event.EventTag()
tag_3.store_number = 1
tag_3.store_index = 2
tag_3.comment = 'This is interesting'
tag_3.tags = ['Malware', 'Benign']
tag_3.color = 'red'
tags_mock.append(tag_3)
store.StoreTagging(tags_mock)
# Add additional tagging, second round.
tag_4 = event.EventTag()
tag_4.store_index = 1
tag_4.store_number = 1
tag_4.tags = ['Interesting']
store.StoreTagging([tag_4])
group_mock.AddGroup(
'Malicious', [(1, 1), (1, 2)], desc='Events that are malicious',
color='red', first=13349402860000000, last=13349615269295969,
cat='Malware')
store.StoreGrouping(group_mock)
store.Close()
read_store = storage.StorageFile(temp_file, read_only=True)
self.assertTrue(read_store.HasTagging())
self.assertTrue(read_store.HasGrouping())
for event_object in read_store.GetEntries(1):
event_objects.append(event_object)
timestamps.append(event_object.timestamp)
if event_object.data_type == 'windows:registry:key_value':
self.assertEquals(event_object.timestamp_desc,
eventdata.EventTimestamp.WRITTEN_TIME)
else:
self.assertEquals(event_object.timestamp_desc,
eventdata.EventTimestamp.WRITTEN_TIME)
for tag in read_store.GetTagging():
event_object = read_store.GetTaggedEvent(tag)
tags.append(event_object)
groups = list(read_store.GetGrouping())
self.assertEquals(len(groups), 1)
group_events = list(read_store.GetEventsFromGroup(groups[0]))
# Read the same events that were put in the group, just to compare
# against.
event_object = read_store.GetEventObject(1, 1)
serialized_event_object = serializer.WriteSerialized(event_object)
same_events.append(serialized_event_object)
event_object = read_store.GetEventObject(1, 2)
serialized_event_object = serializer.WriteSerialized(event_object)
same_events.append(serialized_event_object)
self.assertEquals(len(event_objects), 4)
self.assertEquals(len(tags), 4)
self.assertEquals(tags[0].timestamp, 12389344590000000)
self.assertEquals(tags[0].store_number, 1)
self.assertEquals(tags[0].store_index, 0)
self.assertEquals(tags[0].tag.comment, u'My comment')
self.assertEquals(tags[0].tag.color, u'blue')
msg, _ = eventdata.EventFormatterManager.GetMessageStrings(tags[0])
self.assertEquals(msg[0:10], u'This is a ')
self.assertEquals(tags[1].tag.tags[0], 'Malware')
msg, _ = eventdata.EventFormatterManager.GetMessageStrings(tags[1])
self.assertEquals(msg[0:15], u'[\\HKCU\\Windows\\')
self.assertEquals(tags[2].tag.comment, u'This is interesting')
self.assertEquals(tags[2].tag.tags[0], 'Malware')
self.assertEquals(tags[2].tag.tags[1], 'Benign')
self.assertEquals(tags[2].parser, 'UNKNOWN')
# Test the newly added fourth tag, which should include data from
# the first version as well.
self.assertEquals(tags[3].tag.tags[0], 'Interesting')
self.assertEquals(tags[3].tag.tags[1], 'Malware')
expected_timestamps = [
12389344590000000, 13349402860000000, 13349615269295969,
13359662069295961]
self.assertEquals(timestamps, expected_timestamps)
self.assertEquals(groups[0].name, u'Malicious')
self.assertEquals(groups[0].category, u'Malware')
self.assertEquals(groups[0].color, u'red')
self.assertEquals(groups[0].description, u'Events that are malicious')
self.assertEquals(groups[0].first_timestamp, 13349402860000000)
self.assertEquals(groups[0].last_timestamp, 13349615269295969)
self.assertEquals(len(group_events), 2)
self.assertEquals(group_events[0].timestamp, 13349402860000000)
self.assertEquals(group_events[1].timestamp, 13349615269295969L)
proto_group_events = []
for group_event in group_events:
serialized_event_object = serializer.WriteSerialized(group_event)
proto_group_events.append(serialized_event_object)
self.assertEquals(same_events, proto_group_events)
