def testMergeFromStorage(self):
"""Tests the MergeFromStorage function."""
session = sessions.Session()
storage_writer = fake_storage.FakeStorageWriter(session)
storage_writer.Open()
test_file = self._GetTestFilePath([u'psort_test.json.plaso'])
storage_reader = zip_file.ZIPStorageFileReader(test_file)
storage_writer.MergeFromStorage(storage_reader)
test_file = self._GetTestFilePath([u'pinfo_test.json.plaso'])
storage_reader = zip_file.ZIPStorageFileReader(test_file)
storage_writer.MergeFromStorage(storage_reader)
storage_writer.Close()
def testOutput(self):
"""Tests the Output function."""
test_filename = os.path.join(u'test_data', u'psort_test.json.plaso')
with shared_test_lib.TempDirectory() as temp_directory:
temp_file = os.path.join(temp_directory, u'pstorage.plaso')
storage_file = storage_zip_file.StorageFile(test_filename,
read_only=True)
with storage_zip_file.ZIPStorageFileReader(
storage_file) as storage_reader:
output_mediator = self._CreateOutputMediator(
storage_file=storage_file)
output_module = pstorage.PlasoStorageOutputModule(
output_mediator)
output_module.SetFilePath(temp_file)
with event_buffer.EventBuffer(
output_module, check_dedups=False) as output_buffer:
for event_object in storage_reader.GetEvents():
output_buffer.Append(event_object)
original_zip_file = storage_zip_file.StorageFile(test_filename,
read_only=True)
pstorage_zip_file = storage_zip_file.StorageFile(temp_file,
read_only=True)
original_list = []
pstorage_list = []
event_object_original = original_zip_file.GetSortedEntry()
event_object_pstorage = pstorage_zip_file.GetSortedEntry()
while event_object_original:
original_equality_string = event_object_original.EqualityString(
)
pstorage_equality_string = event_object_pstorage.EqualityString(
)
# Remove the UUID for comparision.
original_equality_string, _, _ = original_equality_string.rpartition(
u'|')
pstorage_equality_string, _, _ = pstorage_equality_string.rpartition(
u'|')
original_list.append(original_equality_string)
pstorage_list.append(pstorage_equality_string)
event_object_original = original_zip_file.GetSortedEntry()
event_object_pstorage = pstorage_zip_file.GetSortedEntry()
self.assertFalse(event_object_pstorage)
for original_str, dump_str in zip(sorted(original_list),
sorted(pstorage_list)):
self.assertEqual(original_str, dump_str)
def CreateStorageReader(self, storage_file_path):
"""Creates a storage reader.
Args:
storage_file_path (str): path of the storage file.
Returns:
StorageReader: storage reader.
"""
return storage_zip_file.ZIPStorageFileReader(storage_file_path)
def CreateStorageReaderForFile(cls, path):
"""Creates a storage reader based on the file.
Args:
path (str): path to the storage file.
Returns:
StorageReader: a storage reader or None if the storage file cannot be
opened or the storage format is not supported.
"""
if storage_sqlite_file.SQLiteStorageFile.CheckSupportedFormat(path):
return storage_sqlite_file.SQLiteStorageFileReader(path)
elif storage_zip_file.ZIPStorageFile.CheckSupportedFormat(path):
return storage_zip_file.ZIPStorageFileReader(path)
def export_storage_file(self, storage_file):
"""Extracts and exports plaso event data and sources into the graph."""
with zip_file.ZIPStorageFileReader(storage_file) as storage_reader:
knowledge_base = KnowledgeBase()
storage_reader.ReadPreprocessingInformation(knowledge_base)
# TODO: Export knowledge base.
for session in storage_reader._storage_file.GetSessions():
self.export_session(session)
for source in storage_reader.GetEventSources():
self.export_event_source(source)
for event in storage_reader.GetEvents():
self.export_event(event)
def testInternalExportEvents(self):
"""Tests the _ExportEvents function."""
knowledge_base_object = knowledge_base.KnowledgeBase()
output_writer = cli_test_lib.TestOutputWriter()
formatter_mediator = formatters_mediator.FormatterMediator()
output_mediator_object = output_mediator.OutputMediator(
knowledge_base_object, formatter_mediator)
output_module = TestOutputModule(output_mediator_object)
output_module.SetOutputWriter(output_writer)
test_engine = psort.PsortMultiProcessEngine()
formatters_manager.FormattersManager.RegisterFormatter(
TestEventFormatter)
with shared_test_lib.TempDirectory() as temp_directory:
temp_file = os.path.join(temp_directory, u'storage.plaso')
self._CreateTestStorageFile(temp_file)
storage_reader = storage_zip_file.ZIPStorageFileReader(temp_file)
storage_reader.ReadPreprocessingInformation(knowledge_base_object)
event_buffer = TestEventBuffer(output_module, check_dedups=False)
test_engine._ExportEvents(storage_reader, event_buffer)
event_buffer.Flush()
formatters_manager.FormattersManager.DeregisterFormatter(
TestEventFormatter)
lines = []
output = output_writer.ReadOutput()
for line in output.split(b'\n'):
lines.append(line)
self.assertEqual(len(lines), 8)
self.assertTrue(b'My text goes along: My text dude. lines' in lines[2])
self.assertTrue(b'LOG/' in lines[2])
self.assertTrue(b'None in Particular' in lines[2])
self.assertEqual(lines[0], (
b'date,time,timezone,MACB,source,sourcetype,type,user,host,short,desc,'
b'version,filename,inode,notes,format,extra'))
def testReadEntries(self):
"""Ensure returned EventObjects from the storage are within time bounds."""
storage_file_path = self._GetTestFilePath([u'psort_test.proto.plaso'])
storage_file = storage_zip_file.StorageFile(storage_file_path,
read_only=True)
time_range = storage_time_range.TimeRange(self._start_timestamp,
self._end_timestamp)
timestamp_list = []
with storage_zip_file.ZIPStorageFileReader(
storage_file) as storage_reader:
for event_object in storage_reader.GetEvents(
time_range=time_range):
timestamp_list.append(event_object.timestamp)
self.assertEqual(len(timestamp_list), 15)
self.assertEqual(timestamp_list[0], self._start_timestamp)
self.assertEqual(timestamp_list[-1], self._end_timestamp)
def testOutput(self):
"""Tests the Output function."""
with shared_test_lib.TempDirectory() as temp_directory:
temp_file = os.path.join(temp_directory, u'pstorage.plaso')
# Copy events to pstorage dump.
storage_file = storage_zip_file.StorageFile(self._test_filename,
read_only=True)
with storage_zip_file.ZIPStorageFileReader(
storage_file) as storage_reader:
output_mediator = self._CreateOutputMediator(
storage_file=storage_file)
output_module = pstorage.PlasoStorageOutputModule(
output_mediator)
output_module.SetFilePath(temp_file)
with event_buffer.EventBuffer(
output_module, check_dedups=False) as output_buffer:
for event_object in storage_reader.GetEvents():
output_buffer.Append(event_object)
# Make sure original and dump have the same events.
original = storage_zip_file.StorageFile(self._test_filename,
read_only=True)
dump = storage_zip_file.StorageFile(temp_file, read_only=True)
event_object_original = original.GetSortedEntry()
event_object_dump = dump.GetSortedEntry()
original_list = []
dump_list = []
while event_object_original:
original_list.append(event_object_original.EqualityString())
dump_list.append(event_object_dump.EqualityString())
event_object_original = original.GetSortedEntry()
event_object_dump = dump.GetSortedEntry()
self.assertFalse(event_object_dump)
for original_str, dump_str in zip(sorted(original_list),
sorted(dump_list)):
self.assertEqual(original_str, dump_str)
def testExportEvents(self):
"""Tests the ExportEvents function."""
storage_file_path = self._GetTestFilePath([u'psort_test.json.plaso'])
knowledge_base_object = knowledge_base.KnowledgeBase()
output_writer = cli_test_lib.TestOutputWriter()
formatter_mediator = formatters_mediator.FormatterMediator()
formatter_mediator.SetPreferredLanguageIdentifier(u'en-US')
output_mediator_object = output_mediator.OutputMediator(
knowledge_base_object, formatter_mediator)
output_module = dynamic.DynamicOutputModule(output_mediator_object)
output_module.SetOutputWriter(output_writer)
storage_reader = storage_zip_file.ZIPStorageFileReader(storage_file_path)
test_engine = psort.PsortMultiProcessEngine()
counter = test_engine.ExportEvents(
knowledge_base_object, storage_reader, output_module)
# TODO: refactor preprocessing object.
self.assertEqual(counter[u'Stored Events'], 0)
lines = []
output = output_writer.ReadOutput()
for line in output.split(b'\n'):
lines.append(line)
self.assertEqual(len(lines), 24)
expected_line = (
u'2016-10-16T15:13:43+00:00,'
u'mtime,'
u'FILE,'
u'OS mtime,'
u'OS:/tmp/test/test_data/syslog Type: file,'
u'filestat,'
u'OS:/tmp/test/test_data/syslog,-')
self.assertEquals(lines[14], expected_line)
def testExportEvents(self):
"""Tests the ExportEvents function."""
storage_file_path = self._GetTestFilePath(['psort_test.json.plaso'])
knowledge_base_object = knowledge_base.KnowledgeBase()
output_writer = cli_test_lib.TestOutputWriter()
formatter_mediator = formatters_mediator.FormatterMediator()
formatter_mediator.SetPreferredLanguageIdentifier('en-US')
output_mediator_object = output_mediator.OutputMediator(
knowledge_base_object, formatter_mediator)
output_module = dynamic.DynamicOutputModule(output_mediator_object)
output_module.SetOutputWriter(output_writer)
storage_reader = storage_zip_file.ZIPStorageFileReader(
storage_file_path)
test_engine = psort.PsortMultiProcessEngine()
counter = test_engine.ExportEvents(knowledge_base_object,
storage_reader, output_module)
self.assertEqual(counter['Stored Events'], 0)
lines = []
output = output_writer.ReadOutput()
for line in output.split(b'\n'):
lines.append(line)
self.assertEqual(len(lines), 22)
expected_line = ('2014-11-18T01:15:43+00:00,'
'Content Modification Time,'
'LOG,'
'Log File,'
'[---] last message repeated 5 times ---,'
'syslog,'
'OS:/tmp/test/test_data/syslog,'
'repeated')
self.assertEqual(lines[14], expected_line)
def testInternalExportEvents(self):
"""Tests the _ExportEvents function."""
knowledge_base_object = knowledge_base.KnowledgeBase()
output_writer = cli_test_lib.TestOutputWriter()
formatter_mediator = formatters_mediator.FormatterMediator()
output_mediator_object = output_mediator.OutputMediator(
knowledge_base_object, formatter_mediator)
output_module = TestOutputModule(output_mediator_object)
output_module.SetOutputWriter(output_writer)
test_engine = psort.PsortMultiProcessEngine()
formatters_manager.FormattersManager.RegisterFormatter(
TestEventFormatter)
with shared_test_lib.TempDirectory() as temp_directory:
temp_file = os.path.join(temp_directory, 'storage.plaso')
self._CreateTestStorageFile(temp_file)
storage_reader = storage_zip_file.ZIPStorageFileReader(temp_file)
storage_reader.ReadPreprocessingInformation(knowledge_base_object)
test_engine._ExportEvents(storage_reader,
output_module,
deduplicate_events=False)
formatters_manager.FormattersManager.DeregisterFormatter(
TestEventFormatter)
lines = []
output = output_writer.ReadOutput()
for line in output.split(b'\n'):
lines.append(line)
self.assertEqual(len(output_module.events), 17)
self.assertEqual(len(output_module.macb_groups), 3)
def testGetEvents(self):
"""Tests the GetEvents function."""
test_file = self._GetTestFilePath([u'psort_test.json.plaso'])
storage_file = zip_file.StorageFile(test_file, read_only=True)
timestamps = []
with zip_file.ZIPStorageFileReader(storage_file) as storage_reader:
for event_object in storage_reader.GetEvents():
timestamps.append(event_object.timestamp)
expected_timestamps = [
1453449153000000, 1453449153000000, 1453449153000000, 1453449153000000,
1453449181000000, 1453449181000000, 1453449241000000, 1453449241000000,
1453449241000000, 1453449241000000, 1453449272000000, 1453449272000000,
1456708543000000, 1456708543000000, 1462105168000000, 1462105168000000,
1462105168000000, 1462105168000000, 1462105169000000, 1462105170000000,
1482083672000000, 1482083672000000, 1490310078000000, 1490310078000000,
1490310078000123, 1490310078000123, 1514742872000000, 1514742872000000,
1542503720000000, 1542503720000000, 1542503743000000, 1542503743000000]
self.assertEqual(len(timestamps), 32)
self.assertEqual(sorted(timestamps), expected_timestamps)
# Test lower bound time range filter.
test_time_range = time_range.TimeRange(
timelib.Timestamp.CopyFromString(u'2016-04-30 06:41:49'),
timelib.Timestamp.CopyFromString(u'2030-12-31 23:59:59'))
storage_file = zip_file.StorageFile(test_file, read_only=True)
timestamps = []
with zip_file.ZIPStorageFileReader(storage_file) as storage_reader:
for event_object in storage_reader.GetEvents(
time_range=test_time_range):
timestamps.append(event_object.timestamp)
expected_timestamps = [
1462105168000000, 1462105168000000, 1462105168000000, 1462105168000000,
1462105169000000, 1462105170000000, 1482083672000000, 1482083672000000,
1490310078000000, 1490310078000000, 1490310078000123, 1490310078000123,
1514742872000000, 1514742872000000, 1542503720000000, 1542503720000000,
1542503743000000, 1542503743000000]
self.assertEqual(sorted(timestamps), expected_timestamps)
# Test upper bound time range filter.
test_time_range = time_range.TimeRange(
timelib.Timestamp.CopyFromString(u'2000-01-01 00:00:00'),
timelib.Timestamp.CopyFromString(u'2016-04-30 06:41:49'))
storage_file = zip_file.StorageFile(test_file, read_only=True)
timestamps = []
with zip_file.ZIPStorageFileReader(storage_file) as storage_reader:
for event_object in storage_reader.GetEvents(
time_range=test_time_range):
timestamps.append(event_object.timestamp)
expected_timestamps = [
1453449153000000, 1453449153000000, 1453449153000000, 1453449153000000,
1453449181000000, 1453449181000000, 1453449241000000, 1453449241000000,
1453449241000000, 1453449241000000, 1453449272000000, 1453449272000000,
1456708543000000, 1456708543000000]
self.assertEqual(sorted(timestamps), expected_timestamps)
def ProcessStorage(self,
output_module,
storage_file,
storage_file_path,
analysis_plugins,
event_queue_producers,
command_line_arguments=None,
deduplicate_events=True,
preferred_encoding=u'utf-8',
time_slice=None,
use_time_slicer=False):
"""Processes a plaso storage file.
Args:
output_module: an output module (instance of OutputModule).
storage_file: the storage file object (instance of StorageFile).
storage_file_path: string containing the path of the storage file.
analysis_plugins: list of analysis plugin objects (instance of
AnalysisPlugin).
event_queue_producers: list of event queue producer objects (instance
of ItemQueueProducer).
command_line_arguments: optional string of the command line arguments or
None if not set.
deduplicate_events: optional boolean value to indicate if the event
objects should be deduplicated.
preferred_encoding: optional preferred encoding.
time_slice: optional time slice object (instance of TimeSlice).
use_time_slicer: optional boolean value to indicate the 'time slicer'
should be used. The 'time slicer' will provide a
context of events around an event of interest.
Returns:
A counter (an instance of collections.Counter) that tracks the number of
events extracted from storage, and the analysis plugin results.
Raises:
RuntimeError: if a non-recoverable situation is encountered.
"""
if time_slice:
if time_slice.event_timestamp is not None:
time_slice = storage_time_range.TimeRange(
time_slice.start_timestamp, time_slice.end_timestamp)
elif use_time_slicer:
self._filter_buffer = bufferlib.CircularBuffer(
time_slice.duration)
# TODO: allow for single processing.
# TODO: add upper queue limit.
analysis_queue_port = None
if self._use_zeromq:
analysis_report_incoming_queue = zeromq_queue.ZeroMQPullBindQueue(
delay_open=False, port=None, linger_seconds=5)
analysis_queue_port = analysis_report_incoming_queue.port
else:
analysis_report_incoming_queue = multi_process.MultiProcessingQueue(
timeout=5)
pre_obj = self._GetLastGoodPreprocess(storage_file)
if pre_obj is None:
pre_obj = event.PreprocessObject()
if analysis_plugins:
self._StartAnalysisPlugins(
storage_file_path,
analysis_plugins,
pre_obj,
analysis_queue_port=analysis_queue_port,
analysis_report_incoming_queue=analysis_report_incoming_queue,
command_line_arguments=command_line_arguments)
# Assign the preprocessing object to the storage.
# This is normally done in the construction of the storage object,
# however we cannot do that here since the preprocessing object is
# stored inside the storage file, so we need to open it first to
# be able to read it in, before we make changes to it. Thus we need
# to access this protected member of the class.
# pylint: disable=protected-access
storage_file._pre_obj = pre_obj
else:
event_queue_producers = []
output_buffer = output_event_buffer.EventBuffer(
output_module, deduplicate_events)
with output_buffer:
storage_reader = storage_zip_file.ZIPStorageFileReader(
storage_file)
counter = self.ProcessEventsFromStorage(
storage_reader,
output_buffer,
analysis_queues=event_queue_producers,
filter_buffer=self._filter_buffer,
my_filter=self._filter_object,
time_slice=time_slice)
for information in storage_file.GetStorageInformation():
if hasattr(information, u'counter'):
counter[u'Stored Events'] += information.counter[u'total']
# Get all reports and tags from analysis plugins.
self._ProcessAnalysisPlugins(analysis_plugins,
analysis_report_incoming_queue,
storage_file,
counter,
pre_obj,
preferred_encoding=preferred_encoding)
if self._filter_object and not counter[u'Limited By']:
counter[u'Filter By Date'] = (counter[u'Stored Events'] -
counter[u'Events Included'] -
counter[u'Events Filtered Out'])
return counter
def AnalyzeEvents(self):
"""Analyzes events from a plaso storage file and generate a report.
Raises:
BadConfigOption: when a configuration parameter fails validation.
RuntimeError: if a non-recoverable situation is encountered.
"""
session = engine.BaseEngine.CreateSession(
command_line_arguments=self._command_line_arguments,
preferred_encoding=self.preferred_encoding)
storage_reader = storage_zip_file.ZIPStorageFileReader(
self._storage_file_path)
self._number_of_analysis_reports = (
storage_reader.GetNumberOfAnalysisReports())
storage_reader.Close()
counter = collections.Counter()
if self._output_format != u'null':
self._status_view.SetMode(self._status_view_mode)
self._status_view.SetStorageFileInformation(
self._storage_file_path)
status_update_callback = (
self._status_view.GetAnalysisStatusUpdateCallback())
storage_reader = storage_zip_file.ZIPStorageFileReader(
self._storage_file_path)
# TODO: add single processing support.
analysis_engine = psort.PsortMultiProcessEngine(
use_zeromq=self._use_zeromq)
# TODO: pass configuration object.
events_counter = analysis_engine.ExportEvents(
self._knowledge_base,
storage_reader,
self._output_module,
deduplicate_events=self._deduplicate_events,
status_update_callback=status_update_callback,
time_slice=self._time_slice,
use_time_slicer=self._use_time_slicer)
counter += events_counter
for item, value in iter(session.analysis_reports_counter.items()):
counter[item] = value
if self._quiet_mode:
return
self._output_writer.Write(u'Processing completed.\n')
table_view = views.ViewsFactory.GetTableView(self._views_format_type,
title=u'Counter')
for element, count in counter.most_common():
if not element:
element = u'N/A'
table_view.AddRow([element, count])
table_view.Write(self._output_writer)
storage_reader = storage_zip_file.ZIPStorageFileReader(
self._storage_file_path)
self._PrintAnalysisReportsDetails(storage_reader,
self._number_of_analysis_reports)
self._output_writer.Write(u'Storage file is {0:s}\n'.format(
self._storage_file_path))
def testOutput(self):
"""Testing if psort can output data."""
formatters_manager.FormattersManager.RegisterFormatter(
PsortTestEventFormatter)
events = [
PsortTestEvent(5134324321),
PsortTestEvent(2134324321),
PsortTestEvent(9134324321),
PsortTestEvent(15134324321),
PsortTestEvent(5134324322),
PsortTestEvent(5134024321)
]
output_writer = cli_test_lib.TestOutputWriter()
with shared_test_lib.TempDirectory() as temp_directory:
temp_file = os.path.join(temp_directory, u'plaso.db')
storage_file = storage_zip_file.StorageFile(temp_file)
storage_file.AddEventObjects(events)
storage_file.Close()
storage_file = storage_zip_file.StorageFile(temp_file,
read_only=True)
with storage_zip_file.ZIPStorageFileReader(
storage_file) as storage_reader:
output_mediator_object = output_mediator.OutputMediator(
self._formatter_mediator)
output_mediator_object.SetStorageFile(storage_file)
output_module = TestOutputModule(output_mediator_object)
output_module.SetOutputWriter(output_writer)
event_buffer = TestEventBuffer(output_module,
check_dedups=False,
store=storage_file)
self._front_end.ProcessEventsFromStorage(
storage_reader, event_buffer)
event_buffer.Flush()
lines = []
output = output_writer.ReadOutput()
for line in output.split(b'\n'):
if line == b'.':
continue
if line:
lines.append(line)
# One more line than events (header row).
self.assertEqual(len(lines), 7)
self.assertTrue(b'My text goes along: My text dude. lines' in lines[2])
self.assertTrue(b'LOG/' in lines[2])
self.assertTrue(b'None in Particular' in lines[2])
self.assertEqual(lines[0], (
b'date,time,timezone,MACB,source,sourcetype,type,user,host,short,desc,'
b'version,filename,inode,notes,format,extra'))
formatters_manager.FormattersManager.DeregisterFormatter(
PsortTestEventFormatter)
def testGetEvents(self):
"""Tests the GetEvents function."""
test_file = self._GetTestFilePath([u'psort_test.proto.plaso'])
storage_file = zip_file.StorageFile(test_file, read_only=True)
timestamps = []
with zip_file.ZIPStorageFileReader(storage_file) as storage_reader:
for event_object in storage_reader.GetEvents():
timestamps.append(event_object.timestamp)
expected_timestamps = [
1343166324000000, 1344270407000000, 1390377153000000,
1390377153000000, 1390377181000000, 1390377241000000,
1390377241000000, 1390377272000000, 1392438730000000,
1418925272000000, 1427151678000000, 1427151678000123,
1451584472000000, 1479431720000000, 1479431743000000
]
self.assertEqual(sorted(timestamps), expected_timestamps)
# Test lower bound time range filter.
test_time_range = time_range.TimeRange(
timelib.Timestamp.CopyFromString(u'2014-02-16 00:00:00'),
timelib.Timestamp.CopyFromString(u'2030-12-31 23:59:59'))
storage_file = zip_file.StorageFile(test_file, read_only=True)
timestamps = []
with zip_file.ZIPStorageFileReader(storage_file) as storage_reader:
for event_object in storage_reader.GetEvents(
time_range=test_time_range):
timestamps.append(event_object.timestamp)
expected_timestamps = [
1418925272000000, 1427151678000000, 1427151678000123,
1451584472000000, 1479431720000000, 1479431743000000
]
self.assertEqual(sorted(timestamps), expected_timestamps)
# Test upper bound time range filter.
test_time_range = time_range.TimeRange(
timelib.Timestamp.CopyFromString(u'2000-01-01 00:00:00'),
timelib.Timestamp.CopyFromString(u'2014-02-16 00:00:00'))
storage_file = zip_file.StorageFile(test_file, read_only=True)
timestamps = []
with zip_file.ZIPStorageFileReader(storage_file) as storage_reader:
for event_object in storage_reader.GetEvents(
time_range=test_time_range):
timestamps.append(event_object.timestamp)
expected_timestamps = [
1343166324000000, 1344270407000000, 1390377153000000,
1390377153000000, 1390377181000000, 1390377241000000,
1390377241000000, 1390377272000000, 1392438730000000
]
self.assertEqual(sorted(timestamps), expected_timestamps)
