def __init__(self, input_reader=None, output_writer=None):
"""Initializes the CLI tool object.
Args:
input_reader: the input reader (instance of InputReader).
The default is None which indicates the use of the stdin
input reader.
output_writer: the output writer (instance of OutputWriter).
The default is None which indicates the use of the stdout
output writer.
"""
super(PsortTool, self).__init__(input_reader=input_reader,
output_writer=output_writer)
self._analysis_plugins = None
self._analysis_plugins_output_format = None
self._command_line_arguments = None
self._deduplicate_events = True
self._filter_expression = None
self._filter_object = None
self._front_end = psort.PsortFrontend()
self._options = None
self._output_format = None
self._time_slice = None
self._use_time_slicer = False
self.list_analysis_plugins = False
self.list_language_identifiers = False
self.list_output_modules = False
def run_plaso(source_file_path, timeline_name, index_name, username=None):
"""Create a Celery task for processing Plaso storage file.
Args:
source_file_path: Path to plaso storage file.
timeline_name: Name of the Timesketch timeline.
index_name: Name of the datastore index.
username: Username of the user who will own the timeline.
Returns:
Dictionary with count of processed events.
"""
plaso_data_location = get_data_location()
flush_interval = 1000 # events to queue before bulk index
doc_type = u'plaso_event' # Document type for Elasticsearch
# Use Plaso psort frontend tool.
frontend = psort.PsortFrontend()
frontend.SetDataLocation(plaso_data_location)
storage_reader = frontend.CreateStorageReader(source_file_path)
# Setup the Timesketch output module.
output_module = frontend.CreateOutputModule(u'timesketch')
output_module.SetIndexName(index_name)
output_module.SetTimelineName(timeline_name)
output_module.SetFlushInterval(flush_interval)
output_module.SetDocType(doc_type)
if username:
output_module.SetUserName(username)
# Start process the Plaso storage file.
counter = frontend.ExportEvents(storage_reader, output_module)
return dict(counter)
def __init__(self, input_reader=None, output_writer=None):
"""Initializes the CLI tool object.
Args:
input_reader (Optional[InputReader]): input reader, where None indicates
that the stdin input reader should be used.
output_writer (Optional[OutputWriter]): output writer, where None
indicates that the stdout output writer should be used.
"""
super(PsortTool, self).__init__(
input_reader=input_reader, output_writer=output_writer)
self._analysis_plugins = None
self._analysis_plugins_output_format = None
self._command_line_arguments = None
self._deduplicate_events = True
self._event_filter = None
self._event_filter_expression = None
self._front_end = psort.PsortFrontend()
self._number_of_analysis_reports = 0
self._options = None
self._output_filename = None
self._output_format = None
self._status_view_mode = u'linear'
self._stdout_output_writer = isinstance(
self._output_writer, cli_tools.StdoutOutputWriter)
self._temporary_directory = None
self._time_slice = None
self._use_time_slicer = False
self._worker_memory_limit = None
self.list_analysis_plugins = False
self.list_language_identifiers = False
self.list_output_modules = False
def run_plaso(source_file_path, timeline_name, index_name):
"""Create a Celery task for processing Plaso storage file.
Args:
source_file_path: Path to plaso storage file.
timeline_name: Name of the Timesketch timeline.
index_name: Name of the datastore index.
Returns:
Dictionary with count of processed events.
"""
plaso_data_location = get_data_location()
analysis_plugins = None
flush_interval_ms = 1000
# Use the Psort frontend for processing.
frontend = psort.PsortFrontend()
frontend.SetDataLocation(plaso_data_location)
storage_file = frontend.OpenStorage(source_file_path, read_only=True)
# Setup the Timesketch output module.
frontend.SetOutputFormat(u'timesketch')
output_module = frontend.GetOutputModule(storage_file)
output_module.SetFlushInterval(flush_interval_ms)
output_module.SetIndex(index_name)
output_module.SetName(timeline_name)
plugins, queue_producers = frontend.GetAnalysisPluginsAndEventQueues(
analysis_plugins)
counter = frontend.ProcessStorage(output_module, storage_file,
source_file_path, plugins,
queue_producers)
return dict(counter)
def __init__(self, input_reader=None, output_writer=None):
"""Initializes the CLI tool object.
Args:
input_reader (InputReader): the input reader.
The default is None which indicates the use of the stdin
input reader.
output_writer (OutputWriter): the output writer.
The default is None which indicates the use of the stdout
output writer.
"""
super(PstealTool, self).__init__(
input_reader=input_reader, output_writer=output_writer)
self._analysis_front_end = psort.PsortFrontend()
self._command_line_arguments = None
self._deduplicate_events = True
self._enable_sigsegv_handler = False
self._extraction_front_end = log2timeline.Log2TimelineFrontend()
self._force_preprocessing = False
self._hasher_names_string = None
self._number_of_extraction_workers = 0
self._options = None
self._output_format = u'dynamic'
self._output_filename = None
self._output_module = None
self._parser_filter_expression = None
self._preferred_year = None
self._single_process_mode = False
self._source_type = None
self._status_view_mode = u'window'
self._time_slice = None
self._use_time_slicer = False
self._yara_rules_string = None
def testProcessStorage(self):
"""Test the ProcessStorage function."""
test_front_end = psort.PsortFrontend()
test_front_end.SetOutputFilename(u'output.txt')
test_front_end.SetOutputFormat(u'dynamic')
test_front_end.SetPreferredLanguageIdentifier(u'en-US')
test_front_end.SetQuietMode(True)
storage_file_path = self._GetTestFilePath([u'psort_test.out'])
storage_file = test_front_end.OpenStorage(storage_file_path, read_only=True)
output_writer = test_lib.StringIOOutputWriter()
output_module = test_front_end.GetOutputModule(storage_file)
output_module.SetOutputWriter(output_writer)
counter = test_front_end.ProcessStorage(output_module, storage_file, [], [])
self.assertEqual(counter[u'Stored Events'], 15)
output_writer.SeekToBeginning()
lines = []
line = output_writer.GetLine()
while line:
lines.append(line)
line = output_writer.GetLine()
self.assertEqual(len(lines), 16)
expected_line = (
u'2015-12-31T17:54:32+00:00,Entry Written,LOG,Log File,[anacron '
u'pid: 1234] : Another one just like this (124 job run),syslog,'
u'OS:syslog,-,6,1\n')
self.assertEquals(lines[13], expected_line)
def setUp(self):
"""Setup sets parameters that will be reused throughout this test."""
self._front_end = psort.PsortFrontend()
# TODO: have sample output generated from the test.
self._test_file = os.path.join(self._TEST_DATA_PATH, 'psort_test.out')
self.first = timelib_test.CopyStringToTimestamp('2012-07-24 21:45:24')
self.last = timelib_test.CopyStringToTimestamp('2016-11-18 01:15:43')
def testGetLastGoodPreprocess(self): """Tests the last good preprocess method.""" test_front_end = psort.PsortFrontend() storage_file_path = self._GetTestFilePath([u'psort_test.json.plaso']) storage_file = test_front_end.OpenStorage(storage_file_path, read_only=True) preprocessor_object = test_front_end._GetLastGoodPreprocess(storage_file) self.assertIsNotNone(preprocessor_object) timezone = getattr(preprocessor_object, u'zone') self.assertEqual(timezone.zone, u'Iceland')
def setUp(self):
"""Makes preparations before running an individual test."""
self._formatter_mediator = formatters_mediator.FormatterMediator()
self._front_end = psort.PsortFrontend()
# TODO: have sample output generated from the test.
self._start_timestamp = timelib.Timestamp.CopyFromString(
u'2012-07-24 21:45:24')
self._end_timestamp = timelib.Timestamp.CopyFromString(
u'2016-11-18 01:15:43')
def setUp(self):
"""Setup sets parameters that will be reused throughout this test."""
self._formatter_mediator = formatters_mediator.FormatterMediator()
self._front_end = psort.PsortFrontend()
# TODO: have sample output generated from the test.
self._test_file_proto = self._GetTestFilePath(
[u'psort_test.proto.plaso'])
self._test_file_json = self._GetTestFilePath(
[u'psort_test.json.plaso'])
self.first = timelib.Timestamp.CopyFromString(u'2012-07-24 21:45:24')
self.last = timelib.Timestamp.CopyFromString(u'2016-11-18 01:15:43')
def testSetAnalysisPluginProcessInformation(self):
"""Test the _SetAnalysisPluginProcessInformation method."""
test_front_end = psort.PsortFrontend()
analysis_plugins = [test_lib.TestAnalysisPlugin(None)]
preprocess_object = event.PreprocessObject()
preprocess_object.SetCollectionInformationValues({})
test_front_end._SetAnalysisPluginProcessInformation(
u'', analysis_plugins, preprocess_object)
self.assertIsNotNone(preprocess_object)
plugin_names = preprocess_object.collection_information[u'plugins']
time_of_run = preprocess_object.collection_information[u'time_of_run']
method = preprocess_object.collection_information[u'method']
for analysis_plugin in analysis_plugins:
self.assertIn(analysis_plugin.NAME, plugin_names)
self.assertAlmostEqual(timelib.Timestamp.GetNow(), time_of_run, 2000000)
self.assertIsNotNone(method)
def ProcessStorage(options):
"""Process a storage file and produce profile results.
Args:
options: the command line arguments (instance of argparse.Namespace).
Returns:
The profiling statistics or None on error.
"""
storage_parameters = options.storage.split()
storage_parameters.append(options.file_to_parse)
if options.filter:
storage_parameters.append(options.filter)
front_end = psort.PsortFrontend()
try:
front_end.ParseOptions(options)
except errors.BadConfigOption as exception:
logging.error(u'{0:s}'.format(exception))
return
if options.verbose:
# TODO: why not move this functionality into psort?
profiler = cProfile.Profile()
profiler.enable()
else:
time_start = time.time()
# Call psort and process output.
_ = front_end.ParseStorage(options)
if options.verbose:
profiler.disable()
else:
time_end = time.time()
if options.verbose:
return GetStats(profiler)
else:
print frontend_utils.FormatHeader('Time Used')
print u'{:>20f}s'.format(time_end - time_start)
def testProcessStorage(self):
"""Test the ProcessStorage function."""
test_front_end = psort.PsortFrontend()
test_front_end.SetOutputFormat(u'dynamic')
test_front_end.SetPreferredLanguageIdentifier(u'en-US')
test_front_end.SetQuietMode(True)
storage_file_path = self._GetTestFilePath([u'psort_test.json.plaso'])
storage_file = test_front_end.OpenStorage(storage_file_path,
read_only=True)
try:
output_writer = test_lib.StringIOOutputWriter()
output_module = test_front_end.GetOutputModule(storage_file)
output_module.SetOutputWriter(output_writer)
counter = test_front_end.ProcessStorage(output_module,
storage_file,
storage_file_path, [], [])
finally:
storage_file.Close()
self.assertEqual(counter[u'Stored Events'], 32)
output_writer.SeekToBeginning()
lines = []
line = output_writer.GetLine()
while line:
lines.append(line)
line = output_writer.GetLine()
self.assertEqual(len(lines), 20)
expected_line = (u'2016-05-01T12:19:30+00:00,'
u'atime,'
u'FILE,'
u'OS atime,'
u'OS:/tmp/test/test_data/syslog Type: file,'
u'filestat,'
u'OS:/tmp/test/test_data/syslog,-\n')
self.assertEquals(lines[13], expected_line)
def Main(arguments=None):
"""Start the tool."""
multiprocessing.freeze_support()
front_end = psort.PsortFrontend()
arg_parser = argparse.ArgumentParser(
description=(u'PSORT - Application to read, filter and process '
u'output from a plaso storage file.'),
add_help=False)
tool_group = arg_parser.add_argument_group(u'Optional arguments For psort')
output_group = arg_parser.add_argument_group(
u'Optional arguments for output modules')
analysis_group = arg_parser.add_argument_group(
u'Optional arguments for analysis modules')
tool_group.add_argument(u'-d',
u'--debug',
action=u'store_true',
dest=u'debug',
default=False,
help=u'Fall back to debug shell if psort fails.')
tool_group.add_argument(u'-q',
u'--quiet',
action=u'store_true',
dest=u'quiet',
default=False,
help=u'Do not print a summary after processing.')
tool_group.add_argument(u'-h',
u'--help',
action=u'help',
help=u'Show this help message and exit.')
tool_group.add_argument(
u'-a',
u'--include_all',
action=u'store_false',
dest=u'dedup',
default=True,
help=(
u'By default the psort removes duplicate entries from the output. '
u'This parameter changes that behavior so all events are included.'
))
tool_group.add_argument(
u'-o',
u'--output_format',
u'--output-format',
metavar=u'FORMAT',
dest=u'output_format',
default=u'dynamic',
help=(u'The output format. Use "-o list" to see a list of available '
u'output formats.'))
tool_group.add_argument(
u'--analysis',
metavar=u'PLUGIN_LIST',
dest=u'analysis_plugins',
default=u'',
action=u'store',
type=unicode,
help=(u'A comma separated list of analysis plugin names to be loaded '
u'or "--analysis list" to see a list of available plugins.'))
tool_group.add_argument(u'--data',
metavar=u'PATH',
dest=u'data_location',
default=u'',
action=u'store',
type=unicode,
help=u'The location of the analysis data.')
tool_group.add_argument(
u'--language',
metavar=u'LANGUAGE',
dest=u'preferred_language',
default=u'en-US',
type=unicode,
help=(
u'The preferred language identifier for Windows Event Log message '
u'strings. Use "--language list" to see a list of available '
u'language identifiers. Note that formatting will fall back on '
u'en-US (LCID 0x0409) if the preferred language is not available '
u'in the database of message string templates.'))
tool_group.add_argument(
u'-z',
u'--zone',
metavar=u'TIMEZONE',
dest=u'timezone',
default=u'UTC',
type=unicode,
help=(u'The timezone in which to represent the date and time values. '
u'Use "-z list" to see a list of available timezones.'))
tool_group.add_argument(u'-w',
u'--write',
metavar=u'OUTPUTFILE',
dest=u'write',
help=u'Output filename, defaults to stdout.')
tool_group.add_argument(
u'--slice',
metavar=u'DATE',
dest=u'slice',
type=str,
default=u'',
action=u'store',
help=
(u'Create a time slice around a certain date. This parameter, if '
u'defined will display all events that happened X minutes before and '
u'after the defined date. X is controlled by the parameter '
u'--slice_size but defaults to 5 minutes.'))
tool_group.add_argument(
u'--slicer',
dest=u'slicer',
action=u'store_true',
default=False,
help=
(u'Create a time slice around every filter match. This parameter, if '
u'defined will save all X events before and after a filter match has '
u'been made. X is defined by the --slice_size parameter.'))
tool_group.add_argument(
u'--slice_size',
dest=u'slice_size',
type=int,
default=5,
action=u'store',
help=(
u'Defines the slice size. In the case of a regular time slice it '
u'defines the number of minutes the slice size should be. In the '
u'case of the --slicer it determines the number of events before '
u'and after a filter match has been made that will be included in '
u'the result set. The default value is 5]. See --slice or --slicer '
u'for more details about this option.'))
tool_group.add_argument(
u'-v',
u'--version',
dest=u'version',
action=u'version',
version=u'log2timeline - psort version {0:s}'.format(
plaso.GetVersion()),
help=u'Show the current version of psort.')
front_end.AddStorageFileOptions(tool_group)
tool_group.add_argument(
u'filter',
nargs=u'?',
action=u'store',
metavar=u'FILTER',
default=None,
type=unicode,
help=(u'A filter that can be used to filter the dataset before it '
u'is written into storage. More information about the filters'
u' and it\'s usage can be found here: http://plaso.kiddaland.'
u'net/usage/filters'))
if arguments is None:
arguments = sys.argv[1:]
# Add the output module options.
if u'-o' in arguments:
argument_index = arguments.index(u'-o') + 1
elif u'--output_format' in arguments:
argument_index = arguments.index(u'--output_format') + 1
elif u'--output-format' in arguments:
argument_index = arguments.index(u'--output-format') + 1
else:
argument_index = 0
if argument_index > 0:
module_names = arguments[argument_index]
front_end.AddOutputModuleOptions(output_group, [module_names])
# Add the analysis plugin options.
if u'--analysis' in arguments:
argument_index = arguments.index(u'--analysis') + 1
# Get the names of the analysis plugins that should be loaded.
plugin_names = arguments[argument_index]
try:
front_end.AddAnalysisPluginOptions(analysis_group, plugin_names)
except errors.BadConfigOption as exception:
arg_parser.print_help()
print u''
logging.error(u'{0:s}'.format(exception))
return False
options = arg_parser.parse_args(args=arguments)
format_str = u'[%(levelname)s] %(message)s'
if getattr(options, u'debug', False):
logging.basicConfig(level=logging.DEBUG, format=format_str)
else:
logging.basicConfig(level=logging.INFO, format=format_str)
have_list_option = False
if options.analysis_plugins == u'list':
front_end.ListAnalysisPlugins()
have_list_option = True
if options.output_format == u'list':
front_end.ListOutputModules()
have_list_option = True
if options.preferred_language == u'list':
front_end.ListLanguageIdentifiers()
have_list_option = True
if options.timezone == u'list':
front_end.ListTimeZones()
have_list_option = True
if have_list_option:
return True
if not getattr(options, u'data_location', None):
# Determine if we are running from the source directory.
options.data_location = os.path.dirname(__file__)
options.data_location = os.path.dirname(options.data_location)
options.data_location = os.path.join(options.data_location, u'data')
if not os.path.exists(options.data_location):
# Otherwise determine if there is shared plaso data location.
options.data_location = os.path.join(sys.prefix, u'share',
u'plaso')
if not os.path.exists(options.data_location):
logging.warning(
u'Unable to automatically determine data location.')
options.data_location = None
try:
front_end.ParseOptions(options)
except errors.BadConfigOption as exception:
arg_parser.print_help()
print u''
logging.error(u'{0:s}'.format(exception))
return False
if front_end.preferred_encoding == u'ascii':
logging.warning(
u'The preferred encoding of your system is ASCII, which is not optimal '
u'for the typically non-ASCII characters that need to be parsed and '
u'processed. The tool will most likely crash and die, perhaps in a way '
u'that may not be recoverable. A five second delay is introduced to '
u'give you time to cancel the runtime and reconfigure your preferred '
u'encoding, otherwise continue at own risk.')
time.sleep(5)
try:
counter = front_end.ProcessStorage(options)
if not options.quiet:
logging.info(frontend_utils.FormatHeader(u'Counter'))
for element, count in counter.most_common():
logging.info(frontend_utils.FormatOutputString(element, count))
except IOError as exception:
# Piping results to "|head" for instance causes an IOError.
if u'Broken pipe' not in exception:
logging.error(
u'Processing stopped early: {0:s}.'.format(exception))
except KeyboardInterrupt:
pass
# Catching every remaining exception in case we are debugging.
except Exception as exception:
if not options.debug:
raise
logging.error(u'{0:s}'.format(exception))
pdb.post_mortem()
return True
