def __init__(self, input_reader=None, output_writer=None): """Initializes the CLI tool object. Args: input_reader: the input reader (instance of InputReader). The default is None which indicates the use of the stdin input reader. output_writer: the output writer (instance of OutputWriter). The default is None which indicates the use of the stdout output writer. """ super(PsortTool, self).__init__(input_reader=input_reader, output_writer=output_writer) self._analysis_plugins = None self._analysis_plugins_output_format = None self._command_line_arguments = None self._deduplicate_events = True self._filter_expression = None self._filter_object = None self._front_end = psort.PsortFrontend() self._options = None self._output_format = None self._time_slice = None self._use_time_slicer = False self.list_analysis_plugins = False self.list_language_identifiers = False self.list_output_modules = False
def run_plaso(source_file_path, timeline_name, index_name, username=None): """Create a Celery task for processing Plaso storage file. Args: source_file_path: Path to plaso storage file. timeline_name: Name of the Timesketch timeline. index_name: Name of the datastore index. username: Username of the user who will own the timeline. Returns: Dictionary with count of processed events. """ plaso_data_location = get_data_location() flush_interval = 1000 # events to queue before bulk index doc_type = u'plaso_event' # Document type for Elasticsearch # Use Plaso psort frontend tool. frontend = psort.PsortFrontend() frontend.SetDataLocation(plaso_data_location) storage_reader = frontend.CreateStorageReader(source_file_path) # Setup the Timesketch output module. output_module = frontend.CreateOutputModule(u'timesketch') output_module.SetIndexName(index_name) output_module.SetTimelineName(timeline_name) output_module.SetFlushInterval(flush_interval) output_module.SetDocType(doc_type) if username: output_module.SetUserName(username) # Start process the Plaso storage file. counter = frontend.ExportEvents(storage_reader, output_module) return dict(counter)
def __init__(self, input_reader=None, output_writer=None): """Initializes the CLI tool object. Args: input_reader (Optional[InputReader]): input reader, where None indicates that the stdin input reader should be used. output_writer (Optional[OutputWriter]): output writer, where None indicates that the stdout output writer should be used. """ super(PsortTool, self).__init__( input_reader=input_reader, output_writer=output_writer) self._analysis_plugins = None self._analysis_plugins_output_format = None self._command_line_arguments = None self._deduplicate_events = True self._event_filter = None self._event_filter_expression = None self._front_end = psort.PsortFrontend() self._number_of_analysis_reports = 0 self._options = None self._output_filename = None self._output_format = None self._status_view_mode = u'linear' self._stdout_output_writer = isinstance( self._output_writer, cli_tools.StdoutOutputWriter) self._temporary_directory = None self._time_slice = None self._use_time_slicer = False self._worker_memory_limit = None self.list_analysis_plugins = False self.list_language_identifiers = False self.list_output_modules = False
def run_plaso(source_file_path, timeline_name, index_name): """Create a Celery task for processing Plaso storage file. Args: source_file_path: Path to plaso storage file. timeline_name: Name of the Timesketch timeline. index_name: Name of the datastore index. Returns: Dictionary with count of processed events. """ plaso_data_location = get_data_location() analysis_plugins = None flush_interval_ms = 1000 # Use the Psort frontend for processing. frontend = psort.PsortFrontend() frontend.SetDataLocation(plaso_data_location) storage_file = frontend.OpenStorage(source_file_path, read_only=True) # Setup the Timesketch output module. frontend.SetOutputFormat(u'timesketch') output_module = frontend.GetOutputModule(storage_file) output_module.SetFlushInterval(flush_interval_ms) output_module.SetIndex(index_name) output_module.SetName(timeline_name) plugins, queue_producers = frontend.GetAnalysisPluginsAndEventQueues( analysis_plugins) counter = frontend.ProcessStorage(output_module, storage_file, source_file_path, plugins, queue_producers) return dict(counter)
def __init__(self, input_reader=None, output_writer=None): """Initializes the CLI tool object. Args: input_reader (InputReader): the input reader. The default is None which indicates the use of the stdin input reader. output_writer (OutputWriter): the output writer. The default is None which indicates the use of the stdout output writer. """ super(PstealTool, self).__init__( input_reader=input_reader, output_writer=output_writer) self._analysis_front_end = psort.PsortFrontend() self._command_line_arguments = None self._deduplicate_events = True self._enable_sigsegv_handler = False self._extraction_front_end = log2timeline.Log2TimelineFrontend() self._force_preprocessing = False self._hasher_names_string = None self._number_of_extraction_workers = 0 self._options = None self._output_format = u'dynamic' self._output_filename = None self._output_module = None self._parser_filter_expression = None self._preferred_year = None self._single_process_mode = False self._source_type = None self._status_view_mode = u'window' self._time_slice = None self._use_time_slicer = False self._yara_rules_string = None
def testProcessStorage(self): """Test the ProcessStorage function.""" test_front_end = psort.PsortFrontend() test_front_end.SetOutputFilename(u'output.txt') test_front_end.SetOutputFormat(u'dynamic') test_front_end.SetPreferredLanguageIdentifier(u'en-US') test_front_end.SetQuietMode(True) storage_file_path = self._GetTestFilePath([u'psort_test.out']) storage_file = test_front_end.OpenStorage(storage_file_path, read_only=True) output_writer = test_lib.StringIOOutputWriter() output_module = test_front_end.GetOutputModule(storage_file) output_module.SetOutputWriter(output_writer) counter = test_front_end.ProcessStorage(output_module, storage_file, [], []) self.assertEqual(counter[u'Stored Events'], 15) output_writer.SeekToBeginning() lines = [] line = output_writer.GetLine() while line: lines.append(line) line = output_writer.GetLine() self.assertEqual(len(lines), 16) expected_line = ( u'2015-12-31T17:54:32+00:00,Entry Written,LOG,Log File,[anacron ' u'pid: 1234] : Another one just like this (124 job run),syslog,' u'OS:syslog,-,6,1\n') self.assertEquals(lines[13], expected_line)
def setUp(self): """Setup sets parameters that will be reused throughout this test.""" self._front_end = psort.PsortFrontend() # TODO: have sample output generated from the test. self._test_file = os.path.join(self._TEST_DATA_PATH, 'psort_test.out') self.first = timelib_test.CopyStringToTimestamp('2012-07-24 21:45:24') self.last = timelib_test.CopyStringToTimestamp('2016-11-18 01:15:43')
def testGetLastGoodPreprocess(self): """Tests the last good preprocess method.""" test_front_end = psort.PsortFrontend() storage_file_path = self._GetTestFilePath([u'psort_test.json.plaso']) storage_file = test_front_end.OpenStorage(storage_file_path, read_only=True) preprocessor_object = test_front_end._GetLastGoodPreprocess(storage_file) self.assertIsNotNone(preprocessor_object) timezone = getattr(preprocessor_object, u'zone') self.assertEqual(timezone.zone, u'Iceland')
def setUp(self): """Makes preparations before running an individual test.""" self._formatter_mediator = formatters_mediator.FormatterMediator() self._front_end = psort.PsortFrontend() # TODO: have sample output generated from the test. self._start_timestamp = timelib.Timestamp.CopyFromString( u'2012-07-24 21:45:24') self._end_timestamp = timelib.Timestamp.CopyFromString( u'2016-11-18 01:15:43')
def setUp(self): """Setup sets parameters that will be reused throughout this test.""" self._formatter_mediator = formatters_mediator.FormatterMediator() self._front_end = psort.PsortFrontend() # TODO: have sample output generated from the test. self._test_file_proto = self._GetTestFilePath( [u'psort_test.proto.plaso']) self._test_file_json = self._GetTestFilePath( [u'psort_test.json.plaso']) self.first = timelib.Timestamp.CopyFromString(u'2012-07-24 21:45:24') self.last = timelib.Timestamp.CopyFromString(u'2016-11-18 01:15:43')
def testSetAnalysisPluginProcessInformation(self): """Test the _SetAnalysisPluginProcessInformation method.""" test_front_end = psort.PsortFrontend() analysis_plugins = [test_lib.TestAnalysisPlugin(None)] preprocess_object = event.PreprocessObject() preprocess_object.SetCollectionInformationValues({}) test_front_end._SetAnalysisPluginProcessInformation( u'', analysis_plugins, preprocess_object) self.assertIsNotNone(preprocess_object) plugin_names = preprocess_object.collection_information[u'plugins'] time_of_run = preprocess_object.collection_information[u'time_of_run'] method = preprocess_object.collection_information[u'method'] for analysis_plugin in analysis_plugins: self.assertIn(analysis_plugin.NAME, plugin_names) self.assertAlmostEqual(timelib.Timestamp.GetNow(), time_of_run, 2000000) self.assertIsNotNone(method)
def ProcessStorage(options): """Process a storage file and produce profile results. Args: options: the command line arguments (instance of argparse.Namespace). Returns: The profiling statistics or None on error. """ storage_parameters = options.storage.split() storage_parameters.append(options.file_to_parse) if options.filter: storage_parameters.append(options.filter) front_end = psort.PsortFrontend() try: front_end.ParseOptions(options) except errors.BadConfigOption as exception: logging.error(u'{0:s}'.format(exception)) return if options.verbose: # TODO: why not move this functionality into psort? profiler = cProfile.Profile() profiler.enable() else: time_start = time.time() # Call psort and process output. _ = front_end.ParseStorage(options) if options.verbose: profiler.disable() else: time_end = time.time() if options.verbose: return GetStats(profiler) else: print frontend_utils.FormatHeader('Time Used') print u'{:>20f}s'.format(time_end - time_start)
def testProcessStorage(self): """Test the ProcessStorage function.""" test_front_end = psort.PsortFrontend() test_front_end.SetOutputFormat(u'dynamic') test_front_end.SetPreferredLanguageIdentifier(u'en-US') test_front_end.SetQuietMode(True) storage_file_path = self._GetTestFilePath([u'psort_test.json.plaso']) storage_file = test_front_end.OpenStorage(storage_file_path, read_only=True) try: output_writer = test_lib.StringIOOutputWriter() output_module = test_front_end.GetOutputModule(storage_file) output_module.SetOutputWriter(output_writer) counter = test_front_end.ProcessStorage(output_module, storage_file, storage_file_path, [], []) finally: storage_file.Close() self.assertEqual(counter[u'Stored Events'], 32) output_writer.SeekToBeginning() lines = [] line = output_writer.GetLine() while line: lines.append(line) line = output_writer.GetLine() self.assertEqual(len(lines), 20) expected_line = (u'2016-05-01T12:19:30+00:00,' u'atime,' u'FILE,' u'OS atime,' u'OS:/tmp/test/test_data/syslog Type: file,' u'filestat,' u'OS:/tmp/test/test_data/syslog,-\n') self.assertEquals(lines[13], expected_line)
def Main(arguments=None): """Start the tool.""" multiprocessing.freeze_support() front_end = psort.PsortFrontend() arg_parser = argparse.ArgumentParser( description=(u'PSORT - Application to read, filter and process ' u'output from a plaso storage file.'), add_help=False) tool_group = arg_parser.add_argument_group(u'Optional arguments For psort') output_group = arg_parser.add_argument_group( u'Optional arguments for output modules') analysis_group = arg_parser.add_argument_group( u'Optional arguments for analysis modules') tool_group.add_argument(u'-d', u'--debug', action=u'store_true', dest=u'debug', default=False, help=u'Fall back to debug shell if psort fails.') tool_group.add_argument(u'-q', u'--quiet', action=u'store_true', dest=u'quiet', default=False, help=u'Do not print a summary after processing.') tool_group.add_argument(u'-h', u'--help', action=u'help', help=u'Show this help message and exit.') tool_group.add_argument( u'-a', u'--include_all', action=u'store_false', dest=u'dedup', default=True, help=( u'By default the psort removes duplicate entries from the output. ' u'This parameter changes that behavior so all events are included.' )) tool_group.add_argument( u'-o', u'--output_format', u'--output-format', metavar=u'FORMAT', dest=u'output_format', default=u'dynamic', help=(u'The output format. Use "-o list" to see a list of available ' u'output formats.')) tool_group.add_argument( u'--analysis', metavar=u'PLUGIN_LIST', dest=u'analysis_plugins', default=u'', action=u'store', type=unicode, help=(u'A comma separated list of analysis plugin names to be loaded ' u'or "--analysis list" to see a list of available plugins.')) tool_group.add_argument(u'--data', metavar=u'PATH', dest=u'data_location', default=u'', action=u'store', type=unicode, help=u'The location of the analysis data.') tool_group.add_argument( u'--language', metavar=u'LANGUAGE', dest=u'preferred_language', default=u'en-US', type=unicode, help=( u'The preferred language identifier for Windows Event Log message ' u'strings. Use "--language list" to see a list of available ' u'language identifiers. Note that formatting will fall back on ' u'en-US (LCID 0x0409) if the preferred language is not available ' u'in the database of message string templates.')) tool_group.add_argument( u'-z', u'--zone', metavar=u'TIMEZONE', dest=u'timezone', default=u'UTC', type=unicode, help=(u'The timezone in which to represent the date and time values. ' u'Use "-z list" to see a list of available timezones.')) tool_group.add_argument(u'-w', u'--write', metavar=u'OUTPUTFILE', dest=u'write', help=u'Output filename, defaults to stdout.') tool_group.add_argument( u'--slice', metavar=u'DATE', dest=u'slice', type=str, default=u'', action=u'store', help= (u'Create a time slice around a certain date. This parameter, if ' u'defined will display all events that happened X minutes before and ' u'after the defined date. X is controlled by the parameter ' u'--slice_size but defaults to 5 minutes.')) tool_group.add_argument( u'--slicer', dest=u'slicer', action=u'store_true', default=False, help= (u'Create a time slice around every filter match. This parameter, if ' u'defined will save all X events before and after a filter match has ' u'been made. X is defined by the --slice_size parameter.')) tool_group.add_argument( u'--slice_size', dest=u'slice_size', type=int, default=5, action=u'store', help=( u'Defines the slice size. In the case of a regular time slice it ' u'defines the number of minutes the slice size should be. In the ' u'case of the --slicer it determines the number of events before ' u'and after a filter match has been made that will be included in ' u'the result set. The default value is 5]. See --slice or --slicer ' u'for more details about this option.')) tool_group.add_argument( u'-v', u'--version', dest=u'version', action=u'version', version=u'log2timeline - psort version {0:s}'.format( plaso.GetVersion()), help=u'Show the current version of psort.') front_end.AddStorageFileOptions(tool_group) tool_group.add_argument( u'filter', nargs=u'?', action=u'store', metavar=u'FILTER', default=None, type=unicode, help=(u'A filter that can be used to filter the dataset before it ' u'is written into storage. More information about the filters' u' and it\'s usage can be found here: http://plaso.kiddaland.' u'net/usage/filters')) if arguments is None: arguments = sys.argv[1:] # Add the output module options. if u'-o' in arguments: argument_index = arguments.index(u'-o') + 1 elif u'--output_format' in arguments: argument_index = arguments.index(u'--output_format') + 1 elif u'--output-format' in arguments: argument_index = arguments.index(u'--output-format') + 1 else: argument_index = 0 if argument_index > 0: module_names = arguments[argument_index] front_end.AddOutputModuleOptions(output_group, [module_names]) # Add the analysis plugin options. if u'--analysis' in arguments: argument_index = arguments.index(u'--analysis') + 1 # Get the names of the analysis plugins that should be loaded. plugin_names = arguments[argument_index] try: front_end.AddAnalysisPluginOptions(analysis_group, plugin_names) except errors.BadConfigOption as exception: arg_parser.print_help() print u'' logging.error(u'{0:s}'.format(exception)) return False options = arg_parser.parse_args(args=arguments) format_str = u'[%(levelname)s] %(message)s' if getattr(options, u'debug', False): logging.basicConfig(level=logging.DEBUG, format=format_str) else: logging.basicConfig(level=logging.INFO, format=format_str) have_list_option = False if options.analysis_plugins == u'list': front_end.ListAnalysisPlugins() have_list_option = True if options.output_format == u'list': front_end.ListOutputModules() have_list_option = True if options.preferred_language == u'list': front_end.ListLanguageIdentifiers() have_list_option = True if options.timezone == u'list': front_end.ListTimeZones() have_list_option = True if have_list_option: return True if not getattr(options, u'data_location', None): # Determine if we are running from the source directory. options.data_location = os.path.dirname(__file__) options.data_location = os.path.dirname(options.data_location) options.data_location = os.path.join(options.data_location, u'data') if not os.path.exists(options.data_location): # Otherwise determine if there is shared plaso data location. options.data_location = os.path.join(sys.prefix, u'share', u'plaso') if not os.path.exists(options.data_location): logging.warning( u'Unable to automatically determine data location.') options.data_location = None try: front_end.ParseOptions(options) except errors.BadConfigOption as exception: arg_parser.print_help() print u'' logging.error(u'{0:s}'.format(exception)) return False if front_end.preferred_encoding == u'ascii': logging.warning( u'The preferred encoding of your system is ASCII, which is not optimal ' u'for the typically non-ASCII characters that need to be parsed and ' u'processed. The tool will most likely crash and die, perhaps in a way ' u'that may not be recoverable. A five second delay is introduced to ' u'give you time to cancel the runtime and reconfigure your preferred ' u'encoding, otherwise continue at own risk.') time.sleep(5) try: counter = front_end.ProcessStorage(options) if not options.quiet: logging.info(frontend_utils.FormatHeader(u'Counter')) for element, count in counter.most_common(): logging.info(frontend_utils.FormatOutputString(element, count)) except IOError as exception: # Piping results to "|head" for instance causes an IOError. if u'Broken pipe' not in exception: logging.error( u'Processing stopped early: {0:s}.'.format(exception)) except KeyboardInterrupt: pass # Catching every remaining exception in case we are debugging. except Exception as exception: if not options.debug: raise logging.error(u'{0:s}'.format(exception)) pdb.post_mortem() return True