def testReadAndWriteSerializedSessionStart(self):
"""Test ReadSerialized and WriteSerialized of SessionStart."""
timestamp = int(time.time() * 1000000)
session_identifier = u'{0:s}'.format(uuid.uuid4().get_hex())
expected_session_start = sessions.SessionStart(
identifier=session_identifier)
expected_session_start.timestamp = timestamp
expected_session_start.product_name = u'plaso'
expected_session_start.product_version = plaso.GetVersion()
json_string = (json_serializer.JSONAttributeContainerSerializer.
WriteSerialized(expected_session_start))
self.assertIsNotNone(json_string)
session_start = (json_serializer.JSONAttributeContainerSerializer.
ReadSerialized(json_string))
self.assertIsNotNone(session_start)
self.assertIsInstance(session_start, sessions.SessionStart)
expected_session_start_dict = {
u'debug_mode': False,
u'identifier': session_identifier,
u'product_name': u'plaso',
u'product_version': plaso.GetVersion(),
u'timestamp': timestamp
}
session_start_dict = session_start.CopyToDict()
self.assertEqual(sorted(session_start_dict.items()),
sorted(expected_session_start_dict.items()))
def GetPluginData(self):
"""Return a dict object with a list of all available parsers and plugins."""
return_dict = {}
# Import all plugins and parsers to print out the necessary information.
# This is not import at top since this is only required if this parameter
# is set, otherwise these libraries get imported in their respected
# locations.
# The reason why some of these libraries are imported as '_' is to make sure
# all appropriate parsers and plugins are registered, yet we don't need to
# directly call these libraries, it is enough to load them up to get them
# registered.
# TODO: remove this hack includes should be a the top if this does not work
# remove the need for implicit behavior on import.
from plaso import filters
from plaso import hashers as _ # pylint: disable=redefined-outer-name
from plaso import parsers as _ # pylint: disable=redefined-outer-name
from plaso import output as _ # pylint: disable=redefined-outer-name
from plaso.frontend import presets
from plaso.output import manager as output_manager
return_dict[u'Versions'] = [(u'plaso engine', plaso.GetVersion()),
(u'python', sys.version)]
return_dict[u'Hashers'] = []
for _, hasher_class in hashers_manager.HashersManager.GetHashers():
description = getattr(hasher_class, u'DESCRIPTION', u'')
return_dict[u'Hashers'].append((hasher_class.NAME, description))
return_dict[u'Parsers'] = []
for _, parser_class in parsers_manager.ParsersManager.GetParsers():
description = getattr(parser_class, u'DESCRIPTION', u'')
return_dict[u'Parsers'].append((parser_class.NAME, description))
return_dict[u'Parser Lists'] = []
for category, parsers in sorted(presets.categories.items()):
return_dict[u'Parser Lists'].append((category, ', '.join(parsers)))
return_dict[u'Output Modules'] = []
for name, description in sorted(
output_manager.OutputManager.GetOutputs()):
return_dict[u'Output Modules'].append((name, description))
return_dict[u'Plugins'] = []
for _, parser_class in parsers_manager.ParsersManager.GetParsers():
if parser_class.SupportsPlugins():
for _, plugin_class in parser_class.GetPlugins():
description = getattr(plugin_class, u'DESCRIPTION', u'')
return_dict[u'Plugins'].append(
(plugin_class.NAME, description))
return_dict[u'Filters'] = []
for filter_obj in sorted(filters.ListFilters()):
doc_string, _, _ = filter_obj.__doc__.partition('\n')
return_dict[u'Filters'].append(
(filter_obj.filter_name, doc_string))
return return_dict
def testReadAndWriteSerializedSession(self):
"""Test ReadSerialized and WriteSerialized of Session."""
parsers_counter = collections.Counter()
parsers_counter[u'filestat'] = 3
parsers_counter[u'total'] = 3
expected_session = sessions.Session()
expected_session.product_name = u'plaso'
expected_session.product_version = plaso.GetVersion()
expected_session.parsers_counter = parsers_counter
json_string = (
json_serializer.JSONAttributeContainerSerializer.WriteSerialized(
expected_session))
self.assertIsNotNone(json_string)
session = (
json_serializer.JSONAttributeContainerSerializer.ReadSerialized(
json_string))
self.assertIsNotNone(session)
self.assertIsInstance(session, sessions.Session)
expected_session_dict = {
u'aborted': False,
u'analysis_reports_counter': session.analysis_reports_counter,
u'debug_mode': False,
u'event_labels_counter': session.event_labels_counter,
u'identifier': session.identifier,
u'parsers_counter': parsers_counter,
u'preferred_encoding': u'utf-8',
u'preferred_time_zone': u'UTC',
u'product_name': u'plaso',
u'product_version': plaso.GetVersion(),
u'start_time': session.start_time
}
session_dict = session.CopyToDict()
self.assertEqual(
sorted(session_dict.items()), sorted(expected_session_dict.items()))
def __init__(self):
"""Initializes a session start attribute container."""
super(SessionStart, self).__init__()
self.command_line_arguments = u''
self.debug_mode = False
self.filter_expression = u''
self.filter_file = u''
self.parser_filter_expression = u''
self.preferred_encoding = u'utf-8'
self.product_name = u'plaso'
self.product_version = plaso.GetVersion()
self.timestamp = int(time.time() * 100000)
def testCopyToDict(self):
"""Tests the CopyToDict function."""
timestamp = int(time.time() * 1000000)
session_identifier = u'{0:s}'.format(uuid.uuid4().get_hex())
session_start = sessions.SessionStart(identifier=session_identifier)
session_start.timestamp = timestamp
session_start.product_name = u'plaso'
session_start.product_version = plaso.GetVersion()
self.assertEquals(session_start.identifier, session_identifier)
expected_dict = {
u'debug_mode': False,
u'identifier': session_start.identifier,
u'product_name': u'plaso',
u'product_version': plaso.GetVersion(),
u'timestamp': timestamp
}
test_dict = session_start.CopyToDict()
self.assertEqual(test_dict, expected_dict)
def _GetPluginData(self):
"""Return a dict object with a list of all available parsers and plugins."""
return_dict = {}
# Import all plugins and parsers to print out the necessary information.
# This is not import at top since this is only required if this parameter
# is set, otherwise these libraries get imported in their respected
# locations.
# The reason why some of these libraries are imported as '_' is to make sure
# all appropriate parsers and plugins are registered, yet we don't need to
# directly call these libraries, it is enough to load them up to get them
# registered.
from plaso import filters
from plaso import parsers as _
from plaso import output as _
from plaso.frontend import presets
from plaso.lib import output
from plaso.lib import plugin
return_dict['Versions'] = [('plaso engine', plaso.GetVersion()),
('python', sys.version)]
return_dict['Parsers'] = []
for parser in sorted(putils.FindAllParsers()['all']):
doc_string, _, _ = parser.__doc__.partition('\n')
return_dict['Parsers'].append((parser.parser_name, doc_string))
return_dict['Parser Lists'] = []
for category, parsers in sorted(presets.categories.items()):
return_dict['Parser Lists'].append((category, ', '.join(parsers)))
return_dict['Output Modules'] = []
for name, description in sorted(output.ListOutputFormatters()):
return_dict['Output Modules'].append((name, description))
return_dict['Plugins'] = []
for plugin, obj in sorted(plugin.BasePlugin.classes.iteritems()):
doc_string, _, _ = obj.__doc__.partition('\n')
return_dict['Plugins'].append((plugin, doc_string))
return_dict['Filters'] = []
for filter_obj in sorted(filters.ListFilters()):
doc_string, _, _ = filter_obj.__doc__.partition('\n')
return_dict['Filters'].append((filter_obj.filter_name, doc_string))
return return_dict
def AddBasicOptions(self, argument_group):
"""Adds the basic options to the argument group.
Args:
argument_group (argparse._ArgumentGroup): argparse argument group.
"""
version_string = u'plaso - {0:s} version {1:s}'.format(
self.NAME, plaso.GetVersion())
# We want a custom help message and not the default argparse one.
argument_group.add_argument(
u'-h', u'--help', action=u'help',
help=u'show this help message and exit.')
argument_group.add_argument(
u'-V', u'--version', dest=u'version', action=u'version',
version=version_string, help=u'show the version information.')
def _PrintStatusUpdate(self, processing_status):
"""Prints the processing status.
Args:
processing_status (ProcessingStatus): processing status.
"""
if self._stdout_output_writer:
self._ClearScreen()
self._output_writer.Write(
u'plaso - {0:s} version {1:s}\n'.format(
self.NAME, plaso.GetVersion()))
self._output_writer.Write(u'\n')
self._PrintStatusHeader()
# TODO: for win32console get current color and set intensity,
# write the header separately then reset intensity.
status_header = u'Identifier\t\tPID\tStatus\t\tEvents\t\tTags\t\tReports'
if not win32console:
status_header = u'\x1b[1m{0:s}\x1b[0m'.format(status_header)
status_table = [status_header]
status_row = self._FormatStatusTableRow(processing_status.foreman_status)
status_table.append(status_row)
for worker_status in processing_status.workers_status:
status_row = self._FormatStatusTableRow(worker_status)
status_table.append(status_row)
status_table.append(u'')
self._output_writer.Write(u'\n'.join(status_table))
self._output_writer.Write(u'\n')
if processing_status.aborted:
self._output_writer.Write(
u'Processing aborted - waiting for clean up.\n\n')
# TODO: remove update flicker. For win32console we could set the cursor
# top left, write the table, clean the remainder of the screen buffer
# and set the cursor at the end of the table.
if self._stdout_output_writer:
# We need to explicitly flush stdout to prevent partial status updates.
sys.stdout.flush()
def GetPluginData(self):
"""Retrieves the version and various plugin information.
Returns:
A dictionary object with lists of available parsers and plugins.
"""
return_dict = {}
return_dict[u'Versions'] = [(u'plaso engine', plaso.GetVersion()),
(u'python', sys.version)]
return_dict[u'Hashers'] = self.GetHashersInformation()
return_dict[u'Parsers'] = self.GetParsersInformation()
return_dict[u'Parser Plugins'] = self.GetParserPluginsInformation()
return_dict[u'Parser Presets'] = self.GetParserPresetsInformation()
return_dict[u'Output Modules'] = self._GetOutputModulesInformation()
return_dict[u'Filters'] = self._GetFiltersInformation()
return return_dict
def __init__(self):
"""Initializes a session attribute container."""
super(Session, self).__init__()
self.aborted = False
self.analysis_reports_counter = collections.Counter()
self.command_line_arguments = None
self.completion_time = None
self.debug_mode = False
self.enabled_parser_names = None
self.event_labels_counter = collections.Counter()
self.filter_expression = None
self.filter_file = None
self.identifier = u'{0:s}'.format(uuid.uuid4().get_hex())
self.parser_filter_expression = None
self.parsers_counter = collections.Counter()
self.preferred_encoding = u'utf-8'
self.preferred_year = None
self.product_name = u'plaso'
self.product_version = plaso.GetVersion()
self.start_time = int(time.time() * 1000000)
def AddBasicOptions(self, argument_group):
"""Adds the basic options to the argument group.
Args:
argument_group: The argparse argument group (instance of
argparse._ArgumentGroup).
"""
version_string = u'plaso - {0:s} version {1:s}'.format(
self.NAME, plaso.GetVersion())
argument_group.add_argument(u'-h',
u'--help',
action=u'help',
help=(u'Show this help message and exit.'))
argument_group.add_argument(u'-v',
u'--version',
dest=u'version',
action=u'version',
version=version_string,
help=u'Show the current version.')
def PrintHeader(options):
"""Print header information, including library versions."""
print frontend_utils.FormatHeader('File Parsed')
print u'{:>20s}'.format(options.file_to_parse)
print frontend_utils.FormatHeader('Versions')
print frontend_utils.FormatOutputString('plaso engine', plaso.GetVersion())
print frontend_utils.FormatOutputString('pyevt', pyevt.get_version())
print frontend_utils.FormatOutputString('pyevtx', pyevtx.get_version())
print frontend_utils.FormatOutputString('pylnk', pylnk.get_version())
print frontend_utils.FormatOutputString('pymsiecf', pymsiecf.get_version())
print frontend_utils.FormatOutputString('pyregf', pyregf.get_version())
if options.filter:
print frontend_utils.FormatHeader('Filter Used')
print frontend_utils.FormatOutputString('Filter String',
options.filter)
if options.parsers:
print frontend_utils.FormatHeader('Parser Filter Used')
print frontend_utils.FormatOutputString('Parser String',
options.parsers)
def testCopyToDict(self):
"""Tests the CopyToDict function."""
session = sessions.Session()
self.assertIsNotNone(session.identifier)
self.assertIsNotNone(session.start_time)
self.assertIsNone(session.completion_time)
expected_dict = {
u'aborted': False,
u'analysis_reports_counter': session.analysis_reports_counter,
u'debug_mode': False,
u'event_labels_counter': session.event_labels_counter,
u'identifier': session.identifier,
u'parsers_counter': session.parsers_counter,
u'preferred_encoding': u'utf-8',
u'product_name': u'plaso',
u'product_version': plaso.GetVersion(),
u'start_time': session.start_time
}
test_dict = session.CopyToDict()
self.assertEqual(test_dict, expected_dict)
def Main(arguments=None):
"""Start the tool."""
multiprocessing.freeze_support()
front_end = psort.PsortFrontend()
arg_parser = argparse.ArgumentParser(
description=(u'PSORT - Application to read, filter and process '
u'output from a plaso storage file.'),
add_help=False)
tool_group = arg_parser.add_argument_group(u'Optional arguments For psort')
output_group = arg_parser.add_argument_group(
u'Optional arguments for output modules')
analysis_group = arg_parser.add_argument_group(
u'Optional arguments for analysis modules')
tool_group.add_argument(u'-d',
u'--debug',
action=u'store_true',
dest=u'debug',
default=False,
help=u'Fall back to debug shell if psort fails.')
tool_group.add_argument(u'-q',
u'--quiet',
action=u'store_true',
dest=u'quiet',
default=False,
help=u'Do not print a summary after processing.')
tool_group.add_argument(u'-h',
u'--help',
action=u'help',
help=u'Show this help message and exit.')
tool_group.add_argument(
u'-a',
u'--include_all',
action=u'store_false',
dest=u'dedup',
default=True,
help=(
u'By default the psort removes duplicate entries from the output. '
u'This parameter changes that behavior so all events are included.'
))
tool_group.add_argument(
u'-o',
u'--output_format',
u'--output-format',
metavar=u'FORMAT',
dest=u'output_format',
default=u'dynamic',
help=(u'The output format. Use "-o list" to see a list of available '
u'output formats.'))
tool_group.add_argument(
u'--analysis',
metavar=u'PLUGIN_LIST',
dest=u'analysis_plugins',
default=u'',
action=u'store',
type=unicode,
help=(u'A comma separated list of analysis plugin names to be loaded '
u'or "--analysis list" to see a list of available plugins.'))
tool_group.add_argument(u'--data',
metavar=u'PATH',
dest=u'data_location',
default=u'',
action=u'store',
type=unicode,
help=u'The location of the analysis data.')
tool_group.add_argument(
u'--language',
metavar=u'LANGUAGE',
dest=u'preferred_language',
default=u'en-US',
type=unicode,
help=(
u'The preferred language identifier for Windows Event Log message '
u'strings. Use "--language list" to see a list of available '
u'language identifiers. Note that formatting will fall back on '
u'en-US (LCID 0x0409) if the preferred language is not available '
u'in the database of message string templates.'))
tool_group.add_argument(
u'-z',
u'--zone',
metavar=u'TIMEZONE',
dest=u'timezone',
default=u'UTC',
type=unicode,
help=(u'The timezone in which to represent the date and time values. '
u'Use "-z list" to see a list of available timezones.'))
tool_group.add_argument(u'-w',
u'--write',
metavar=u'OUTPUTFILE',
dest=u'write',
help=u'Output filename, defaults to stdout.')
tool_group.add_argument(
u'--slice',
metavar=u'DATE',
dest=u'slice',
type=str,
default=u'',
action=u'store',
help=
(u'Create a time slice around a certain date. This parameter, if '
u'defined will display all events that happened X minutes before and '
u'after the defined date. X is controlled by the parameter '
u'--slice_size but defaults to 5 minutes.'))
tool_group.add_argument(
u'--slicer',
dest=u'slicer',
action=u'store_true',
default=False,
help=
(u'Create a time slice around every filter match. This parameter, if '
u'defined will save all X events before and after a filter match has '
u'been made. X is defined by the --slice_size parameter.'))
tool_group.add_argument(
u'--slice_size',
dest=u'slice_size',
type=int,
default=5,
action=u'store',
help=(
u'Defines the slice size. In the case of a regular time slice it '
u'defines the number of minutes the slice size should be. In the '
u'case of the --slicer it determines the number of events before '
u'and after a filter match has been made that will be included in '
u'the result set. The default value is 5]. See --slice or --slicer '
u'for more details about this option.'))
tool_group.add_argument(
u'-v',
u'--version',
dest=u'version',
action=u'version',
version=u'log2timeline - psort version {0:s}'.format(
plaso.GetVersion()),
help=u'Show the current version of psort.')
front_end.AddStorageFileOptions(tool_group)
tool_group.add_argument(
u'filter',
nargs=u'?',
action=u'store',
metavar=u'FILTER',
default=None,
type=unicode,
help=(u'A filter that can be used to filter the dataset before it '
u'is written into storage. More information about the filters'
u' and it\'s usage can be found here: http://plaso.kiddaland.'
u'net/usage/filters'))
if arguments is None:
arguments = sys.argv[1:]
# Add the output module options.
if u'-o' in arguments:
argument_index = arguments.index(u'-o') + 1
elif u'--output_format' in arguments:
argument_index = arguments.index(u'--output_format') + 1
elif u'--output-format' in arguments:
argument_index = arguments.index(u'--output-format') + 1
else:
argument_index = 0
if argument_index > 0:
module_names = arguments[argument_index]
front_end.AddOutputModuleOptions(output_group, [module_names])
# Add the analysis plugin options.
if u'--analysis' in arguments:
argument_index = arguments.index(u'--analysis') + 1
# Get the names of the analysis plugins that should be loaded.
plugin_names = arguments[argument_index]
try:
front_end.AddAnalysisPluginOptions(analysis_group, plugin_names)
except errors.BadConfigOption as exception:
arg_parser.print_help()
print u''
logging.error(u'{0:s}'.format(exception))
return False
options = arg_parser.parse_args(args=arguments)
format_str = u'[%(levelname)s] %(message)s'
if getattr(options, u'debug', False):
logging.basicConfig(level=logging.DEBUG, format=format_str)
else:
logging.basicConfig(level=logging.INFO, format=format_str)
have_list_option = False
if options.analysis_plugins == u'list':
front_end.ListAnalysisPlugins()
have_list_option = True
if options.output_format == u'list':
front_end.ListOutputModules()
have_list_option = True
if options.preferred_language == u'list':
front_end.ListLanguageIdentifiers()
have_list_option = True
if options.timezone == u'list':
front_end.ListTimeZones()
have_list_option = True
if have_list_option:
return True
if not getattr(options, u'data_location', None):
# Determine if we are running from the source directory.
options.data_location = os.path.dirname(__file__)
options.data_location = os.path.dirname(options.data_location)
options.data_location = os.path.join(options.data_location, u'data')
if not os.path.exists(options.data_location):
# Otherwise determine if there is shared plaso data location.
options.data_location = os.path.join(sys.prefix, u'share',
u'plaso')
if not os.path.exists(options.data_location):
logging.warning(
u'Unable to automatically determine data location.')
options.data_location = None
try:
front_end.ParseOptions(options)
except errors.BadConfigOption as exception:
arg_parser.print_help()
print u''
logging.error(u'{0:s}'.format(exception))
return False
if front_end.preferred_encoding == u'ascii':
logging.warning(
u'The preferred encoding of your system is ASCII, which is not optimal '
u'for the typically non-ASCII characters that need to be parsed and '
u'processed. The tool will most likely crash and die, perhaps in a way '
u'that may not be recoverable. A five second delay is introduced to '
u'give you time to cancel the runtime and reconfigure your preferred '
u'encoding, otherwise continue at own risk.')
time.sleep(5)
try:
counter = front_end.ProcessStorage(options)
if not options.quiet:
logging.info(frontend_utils.FormatHeader(u'Counter'))
for element, count in counter.most_common():
logging.info(frontend_utils.FormatOutputString(element, count))
except IOError as exception:
# Piping results to "|head" for instance causes an IOError.
if u'Broken pipe' not in exception:
logging.error(
u'Processing stopped early: {0:s}.'.format(exception))
except KeyboardInterrupt:
pass
# Catching every remaining exception in case we are debugging.
except Exception as exception:
if not options.debug:
raise
logging.error(u'{0:s}'.format(exception))
pdb.post_mortem()
return True
def _PreprocessSetCollectionInformation(self,
pre_obj,
source_type,
unused_engine,
filter_file=None,
parser_filter_string=None,
preferred_encoding=u'utf-8'):
"""Sets the collection information as part of the preprocessing.
Args:
pre_obj: the preprocess object (instance of PreprocessObject).
source_type: the dfVFS source type definition.
engine: the engine object (instance of BaseEngine).
filter_file: a path to a file that contains find specifications.
The default is None.
parser_filter_string: optional parser filter string. The default is None.
preferred_encoding: optional preferred encoding. The default is UTF-8.
"""
collection_information = {}
# TODO: informational values.
collection_information[u'version'] = plaso.GetVersion()
collection_information[u'debug'] = self._debug_mode
# TODO: extraction preferences:
if not parser_filter_string:
parser_filter_string = u'(no list set)'
collection_information[u'parser_selection'] = parser_filter_string
collection_information[u'preferred_encoding'] = preferred_encoding
# TODO: extraction info:
collection_information[u'configured_zone'] = pre_obj.zone
collection_information[u'parsers'] = self._parser_names
collection_information[u'preprocess'] = self._enable_preprocessing
if self._filter_expression:
collection_information[u'filter'] = self._filter_expression
if filter_file and os.path.isfile(filter_file):
filters = []
with open(filter_file, 'rb') as file_object:
for line in file_object.readlines():
filters.append(line.rstrip())
collection_information[u'file_filter'] = u', '.join(filters)
if self._operating_system:
collection_information[u'os_detected'] = self._operating_system
else:
collection_information[u'os_detected'] = u'N/A'
# TODO: processing settings:
collection_information[u'protobuf_size'] = self._buffer_size
collection_information[u'time_of_run'] = timelib.Timestamp.GetNow()
if self._single_process_mode:
collection_information[u'runtime'] = u'single process mode'
else:
collection_information[u'runtime'] = u'multi process mode'
# TODO: retrieve this value from the multi-process engine.
# refactor engine to set number_of_extraction_workers
# before ProcessSources.
collection_information[u'workers'] = 0
# TODO: output/storage settings:
collection_information[u'output_file'] = self._storage_file_path
# TODO: source settings:
# TODO: move source_scanner.SourceScannerContext.SOURCE_TYPE_
# to definitions.SOURCE_TYPE_.
if source_type == source_scanner.SourceScannerContext.SOURCE_TYPE_DIRECTORY:
recursive = True
else:
recursive = False
# TODO: replace by scan node.
# collection_information[u'file_processed'] = self._source_path
collection_information[u'recursive'] = recursive
# TODO: replace by scan node.
# collection_information[u'vss parsing'] = bool(self.vss_stores)
# TODO: move source_scanner.SourceScannerContext.SOURCE_TYPE_
# to definitions.SOURCE_TYPE_.
if source_type in [
source_scanner.SourceScannerContext.
SOURCE_TYPE_STORAGE_MEDIA_DEVICE, source_scanner.
SourceScannerContext.SOURCE_TYPE_STORAGE_MEDIA_IMAGE
]:
collection_information[u'method'] = u'imaged processed'
# TODO: replace by scan node.
# collection_information[u'image_offset'] = self.partition_offset
else:
collection_information[u'method'] = u'OS collection'
pre_obj.collection_information = collection_information
def Process(self, persistent_disk_name, project_name, disk_name):
"""Configure and run Plaso processing.
Args:
persistent_disk_name: Name of disk to process (created from snapshot).
project_name: Project name of remote project being processed.
disk_name: Name of remote disk that the snapshot was created from.
"""
compute_client = CreateServiceClient(u'compute')
path = u'/dev/disk/by-id/google-' + config.DEVICE_NAME
if IsBlockDevice(path):
WriteToStdOut(u'Disk already attached!')
self._DetachDisk(compute_client)
# Mount the disk
disk_path = (u'projects/' + config.PROJECT + u'/zones/' + config.ZONE +
u'/disks/' + persistent_disk_name)
self._AttachDisk(compute_client, disk_path)
output_file_basename = project_name + disk_name + GetCurrentTimeUTC()
# Make sure we have a proper block device
_RETRY_MAX = 10
_RETRY_COUNT = 0
while _RETRY_COUNT < _RETRY_MAX:
if IsBlockDevice(path):
WriteToStdOut(u'Block device: OK')
break
if os.path.exists(path):
WriteToStdOut(u'Block device: Current mode is {0}'.format(
os.stat(path).st_mode))
_RETRY_COUNT += 1
time.sleep(1)
# Configure Log2Timeline
tool = log2timeline.Log2TimelineTool()
options = argparse.Namespace()
options.debug = True
options.hashers = u'all'
options.dependencies_check = False
options.serializer_format = u'json'
options.status_view_mode = u'none'
options.vss_stores = u'all'
options.partition_number = u'all'
options.log_file = os.path.join(config.SCRATCH_PATH,
output_file_basename + u'.log')
# Let plaso choose the appropriate number of workers
options.workers = 0
options.source = u'/dev/disk/by-id/google-' + config.DEVICE_NAME
options.output = os.path.join(config.SCRATCH_PATH,
output_file_basename + u'.plaso')
tool.ParseOptions(options)
WriteToStdOut(u'Plaso {0:s} {1:s} START'.format(
plaso.GetVersion(), project_name))
tool.ProcessSources()
WriteToStdOut(u'Plaso {0:s} {1:s} END'.format(plaso.GetVersion(),
project_name))
# Add to timesketch
timesketch_client = TimesketchApiClient(config.TIMESKETCH_HOST,
config.TIMESKETCH_USER,
config.TIMESKETCH_PASSWORD)
# Create sketch
sketch_id = timesketch_client.CreateSketch(name=sketch_name,
description=sketch_name)
# Create index from Plaso storage file
index_id = timesketch_client.UploadTimeline(
timeline_name=timeline_name, plaso_storage_path=storage_file)
# Create sketch
timesketch_client.AddTimelineToSketch(sketch_id, index_id)
# Upload to GCS
storage_client = CreateServiceClient(u'storage')
self._CopyFileToBucket(storage_client, options.log_file,
output_file_basename + u'.log')
self._CopyFileToBucket(storage_client, options.output,
output_file_basename + u'.plaso')
self._DetachDisk(compute_client)
def testPrintStatusUpdate(self):
"""Tests the PrintStatusUpdate function."""
input_reader = tools.StdinInputReader(encoding=u'ascii')
output_writer = test_lib.TestOutputWriter()
status_view_tool = TestStatusViewTool(
input_reader=input_reader, output_writer=output_writer)
status_view_tool._PrintStatusHeader()
process_status = processing_status.ProcessingStatus()
process_status.UpdateForemanStatus(
u'f_identifier', u'f_status', 123, 0,
u'f_test_file', 1, 29, 3, 456, 5, 6, 7,
8, 9, 10)
status_view_tool._PrintStatusUpdate(process_status)
string = output_writer.ReadOutput()
plaso_version = plaso.GetVersion()
table_header = (
b'Identifier\tPID\tStatus\t\tMemory\t\tSources\t\tEvents\t\tFile')
if not sys.platform.startswith(u'win'):
table_header = b'\x1b[1m{0:s}\x1b[0m'.format(table_header)
expected_lines = [
b'Source path\t: /test/source/path',
b'Source type\t: TESTSOURCE',
b'',
b'plaso - version {0:s}'.format(plaso_version),
b'',
b'Source path\t: /test/source/path',
b'Source type\t: TESTSOURCE',
b'',
table_header,
(b'f_identifier\t123\tf_status\t0 B\t\t29 (29)\t\t456 (456)\t'
b'f_test_file'),
b'',
b'']
self.assertEqual(string.split(b'\n'), expected_lines)
process_status.UpdateWorkerStatus(
u'w_identifier', u'w_status', 123, 0,
u'w_test_file', 1, 2, 3, 4, 5, 6, 7, 8, 9,
10)
status_view_tool._PrintStatusUpdate(process_status)
string = output_writer.ReadOutput()
expected_lines = [
b'plaso - version {0:s}'.format(plaso_version),
b'',
b'Source path\t: /test/source/path',
b'Source type\t: TESTSOURCE',
b'',
table_header,
(b'f_identifier\t123\tf_status\t0 B\t\t29 (29)\t\t456 (456)\t'
b'f_test_file'),
b'w_identifier\t123\tw_status\t0 B\t\t2 (2)\t\t4 (4)\t\tw_test_file',
b'',
b'']
self.assertEqual(string.split(b'\n'), expected_lines)
def Main():
"""Start the tool."""
multiprocessing.freeze_support()
front_end = log2timeline.Log2TimelineFrontend()
epilog = u'\n'.join([
u'', u'Example usage:', u'',
u'Run the tool against an image (full kitchen sink)',
u' log2timeline.py /cases/mycase/plaso.dump ímynd.dd', u'',
u'Instead of answering questions, indicate some of the options on the',
u'command line (including data from particular VSS stores).',
(u' log2timeline.py -o 63 --vss_stores 1,2 /cases/plaso_vss.dump '
u'image.E01'), u'',
u'And that\'s how you build a timeline using log2timeline...', u''
])
description = u'\n'.join([
u'',
u'log2timeline is the main front-end to the plaso back-end, used to',
u'collect and correlate events extracted from a filesystem.', u'',
u'More information can be gathered from here:',
u' http://plaso.kiddaland.net/usage/log2timeline', u''
])
arg_parser = argparse.ArgumentParser(
description=textwrap.dedent(description),
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=textwrap.dedent(epilog),
add_help=False)
# Create few argument groups to make formatting help messages clearer.
info_group = arg_parser.add_argument_group('Informational Arguments')
function_group = arg_parser.add_argument_group('Functional Arguments')
deep_group = arg_parser.add_argument_group('Deep Analysis Arguments')
performance_group = arg_parser.add_argument_group('Performance Arguments')
function_group.add_argument(
'-z',
'--zone',
'--timezone',
dest='timezone',
action='store',
type=str,
default='UTC',
help=
(u'Define the timezone of the IMAGE (not the output). This is usually '
u'discovered automatically by preprocessing but might need to be '
u'specifically set if preprocessing does not properly detect or to '
u'overwrite the detected time zone.'))
function_group.add_argument(
'-t',
'--text',
dest='text_prepend',
action='store',
type=unicode,
default=u'',
metavar='TEXT',
help=(u'Define a free form text string that is prepended to each path '
u'to make it easier to distinguish one record from another in a '
u'timeline (like c:\\, or host_w_c:\\)'))
function_group.add_argument(
'--parsers',
dest='parsers',
type=str,
action='store',
default='',
metavar='PARSER_LIST',
help=
(u'Define a list of parsers to use by the tool. This is a comma '
u'separated list where each entry can be either a name of a parser '
u'or a parser list. Each entry can be prepended with a minus sign '
u'to negate the selection (exclude it). The list match is an '
u'exact match while an individual parser matching is a case '
u'insensitive substring match, with support for glob patterns. '
u'Examples would be: "reg" that matches the substring "reg" in '
u'all parser names or the glob pattern "sky[pd]" that would match '
u'all parsers that have the string "skyp" or "skyd" in it\'s name. '
u'All matching is case insensitive.'))
function_group.add_argument(
'--hashers',
dest='hashers',
type=str,
action='store',
default='',
metavar='HASHER_LIST',
help=(
u'Define a list of hashers to use by the tool. This is a comma '
u'separated list where each entry is the name of a hasher. eg. md5,'
u'sha256.'))
info_group.add_argument('-h',
'--help',
action='help',
help=u'Show this help message and exit.')
info_group.add_argument(
'--logfile',
action='store',
metavar='FILENAME',
dest='logfile',
type=unicode,
default=u'',
help=(u'If defined all log messages will be redirected to this file '
u'instead the default STDERR.'))
function_group.add_argument(
'-p',
'--preprocess',
dest='preprocess',
action='store_true',
default=False,
help=(u'Turn on preprocessing. Preprocessing is turned on by default '
u'when parsing image files, however if a mount point is being '
u'parsed then this parameter needs to be set manually.'))
front_end.AddPerformanceOptions(performance_group)
performance_group.add_argument(
'--workers',
dest='workers',
action='store',
type=int,
default=0,
help=(u'The number of worker threads [defaults to available system '
u'CPU\'s minus three].'))
# TODO: seems to be no longer used, remove.
# function_group.add_argument(
# '-i', '--image', dest='image', action='store_true', default=False,
# help=(
# 'Indicates that this is an image instead of a regular file. It is '
# 'not necessary to include this option if -o (offset) is used, then '
# 'this option is assumed. Use this when parsing an image with an '
# 'offset of zero.'))
front_end.AddVssProcessingOptions(deep_group)
performance_group.add_argument(
'--single_thread',
'--single-thread',
'--single_process',
'--single-process',
dest='single_process',
action='store_true',
default=False,
help=(u'Indicate that the tool should run in a single process.'))
function_group.add_argument(
'-f',
'--file_filter',
'--file-filter',
dest='file_filter',
action='store',
type=unicode,
default=None,
help=(
u'List of files to include for targeted collection of files to '
u'parse, one line per file path, setup is /path|file - where each '
u'element can contain either a variable set in the preprocessing '
u'stage or a regular expression.'))
deep_group.add_argument('--scan_archives',
dest='scan_archives',
action='store_true',
default=False,
help=argparse.SUPPRESS)
# This option is "hidden" for the time being, still left in there for testing
# purposes, but hidden from the tool usage and help messages.
# help=('Indicate that the tool should try to open files to extract embedd'
# 'ed files within them, for instance to extract files from compress'
# 'ed containers, etc. Be AWARE THAT THIS IS EXTREMELY SLOW.'))
front_end.AddImageOptions(function_group)
function_group.add_argument(
'--partition',
dest='partition_number',
action='store',
type=int,
default=None,
help=(u'Choose a partition number from a disk image. This partition '
u'number should correspond to the partion number on the disk '
u'image, starting from partition 1.'))
# Build the version information.
version_string = u'log2timeline - plaso back-end {0:s}'.format(
plaso.GetVersion())
info_group.add_argument('-v',
'--version',
action='version',
version=version_string,
help=u'Show the current version of the back-end.')
info_group.add_argument(
'--info',
dest='show_info',
action='store_true',
default=False,
help=u'Print out information about supported plugins and parsers.')
info_group.add_argument(
'--show_memory_usage',
'--show-memory-usage',
action='store_true',
default=False,
dest='foreman_verbose',
help=(u'Indicates that basic memory usage should be included in the '
u'output of the process monitor. If this option is not set the '
u'tool only displays basic status and counter information.'))
info_group.add_argument(
'--disable_worker_monitor',
'--disable-worker-monitor',
action='store_false',
default=True,
dest='foreman_enabled',
help=
(u'Turn off the foreman. The foreman monitors all worker processes '
u'and periodically prints out information about all running workers.'
u'By default the foreman is run, but it can be turned off using this '
u'parameter.'))
front_end.AddExtractionOptions(function_group)
function_group.add_argument(
'--output',
dest='output_module',
action='store',
type=unicode,
default='',
help=(
u'Bypass the storage module directly storing events according to '
u'the output module. This means that the output will not be in the '
u'pstorage format but in the format chosen by the output module. '
u'[Please not this feature is EXPERIMENTAL at this time, use at '
u'own risk (eg. sqlite output does not yet work)]'))
function_group.add_argument(
'--serializer-format',
'--serializer_format',
dest='serializer_format',
action='store',
default='proto',
metavar='FORMAT',
help=(u'By default the storage uses protobufs for serializing event '
u'objects. This parameter can be used to change that behavior. '
u'The choices are "proto" and "json".'))
front_end.AddInformationalOptions(info_group)
arg_parser.add_argument(
'output',
action='store',
metavar='STORAGE_FILE',
nargs='?',
type=unicode,
help=(u'The path to the output file, if the file exists it will get '
u'appended to.'))
arg_parser.add_argument(
'source',
action='store',
metavar='SOURCE',
nargs='?',
type=unicode,
help=
(u'The path to the source device, file or directory. If the source is '
u'a supported storage media device or image file, archive file or '
u'a directory, the files within are processed recursively.'))
arg_parser.add_argument(
'filter',
action='store',
metavar='FILTER',
nargs='?',
default=None,
type=unicode,
help=(u'A filter that can be used to filter the dataset before it '
u'is written into storage. More information about the filters '
u'and it\'s usage can be found here: http://plaso.kiddaland.'
u'net/usage/filters'))
# Properly prepare the attributes according to local encoding.
if front_end.preferred_encoding == 'ascii':
logging.warning(
u'The preferred encoding of your system is ASCII, which is not optimal '
u'for the typically non-ASCII characters that need to be parsed and '
u'processed. The tool will most likely crash and die, perhaps in a way '
u'that may not be recoverable. A five second delay is introduced to '
u'give you time to cancel the runtime and reconfigure your preferred '
u'encoding, otherwise continue at own risk.')
time.sleep(5)
u_argv = [x.decode(front_end.preferred_encoding) for x in sys.argv]
sys.argv = u_argv
try:
options = arg_parser.parse_args()
except UnicodeEncodeError:
# If we get here we are attempting to print help in a "dumb" terminal.
print arg_parser.format_help().encode(front_end.preferred_encoding)
return False
if options.timezone == 'list':
front_end.ListTimeZones()
return True
if options.show_info:
front_end.ListPluginInformation()
return True
format_str = (
u'%(asctime)s [%(levelname)s] (%(processName)-10s) PID:%(process)d '
u'<%(module)s> %(message)s')
if options.debug:
if options.logfile:
logging.basicConfig(level=logging.DEBUG,
format=format_str,
filename=options.logfile)
else:
logging.basicConfig(level=logging.DEBUG, format=format_str)
logging_filter = log2timeline.LoggingFilter()
root_logger = logging.getLogger()
root_logger.addFilter(logging_filter)
elif options.logfile:
logging.basicConfig(level=logging.INFO,
format=format_str,
filename=options.logfile)
else:
logging.basicConfig(level=logging.INFO, format=format_str)
if not options.output:
arg_parser.print_help()
print u''
arg_parser.print_usage()
print u''
logging.error(u'Wrong usage: need to define an output.')
return False
try:
front_end.ParseOptions(options)
front_end.SetStorageFile(options.output)
except errors.BadConfigOption as exception:
arg_parser.print_help()
print u''
logging.error(u'{0:s}'.format(exception))
return False
# Configure the foreman (monitors workers).
front_end.SetShowMemoryInformation(show_memory=options.foreman_verbose)
front_end.SetRunForeman(run_foreman=options.foreman_enabled)
try:
front_end.ProcessSource(options)
logging.info(u'Processing completed.')
except (KeyboardInterrupt, errors.UserAbort):
logging.warning(u'Aborted by user.')
return False
except errors.SourceScannerError as exception:
logging.warning(
(u'Unable to scan for a supported filesystem with error: {0:s}\n'
u'Most likely the image format is not supported by the '
u'tool.').format(exception))
return False
return True
master_doc = 'index' # General information about the project. project = u'Plaso' # This is a built in, but also apparently a sphinx config option. # pylint: disable=redefined-builtin copyright = u'The Plaso Project Authors' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = plaso.__version__ # The full version, including alpha/beta/rc tags. release = plaso.GetVersion() # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ['_build']
encoding = locale.getpreferredencoding()
# Make sure the default encoding is set correctly otherwise
# setup.py sdist will fail to include filenames with Unicode characters.
reload(sys)
sys.setdefaultencoding(encoding)
# Unicode in the description will break python-setuptools, hence
# "Plaso Langar Að Safna Öllu" was removed.
plaso_description = (
u'plaso is a tool designed to extract timestamps from various files found '
u'on a typical computer system(s) and aggregate them.')
setup(
name='plaso',
version=plaso.GetVersion(),
description=plaso_description,
long_description=plaso_description,
license='Apache License, Version 2.0',
url='https://sites.google.com/a/kiddaland.net/plaso',
maintainer='Plaso development team',
maintainer_email='*****@*****.**',
scripts=GetScripts(),
cmdclass={'test': TestCommand},
classifiers=[
'Development Status :: 4 - Beta',
'Environment :: Console',
'Operating System :: OS Independent',
'Programming Language :: Python',
],
packages=find_packages('.', exclude=[u'tests', u'tests.*', u'tools']),
def _PreprocessSetCollectionInformation(
self, pre_obj, unused_engine, filter_file=None,
parser_filter_string=None):
"""Sets the collection information as part of the preprocessing.
Args:
pre_obj: the preprocess object (instance of PreprocessObject).
engine: the engine object (instance of BaseEngine).
filter_file: a path to a file that contains find specifications.
The default is None.
parser_filter_string: optional parser filter string. The default is None.
"""
collection_information = {}
# TODO: informational values.
collection_information[u'version'] = plaso.GetVersion()
collection_information[u'debug'] = self._debug_mode
# TODO: extraction preferences:
if not parser_filter_string:
parser_filter_string = u'(no list set)'
collection_information[u'parser_selection'] = parser_filter_string
collection_information[u'preferred_encoding'] = self.preferred_encoding
# TODO: extraction info:
collection_information[u'configured_zone'] = pre_obj.zone
collection_information[u'parsers'] = self._parser_names
collection_information[u'preprocess'] = self._preprocess
if self._filter_expression:
collection_information[u'filter'] = self._filter_expression
if filter_file and os.path.isfile(filter_file):
filters = []
with open(filter_file, 'rb') as file_object:
for line in file_object.readlines():
filters.append(line.rstrip())
collection_information[u'file_filter'] = u', '.join(filters)
if self._operating_system:
collection_information[u'os_detected'] = self._operating_system
else:
collection_information[u'os_detected'] = 'N/A'
# TODO: processing settings:
collection_information[u'protobuf_size'] = self._buffer_size
collection_information[u'time_of_run'] = timelib.Timestamp.GetNow()
if self._single_process_mode:
collection_information[u'runtime'] = u'single process mode'
else:
collection_information[u'runtime'] = u'multi process mode'
# TODO: retrieve this value from the multi-process engine.
# refactor engine to set number_of_extraction_workers
# before ProcessSource.
collection_information[u'workers'] = 0
# TODO: output/storage settings:
collection_information[u'output_file'] = self._storage_file_path
# TODO: source settings:
if self.SourceIsDirectory():
recursive = True
else:
recursive = False
collection_information[u'file_processed'] = self._source_path
collection_information[u'recursive'] = recursive
collection_information[u'vss parsing'] = bool(self.vss_stores)
if self.SourceIsStorageMediaImage():
collection_information[u'method'] = u'imaged processed'
collection_information[u'image_offset'] = self.partition_offset
else:
collection_information[u'method'] = u'OS collection'
pre_obj.collection_information = collection_information
def _PreprocessSetCollectionInformation(self, options, pre_obj):
"""Sets the collection information as part of the preprocessing.
Args:
options: the command line arguments (instance of argparse.Namespace).
pre_obj: the preprocess object (instance of PreprocessObject).
"""
collection_information = {}
collection_information['version'] = plaso.GetVersion()
collection_information['configured_zone'] = self._timezone
collection_information['file_processed'] = self._source_path
collection_information['output_file'] = self._storage_file_path
collection_information['protobuf_size'] = self._buffer_size
collection_information['parser_selection'] = getattr(
options, 'parsers', '(no list set)')
collection_information['preferred_encoding'] = self.preferred_encoding
collection_information['time_of_run'] = timelib.Timestamp.GetNow()
collection_information['parsers'] = self._parser_names
collection_information['preprocess'] = self._preprocess
if self._scan_context.source_type in [
self._scan_context.SOURCE_TYPE_DIRECTORY]:
recursive = True
else:
recursive = False
collection_information['recursive'] = recursive
collection_information['debug'] = self._debug_mode
collection_information['vss parsing'] = bool(self._vss_stores)
if self._filter_expression:
collection_information['filter'] = self._filter_expression
filter_file = getattr(options, 'file_filter', None)
if filter_file:
if os.path.isfile(filter_file):
filters = []
with open(filter_file, 'rb') as fh:
for line in fh:
filters.append(line.rstrip())
collection_information['file_filter'] = ', '.join(filters)
if self._operating_system:
collection_information['os_detected'] = self._operating_system
else:
collection_information['os_detected'] = 'N/A'
if self._scan_context.source_type in [
self._scan_context.SOURCE_TYPE_STORAGE_MEDIA_DEVICE,
self._scan_context.SOURCE_TYPE_STORAGE_MEDIA_IMAGE]:
collection_information['method'] = 'imaged processed'
collection_information['image_offset'] = self._partition_offset
else:
collection_information['method'] = 'OS collection'
if self._single_process_mode:
collection_information['runtime'] = 'single process mode'
else:
collection_information['runtime'] = 'multi process mode'
collection_information['workers'] = self._number_of_worker_processes
pre_obj.collection_information = collection_information
def _PrintStatusUpdate(self, processing_status):
"""Prints the processing status.
Args:
processing_status: the processing status (instance of ProcessingStatus).
"""
if self._stdout_output_writer:
self._ClearScreen()
self._output_writer.Write(u'plaso - {0:s} version {1:s}\n'.format(
self.NAME, plaso.GetVersion()))
self._output_writer.Write(u'\n')
self._PrintStatusHeader()
# TODO: for win32console get current color and set intensity,
# write the header separately then reset intensity.
status_header = u'Identifier\tPID\tStatus\t\tEvents\t\tFile'
if not win32console:
status_header = u'\x1b[1m{0:s}\x1b[0m'.format(status_header)
status_table = [status_header]
status_row = self._FormatStatusTableRow(
processing_status.collector.identifier,
processing_status.collector.pid,
processing_status.collector.status,
processing_status.collector.process_status, None, None, u'')
status_table.append(status_row)
for extraction_worker_status in processing_status.extraction_workers:
status_row = self._FormatStatusTableRow(
extraction_worker_status.identifier,
extraction_worker_status.pid, extraction_worker_status.status,
extraction_worker_status.process_status,
extraction_worker_status.number_of_events,
extraction_worker_status.number_of_events_delta,
extraction_worker_status.display_name)
status_table.append(status_row)
if processing_status.storage_writer:
status_row = self._FormatStatusTableRow(
processing_status.storage_writer.identifier,
processing_status.storage_writer.pid,
processing_status.storage_writer.status,
processing_status.storage_writer.process_status,
processing_status.storage_writer.number_of_events,
processing_status.storage_writer.number_of_events_delta, u'')
status_table.append(status_row)
status_table.append(u'')
self._output_writer.Write(u'\n'.join(status_table))
self._output_writer.Write(u'\n')
if processing_status.GetExtractionCompleted():
self._output_writer.Write(
u'All extraction workers completed - waiting for storage.\n')
self._output_writer.Write(u'\n')
# TODO: remove update flicker. For win32console we could set the cursor
# top left, write the table, clean the remainder of the screen buffer
# and set the cursor at the end of the table.
if self._stdout_output_writer:
# We need to explicitly flush stdout to prevent partial status updates.
sys.stdout.flush()