def ParseArguments(self):
"""Parses the command line arguments.
Returns:
bool: True if the arguments were successfully parsed.
"""
loggers.ConfigureLogging()
argument_parser = argparse.ArgumentParser(
description=self.DESCRIPTION, add_help=False,
formatter_class=argparse.RawDescriptionHelpFormatter)
self.AddBasicOptions(argument_parser)
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
argument_parser, names=['storage_file'])
argument_parser.add_argument(
'-v', '--verbose', dest='verbose', action='store_true',
default=False, help='Print verbose output.')
argument_parser.add_argument(
'--compare', dest='compare_storage_file', type=str,
action='store', default='', metavar='STORAGE_FILE', help=(
'The path of the storage file to compare against.'))
argument_parser.add_argument(
'-w', '--write', metavar='OUTPUTFILE', dest='write',
help='Output filename.')
argument_parser.add_argument(
'--output-format', dest='output_format', type=str,
choices=['text', 'json'], action='store', default='text',
metavar='FORMAT', help='Type of output to produce')
try:
options = argument_parser.parse_args()
except UnicodeEncodeError:
# If we get here we are attempting to print help in a non-Unicode
# terminal.
self._output_writer.Write('\n')
self._output_writer.Write(argument_parser.format_help())
return False
try:
self.ParseOptions(options)
except errors.BadConfigOption as exception:
self._output_writer.Write('ERROR: {0!s}\n'.format(exception))
self._output_writer.Write('\n')
self._output_writer.Write(argument_parser.format_usage())
return False
loggers.ConfigureLogging(
debug_output=self._debug_mode, filename=self._log_file,
quiet_mode=self._quiet_mode)
return True
def testConfigureLogging(self):
"""Tests the ConfigureLogging function."""
with shared_test_lib.TempDirectory() as temp_directory:
filename = os.path.join(temp_directory, 'test.log.gz')
loggers.ConfigureLogging(debug_output=True, filename=filename)
filename = os.path.join(temp_directory, 'test.log')
loggers.ConfigureLogging(filename=filename, quiet_mode=True)
loggers.ConfigureLogging()
def _StartWorkerProcess(self, process_name):
"""Creates, starts, monitors and registers a worker process.
Args:
process_name (str): process name.
Returns:
MultiProcessWorkerProcess: extraction worker process or None if the
process could not be started.
"""
logger.debug('Starting worker process {0:s}'.format(process_name))
queue_name = '{0:s} task queue'.format(process_name)
task_queue = zeromq_queue.ZeroMQRequestConnectQueue(
delay_open=True,
linger_seconds=0,
name=queue_name,
port=self._task_queue_port,
timeout_seconds=self._TASK_QUEUE_TIMEOUT_SECONDS)
process = extraction_process.ExtractionWorkerProcess(
task_queue,
self.collection_filters_helper,
self.knowledge_base,
self._processing_configuration,
enable_sigsegv_handler=self._enable_sigsegv_handler,
name=process_name)
# Remove all possible log handlers to prevent a child process from logging
# to the main process log file and garbling the log. The log handlers are
# recreated after the worker process has been started.
for handler in logging.root.handlers:
logging.root.removeHandler(handler)
handler.close()
process.start()
loggers.ConfigureLogging(debug_output=self._debug_output,
filename=self._log_filename,
mode='a',
quiet_mode=self._quiet_mode)
try:
self._StartMonitoringProcess(process)
except (IOError, KeyError) as exception:
pid = process.pid
logger.error(
('Unable to monitor replacement worker process: {0:s} '
'(PID: {1:d}) with error: {2!s}').format(
process_name, pid, exception))
self._TerminateProcess(process)
return None
self._RegisterProcess(process)
self._last_worker_number += 1
return process
def _RunProcess(self):
"""Runs the process."""
# Prevent the KeyboardInterrupt being raised inside the process.
# This will prevent a process from generating a traceback when interrupted.
signal.signal(signal.SIGINT, signal.SIG_IGN)
# A SIGTERM signal handler is necessary to make sure IPC is cleaned up
# correctly on terminate.
signal.signal(signal.SIGTERM, self._SigTermHandler)
# A SIGSEGV signal handler is necessary to try to indicate where
# worker failed.
# WARNING the SIGSEGV handler will deadlock the process on a real segfault.
if self._enable_sigsegv_handler:
self._original_sigsegv_handler = signal.signal(
signal.SIGSEGV, self._SigSegvHandler)
self._pid = os.getpid()
self._process_information = process_info.ProcessInfo(self._pid)
# We need to set the is running status explicitly to True in case
# the process completes before the engine is able to determine
# the status of the process, such as in the unit tests.
self._status_is_running = True
# Logging needs to be configured before the first output otherwise we
# mess up the logging of the parent process.
loggers.ConfigureLogging(
debug_output=self._debug_output, filename=self._log_filename,
quiet_mode=self._quiet_mode)
logger.debug('Process: {0!s} (PID: {1:d}) started'.format(
self._name, self._pid))
self._StartProcessStatusRPCServer()
logger.debug('Process: {0!s} (PID: {1:d}) enter main'.format(
self._name, self._pid))
self._Main()
logger.debug('Process: {0!s} (PID: {1:d}) exit main'.format(
self._name, self._pid))
self._StopProcessStatusRPCServer()
logger.debug('Process: {0!s} (PID: {1:d}) stopped'.format(
self._name, self._pid))
# Make sure log files are cleanly closed.
logging.shutdown()
self._status_is_running = False
def ParseArguments(self):
"""Parses the command line arguments.
Returns:
bool: True if the arguments were successfully parsed.
"""
loggers.ConfigureLogging()
argument_parser = argparse.ArgumentParser(
description=self.DESCRIPTION, epilog=self.EPILOG, add_help=False,
formatter_class=argparse.RawDescriptionHelpFormatter)
self.AddBasicOptions(argument_parser)
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
argument_parser, names=['storage_file'])
data_location_group = argument_parser.add_argument_group(
'data location arguments')
argument_helper_names = ['artifact_definitions', 'data_location']
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
data_location_group, names=argument_helper_names)
extraction_group = argument_parser.add_argument_group(
'extraction arguments')
argument_helper_names = [
'extraction', 'filter_file', 'hashers', 'parsers', 'yara_rules']
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
extraction_group, names=argument_helper_names)
self.AddStorageMediaImageOptions(extraction_group)
self.AddTimeZoneOption(extraction_group)
self.AddVSSProcessingOptions(extraction_group)
self.AddCredentialOptions(extraction_group)
info_group = argument_parser.add_argument_group('informational arguments')
self.AddInformationalOptions(info_group)
info_group.add_argument(
'--info', dest='show_info', action='store_true', default=False,
help='Print out information about supported plugins and parsers.')
info_group.add_argument(
'--use_markdown', '--use-markdown', dest='use_markdown',
action='store_true', default=False, help=(
'Output lists in Markdown format use in combination with '
'"--hashers list", "--parsers list" or "--timezone list"'))
info_group.add_argument(
'--no_dependencies_check', '--no-dependencies-check',
dest='dependencies_check', action='store_false', default=True,
help='Disable the dependencies check.')
self.AddLogFileOptions(info_group)
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
info_group, names=['status_view'])
output_group = argument_parser.add_argument_group('output arguments')
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
output_group, names=['text_prepend'])
processing_group = argument_parser.add_argument_group(
'processing arguments')
self.AddPerformanceOptions(processing_group)
self.AddProcessingOptions(processing_group)
processing_group.add_argument(
'--sigsegv_handler', '--sigsegv-handler', dest='sigsegv_handler',
action='store_true', default=False, help=(
'Enables the SIGSEGV handler. WARNING this functionality is '
'experimental and will a deadlock worker process if a real '
'segfault is caught, but not signal SIGSEGV. This functionality '
'is therefore primarily intended for debugging purposes'))
profiling_group = argument_parser.add_argument_group('profiling arguments')
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
profiling_group, names=['profiling'])
storage_group = argument_parser.add_argument_group('storage arguments')
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
storage_group, names=['storage_format'])
argument_parser.add_argument(
self._SOURCE_OPTION, action='store', metavar='SOURCE', nargs='?',
default=None, type=str, help=(
'Path to a source device, file or directory. If the source is '
'a supported storage media device or image file, archive file '
'or a directory, the files within are processed recursively.'))
try:
options = argument_parser.parse_args()
except UnicodeEncodeError:
# If we get here we are attempting to print help in a non-Unicode
# terminal.
self._output_writer.Write('\n')
self._output_writer.Write(argument_parser.format_help())
return False
# Properly prepare the attributes according to local encoding.
if self.preferred_encoding == 'ascii':
logger.warning(
'The preferred encoding of your system is ASCII, which is not '
'optimal for the typically non-ASCII characters that need to be '
'parsed and processed. The tool will most likely crash and die, '
'perhaps in a way that may not be recoverable. A five second delay '
'is introduced to give you time to cancel the runtime and '
'reconfigure your preferred encoding, otherwise continue at own '
'risk.')
time.sleep(5)
if self._process_archives:
logger.warning(
'Scanning archive files currently can cause deadlock. Continue at '
'your own risk.')
time.sleep(5)
try:
self.ParseOptions(options)
except errors.BadConfigOption as exception:
self._output_writer.Write('ERROR: {0!s}\n'.format(exception))
self._output_writer.Write('\n')
self._output_writer.Write(argument_parser.format_usage())
return False
self._command_line_arguments = self.GetCommandLineArguments()
loggers.ConfigureLogging(
debug_output=self._debug_mode, filename=self._log_file,
quiet_mode=self._quiet_mode)
return True
def ParseArguments(self):
"""Parses the command line arguments.
Returns:
bool: True if the arguments were successfully parsed.
"""
loggers.ConfigureLogging()
argument_parser = argparse.ArgumentParser(
description=self.DESCRIPTION,
epilog=self.EPILOG,
add_help=False,
formatter_class=argparse.RawDescriptionHelpFormatter)
self.AddBasicOptions(argument_parser)
self.AddInformationalOptions(argument_parser)
argument_helper_names = ['artifact_definitions', 'data_location']
if self._CanEnforceProcessMemoryLimit():
argument_helper_names.append('process_resources')
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
argument_parser, names=argument_helper_names)
self.AddLogFileOptions(argument_parser)
self.AddStorageMediaImageOptions(argument_parser)
self.AddVSSProcessingOptions(argument_parser)
self.AddFilterOptions(argument_parser)
argument_parser.add_argument(
'-w',
'--write',
action='store',
dest='path',
type=str,
metavar='PATH',
default='export',
help=('The directory in which extracted files should be stored.'))
argument_parser.add_argument(
'--include_duplicates',
dest='include_duplicates',
action='store_true',
default=False,
help=(
'If extraction from VSS is enabled, by default a digest hash '
'is calculated for each file. These hashes are compared to the '
'previously exported files and duplicates are skipped. Use '
'this option to include duplicate files in the export.'))
argument_parser.add_argument(
self._SOURCE_OPTION,
nargs='?',
action='store',
metavar='IMAGE',
default=None,
type=str,
help=(
'The full path to the image file that we are about to extract '
'files from, it should be a raw image or another image that '
'plaso supports.'))
try:
options = argument_parser.parse_args()
except UnicodeEncodeError:
# If we get here we are attempting to print help in a non-Unicode
# terminal.
self._output_writer.Write('')
self._output_writer.Write(argument_parser.format_help())
return False
try:
self.ParseOptions(options)
except errors.BadConfigOption as exception:
self._output_writer.Write('ERROR: {0!s}\n'.format(exception))
self._output_writer.Write('')
self._output_writer.Write(argument_parser.format_usage())
return False
loggers.ConfigureLogging(debug_output=self._debug_mode,
filename=self._log_file,
quiet_mode=self._quiet_mode)
return True
def ParseArguments(self):
"""Parses the command line arguments.
Returns:
bool: True if the arguments were successfully parsed.
"""
loggers.ConfigureLogging()
argument_parser = argparse.ArgumentParser(
description=self.DESCRIPTION, add_help=False,
conflict_handler='resolve',
formatter_class=argparse.RawDescriptionHelpFormatter)
self.AddBasicOptions(argument_parser)
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
argument_parser, names=['storage_file'])
analysis_group = argument_parser.add_argument_group('Analysis Arguments')
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
analysis_group, names=['analysis_plugins'])
processing_group = argument_parser.add_argument_group('Processing')
self.AddProcessingOptions(processing_group)
info_group = argument_parser.add_argument_group('Informational Arguments')
self.AddLogFileOptions(info_group)
self.AddInformationalOptions(info_group)
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
info_group, names=['status_view'])
filter_group = argument_parser.add_argument_group('Filter Arguments')
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
filter_group, names=['event_filters'])
input_group = argument_parser.add_argument_group('Input Arguments')
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
input_group, names=['data_location'])
output_group = argument_parser.add_argument_group('Output Arguments')
output_group.add_argument(
'-a', '--include_all', '--include-all', action='store_false',
dest='dedup', default=True, help=(
'By default the psort removes duplicate entries from the '
'output. This parameter changes that behavior so all events '
'are included.'))
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
output_group, names=['language'])
self.AddTimeZoneOption(output_group)
output_format_group = argument_parser.add_argument_group(
'Output Format Arguments')
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
output_format_group, names=['output_modules'])
profiling_group = argument_parser.add_argument_group('profiling arguments')
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
profiling_group, names=['profiling'])
try:
# TODO: refactor how arguments is used in a more argparse way.
options = argument_parser.parse_args()
except UnicodeEncodeError:
# If we get here we are attempting to print help in a non-Unicode
# terminal.
self._output_writer.Write('\n')
self._output_writer.Write(argument_parser.format_help())
return False
# Properly prepare the attributes according to local encoding.
if self.preferred_encoding == 'ascii':
logger.warning(
'The preferred encoding of your system is ASCII, which is not '
'optimal for the typically non-ASCII characters that need to be '
'parsed and processed. The tool will most likely crash and die, '
'perhaps in a way that may not be recoverable. A five second delay '
'is introduced to give you time to cancel the runtime and '
'reconfigure your preferred encoding, otherwise continue at own '
'risk.')
time.sleep(5)
try:
self.ParseOptions(options)
except errors.BadConfigOption as exception:
self._output_writer.Write('ERROR: {0!s}\n'.format(exception))
self._output_writer.Write('\n')
self._output_writer.Write(argument_parser.format_usage())
return False
loggers.ConfigureLogging(
debug_output=self._debug_mode, filename=self._log_file,
quiet_mode=self._quiet_mode)
return True
def ParseArguments(self, arguments):
"""Parses the command line arguments.
Args:
arguments (list[str]): command line arguments.
Returns:
bool: True if the arguments were successfully parsed.
"""
loggers.ConfigureLogging()
argument_parser = argparse.ArgumentParser(
description=self.DESCRIPTION, epilog=self.EPILOG, add_help=False,
formatter_class=argparse.RawDescriptionHelpFormatter)
self.AddBasicOptions(argument_parser)
data_location_group = argument_parser.add_argument_group(
'data location arguments')
argument_helper_names = ['artifact_definitions', 'data_location']
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
data_location_group, names=argument_helper_names)
extraction_group = argument_parser.add_argument_group(
'extraction arguments')
argument_helper_names = ['extraction', 'hashers', 'parsers']
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
extraction_group, names=argument_helper_names)
extraction_group.add_argument(
'--storage_file', '--storage-file', metavar='PATH', type=str,
default=None, help=(
'The path of the storage file. If not specified, one will be made '
'in the form <timestamp>-<source>.plaso'))
self.AddStorageMediaImageOptions(extraction_group)
self.AddTimeZoneOption(extraction_group)
self.AddVSSProcessingOptions(extraction_group)
self.AddCredentialOptions(extraction_group)
info_group = argument_parser.add_argument_group('informational arguments')
self.AddInformationalOptions(info_group)
info_group.add_argument(
'--no_dependencies_check', '--no-dependencies-check',
dest='dependencies_check', action='store_false', default=True,
help='Disable the dependencies check.')
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
info_group, names=['status_view'])
input_group = argument_parser.add_argument_group('input arguments')
input_group.add_argument(
'--source', dest='source', action='store',
type=str, help='The source to process')
output_group = argument_parser.add_argument_group('output arguments')
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
output_group, names=['language'])
self.AddOutputTimeZoneOption(output_group)
output_format_group = argument_parser.add_argument_group(
'output format arguments')
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
output_format_group, names=['output_modules'])
processing_group = argument_parser.add_argument_group(
'processing arguments')
self.AddPerformanceOptions(processing_group)
self.AddProcessingOptions(processing_group)
try:
options = argument_parser.parse_args(arguments)
except UnicodeEncodeError:
# If we get here we are attempting to print help in a non-Unicode
# terminal.
self._output_writer.Write('\n')
self._output_writer.Write(argument_parser.format_help())
return False
try:
self.ParseOptions(options)
except errors.BadConfigOption as exception:
self._output_writer.Write('ERROR: {0!s}\n'.format(exception))
self._output_writer.Write('\n')
self._output_writer.Write(argument_parser.format_usage())
return False
loggers.ConfigureLogging(
debug_output=self._debug_mode, filename=self._log_file,
quiet_mode=self._quiet_mode)
return True
def ParseArguments(self, arguments):
"""Parses the command line arguments.
Args:
arguments (list[str]): command line arguments.
Returns:
bool: True if the arguments were successfully parsed.
"""
loggers.ConfigureLogging()
argument_parser = argparse.ArgumentParser(
description=self.DESCRIPTION, add_help=False,
formatter_class=argparse.RawDescriptionHelpFormatter)
self.AddBasicOptions(argument_parser)
argument_helper_names = ['storage_file']
if self._CanEnforceProcessMemoryLimit():
argument_helper_names.append('process_resources')
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
argument_parser, names=argument_helper_names)
argument_parser.add_argument(
'--compare', dest='compare_storage_file', type=str,
action='store', default='', metavar='STORAGE_FILE', help=(
'The path of the storage file to compare against.'))
argument_parser.add_argument(
'--output_format', '--output-format', dest='output_format', type=str,
choices=['text', 'json'], action='store', default='text',
metavar='FORMAT', help=(
'Format of the output, the default is: text. Supported options: '
'json, text.'))
argument_parser.add_argument(
'--sections', dest='sections', type=str, action='store', default='all',
metavar='SECTIONS_LIST', help=(
'List of sections to output. This is a comma separated list where '
'each entry is the name of a section. Use "--sections list" to '
'list the available sections and "--sections all" to show all '
'available sections.'))
argument_parser.add_argument(
'-v', '--verbose', dest='verbose', action='store_true',
default=False, help='Print verbose output.')
argument_parser.add_argument(
'-w', '--write', metavar='OUTPUTFILE', dest='write',
help='Output filename.')
try:
options = argument_parser.parse_args(arguments)
except UnicodeEncodeError:
# If we get here we are attempting to print help in a non-Unicode
# terminal.
self._output_writer.Write('\n')
self._output_writer.Write(argument_parser.format_help())
return False
try:
self.ParseOptions(options)
except errors.BadConfigOption as exception:
self._output_writer.Write('ERROR: {0!s}\n'.format(exception))
self._output_writer.Write('\n')
self._output_writer.Write(argument_parser.format_usage())
return False
loggers.ConfigureLogging(
debug_output=self._debug_mode, filename=self._log_file,
quiet_mode=self._quiet_mode)
return True
def ParseArguments(self, arguments):
"""Parses the command line arguments.
Args:
arguments (list[str]): command line arguments.
Returns:
bool: True if the arguments were successfully parsed.
"""
loggers.ConfigureLogging()
argument_parser = argparse.ArgumentParser(
description=self.DESCRIPTION, epilog=self.EPILOG, add_help=False,
formatter_class=argparse.RawDescriptionHelpFormatter)
self.AddBasicOptions(argument_parser)
data_location_group = argument_parser.add_argument_group(
'data location arguments')
argument_helper_names = ['artifact_definitions', 'data_location']
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
data_location_group, names=argument_helper_names)
extraction_group = argument_parser.add_argument_group(
'extraction arguments')
argument_helper_names = ['extraction', 'hashers', 'parsers']
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
extraction_group, names=argument_helper_names)
self.AddStorageOptions(extraction_group)
self.AddStorageMediaImageOptions(extraction_group)
self.AddTimeZoneOption(extraction_group)
self.AddVSSProcessingOptions(extraction_group)
self.AddCredentialOptions(extraction_group)
info_group = argument_parser.add_argument_group('informational arguments')
self.AddInformationalOptions(info_group)
info_group.add_argument(
'--no_dependencies_check', '--no-dependencies-check',
dest='dependencies_check', action='store_false', default=True,
help='Disable the dependencies check.')
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
info_group, names=['status_view'])
input_group = argument_parser.add_argument_group('input arguments')
input_group.add_argument(
'--source', dest='source', action='store',
type=str, help='The source to process')
output_group = argument_parser.add_argument_group('output arguments')
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
output_group, names=['language'])
self.AddOutputOptions(output_group)
output_format_group = argument_parser.add_argument_group(
'output format arguments')
helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
output_format_group, names=['output_modules'])
processing_group = argument_parser.add_argument_group(
'processing arguments')
self.AddPerformanceOptions(processing_group)
self.AddProcessingOptions(processing_group)
try:
options = argument_parser.parse_args(arguments)
except UnicodeEncodeError:
# If we get here we are attempting to print help in a non-Unicode
# terminal.
self._output_writer.Write('\n')
self._output_writer.Write(argument_parser.format_help())
return False
# Properly prepare the attributes according to local encoding.
if self.preferred_encoding == 'ascii':
self._PrintUserWarning((
'the preferred encoding of your system is ASCII, which is not '
'optimal for the typically non-ASCII characters that need to be '
'parsed and processed. This will most likely result in an error.'))
try:
self.ParseOptions(options)
except errors.BadConfigOption as exception:
self._output_writer.Write('ERROR: {0!s}\n'.format(exception))
self._output_writer.Write('\n')
self._output_writer.Write(argument_parser.format_usage())
return False
self._WaitUserWarning()
loggers.ConfigureLogging(
debug_output=self._debug_mode, filename=self._log_file,
quiet_mode=self._quiet_mode)
return True
