def _SetupModuleThread(self, module_definition):
"""Calls the module's SetUp() function and sets a threading event for it.
Callback for _InvokeModulesInThreads.
Args:
module_definition (dict[str, str]): recipe module definition.
"""
module_name = module_definition['name']
logger.info('Setting up module: {0:s}'.format(module_name))
new_args = utils.ImportArgsFromDict(
module_definition['args'], self.command_line_options, self.config)
module = self._module_pool[module_name]
try:
module.SetUp(**new_args)
except errors.DFTimewolfError as exception:
msg = "A critical error occurred in module {0:s}, aborting execution."
logger.critical(msg.format(module.name))
except Exception as exception: # pylint: disable=broad-except
msg = 'An unknown error occurred in module {0:s}: {1!s}'.format(
module.name, exception)
logger.critical(msg)
# We're catching any exception that is not a DFTimewolfError, so we want
# to generate an error for further reporting.
error = errors.DFTimewolfError(
message=msg, name='state', stacktrace=traceback.format_exc(),
critical=True, unexpected=True)
self.AddError(error)
self._threading_event_per_module[module_name] = threading.Event()
self.CleanUp()
def _SetupModuleThread(self, module_definition):
"""Calls the module's SetUp() function and sets a threading event for it.
Callback for _InvokeModulesInThreads.
Args:
module_definition (dict[str, str]): recipe module definition.
"""
module_name = module_definition['name']
new_args = utils.ImportArgsFromDict(module_definition['args'],
self.command_line_options,
self.config)
module = self._module_pool[module_name]
try:
module.SetUp(**new_args)
except Exception as exception: # pylint: disable=broad-except
self.AddError(
'An unknown error occurred: {0!s}\nFull traceback:\n{1:s}'.
format(exception, traceback.format_exc()),
critical=True)
self._threading_event_per_module[module_name] = threading.Event()
self.CleanUp()
def test_nonexistent_arg(self):
"""Makes sure that an exception is raised for unknown @ variables.
A recipe writer needs to define a parsing tuple for each @ variable used by
the recipe.
"""
recipe_args = {
'recipe_arg1': 'This should be replaced: @parameterone',
'recipe_arg2': 'This arg cannot be replaced @nonexistent',
}
parser.set_defaults(**config.Config.GetExtra())
args = parser.parse_args([
'value_for_param_one',
'--optional_param',
'3',
'--spaces_param',
'So unique!',
'BOOM',
])
with self.assertRaises(ValueError):
imported = dftw_utils.ImportArgsFromDict(recipe_args, vars(args),
config.Config)
self._CheckPlaceholders(imported)
def test_import_args_from_cli(self):
"""Tries parsing the CLI arguments and updating a recipe dictionary."""
recipe_args = {
'recipe_arg1': 'This should remain intact',
'recipe_arg2': 'This should be replaced: @parameterone',
'recipe_arg3': 'This should be replaced by @optional_param',
'recipe_arg4': 'This includes spaces: @spaces_param',
'recipe_arg5': 'Characters after param: @explosion!',
}
expected_args = {
'recipe_arg1': 'This should remain intact',
'recipe_arg2': 'This should be replaced: value_for_param_one',
'recipe_arg3': 'This should be replaced by 3',
'recipe_arg4': 'This includes spaces: S P A C E',
'recipe_arg5': 'Characters after param: BOOM!',
}
parser.set_defaults(**config.Config.GetExtra())
args = parser.parse_args([
'value_for_param_one',
'--optional_param',
'3',
'--spaces_param',
'S P A C E',
'BOOM',
])
imported_args = dftw_utils.ImportArgsFromDict(recipe_args, vars(args),
config.Config)
self.assertEqual(imported_args, expected_args)
def RunPreflights(self):
"""Runs preflight modules."""
for preflight_definition in self.recipe.get('preflights', []):
preflight_name = preflight_definition['name']
args = preflight_definition.get('args', {})
new_args = utils.ImportArgsFromDict(
args, self.command_line_options, self.config)
preflight = self._module_pool[preflight_name]
try:
preflight.SetUp(**new_args)
preflight.Process()
finally:
self.CheckErrors(is_global=True)
def test_config_fills_missing_args(self):
"""Tests that a configuration file will fill-in arguments that are missing
from the CLI."""
provided_args = {'arg1': 'This should remain intact', 'arg2': '@config'}
expected_args = {
'arg1': 'This should remain intact',
'arg2': 'A config arg',
}
config.Config.LoadExtraData('{"config": "A config arg"}')
parser.set_defaults(**config.Config.GetExtra())
args = parser.parse_args(['a', 'b'])
imported_args = dftw_utils.ImportArgsFromDict(
provided_args, vars(args), config.Config)
self.assertEqual(imported_args, expected_args)
def test_cli_precedence_over_config(self):
"""Tests that the same argument provided via the CLI overrides the one
specified in the config file."""
provided_args = {
'arg1': 'I want whatever CLI says: @parameterone',
}
expected_args = {
'arg1': 'I want whatever CLI says: CLI WINS!',
}
config.Config.LoadExtraData('{"parameterone": "CONFIG WINS!"}')
parser.set_defaults(**config.Config.GetExtra())
args = parser.parse_args(['CLI WINS!', 'BOOM'])
imported_args = dftw_utils.ImportArgsFromDict(
provided_args, vars(args), config.Config)
self.assertEqual(imported_args, expected_args)
def _SetupModuleThread(module_description):
"""Calls the module's SetUp() function and sets an Event object for it.
Args:
module_description (dict): Corresponding recipe module description.
"""
new_args = utils.ImportArgsFromDict(module_description['args'],
vars(args), self.config)
module = self._module_pool[module_description['name']]
try:
module.SetUp(**new_args)
except Exception as exception: # pylint: disable=broad-except
self.AddError(
'An unknown error occurred: {0!s}\nFull traceback:\n{1:s}'.
format(exception, traceback.format_exc()),
critical=True)
self.events[module_description['name']] = threading.Event()
self.CleanUp()
def LogExecutionPlan(self):
"""Logs loaded modules and their corresponding arguments to stdout."""
maxlen = 0
for module in self.recipe['modules']:
if not module['args']:
continue
spacing = len(max(module['args'].keys(), key=len))
maxlen = maxlen if maxlen > spacing else spacing
for module in self.recipe['modules']:
logger.debug('{0:s}:'.format(module['name']))
new_args = utils.ImportArgsFromDict(module['args'],
self.command_line_options,
self.config)
if not new_args:
logger.debug(' *No params*')
for key, value in new_args.items():
logger.debug(' {0:s}{1:s}'.format(key.ljust(maxlen + 3),
repr(value)))
