def __init__(self, verbose):
"""Initialize the base collector object.
Args:
verbose: (Optional[bool]) whether verbose output is desired.
"""
super(BaseCollector, self).__init__()
self.console_out = utils.DFTimewolfConsoleOutput(
sender=self.__class__.__name__, verbose=verbose)
self.results = []
def __init__(self, verbose):
super(BaseArtifactProcessor, self).__init__()
self.console_out = utils.DFTimewolfConsoleOutput(
sender=self.__class__.__name__, verbose=verbose)
self.errors = []
def __init__(self, verbose):
super(BaseExporter, self).__init__()
self.console_out = utils.DFTimewolfConsoleOutput(
sender=self.__class__.__name__, verbose=verbose)
def main():
"""Main function for DFTimewolf."""
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(
title=u'Available recipes',
description=u'List of currently loaded recipes',
help=u'Recipe-specific help')
for recipe, recipe_args in config.Config.get_registered_recipes():
subparser = subparsers.add_parser(recipe[u'name'],
description=u'{0:s}'.format(
recipe.__doc__))
subparser.set_defaults(recipe=recipe)
for switch, help_text in recipe_args:
subparser.add_argument(switch, help=help_text)
args = parser.parse_args()
recipe = args.recipe
console_out = dftw_utils.DFTimewolfConsoleOutput(sender=u'DFTimewolfCli',
verbose=True)
# COLLECTORS
# Thread collectors
console_out.StdOut(u'Collectors:')
for collector in recipe['collectors']:
console_out.StdOut(u' {0:s}'.format(collector[u'name']))
collector_objs = []
for collector in recipe[u'collectors']:
new_args = dftw_utils.import_args_from_dict(collector[u'args'],
vars(args))
collector_cls = config.Config.get_collector(collector[u'name'])
collector_objs.extend(collector_cls.launch_collector(**new_args))
# Wait for collectors to finish and collect output
collector_output = []
for collector_obj in collector_objs:
collector_obj.join()
collector_output.extend(collector_obj.results)
if recipe[u'processors']:
# PROCESSORS
# Thread processors
console_out.StdOut(u'Processors:')
for processor in recipe[u'processors']:
console_out.StdOut(u' {0:s}'.format(processor[u'name']))
processor_objs = []
for processor in recipe[u'processors']:
new_args = dftw_utils.import_args_from_dict(
processor[u'args'], vars(args))
new_args[u'collector_output'] = collector_output
processor_class = config.Config.get_processor(processor[u'name'])
processor_objs.extend(processor_class.launch_processor(**new_args))
# Wait for processors to finish and collect output
processor_output = []
for processor in processor_objs:
processor.join()
processor_output.extend(processor.output)
else:
processor_output = collector_output
# EXPORTERS
# Thread exporters
console_out.StdOut(u'Exporters:')
for exporter in recipe[u'exporters']:
console_out.StdOut(u' {0:s}'.format(exporter[u'name']))
exporter_objs = []
for exporter in recipe[u'exporters']:
new_args = dftw_utils.import_args_from_dict(exporter[u'args'],
vars(args))
new_args[u'processor_output'] = processor_output
exporter_class = config.Config.get_exporter(exporter[u'name'])
exporter_objs.extend(exporter_class.launch_exporter(**new_args))
# Wait for exporters to finish
exporter_output = []
for exporter in exporter_objs:
exporter.join()
exporter_output.extend(exporter.output)
console_out.StdOut(u'Recipe {0:s} executed successfully'.format(
recipe[u'name']))
def main():
"""Main function for DFTimewolf."""
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(
title='Available recipes',
description='List of currently loaded recipes',
help=
'Recipe-specific help. Run dftimewolf <RECIPE_NAME> -h for details.')
for registered_recipe in config.Config.get_registered_recipes():
recipe, recipe_args, documentation = registered_recipe
subparser = subparsers.add_parser(
recipe['name'], description='{0:s}'.format(documentation))
subparser.set_defaults(recipe=recipe)
for switch, help_text, default in recipe_args:
subparser.add_argument(switch, help=help_text, default=default)
args = parser.parse_args()
recipe = args.recipe
console_out = dftw_utils.DFTimewolfConsoleOutput(sender='DFTimewolfCli',
verbose=True)
# Thread all collectors.
console_out.StdOut('Collectors:')
for collector in recipe['collectors']:
console_out.StdOut(' {0:s}'.format(collector['name']))
collector_objects = []
for collector in recipe['collectors']:
new_args = dftw_utils.import_args_from_dict(collector['args'],
vars(args), config.Config)
collector_cls = config.Config.get_collector(collector['name'])
collector_objects.extend(collector_cls.launch_collector(**new_args))
# global_errors will contain any errors generated along the way by collectors,
# producers or exporters.
global_errors = []
# Wait for collectors to finish and collect output.
collector_output = []
for collector_obj in collector_objects:
collector_obj.join()
collector_output.extend(collector_obj.results)
if collector_obj.errors:
#TODO(tomchop): Add name attributes in module objects
error = (collector_obj.__class__.__name__,
", ".join(collector_obj.errors))
global_errors.append(error)
console_out.StdErr("ERROR:{0:s}:{1:s}\n".format(*error))
if recipe['processors']:
# Thread processors.
console_out.StdOut('Processors:')
for processor in recipe['processors']:
console_out.StdOut(' {0:s}'.format(processor['name']))
processor_objs = []
for processor in recipe['processors']:
new_args = dftw_utils.import_args_from_dict(
processor['args'], vars(args), config.Config)
new_args['collector_output'] = collector_output
processor_class = config.Config.get_processor(processor['name'])
processor_objs.extend(processor_class.launch_processor(**new_args))
# Wait for processors to finish and collect output
processor_output = []
for processor in processor_objs:
processor.join()
processor_output.extend(processor.output)
if processor.errors:
# Note: Should we fail if modules produce errors, or is warning the user
# enough?
# TODO(tomchop): Add name attributes in module objects.
error = (processor.__class__.__name__,
", ".join(processor.errors))
global_errors.append(error)
console_out.StdErr("ERROR:{0:s}:{1:s}\n".format(*error))
else:
processor_output = collector_output
# Thread all exporters.
if recipe['exporters']:
console_out.StdOut('Exporters:')
for exporter in recipe['exporters']:
console_out.StdOut(' {0:s}'.format(exporter['name']))
exporter_objs = []
for exporter in recipe['exporters']:
new_args = dftw_utils.import_args_from_dict(
exporter['args'], vars(args), config.Config)
new_args['processor_output'] = processor_output
exporter_class = config.Config.get_exporter(exporter['name'])
exporter_objs.extend(exporter_class.launch_exporter(**new_args))
# Wait for exporters to finish.
exporter_output = []
for exporter in exporter_objs:
exporter.join()
exporter_output.extend(exporter.output)
if exporter.errors:
#TODO(tomchop): Add name attributes in module objects
error = (exporter.__class__.__name__,
", ".join(exporter.errors))
global_errors.append(error)
console_out.StdErr("ERROR:{0:s}:{1:s}\n".format(*error))
else:
exporter_output = processor_output
if not global_errors:
console_out.StdOut('Recipe {0:s} executed successfully'.format(
recipe['name']))
else:
console_out.StdOut('Recipe {0:s} executed with {1:d} errors:'.format(
recipe['name'], len(global_errors)))
for error in global_errors:
console_out.StdOut(' {0:s}: {1:s}'.format(*error))