def test_list_by_kind(self):
loader = ExtensionLoader(paths=[
EXTDIR,
], load_defaults=False)
exts = loader.list_devices()
assert_equal(len(exts), 1)
assert_equal(exts[0].name, 'test-device')
def test_clear_and_reload(self):
loader = ExtensionLoader()
assert_greater(len(loader.list_devices()), 1)
loader.clear()
loader.update(paths=[EXTDIR, ])
devices = loader.list_devices()
assert_equal(len(devices), 1)
assert_equal(devices[0].name, 'test-device')
assert_equal(len(loader.list_extensions()), 1)
def test_clear_and_reload(self):
loader = ExtensionLoader()
assert_greater(len(loader.list_devices()), 1)
loader.clear()
loader.update(paths=[EXTDIR, ])
devices = loader.list_devices()
assert_equal(len(devices), 1)
assert_equal(devices[0].name, 'test-device')
assert_equal(len(loader.list_extensions()), 1)
def initialize(self, context):
"""
Initialization that is performed at the begining of the run (after the device has
been connecte).
"""
loader = ExtensionLoader()
for module_spec in self.dynamic_modules:
module = self._load_module(loader, module_spec)
if not hasattr(module, 'probe'):
message = 'Module {} does not have "probe" attribute; cannot be loaded dynamically'
raise ValueError(message.format(module.name))
if module.probe(self):
self.logger.debug('Installing module "{}"'.format(module.name))
self._install_module(module)
else:
self.logger.debug('Module "{}" is not supported by the device'.format(module.name))
def execute(self, agenda, selectors=None): # NOQA
"""
Execute the run specified by an agenda. Optionally, selectors may be used to only
selecute a subset of the specified agenda.
Params::
:agenda: an ``Agenda`` instance to be executed.
:selectors: A dict mapping selector name to the coresponding values.
**Selectors**
Currently, the following seectors are supported:
ids
The value must be a sequence of workload specfication IDs to be executed. Note
that if sections are specified inthe agenda, the workload specifacation ID will
be a combination of the section and workload IDs.
"""
signal.connect(self._error_signalled_callback, signal.ERROR_LOGGED)
signal.connect(self._warning_signalled_callback, signal.WARNING_LOGGED)
self.logger.info('Initializing')
self.ext_loader = ExtensionLoader(packages=settings.extension_packages,
paths=settings.extension_paths)
self.logger.debug('Loading run configuration.')
self.config = RunConfiguration(self.ext_loader)
for filepath in settings.get_config_paths():
self.config.load_config(filepath)
self.config.set_agenda(agenda, selectors)
self.config.finalize()
config_outfile = os.path.join(settings.meta_directory,
'run_config.json')
with open(config_outfile, 'w') as wfh:
self.config.serialize(wfh)
self.logger.debug('Initialising device configuration.')
if not self.config.device:
raise ConfigError('Make sure a device is specified in the config.')
self.device = self.ext_loader.get_device(self.config.device,
**self.config.device_config)
self.device.validate()
self.context = ExecutionContext(self.device, self.config)
self.logger.debug('Loading resource discoverers.')
self.context.initialize()
self.context.resolver.load()
self.context.add_artifact('run_config', config_outfile, 'meta')
self.logger.debug('Installing instrumentation')
for name, params in self.config.instrumentation.iteritems():
instrument = self.ext_loader.get_instrument(
name, self.device, **params)
instrumentation.install(instrument)
instrumentation.validate()
self.logger.debug('Installing result processors')
result_manager = ResultManager()
for name, params in self.config.result_processors.iteritems():
processor = self.ext_loader.get_result_processor(name, **params)
result_manager.install(processor)
result_manager.validate()
self.logger.debug('Loading workload specs')
for workload_spec in self.config.workload_specs:
workload_spec.load(self.device, self.ext_loader)
workload_spec.workload.init_resources(self.context)
workload_spec.workload.validate()
if self.config.flashing_config:
if not self.device.flasher:
msg = 'flashing_config specified for {} device that does not support flashing.'
raise ConfigError(msg.format(self.device.name))
self.logger.debug('Flashing the device')
self.device.flasher.flash(self.device)
self.logger.info('Running workloads')
runner = self._get_runner(result_manager)
runner.init_queue(self.config.workload_specs)
runner.run()
self.execute_postamble()
class Executor(object):
"""
The ``Executor``'s job is to set up the execution context and pass to a ``Runner``
along with a loaded run specification. Once the ``Runner`` has done its thing,
the ``Executor`` performs some final reporint before returning.
The initial context set up involves combining configuration from various sources,
loading of requided workloads, loading and installation of instruments and result
processors, etc. Static validation of the combined configuration is also performed.
"""
# pylint: disable=R0915
def __init__(self):
self.logger = logging.getLogger('Executor')
self.error_logged = False
self.warning_logged = False
self.config = None
self.ext_loader = None
self.device = None
self.context = None
def execute(self, agenda, selectors=None): # NOQA
"""
Execute the run specified by an agenda. Optionally, selectors may be used to only
selecute a subset of the specified agenda.
Params::
:agenda: an ``Agenda`` instance to be executed.
:selectors: A dict mapping selector name to the coresponding values.
**Selectors**
Currently, the following seectors are supported:
ids
The value must be a sequence of workload specfication IDs to be executed. Note
that if sections are specified inthe agenda, the workload specifacation ID will
be a combination of the section and workload IDs.
"""
signal.connect(self._error_signalled_callback, signal.ERROR_LOGGED)
signal.connect(self._warning_signalled_callback, signal.WARNING_LOGGED)
self.logger.info('Initializing')
self.ext_loader = ExtensionLoader(packages=settings.extension_packages,
paths=settings.extension_paths)
self.logger.debug('Loading run configuration.')
self.config = RunConfiguration(self.ext_loader)
for filepath in settings.get_config_paths():
self.config.load_config(filepath)
self.config.set_agenda(agenda, selectors)
self.config.finalize()
config_outfile = os.path.join(settings.meta_directory,
'run_config.json')
with open(config_outfile, 'w') as wfh:
self.config.serialize(wfh)
self.logger.debug('Initialising device configuration.')
if not self.config.device:
raise ConfigError('Make sure a device is specified in the config.')
self.device = self.ext_loader.get_device(self.config.device,
**self.config.device_config)
self.device.validate()
self.context = ExecutionContext(self.device, self.config)
self.logger.debug('Loading resource discoverers.')
self.context.initialize()
self.context.resolver.load()
self.context.add_artifact('run_config', config_outfile, 'meta')
self.logger.debug('Installing instrumentation')
for name, params in self.config.instrumentation.iteritems():
instrument = self.ext_loader.get_instrument(
name, self.device, **params)
instrumentation.install(instrument)
instrumentation.validate()
self.logger.debug('Installing result processors')
result_manager = ResultManager()
for name, params in self.config.result_processors.iteritems():
processor = self.ext_loader.get_result_processor(name, **params)
result_manager.install(processor)
result_manager.validate()
self.logger.debug('Loading workload specs')
for workload_spec in self.config.workload_specs:
workload_spec.load(self.device, self.ext_loader)
workload_spec.workload.init_resources(self.context)
workload_spec.workload.validate()
if self.config.flashing_config:
if not self.device.flasher:
msg = 'flashing_config specified for {} device that does not support flashing.'
raise ConfigError(msg.format(self.device.name))
self.logger.debug('Flashing the device')
self.device.flasher.flash(self.device)
self.logger.info('Running workloads')
runner = self._get_runner(result_manager)
runner.init_queue(self.config.workload_specs)
runner.run()
self.execute_postamble()
def execute_postamble(self):
"""
This happens after the run has completed. The overall results of the run are
summarised to the user.
"""
result = self.context.run_result
counter = Counter()
for ir in result.iteration_results:
counter[ir.status] += 1
self.logger.info('Done.')
self.logger.info('Run duration: {}'.format(
format_duration(self.context.run_info.duration)))
status_summary = 'Ran a total of {} iterations: '.format(
sum(self.context.job_iteration_counts.values()))
parts = []
for status in IterationResult.values:
if status in counter:
parts.append('{} {}'.format(counter[status], status))
self.logger.info(status_summary + ', '.join(parts))
self.logger.info('Results can be found in {}'.format(
settings.output_directory))
if self.error_logged:
self.logger.warn('There were errors during execution.')
self.logger.warn('Please see {}'.format(settings.log_file))
elif self.warning_logged:
self.logger.warn('There were warnings during execution.')
self.logger.warn('Please see {}'.format(settings.log_file))
def _get_runner(self, result_manager):
if not self.config.execution_order or self.config.execution_order == 'by_iteration':
if self.config.reboot_policy == 'each_spec':
self.logger.info(
'each_spec reboot policy with the default by_iteration execution order is '
'equivalent to each_iteration policy.')
runnercls = ByIterationRunner
elif self.config.execution_order in ['classic', 'by_spec']:
runnercls = BySpecRunner
elif self.config.execution_order == 'by_section':
runnercls = BySectionRunner
elif self.config.execution_order == 'random':
runnercls = RandomRunner
else:
raise ConfigError('Unexpected execution order: {}'.format(
self.config.execution_order))
return runnercls(self.device, self.context, result_manager)
def _error_signalled_callback(self):
self.error_logged = True
signal.disconnect(self._error_signalled_callback, signal.ERROR_LOGGED)
def _warning_signalled_callback(self):
self.warning_logged = True
signal.disconnect(self._warning_signalled_callback,
signal.WARNING_LOGGED)
def execute(self, agenda, selectors=None): # NOQA
"""
Execute the run specified by an agenda. Optionally, selectors may be used to only
selecute a subset of the specified agenda.
Params::
:agenda: an ``Agenda`` instance to be executed.
:selectors: A dict mapping selector name to the coresponding values.
**Selectors**
Currently, the following seectors are supported:
ids
The value must be a sequence of workload specfication IDs to be executed. Note
that if sections are specified inthe agenda, the workload specifacation ID will
be a combination of the section and workload IDs.
"""
signal.connect(self._error_signalled_callback, signal.ERROR_LOGGED)
signal.connect(self._warning_signalled_callback, signal.WARNING_LOGGED)
self.logger.info('Initializing')
self.ext_loader = ExtensionLoader(packages=settings.extension_packages,
paths=settings.extension_paths)
self.logger.debug('Loading run configuration.')
self.config = RunConfiguration(self.ext_loader)
for filepath in settings.get_config_paths():
self.config.load_config(filepath)
self.config.set_agenda(agenda, selectors)
self.config.finalize()
config_outfile = os.path.join(settings.meta_directory, 'run_config.json')
with open(config_outfile, 'w') as wfh:
self.config.serialize(wfh)
self.logger.debug('Initialising device configuration.')
if not self.config.device:
raise ConfigError('Make sure a device is specified in the config.')
self.device = self.ext_loader.get_device(self.config.device, **self.config.device_config)
self.device.validate()
self.context = ExecutionContext(self.device, self.config)
self.logger.debug('Loading resource discoverers.')
self.context.initialize()
self.context.resolver.load()
self.context.add_artifact('run_config', config_outfile, 'meta')
self.logger.debug('Installing instrumentation')
for name, params in self.config.instrumentation.iteritems():
instrument = self.ext_loader.get_instrument(name, self.device, **params)
instrumentation.install(instrument)
instrumentation.validate()
self.logger.debug('Installing result processors')
result_manager = ResultManager()
for name, params in self.config.result_processors.iteritems():
processor = self.ext_loader.get_result_processor(name, **params)
result_manager.install(processor)
result_manager.validate()
self.logger.debug('Loading workload specs')
for workload_spec in self.config.workload_specs:
workload_spec.load(self.device, self.ext_loader)
workload_spec.workload.init_resources(self.context)
workload_spec.workload.validate()
if self.config.flashing_config:
if not self.device.flasher:
msg = 'flashing_config specified for {} device that does not support flashing.'
raise ConfigError(msg.format(self.device.name))
self.logger.debug('Flashing the device')
self.device.flasher.flash(self.device)
self.logger.info('Running workloads')
runner = self._get_runner(result_manager)
runner.init_queue(self.config.workload_specs)
runner.run()
self.execute_postamble()
class Executor(object):
"""
The ``Executor``'s job is to set up the execution context and pass to a ``Runner``
along with a loaded run specification. Once the ``Runner`` has done its thing,
the ``Executor`` performs some final reporint before returning.
The initial context set up involves combining configuration from various sources,
loading of requided workloads, loading and installation of instruments and result
processors, etc. Static validation of the combined configuration is also performed.
"""
# pylint: disable=R0915
def __init__(self):
self.logger = logging.getLogger('Executor')
self.error_logged = False
self.warning_logged = False
self.config = None
self.ext_loader = None
self.device = None
self.context = None
def execute(self, agenda, selectors=None): # NOQA
"""
Execute the run specified by an agenda. Optionally, selectors may be used to only
selecute a subset of the specified agenda.
Params::
:agenda: an ``Agenda`` instance to be executed.
:selectors: A dict mapping selector name to the coresponding values.
**Selectors**
Currently, the following seectors are supported:
ids
The value must be a sequence of workload specfication IDs to be executed. Note
that if sections are specified inthe agenda, the workload specifacation ID will
be a combination of the section and workload IDs.
"""
signal.connect(self._error_signalled_callback, signal.ERROR_LOGGED)
signal.connect(self._warning_signalled_callback, signal.WARNING_LOGGED)
self.logger.info('Initializing')
self.ext_loader = ExtensionLoader(packages=settings.extension_packages,
paths=settings.extension_paths)
self.logger.debug('Loading run configuration.')
self.config = RunConfiguration(self.ext_loader)
for filepath in settings.get_config_paths():
self.config.load_config(filepath)
self.config.set_agenda(agenda, selectors)
self.config.finalize()
config_outfile = os.path.join(settings.meta_directory, 'run_config.json')
with open(config_outfile, 'w') as wfh:
self.config.serialize(wfh)
self.logger.debug('Initialising device configuration.')
if not self.config.device:
raise ConfigError('Make sure a device is specified in the config.')
self.device = self.ext_loader.get_device(self.config.device, **self.config.device_config)
self.device.validate()
self.context = ExecutionContext(self.device, self.config)
self.logger.debug('Loading resource discoverers.')
self.context.initialize()
self.context.resolver.load()
self.context.add_artifact('run_config', config_outfile, 'meta')
self.logger.debug('Installing instrumentation')
for name, params in self.config.instrumentation.iteritems():
instrument = self.ext_loader.get_instrument(name, self.device, **params)
instrumentation.install(instrument)
instrumentation.validate()
self.logger.debug('Installing result processors')
result_manager = ResultManager()
for name, params in self.config.result_processors.iteritems():
processor = self.ext_loader.get_result_processor(name, **params)
result_manager.install(processor)
result_manager.validate()
self.logger.debug('Loading workload specs')
for workload_spec in self.config.workload_specs:
workload_spec.load(self.device, self.ext_loader)
workload_spec.workload.init_resources(self.context)
workload_spec.workload.validate()
if self.config.flashing_config:
if not self.device.flasher:
msg = 'flashing_config specified for {} device that does not support flashing.'
raise ConfigError(msg.format(self.device.name))
self.logger.debug('Flashing the device')
self.device.flasher.flash(self.device)
self.logger.info('Running workloads')
runner = self._get_runner(result_manager)
runner.init_queue(self.config.workload_specs)
runner.run()
self.execute_postamble()
def execute_postamble(self):
"""
This happens after the run has completed. The overall results of the run are
summarised to the user.
"""
result = self.context.run_result
counter = Counter()
for ir in result.iteration_results:
counter[ir.status] += 1
self.logger.info('Done.')
self.logger.info('Run duration: {}'.format(format_duration(self.context.run_info.duration)))
status_summary = 'Ran a total of {} iterations: '.format(sum(self.context.job_iteration_counts.values()))
parts = []
for status in IterationResult.values:
if status in counter:
parts.append('{} {}'.format(counter[status], status))
self.logger.info(status_summary + ', '.join(parts))
self.logger.info('Results can be found in {}'.format(settings.output_directory))
if self.error_logged:
self.logger.warn('There were errors during execution.')
self.logger.warn('Please see {}'.format(settings.log_file))
elif self.warning_logged:
self.logger.warn('There were warnings during execution.')
self.logger.warn('Please see {}'.format(settings.log_file))
def _get_runner(self, result_manager):
if not self.config.execution_order or self.config.execution_order == 'by_iteration':
if self.config.reboot_policy == 'each_spec':
self.logger.info('each_spec reboot policy with the default by_iteration execution order is '
'equivalent to each_iteration policy.')
runnercls = ByIterationRunner
elif self.config.execution_order in ['classic', 'by_spec']:
runnercls = BySpecRunner
elif self.config.execution_order == 'by_section':
runnercls = BySectionRunner
elif self.config.execution_order == 'random':
runnercls = RandomRunner
else:
raise ConfigError('Unexpected execution order: {}'.format(self.config.execution_order))
return runnercls(self.device, self.context, result_manager)
def _error_signalled_callback(self):
self.error_logged = True
signal.disconnect(self._error_signalled_callback, signal.ERROR_LOGGED)
def _warning_signalled_callback(self):
self.warning_logged = True
signal.disconnect(self._warning_signalled_callback, signal.WARNING_LOGGED)
def load_commands(subparsers):
ext_loader = ExtensionLoader(paths=settings.extension_paths)
for command in ext_loader.list_commands():
settings.commands[command.name] = ext_loader.get_command(command.name, subparsers=subparsers)
def test_list_by_kind(self):
loader = ExtensionLoader(paths=[EXTDIR, ], load_defaults=False)
exts = loader.list_devices()
assert_equal(len(exts), 1)
assert_equal(exts[0].name, 'test-device')
def test_load_device(self):
loader = ExtensionLoader(paths=[EXTDIR, ], load_defaults=False)
device = loader.get_device('test-device')
assert_equal(device.name, 'test-device')
def test_load_device(self):
loader = ExtensionLoader(paths=[EXTDIR, ], load_defaults=False)
device = loader.get_device('test-device')
assert_equal(device.name, 'test-device')