def initialize_jobs(self):
new_queue = []
failed_ids = []
for job in self.job_queue:
if job.id in failed_ids:
# Don't try to initialize a job if another job with the same ID
# (i.e. same job spec) has failed - we can assume it will fail
# too.
self.skip_job(job)
continue
try:
job.initialize(self)
except WorkloadError as e:
self.set_job_status(job, Status.FAILED, write=False)
log.log_error(e, self.logger)
failed_ids.append(job.id)
if self.cm.run_config.bail_on_init_failure:
raise
else:
new_queue.append(job)
self.job_queue = new_queue
self.write_state()
def run(self):
try:
self.initialize_run()
self.send(signal.RUN_INITIALIZED)
with signal.wrap('JOB_QUEUE_EXECUTION', self, self.context):
while self.context.job_queue:
if self.context.run_interrupted:
raise KeyboardInterrupt()
self.run_next_job(self.context)
except KeyboardInterrupt as e:
log.log_error(e, self.logger)
self.logger.info('Skipping remaining jobs.')
self.context.skip_remaining_jobs()
except Exception as e:
message = e.args[0] if e.args else str(e)
log.log_error(e, self.logger)
self.logger.error(
'Skipping remaining jobs due to "{}".'.format(message))
self.context.skip_remaining_jobs()
raise e
finally:
self.finalize_run()
self.send(signal.RUN_FINALIZED)
def do_for_each_proc(self, method_name, message, *args):
with indentcontext():
for proc in self.processors:
if proc.is_enabled:
proc_func = getattr(proc, method_name, None)
if proc_func is None:
continue
try:
self.logger.info(message.format(proc.name))
proc_func(*args)
except Exception as e: # pylint: disable=broad-except
if isinstance(e, KeyboardInterrupt):
raise
log_error(e, self.logger)
def do_for_each_proc(self, method_name, message, *args):
with indentcontext():
for proc in self.processors:
if proc.is_enabled:
proc_func = getattr(proc, method_name, None)
if proc_func is None:
continue
try:
self.logger.info(message.format(proc.name))
proc_func(*args)
except Exception as e: # pylint: disable=broad-except
if isinstance(e, KeyboardInterrupt):
raise
log_error(e, self.logger)
def execute(self, config_manager, output):
"""
Execute the run specified by an agenda. Optionally, selectors may be
used to only execute a subset of the specified agenda.
Params::
:state: a ``ConfigManager`` containing processed configuration
:output: an initialized ``RunOutput`` that will be used to
store the results.
"""
signal.connect(self._error_signalled_callback, signal.ERROR_LOGGED)
signal.connect(self._warning_signalled_callback, signal.WARNING_LOGGED)
self.logger.info('Initializing run')
self.logger.debug('Finalizing run configuration.')
config = config_manager.finalize()
output.write_config(config)
self.target_manager = TargetManager(config.run_config.device,
config.run_config.device_config,
output.basepath)
self.logger.info('Initializing execution context')
context = ExecutionContext(config_manager, self.target_manager, output)
try:
self.do_execute(context)
except KeyboardInterrupt as e:
context.run_output.status = Status.ABORTED
log.log_error(e, self.logger)
context.write_output()
raise
except Exception as e:
context.run_output.status = Status.FAILED
log.log_error(e, self.logger)
context.write_output()
raise
finally:
context.finalize()
self.execute_postamble(context, output)
signal.send(signal.RUN_COMPLETED, self, context)
def execute(self, config_manager, output):
"""
Execute the run specified by an agenda. Optionally, selectors may be
used to only selecute a subset of the specified agenda.
Params::
:state: a ``ConfigManager`` containing processed configuration
:output: an initialized ``RunOutput`` that will be used to
store the results.
"""
signal.connect(self._error_signalled_callback, signal.ERROR_LOGGED)
signal.connect(self._warning_signalled_callback, signal.WARNING_LOGGED)
self.logger.info('Initializing run')
self.logger.debug('Finalizing run configuration.')
config = config_manager.finalize()
output.write_config(config)
self.target_manager = TargetManager(config.run_config.device,
config.run_config.device_config,
output.basepath)
self.logger.info('Initializing execution context')
context = ExecutionContext(config_manager, self.target_manager, output)
try:
self.do_execute(context)
except KeyboardInterrupt as e:
context.run_output.status = 'ABORTED'
log.log_error(e, self.logger)
context.write_output()
raise
except Exception as e:
context.run_output.status = 'FAILED'
log.log_error(e, self.logger)
context.write_output()
raise
finally:
context.finalize()
self.execute_postamble(context, output)
signal.send(signal.RUN_COMPLETED, self, context)
def run_next_job(self, context):
job = context.start_job()
self.logger.info('Running job {}'.format(job.id))
try:
log.indent()
if self.context.reboot_policy.reboot_on_each_job:
self.logger.info('Rebooting on new job.')
self.context.tm.reboot(context)
elif self.context.reboot_policy.reboot_on_each_spec and context.spec_changed:
self.logger.info('Rebooting on new spec.')
self.context.tm.reboot(context)
with signal.wrap('JOB', self, context):
context.tm.start()
self.do_run_job(job, context)
context.set_job_status(job, Status.OK)
except (Exception, KeyboardInterrupt) as e: # pylint: disable=broad-except
log.log_error(e, self.logger)
if isinstance(e, KeyboardInterrupt):
context.run_interrupted = True
context.set_job_status(job, Status.ABORTED)
raise e
else:
context.set_job_status(job, Status.FAILED)
if isinstance(e, TargetNotRespondingError):
raise e
elif isinstance(e, TargetError):
context.tm.verify_target_responsive(context)
finally:
self.logger.info('Completing job {}'.format(job.id))
self.send(signal.JOB_COMPLETED)
context.tm.stop()
context.end_job()
log.dedent()
self.check_job(job)
def run_next_job(self, context):
job = context.start_job()
self.logger.info('Running job {}'.format(job.id))
try:
log.indent()
if self.context.reboot_policy.reboot_on_each_job:
self.logger.info('Rebooting on new job.')
self.context.tm.reboot(context)
elif self.context.reboot_policy.reboot_on_each_spec and context.spec_changed:
self.logger.info('Rebooting on new spec.')
self.context.tm.reboot(context)
with signal.wrap('JOB', self, context):
context.tm.start()
self.do_run_job(job, context)
job.set_status(Status.OK)
except (Exception, KeyboardInterrupt) as e: # pylint: disable=broad-except
log.log_error(e, self.logger)
if isinstance(e, KeyboardInterrupt):
context.run_interrupted = True
job.set_status(Status.ABORTED)
raise e
else:
job.set_status(Status.FAILED)
if isinstance(e, TargetNotRespondingError):
raise e
elif isinstance(e, TargetError):
context.tm.verify_target_responsive(context)
finally:
self.logger.info('Completing job {}'.format(job.id))
self.send(signal.JOB_COMPLETED)
context.tm.stop()
context.end_job()
log.dedent()
self.check_job(job)
def __call__(self, context):
if self.instrument.is_enabled:
try:
if not context.tm.is_responsive and not self.is_hostside:
logger.debug("Target unresponsive; skipping callback {}".format(self.callback))
return
self.callback(context)
except (KeyboardInterrupt, TargetNotRespondingError, TimeoutError): # pylint: disable=W0703
raise
except Exception as e: # pylint: disable=W0703
logger.error('Error in instrument {}'.format(self.instrument.name))
global failures_detected # pylint: disable=W0603
failures_detected = True
log_error(e, logger)
context.add_event(e.args[0] if e.args else str(e))
if isinstance(e, WorkloadError):
context.set_status('FAILED')
elif isinstance(e, (TargetError, TimeoutError)):
context.tm.verify_target_responsive(context)
else:
if context.current_job:
context.set_status('PARTIAL')
else:
raise
def run(self):
try:
self.initialize_run()
self.send(signal.RUN_INITIALIZED)
with signal.wrap('JOB_QUEUE_EXECUTION', self, self.context):
while self.context.job_queue:
if self.context.run_interrupted:
raise KeyboardInterrupt()
self.run_next_job(self.context)
except KeyboardInterrupt as e:
log.log_error(e, self.logger)
self.logger.info('Skipping remaining jobs.')
self.context.skip_remaining_jobs()
except Exception as e:
message = e.args[0] if e.args else str(e)
log.log_error(e, self.logger)
self.logger.error('Skipping remaining jobs due to "{}".'.format(message))
self.context.skip_remaining_jobs()
raise e
finally:
self.finalize_run()
self.send(signal.RUN_FINALIZED)
def initialize_jobs(self):
new_queue = []
failed_ids = []
for job in self.job_queue:
if job.id in failed_ids:
# Don't try to initialize a job if another job with the same ID
# (i.e. same job spec) has failed - we can assume it will fail
# too.
self.skip_job(job)
continue
try:
job.initialize(self)
except WorkloadError as e:
job.set_status(Status.FAILED)
log.log_error(e, self.logger)
failed_ids.append(job.id)
if self.cm.run_config.bail_on_init_failure:
raise
else:
new_queue.append(job)
self.job_queue = new_queue
def do_run_job(self, job, context):
# pylint: disable=too-many-branches,too-many-statements
rc = self.context.cm.run_config
if job.workload.phones_home and not rc.allow_phone_home:
self.logger.warning(
'Skipping job {} ({}) due to allow_phone_home=False'.format(
job.id, job.workload.name))
self.context.skip_job(job)
return
context.set_job_status(job, Status.RUNNING)
self.send(signal.JOB_STARTED)
job.configure_augmentations(context, self.pm)
with signal.wrap('JOB_TARGET_CONFIG', self, context):
job.configure_target(context)
try:
job.setup(context)
except Exception as e:
context.set_job_status(job, Status.FAILED)
log.log_error(e, self.logger)
if isinstance(e, (TargetError, TimeoutError)):
context.tm.verify_target_responsive(context)
self.context.record_ui_state('setup-error')
raise e
try:
try:
job.run(context)
except KeyboardInterrupt:
context.run_interrupted = True
context.set_job_status(job, Status.ABORTED)
raise
except Exception as e:
context.set_job_status(job, Status.FAILED)
log.log_error(e, self.logger)
if isinstance(e, (TargetError, TimeoutError)):
context.tm.verify_target_responsive(context)
self.context.record_ui_state('run-error')
raise e
finally:
try:
with signal.wrap('JOB_OUTPUT_PROCESSED', self, context):
job.process_output(context)
self.pm.process_job_output(context)
self.pm.export_job_output(context)
except Exception as e:
context.set_job_status(job, Status.PARTIAL)
if isinstance(e, (TargetError, TimeoutError)):
context.tm.verify_target_responsive(context)
self.context.record_ui_state('output-error')
raise
except KeyboardInterrupt:
context.run_interrupted = True
context.set_status(Status.ABORTED)
raise
finally:
# If setup was successfully completed, teardown must
# run even if the job failed
job.teardown(context)
def main():
if not os.path.exists(settings.user_directory):
init_user_directory()
if not os.path.exists(os.path.join(settings.user_directory,
'config.yaml')):
init_config()
try:
description = (
"Execute automated workloads on a remote device and process "
"the resulting output.\n\nUse \"wa <subcommand> -h\" to see "
"help for individual subcommands.")
parser = argparse.ArgumentParser(
description=format_body(description, 80),
prog='wa',
formatter_class=argparse.RawDescriptionHelpFormatter,
)
init_argument_parser(parser)
# load_commands will trigger plugin enumeration, and we want logging
# to be enabled for that, which requires the verbosity setting; however
# full argument parsing cannot be completed until the commands are loaded; so
# parse just the base args for now so we can get verbosity.
argv = split_joined_options(sys.argv[1:])
# 'Parse_known_args' automatically displays the default help and exits
# if '-h' or '--help' is detected, we want our custom help messages so
# ensure these are never passed as parameters.
filtered_argv = list(argv)
if '-h' in filtered_argv:
filtered_argv.remove('-h')
elif '--help' in filtered_argv:
filtered_argv.remove('--help')
args, _ = parser.parse_known_args(filtered_argv)
settings.set("verbosity", args.verbose)
log.init(settings.verbosity)
logger.debug('Version: {}'.format(get_wa_version_with_commit()))
logger.debug('devlib version: {}'.format(devlib.__full_version__))
logger.debug('Command Line: {}'.format(' '.join(sys.argv)))
check_devlib_version()
# each command will add its own subparser
subparsers = parser.add_subparsers(dest='command')
subparsers.required = True
commands = load_commands(subparsers)
args = parser.parse_args(argv)
config = ConfigManager()
config.load_config_file(settings.user_config_file)
for config_file in args.config:
if not os.path.exists(config_file):
raise ConfigError(
"Config file {} not found".format(config_file))
config.load_config_file(config_file)
command = commands[args.command]
sys.exit(command.execute(config, args))
except KeyboardInterrupt as e:
log.log_error(e, logger)
sys.exit(3)
except Exception as e: # pylint: disable=broad-except
log.log_error(e, logger)
sys.exit(2)
def do_run_job(self, job, context):
# pylint: disable=too-many-branches,too-many-statements
rc = self.context.cm.run_config
if job.workload.phones_home and not rc.allow_phone_home:
self.logger.warning('Skipping job {} ({}) due to allow_phone_home=False'
.format(job.id, job.workload.name))
self.context.skip_job(job)
return
job.set_status(Status.RUNNING)
self.send(signal.JOB_STARTED)
job.configure_augmentations(context, self.pm)
with signal.wrap('JOB_TARGET_CONFIG', self, context):
job.configure_target(context)
try:
job.setup(context)
except Exception as e:
job.set_status(Status.FAILED)
log.log_error(e, self.logger)
if isinstance(e, (TargetError, TimeoutError)):
context.tm.verify_target_responsive(context)
self.context.record_ui_state('setup-error')
raise e
try:
try:
job.run(context)
except KeyboardInterrupt:
context.run_interrupted = True
job.set_status(Status.ABORTED)
raise
except Exception as e:
job.set_status(Status.FAILED)
log.log_error(e, self.logger)
if isinstance(e, (TargetError, TimeoutError)):
context.tm.verify_target_responsive(context)
self.context.record_ui_state('run-error')
raise e
finally:
try:
with signal.wrap('JOB_OUTPUT_PROCESSED', self, context):
job.process_output(context)
self.pm.process_job_output(context)
self.pm.export_job_output(context)
except Exception as e:
job.set_status(Status.PARTIAL)
if isinstance(e, (TargetError, TimeoutError)):
context.tm.verify_target_responsive(context)
self.context.record_ui_state('output-error')
raise
except KeyboardInterrupt:
context.run_interrupted = True
job.set_status(Status.ABORTED)
raise
finally:
# If setup was successfully completed, teardown must
# run even if the job failed
job.teardown(context)
def main():
if not os.path.exists(settings.user_directory):
init_user_directory()
if not os.path.exists(os.path.join(settings.user_directory, 'config.yaml')):
init_config()
try:
description = ("Execute automated workloads on a remote device and process "
"the resulting output.\n\nUse \"wa <subcommand> -h\" to see "
"help for individual subcommands.")
parser = argparse.ArgumentParser(description=format_body(description, 80),
prog='wa',
formatter_class=argparse.RawDescriptionHelpFormatter,
)
init_argument_parser(parser)
# load_commands will trigger plugin enumeration, and we want logging
# to be enabled for that, which requires the verbosity setting; however
# full argument parsing cannot be completed until the commands are loaded; so
# parse just the base args for now so we can get verbosity.
argv = split_joined_options(sys.argv[1:])
# 'Parse_known_args' automatically displays the default help and exits
# if '-h' or '--help' is detected, we want our custom help messages so
# ensure these are never passed as parameters.
filtered_argv = list(argv)
if '-h' in filtered_argv:
filtered_argv.remove('-h')
elif '--help' in filtered_argv:
filtered_argv.remove('--help')
args, _ = parser.parse_known_args(filtered_argv)
settings.set("verbosity", args.verbose)
log.init(settings.verbosity)
logger.debug('Version: {}'.format(get_wa_version_with_commit()))
logger.debug('devlib version: {}'.format(devlib.__full_version__))
logger.debug('Command Line: {}'.format(' '.join(sys.argv)))
# each command will add its own subparser
subparsers = parser.add_subparsers(dest='command')
subparsers.required = True
commands = load_commands(subparsers)
args = parser.parse_args(argv)
config = ConfigManager()
config.load_config_file(settings.user_config_file)
for config_file in args.config:
if not os.path.exists(config_file):
raise ConfigError("Config file {} not found".format(config_file))
config.load_config_file(config_file)
command = commands[args.command]
sys.exit(command.execute(config, args))
except KeyboardInterrupt as e:
log.log_error(e, logger)
sys.exit(3)
except Exception as e: # pylint: disable=broad-except
log.log_error(e, logger)
sys.exit(2)
