def process_output(self, context):
if not context.tm.is_responsive:
self.logger.info('Target unresponsive; not processing job output.')
return
self.logger.info('Processing output for job {}'.format(self))
with indentcontext():
if self.status != Status.FAILED:
with signal.wrap('WORKLOAD_RESULT_EXTRACTION', self, context):
self.workload.extract_results(context)
context.extract_results()
with signal.wrap('WORKLOAD_OUTPUT_UPDATE', self, context):
self.workload.update_output(context)
def process_output(self, context):
if not context.tm.is_responsive:
self.logger.info('Target unresponsive; not processing job output.')
return
self.logger.info('Processing output for job {}'.format(self))
with indentcontext():
if self.status != Status.FAILED:
with signal.wrap('WORKLOAD_RESULT_EXTRACTION', self, context):
self.workload.extract_results(context)
context.extract_results()
with signal.wrap('WORKLOAD_OUTPUT_UPDATE', self, context):
self.workload.update_output(context)
def test_wrap_propagate(self):
d = {'before': False, 'after': False, 'success': False}
def before():
d['before'] = True
def after():
d['after'] = True
def success():
d['success'] = True
signal.connect(before, signal.BEFORE_WORKLOAD_SETUP)
signal.connect(after, signal.AFTER_WORKLOAD_SETUP)
signal.connect(success, signal.SUCCESSFUL_WORKLOAD_SETUP)
caught = False
try:
with signal.wrap('WORKLOAD_SETUP'):
raise RuntimeError()
except RuntimeError:
caught = True
assert_true(d['before'])
assert_true(d['after'])
assert_true(caught)
assert_false(d['success'])
def run(self):
try:
self.initialize_run()
self.send(signal.RUN_INITIALIZED)
with signal.wrap('JOB_QUEUE_EXECUTION', self, self.context):
while self.context.job_queue:
if self.context.run_interrupted:
raise KeyboardInterrupt()
self.run_next_job(self.context)
except KeyboardInterrupt as e:
log.log_error(e, self.logger)
self.logger.info('Skipping remaining jobs.')
self.context.skip_remaining_jobs()
except Exception as e:
message = e.args[0] if e.args else str(e)
log.log_error(e, self.logger)
self.logger.error(
'Skipping remaining jobs due to "{}".'.format(message))
self.context.skip_remaining_jobs()
raise e
finally:
self.finalize_run()
self.send(signal.RUN_FINALIZED)
def teardown(self, context):
if not context.tm.is_responsive:
self.logger.info('Target unresponsive; not tearing down.')
return
self.logger.info('Tearing down job {}'.format(self))
with indentcontext():
with signal.wrap('WORKLOAD_TEARDOWN', self, context):
self.workload.teardown(context)
def initialize(self, context):
self.logger.info('Initializing job {}'.format(self))
with indentcontext():
with signal.wrap('WORKLOAD_INITIALIZED', self, context):
self.workload.logger.context = context
self.workload.initialize(context)
self.set_status(Status.PENDING)
self._has_been_initialized = True
def teardown(self, context):
if not context.tm.is_responsive:
self.logger.info('Target unresponsive; not tearing down.')
return
self.logger.info('Tearing down job {}'.format(self))
with indentcontext():
with signal.wrap('WORKLOAD_TEARDOWN', self, context):
self.workload.teardown(context)
def finalize(self):
if not self.target:
return
if self.disconnect or isinstance(self.target.platform,
Gem5SimulationPlatform):
self.logger.info('Disconnecting from the device')
with signal.wrap('TARGET_DISCONNECT'):
self.target.disconnect()
def run(self, context):
self.logger.info('Running job {}'.format(self))
with indentcontext():
with signal.wrap('WORKLOAD_EXECUTION', self, context):
start_time = datetime.utcnow()
try:
self.workload.run(context)
finally:
self.run_time = datetime.utcnow() - start_time
def initialize(self, context):
self.logger.info('Initializing job {}'.format(self))
with indentcontext():
with signal.wrap('WORKLOAD_INITIALIZED', self, context):
self.workload.logger.context = context
self.workload.initialize(context)
self.set_status(Status.PENDING)
self._has_been_initialized = True
context.update_job_state(self)
def run(self, context):
self.logger.info('Running job {}'.format(self))
with indentcontext():
with signal.wrap('WORKLOAD_EXECUTION', self, context):
start_time = datetime.utcnow()
try:
self.workload.run(context)
finally:
self.run_time = datetime.utcnow() - start_time
def finalize(self, context):
if not self._has_been_initialized:
return
if not context.tm.is_responsive:
self.logger.info('Target unresponsive; not finalizing.')
return
self.logger.info('Finalizing job {} '.format(self))
with indentcontext():
with signal.wrap('WORKLOAD_FINALIZED', self, context):
self.workload.finalize(context)
def finalize(self, context):
if not self._has_been_initialized:
return
if not context.tm.is_responsive:
self.logger.info('Target unresponsive; not finalizing.')
return
self.logger.info('Finalizing job {} '.format(self))
with indentcontext():
with signal.wrap('WORKLOAD_FINALIZED', self, context):
self.workload.finalize(context)
def finalize_run(self):
self.logger.info('Run completed')
with log.indentcontext():
for job in self.context.completed_jobs:
job.finalize(self.context)
self.logger.info('Finalizing run')
self.context.end_run()
self.pm.enable_all()
with signal.wrap('RUN_OUTPUT_PROCESSED', self):
self.pm.process_run_output(self.context)
self.pm.export_run_output(self.context)
self.pm.finalize(self.context)
signal.disconnect(self._error_signalled_callback, signal.ERROR_LOGGED)
signal.disconnect(self._warning_signalled_callback, signal.WARNING_LOGGED)
def finalize_run(self):
self.logger.info('Run completed')
with log.indentcontext():
for job in self.context.completed_jobs:
job.finalize(self.context)
self.logger.info('Finalizing run')
self.context.end_run()
self.pm.enable_all()
with signal.wrap('RUN_OUTPUT_PROCESSED', self):
self.pm.process_run_output(self.context)
self.pm.export_run_output(self.context)
self.pm.finalize(self.context)
signal.disconnect(self._error_signalled_callback, signal.ERROR_LOGGED)
signal.disconnect(self._warning_signalled_callback, signal.WARNING_LOGGED)
def _init_target(self):
tdesc = get_target_description(self.target_name)
extra_plat_params = {}
if tdesc.platform is Gem5SimulationPlatform:
extra_plat_params['host_output_dir'] = self.outdir
self.logger.debug('Creating {} target'.format(self.target_name))
self.target = instantiate_target(tdesc, self.parameters, connect=False,
extra_platform_params=extra_plat_params)
self.is_responsive = True
with signal.wrap('TARGET_CONNECT'):
self.target.connect()
self.logger.info('Setting up target')
self.target.setup()
self.assistant = instantiate_assistant(tdesc, self.parameters, self.target)
def test_wrap_propagate(self):
d = {'before': False, 'after': False, 'success': False}
def before():
d['before'] = True
def after():
d['after'] = True
def success():
d['success'] = True
signal.connect(before, signal.BEFORE_WORKLOAD_SETUP)
signal.connect(after, signal.AFTER_WORKLOAD_SETUP)
signal.connect(success, signal.SUCCESSFUL_WORKLOAD_SETUP)
caught = False
try:
with signal.wrap('WORKLOAD_SETUP'):
raise RuntimeError()
except RuntimeError:
caught=True
assert_true(d['before'])
assert_true(d['after'])
assert_true(caught)
assert_false(d['success'])
def run_next_job(self, context):
job = context.start_job()
self.logger.info('Running job {}'.format(job.id))
try:
log.indent()
if self.context.reboot_policy.reboot_on_each_job:
self.logger.info('Rebooting on new job.')
self.context.tm.reboot(context)
elif self.context.reboot_policy.reboot_on_each_spec and context.spec_changed:
self.logger.info('Rebooting on new spec.')
self.context.tm.reboot(context)
with signal.wrap('JOB', self, context):
context.tm.start()
self.do_run_job(job, context)
context.set_job_status(job, Status.OK)
except (Exception, KeyboardInterrupt) as e: # pylint: disable=broad-except
log.log_error(e, self.logger)
if isinstance(e, KeyboardInterrupt):
context.run_interrupted = True
context.set_job_status(job, Status.ABORTED)
raise e
else:
context.set_job_status(job, Status.FAILED)
if isinstance(e, TargetNotRespondingError):
raise e
elif isinstance(e, TargetError):
context.tm.verify_target_responsive(context)
finally:
self.logger.info('Completing job {}'.format(job.id))
self.send(signal.JOB_COMPLETED)
context.tm.stop()
context.end_job()
log.dedent()
self.check_job(job)
def run_next_job(self, context):
job = context.start_job()
self.logger.info('Running job {}'.format(job.id))
try:
log.indent()
if self.context.reboot_policy.reboot_on_each_job:
self.logger.info('Rebooting on new job.')
self.context.tm.reboot(context)
elif self.context.reboot_policy.reboot_on_each_spec and context.spec_changed:
self.logger.info('Rebooting on new spec.')
self.context.tm.reboot(context)
with signal.wrap('JOB', self, context):
context.tm.start()
self.do_run_job(job, context)
job.set_status(Status.OK)
except (Exception, KeyboardInterrupt) as e: # pylint: disable=broad-except
log.log_error(e, self.logger)
if isinstance(e, KeyboardInterrupt):
context.run_interrupted = True
job.set_status(Status.ABORTED)
raise e
else:
job.set_status(Status.FAILED)
if isinstance(e, TargetNotRespondingError):
raise e
elif isinstance(e, TargetError):
context.tm.verify_target_responsive(context)
finally:
self.logger.info('Completing job {}'.format(job.id))
self.send(signal.JOB_COMPLETED)
context.tm.stop()
context.end_job()
log.dedent()
self.check_job(job)
def run(self):
try:
self.initialize_run()
self.send(signal.RUN_INITIALIZED)
with signal.wrap('JOB_QUEUE_EXECUTION', self, self.context):
while self.context.job_queue:
if self.context.run_interrupted:
raise KeyboardInterrupt()
self.run_next_job(self.context)
except KeyboardInterrupt as e:
log.log_error(e, self.logger)
self.logger.info('Skipping remaining jobs.')
self.context.skip_remaining_jobs()
except Exception as e:
message = e.args[0] if e.args else str(e)
log.log_error(e, self.logger)
self.logger.error('Skipping remaining jobs due to "{}".'.format(message))
self.context.skip_remaining_jobs()
raise e
finally:
self.finalize_run()
self.send(signal.RUN_FINALIZED)
def reboot(self, context, hard=False):
with signal.wrap('REBOOT', self, context):
self.target.reboot(hard)
def do_run_job(self, job, context):
# pylint: disable=too-many-branches,too-many-statements
rc = self.context.cm.run_config
if job.workload.phones_home and not rc.allow_phone_home:
self.logger.warning(
'Skipping job {} ({}) due to allow_phone_home=False'.format(
job.id, job.workload.name))
self.context.skip_job(job)
return
context.set_job_status(job, Status.RUNNING)
self.send(signal.JOB_STARTED)
job.configure_augmentations(context, self.pm)
with signal.wrap('JOB_TARGET_CONFIG', self, context):
job.configure_target(context)
try:
job.setup(context)
except Exception as e:
context.set_job_status(job, Status.FAILED)
log.log_error(e, self.logger)
if isinstance(e, (TargetError, TimeoutError)):
context.tm.verify_target_responsive(context)
self.context.record_ui_state('setup-error')
raise e
try:
try:
job.run(context)
except KeyboardInterrupt:
context.run_interrupted = True
context.set_job_status(job, Status.ABORTED)
raise
except Exception as e:
context.set_job_status(job, Status.FAILED)
log.log_error(e, self.logger)
if isinstance(e, (TargetError, TimeoutError)):
context.tm.verify_target_responsive(context)
self.context.record_ui_state('run-error')
raise e
finally:
try:
with signal.wrap('JOB_OUTPUT_PROCESSED', self, context):
job.process_output(context)
self.pm.process_job_output(context)
self.pm.export_job_output(context)
except Exception as e:
context.set_job_status(job, Status.PARTIAL)
if isinstance(e, (TargetError, TimeoutError)):
context.tm.verify_target_responsive(context)
self.context.record_ui_state('output-error')
raise
except KeyboardInterrupt:
context.run_interrupted = True
context.set_status(Status.ABORTED)
raise
finally:
# If setup was successfully completed, teardown must
# run even if the job failed
job.teardown(context)
def do_run_job(self, job, context):
# pylint: disable=too-many-branches,too-many-statements
rc = self.context.cm.run_config
if job.workload.phones_home and not rc.allow_phone_home:
self.logger.warning('Skipping job {} ({}) due to allow_phone_home=False'
.format(job.id, job.workload.name))
self.context.skip_job(job)
return
job.set_status(Status.RUNNING)
self.send(signal.JOB_STARTED)
job.configure_augmentations(context, self.pm)
with signal.wrap('JOB_TARGET_CONFIG', self, context):
job.configure_target(context)
try:
job.setup(context)
except Exception as e:
job.set_status(Status.FAILED)
log.log_error(e, self.logger)
if isinstance(e, (TargetError, TimeoutError)):
context.tm.verify_target_responsive(context)
self.context.record_ui_state('setup-error')
raise e
try:
try:
job.run(context)
except KeyboardInterrupt:
context.run_interrupted = True
job.set_status(Status.ABORTED)
raise
except Exception as e:
job.set_status(Status.FAILED)
log.log_error(e, self.logger)
if isinstance(e, (TargetError, TimeoutError)):
context.tm.verify_target_responsive(context)
self.context.record_ui_state('run-error')
raise e
finally:
try:
with signal.wrap('JOB_OUTPUT_PROCESSED', self, context):
job.process_output(context)
self.pm.process_job_output(context)
self.pm.export_job_output(context)
except Exception as e:
job.set_status(Status.PARTIAL)
if isinstance(e, (TargetError, TimeoutError)):
context.tm.verify_target_responsive(context)
self.context.record_ui_state('output-error')
raise
except KeyboardInterrupt:
context.run_interrupted = True
job.set_status(Status.ABORTED)
raise
finally:
# If setup was successfully completed, teardown must
# run even if the job failed
job.teardown(context)
def setup(self, context):
self.logger.info('Setting up job {}'.format(self))
with indentcontext():
with signal.wrap('WORKLOAD_SETUP', self, context):
self.workload.setup(context)
def setup(self, context):
self.logger.info('Setting up job {}'.format(self))
with indentcontext():
with signal.wrap('WORKLOAD_SETUP', self, context):
self.workload.setup(context)