def test_enable_disable(self):
instrument = _instantiate(MockInstrument)
instrument2 = _instantiate(MockInstrument2)
instrumentation.install(instrument)
instrumentation.install(instrument2)
instrumentation.disable_all()
signal.send(signal.BEFORE_WORKLOAD_EXECUTION, self, context=None)
signal.send(signal.AFTER_WORKLOAD_EXECUTION, self, context=None)
signal.send(signal.AFTER_WORKLOAD_RESULT_UPDATE, self, context=None)
assert_equal(instrument.before, 0)
assert_equal(instrument.after, 0)
assert_equal(instrument2.before, 0)
assert_equal(instrument2.after, 0)
assert_equal(instrument2.result, 0)
instrumentation.enable(instrument)
signal.send(signal.BEFORE_WORKLOAD_EXECUTION, self, context=None)
signal.send(signal.AFTER_WORKLOAD_EXECUTION, self, context=None)
signal.send(signal.AFTER_WORKLOAD_RESULT_UPDATE, self, context=None)
assert_equal(instrument.before, 1)
assert_equal(instrument.after, 1)
assert_equal(instrument2.before, 0)
assert_equal(instrument2.after, 0)
assert_equal(instrument2.result, 0)
instrumentation.enable_all()
signal.send(signal.BEFORE_WORKLOAD_EXECUTION, self, context=None)
signal.send(signal.AFTER_WORKLOAD_EXECUTION, self, context=None)
signal.send(signal.AFTER_WORKLOAD_RESULT_UPDATE, self, context=None)
assert_equal(instrument.before, 2)
assert_equal(instrument.after, 2)
assert_equal(instrument2.before, 1)
assert_equal(instrument2.after, 1)
assert_equal(instrument2.result, 1)
def test_enable_disable(self):
instrument = _instantiate(MockInstrument)
instrument2 = _instantiate(MockInstrument2)
instrumentation.install(instrument)
instrumentation.install(instrument2)
instrumentation.disable_all()
signal.send(signal.BEFORE_WORKLOAD_EXECUTION, self, context=None)
signal.send(signal.AFTER_WORKLOAD_EXECUTION, self, context=None)
signal.send(signal.AFTER_WORKLOAD_RESULT_UPDATE, self, context=None)
assert_equal(instrument.before, 0)
assert_equal(instrument.after, 0)
assert_equal(instrument2.before, 0)
assert_equal(instrument2.after, 0)
assert_equal(instrument2.result, 0)
instrumentation.enable(instrument)
signal.send(signal.BEFORE_WORKLOAD_EXECUTION, self, context=None)
signal.send(signal.AFTER_WORKLOAD_EXECUTION, self, context=None)
signal.send(signal.AFTER_WORKLOAD_RESULT_UPDATE, self, context=None)
assert_equal(instrument.before, 1)
assert_equal(instrument.after, 1)
assert_equal(instrument2.before, 0)
assert_equal(instrument2.after, 0)
assert_equal(instrument2.result, 0)
instrumentation.enable_all()
signal.send(signal.BEFORE_WORKLOAD_EXECUTION, self, context=None)
signal.send(signal.AFTER_WORKLOAD_EXECUTION, self, context=None)
signal.send(signal.AFTER_WORKLOAD_RESULT_UPDATE, self, context=None)
assert_equal(instrument.before, 2)
assert_equal(instrument.after, 2)
assert_equal(instrument2.before, 1)
assert_equal(instrument2.after, 1)
assert_equal(instrument2.result, 1)
def test_check_enabled(self):
instrument = _instantiate(MockInstrument)
instrumentation.install(instrument)
instrumentation.enable(instrument)
assert_true(instrument_is_enabled(instrument))
assert_true(instrument_is_enabled(instrument.name))
instrumentation.disable(instrument)
assert_false(instrument_is_enabled(instrument))
assert_false(instrument_is_enabled(instrument.name))
def test_check_enabled(self):
instrument = _instantiate(MockInstrument)
instrumentation.install(instrument)
instrumentation.enable(instrument)
assert_true(instrument_is_enabled(instrument))
assert_true(instrument_is_enabled(instrument.name))
instrumentation.disable(instrument)
assert_false(instrument_is_enabled(instrument))
assert_false(instrument_is_enabled(instrument.name))
def signal_check(self, expected_signals, workloads, reboot_policy="never", runner_class=BySpecRunner):
context = Mock()
context.reboot_policy = RebootPolicy(reboot_policy)
context.config.workload_specs = workloads
context.config.retry_on_status = []
instrument = _instantiate(SignalCatcher)
instrumentation.install(instrument)
runner = runner_class(Mock(), context, Mock())
runner.init_queue(context.config.workload_specs)
try:
runner.run()
finally:
instrumentation.uninstall(instrument)
assert_equal(instrument.signals_received, expected_signals)
def signal_check(self, expected_signals, workloads, reboot_policy="never", runner_class=BySpecRunner):
context = Mock()
context.reboot_policy = RebootPolicy(reboot_policy)
context.config.workload_specs = workloads
context.config.retry_on_status = []
instrument = _instantiate(SignalCatcher)
instrumentation.install(instrument)
runner = runner_class(Mock(), context, Mock())
runner.init_queue(context.config.workload_specs)
try:
runner.run()
finally:
instrumentation.uninstall(instrument)
assert_equal(instrument.signals_received, expected_signals)
def test_priority_prefix_instrument(self):
global counter
counter = 0
instrument1 = _instantiate(MockInstrument4)
instrument2 = _instantiate(MockInstrument5)
instrument3 = _instantiate(MockInstrument6)
instrumentation.install(instrument1)
instrumentation.install(instrument2)
instrumentation.install(instrument3)
signal.send(signal.BEFORE_FIRST_ITERATION_BOOT, self, context=None)
assert_equal(counter, 42)
def test_priority_prefix_instrument(self):
global counter
counter = 0
instrument1 = _instantiate(MockInstrument4)
instrument2 = _instantiate(MockInstrument5)
instrument3 = _instantiate(MockInstrument6)
instrumentation.install(instrument1)
instrumentation.install(instrument2)
instrumentation.install(instrument3)
signal.send(signal.BEFORE_FIRST_ITERATION_BOOT, self, context=None)
assert_equal(counter, 42)
def test_CTRL_C(self):
workloads = [
WorkloadRunSpec(id='1',
number_of_iterations=2,
instrumentation=['Signal Catcher']),
WorkloadRunSpec(id='2',
number_of_iterations=2,
instrumentation=['Signal Catcher']),
WorkloadRunSpec(id='3',
number_of_iterations=2,
instrumentation=['Signal Catcher']),
WorkloadRunSpec(id='4',
number_of_iterations=2,
instrumentation=['Signal Catcher']),
]
workloads[0]._workload = BadWorkload(KeyboardInterrupt, ["setup"])
workloads[1]._workload = BadWorkload(KeyboardInterrupt, ["run"])
workloads[2]._workload = BadWorkload(KeyboardInterrupt,
["update_result"])
workloads[3]._workload = BadWorkload(KeyboardInterrupt, ["teardown"])
expected_status = [IterationResult.ABORTED, IterationResult.ABORTED]
expected_signals = [
[
signal.RUN_START.name, signal.RUN_INIT.name,
signal.WORKLOAD_SPEC_START.name, signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name, signal.ITERATION_END.name,
signal.WORKLOAD_SPEC_END.name, signal.RUN_FIN.name,
signal.BEFORE_OVERALL_RESULTS_PROCESSING.name,
signal.SUCCESSFUL_OVERALL_RESULTS_PROCESSING.name,
signal.AFTER_OVERALL_RESULTS_PROCESSING.name,
signal.RUN_END.name
],
[
signal.RUN_START.name, signal.RUN_INIT.name,
signal.WORKLOAD_SPEC_START.name, signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.SUCCESSFUL_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name, signal.ITERATION_END.name,
signal.WORKLOAD_SPEC_END.name, signal.RUN_FIN.name,
signal.BEFORE_OVERALL_RESULTS_PROCESSING.name,
signal.SUCCESSFUL_OVERALL_RESULTS_PROCESSING.name,
signal.AFTER_OVERALL_RESULTS_PROCESSING.name,
signal.RUN_END.name
],
[
signal.RUN_START.name, signal.RUN_INIT.name,
signal.WORKLOAD_SPEC_START.name, signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.SUCCESSFUL_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.SUCCESSFUL_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name, signal.ITERATION_END.name,
signal.WORKLOAD_SPEC_END.name, signal.RUN_FIN.name,
signal.BEFORE_OVERALL_RESULTS_PROCESSING.name,
signal.SUCCESSFUL_OVERALL_RESULTS_PROCESSING.name,
signal.AFTER_OVERALL_RESULTS_PROCESSING.name,
signal.RUN_END.name
],
[
signal.RUN_START.name, signal.RUN_INIT.name,
signal.WORKLOAD_SPEC_START.name, signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.SUCCESSFUL_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE.name,
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name, signal.ITERATION_END.name,
signal.WORKLOAD_SPEC_END.name, signal.RUN_FIN.name,
signal.BEFORE_OVERALL_RESULTS_PROCESSING.name,
signal.SUCCESSFUL_OVERALL_RESULTS_PROCESSING.name,
signal.AFTER_OVERALL_RESULTS_PROCESSING.name,
signal.RUN_END.name
],
]
for i in xrange(0, len(workloads)):
context = Mock()
context.reboot_policy = RebootPolicy("never")
context.config.workload_specs = [workloads[i]]
runner = BySpecRunner(Mock(), context, Mock())
runner.init_queue(context.config.workload_specs)
instrument = _instantiate(SignalCatcher)
instrumentation.install(instrument)
try:
runner.run()
finally:
instrumentation.uninstall(instrument)
#Check queue was handled correctly
assert_equal(len(runner.completed_jobs), 2)
assert_equal(len(runner.job_queue), 0)
#check correct signals were sent
assert_equal(expected_signals[i], instrument.signals_received)
#Check job status'
for j in range(0, len(runner.completed_jobs)):
assert_equal(runner.completed_jobs[j].result.status,
expected_status[j])
def test_bad_workload_status(self):
workloads = [
WorkloadRunSpec(id='1',
number_of_iterations=2,
instrumentation=['Signal Catcher']),
WorkloadRunSpec(id='2',
number_of_iterations=2,
instrumentation=['Signal Catcher']),
WorkloadRunSpec(id='3',
number_of_iterations=2,
instrumentation=['Signal Catcher']),
WorkloadRunSpec(id='4',
number_of_iterations=2,
instrumentation=['Signal Catcher']),
WorkloadRunSpec(id='5',
number_of_iterations=2,
instrumentation=['Signal Catcher'])
]
workloads[0]._workload = BadWorkload(Exception, ["setup"])
workloads[1]._workload = BadWorkload(Exception, ["run"])
workloads[2]._workload = BadWorkload(Exception, ["update_result"])
workloads[3]._workload = BadWorkload(Exception, ["teardown"])
workloads[4]._workload = Mock()
context = Mock()
context.reboot_policy = RebootPolicy("never")
context.config.workload_specs = workloads
runner = BySpecRunner(Mock(), context, Mock())
runner.init_queue(context.config.workload_specs)
instrument = _instantiate(SignalCatcher)
instrumentation.install(instrument)
try:
runner.run()
finally:
instrumentation.uninstall(instrument)
#Check queue was handled correctly
assert_equal(len(runner.completed_jobs), 10)
assert_equal(len(runner.job_queue), 0)
#Check job status'
expected_status = [
IterationResult.FAILED, IterationResult.SKIPPED,
IterationResult.FAILED, IterationResult.FAILED,
IterationResult.PARTIAL, IterationResult.PARTIAL,
IterationResult.NONCRITICAL, IterationResult.NONCRITICAL,
IterationResult.OK, IterationResult.OK
]
for i in range(0, len(runner.completed_jobs)):
assert_equal(runner.completed_jobs[i].result.status,
expected_status[i])
#Check signals were sent correctly
expected_signals = [
signal.RUN_START.name,
signal.RUN_INIT.name,
signal.WORKLOAD_SPEC_START.name, #Fail Setup
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.ITERATION_END.name,
#Skipped iteration
signal.WORKLOAD_SPEC_END.name,
signal.WORKLOAD_SPEC_START.name, #Fail Run
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
#signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE.name, - not sent because run failed
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.SUCCESSFUL_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
#signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE.name, - not sent because run failed
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.SUCCESSFUL_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.WORKLOAD_SPEC_END.name,
signal.WORKLOAD_SPEC_START.name, # Fail Result Update
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.SUCCESSFUL_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.SUCCESSFUL_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.SUCCESSFUL_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.SUCCESSFUL_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.WORKLOAD_SPEC_END.name,
signal.WORKLOAD_SPEC_START.name, # Fail Teardown
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.SUCCESSFUL_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE.name,
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.SUCCESSFUL_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE.name,
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.WORKLOAD_SPEC_END.name,
signal.WORKLOAD_SPEC_START.name, #OK
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.SUCCESSFUL_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE.name,
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.SUCCESSFUL_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.SUCCESSFUL_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE.name,
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.SUCCESSFUL_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.WORKLOAD_SPEC_END.name,
signal.RUN_FIN.name,
signal.BEFORE_OVERALL_RESULTS_PROCESSING.name,
signal.SUCCESSFUL_OVERALL_RESULTS_PROCESSING.name,
signal.AFTER_OVERALL_RESULTS_PROCESSING.name,
signal.RUN_END.name
]
assert_equal(expected_signals, instrument.signals_received)
def test_duplicate_install(self):
instrument = _instantiate(MockInstrument)
instrument2 = _instantiate(MockInstrument)
instrumentation.install(instrument)
instrumentation.install(instrument2)
def install_local_instrument(self):
instrument = _instantiate(MockInstrument3)
instrumentation.install(instrument)
def test_check_installed(self):
instrumentation.install(_instantiate(MockInstrument))
assert_true(instrument_is_installed('mock'))
assert_true(instrument_is_installed(MockInstrument))
assert_false(instrument_is_installed(MockInstrument2))
def test_bad_argspec(self):
instrument = _instantiate(BadInstrument)
instrumentation.install(instrument)
def test_check_installed(self):
instrumentation.install(_instantiate(MockInstrument))
assert_true(instrument_is_installed('mock'))
assert_true(instrument_is_installed(MockInstrument))
assert_false(instrument_is_installed(MockInstrument2))
def test_bad_workload_status(self):
workloads = [
WorkloadRunSpec(id='1', number_of_iterations=2, instrumentation=['Signal Catcher']),
WorkloadRunSpec(id='2', number_of_iterations=2, instrumentation=['Signal Catcher']),
WorkloadRunSpec(id='3', number_of_iterations=2, instrumentation=['Signal Catcher']),
WorkloadRunSpec(id='4', number_of_iterations=2, instrumentation=['Signal Catcher']),
WorkloadRunSpec(id='5', number_of_iterations=2, instrumentation=['Signal Catcher'])
]
workloads[0]._workload = BadWorkload(Exception, ["setup"])
workloads[1]._workload = BadWorkload(Exception, ["run"])
workloads[2]._workload = BadWorkload(Exception, ["update_result"])
workloads[3]._workload = BadWorkload(Exception, ["teardown"])
workloads[4]._workload = Mock()
context = Mock()
context.reboot_policy = RebootPolicy("never")
context.config.workload_specs = workloads
runner = BySpecRunner(Mock(), context, Mock())
runner.init_queue(context.config.workload_specs)
instrument = _instantiate(SignalCatcher)
instrumentation.install(instrument)
try:
runner.run()
finally:
instrumentation.uninstall(instrument)
#Check queue was handled correctly
assert_equal(len(runner.completed_jobs), 10)
assert_equal(len(runner.job_queue), 0)
#Check job status'
expected_status = [
IterationResult.FAILED, IterationResult.SKIPPED,
IterationResult.FAILED, IterationResult.FAILED,
IterationResult.PARTIAL, IterationResult.PARTIAL,
IterationResult.NONCRITICAL, IterationResult.NONCRITICAL,
IterationResult.OK, IterationResult.OK
]
for i in range(0, len(runner.completed_jobs)):
assert_equal(runner.completed_jobs[i].result.status, expected_status[i])
#Check signals were sent correctly
expected_signals = [
signal.RUN_START.name,
signal.RUN_INIT.name,
signal.WORKLOAD_SPEC_START.name, #Fail Setup
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.ITERATION_END.name,
#Skipped iteration
signal.WORKLOAD_SPEC_END.name,
signal.WORKLOAD_SPEC_START.name, #Fail Run
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
#signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE.name, - not sent because run failed
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.SUCCESSFUL_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
#signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE.name, - not sent because run failed
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.SUCCESSFUL_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.WORKLOAD_SPEC_END.name,
signal.WORKLOAD_SPEC_START.name, # Fail Result Update
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.SUCCESSFUL_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.SUCCESSFUL_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.SUCCESSFUL_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.SUCCESSFUL_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.WORKLOAD_SPEC_END.name,
signal.WORKLOAD_SPEC_START.name, # Fail Teardown
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.SUCCESSFUL_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE.name,
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.SUCCESSFUL_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE.name,
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.WORKLOAD_SPEC_END.name,
signal.WORKLOAD_SPEC_START.name, #OK
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.SUCCESSFUL_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE.name,
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.SUCCESSFUL_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.SUCCESSFUL_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE.name,
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.SUCCESSFUL_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.WORKLOAD_SPEC_END.name,
signal.RUN_FIN.name,
signal.BEFORE_OVERALL_RESULTS_PROCESSING.name,
signal.SUCCESSFUL_OVERALL_RESULTS_PROCESSING.name,
signal.AFTER_OVERALL_RESULTS_PROCESSING.name,
signal.RUN_END.name
]
assert_equal(expected_signals, instrument.signals_received)
def execute(self, agenda, selectors=None): # NOQA
"""
Execute the run specified by an agenda. Optionally, selectors may be used to only
selecute a subset of the specified agenda.
Params::
:agenda: an ``Agenda`` instance to be executed.
:selectors: A dict mapping selector name to the coresponding values.
**Selectors**
Currently, the following seectors are supported:
ids
The value must be a sequence of workload specfication IDs to be executed. Note
that if sections are specified inthe agenda, the workload specifacation ID will
be a combination of the section and workload IDs.
"""
signal.connect(self._error_signalled_callback, signal.ERROR_LOGGED)
signal.connect(self._warning_signalled_callback, signal.WARNING_LOGGED)
self.logger.info('Initializing')
self.ext_loader = ExtensionLoader(packages=settings.extension_packages,
paths=settings.extension_paths)
self.logger.debug('Loading run configuration.')
self.config = RunConfiguration(self.ext_loader)
for filepath in settings.get_config_paths():
self.config.load_config(filepath)
self.config.set_agenda(agenda, selectors)
self.config.finalize()
config_outfile = os.path.join(settings.meta_directory,
'run_config.json')
with open(config_outfile, 'w') as wfh:
self.config.serialize(wfh)
self.logger.debug('Initialising device configuration.')
if not self.config.device:
raise ConfigError('Make sure a device is specified in the config.')
self.device = self.ext_loader.get_device(self.config.device,
**self.config.device_config)
self.device.validate()
self.context = ExecutionContext(self.device, self.config)
self.logger.debug('Loading resource discoverers.')
self.context.initialize()
self.context.resolver.load()
self.context.add_artifact('run_config', config_outfile, 'meta')
self.logger.debug('Installing instrumentation')
for name, params in self.config.instrumentation.iteritems():
instrument = self.ext_loader.get_instrument(
name, self.device, **params)
instrumentation.install(instrument)
instrumentation.validate()
self.logger.debug('Installing result processors')
result_manager = ResultManager()
for name, params in self.config.result_processors.iteritems():
processor = self.ext_loader.get_result_processor(name, **params)
result_manager.install(processor)
result_manager.validate()
self.logger.debug('Loading workload specs')
for workload_spec in self.config.workload_specs:
workload_spec.load(self.device, self.ext_loader)
workload_spec.workload.init_resources(self.context)
workload_spec.workload.validate()
if self.config.flashing_config:
if not self.device.flasher:
msg = 'flashing_config specified for {} device that does not support flashing.'
raise ConfigError(msg.format(self.device.name))
self.logger.debug('Flashing the device')
self.device.flasher.flash(self.device)
self.logger.info('Running workloads')
runner = self._get_runner(result_manager)
runner.init_queue(self.config.workload_specs)
runner.run()
self.execute_postamble()
def execute(self, agenda, selectors=None): # NOQA
"""
Execute the run specified by an agenda. Optionally, selectors may be used to only
selecute a subset of the specified agenda.
Params::
:agenda: an ``Agenda`` instance to be executed.
:selectors: A dict mapping selector name to the coresponding values.
**Selectors**
Currently, the following seectors are supported:
ids
The value must be a sequence of workload specfication IDs to be executed. Note
that if sections are specified inthe agenda, the workload specifacation ID will
be a combination of the section and workload IDs.
"""
signal.connect(self._error_signalled_callback, signal.ERROR_LOGGED)
signal.connect(self._warning_signalled_callback, signal.WARNING_LOGGED)
self.logger.info('Initializing')
self.ext_loader = ExtensionLoader(packages=settings.extension_packages,
paths=settings.extension_paths)
self.logger.debug('Loading run configuration.')
self.config = RunConfiguration(self.ext_loader)
for filepath in settings.get_config_paths():
self.config.load_config(filepath)
self.config.set_agenda(agenda, selectors)
self.config.finalize()
config_outfile = os.path.join(settings.meta_directory, 'run_config.json')
with open(config_outfile, 'w') as wfh:
self.config.serialize(wfh)
self.logger.debug('Initialising device configuration.')
if not self.config.device:
raise ConfigError('Make sure a device is specified in the config.')
self.device = self.ext_loader.get_device(self.config.device, **self.config.device_config)
self.device.validate()
self.context = ExecutionContext(self.device, self.config)
self.logger.debug('Loading resource discoverers.')
self.context.initialize()
self.context.resolver.load()
self.context.add_artifact('run_config', config_outfile, 'meta')
self.logger.debug('Installing instrumentation')
for name, params in self.config.instrumentation.iteritems():
instrument = self.ext_loader.get_instrument(name, self.device, **params)
instrumentation.install(instrument)
instrumentation.validate()
self.logger.debug('Installing result processors')
result_manager = ResultManager()
for name, params in self.config.result_processors.iteritems():
processor = self.ext_loader.get_result_processor(name, **params)
result_manager.install(processor)
result_manager.validate()
self.logger.debug('Loading workload specs')
for workload_spec in self.config.workload_specs:
workload_spec.load(self.device, self.ext_loader)
workload_spec.workload.init_resources(self.context)
workload_spec.workload.validate()
if self.config.flashing_config:
if not self.device.flasher:
msg = 'flashing_config specified for {} device that does not support flashing.'
raise ConfigError(msg.format(self.device.name))
self.logger.debug('Flashing the device')
self.device.flasher.flash(self.device)
self.logger.info('Running workloads')
runner = self._get_runner(result_manager)
runner.init_queue(self.config.workload_specs)
runner.run()
self.execute_postamble()
def test_duplicate_install(self):
instrument = _instantiate(MockInstrument)
instrument2 = _instantiate(MockInstrument)
instrumentation.install(instrument)
instrumentation.install(instrument2)
def install_local_instrument(self):
instrument = _instantiate(MockInstrument3)
instrumentation.install(instrument)
def test_CTRL_C(self):
workloads = [
WorkloadRunSpec(id='1', number_of_iterations=2, instrumentation=['Signal Catcher']),
WorkloadRunSpec(id='2', number_of_iterations=2, instrumentation=['Signal Catcher']),
WorkloadRunSpec(id='3', number_of_iterations=2, instrumentation=['Signal Catcher']),
WorkloadRunSpec(id='4', number_of_iterations=2, instrumentation=['Signal Catcher']),
]
workloads[0]._workload = BadWorkload(KeyboardInterrupt, ["setup"])
workloads[1]._workload = BadWorkload(KeyboardInterrupt, ["run"])
workloads[2]._workload = BadWorkload(KeyboardInterrupt, ["update_result"])
workloads[3]._workload = BadWorkload(KeyboardInterrupt, ["teardown"])
expected_status = [IterationResult.ABORTED, IterationResult.ABORTED]
expected_signals = [
[
signal.RUN_START.name,
signal.RUN_INIT.name,
signal.WORKLOAD_SPEC_START.name,
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.ITERATION_END.name,
signal.WORKLOAD_SPEC_END.name,
signal.RUN_FIN.name,
signal.BEFORE_OVERALL_RESULTS_PROCESSING.name,
signal.SUCCESSFUL_OVERALL_RESULTS_PROCESSING.name,
signal.AFTER_OVERALL_RESULTS_PROCESSING.name,
signal.RUN_END.name
],
[
signal.RUN_START.name,
signal.RUN_INIT.name,
signal.WORKLOAD_SPEC_START.name,
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.SUCCESSFUL_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.WORKLOAD_SPEC_END.name,
signal.RUN_FIN.name,
signal.BEFORE_OVERALL_RESULTS_PROCESSING.name,
signal.SUCCESSFUL_OVERALL_RESULTS_PROCESSING.name,
signal.AFTER_OVERALL_RESULTS_PROCESSING.name,
signal.RUN_END.name
],
[
signal.RUN_START.name,
signal.RUN_INIT.name,
signal.WORKLOAD_SPEC_START.name,
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.SUCCESSFUL_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.SUCCESSFUL_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.WORKLOAD_SPEC_END.name,
signal.RUN_FIN.name,
signal.BEFORE_OVERALL_RESULTS_PROCESSING.name,
signal.SUCCESSFUL_OVERALL_RESULTS_PROCESSING.name,
signal.AFTER_OVERALL_RESULTS_PROCESSING.name,
signal.RUN_END.name
],
[
signal.RUN_START.name,
signal.RUN_INIT.name,
signal.WORKLOAD_SPEC_START.name,
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.SUCCESSFUL_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE.name,
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.WORKLOAD_SPEC_END.name,
signal.RUN_FIN.name,
signal.BEFORE_OVERALL_RESULTS_PROCESSING.name,
signal.SUCCESSFUL_OVERALL_RESULTS_PROCESSING.name,
signal.AFTER_OVERALL_RESULTS_PROCESSING.name,
signal.RUN_END.name
],
]
for i in xrange(0, len(workloads)):
context = Mock()
context.reboot_policy = RebootPolicy("never")
context.config.workload_specs = [workloads[i]]
runner = BySpecRunner(Mock(), context, Mock())
runner.init_queue(context.config.workload_specs)
instrument = _instantiate(SignalCatcher)
instrumentation.install(instrument)
try:
runner.run()
finally:
instrumentation.uninstall(instrument)
#Check queue was handled correctly
assert_equal(len(runner.completed_jobs), 2)
assert_equal(len(runner.job_queue), 0)
#check correct signals were sent
assert_equal(expected_signals[i], instrument.signals_received)
#Check job status'
for j in range(0, len(runner.completed_jobs)):
assert_equal(runner.completed_jobs[j].result.status, expected_status[j])
def test_bad_argspec(self):
instrument = _instantiate(BadInstrument)
instrumentation.install(instrument)
