def bad_device(self, method):
workloads = [WorkloadRunSpec(id='1', number_of_iterations=1, instrumentation=[])]
workloads[0]._workload = Mock()
context = Mock()
context.reboot_policy = RebootPolicy("never")
context.config.workload_specs = workloads
runner = BySpecRunner(BadDevice(method), context, Mock())
runner.init_queue(context.config.workload_specs)
runner.run()
def signal_check(self, expected_signals, workloads, reboot_policy="never", runner_class=BySpecRunner):
context = Mock()
context.reboot_policy = RebootPolicy(reboot_policy)
context.config.workload_specs = workloads
context.config.retry_on_status = []
instrument = _instantiate(SignalCatcher)
instrumentation.install(instrument)
runner = runner_class(Mock(), context, Mock())
runner.init_queue(context.config.workload_specs)
try:
runner.run()
finally:
instrumentation.uninstall(instrument)
assert_equal(instrument.signals_received, expected_signals)
def test_CTRL_C(self):
workloads = [
WorkloadRunSpec(id='1',
number_of_iterations=2,
instrumentation=['Signal Catcher']),
WorkloadRunSpec(id='2',
number_of_iterations=2,
instrumentation=['Signal Catcher']),
WorkloadRunSpec(id='3',
number_of_iterations=2,
instrumentation=['Signal Catcher']),
WorkloadRunSpec(id='4',
number_of_iterations=2,
instrumentation=['Signal Catcher']),
]
workloads[0]._workload = BadWorkload(KeyboardInterrupt, ["setup"])
workloads[1]._workload = BadWorkload(KeyboardInterrupt, ["run"])
workloads[2]._workload = BadWorkload(KeyboardInterrupt,
["update_result"])
workloads[3]._workload = BadWorkload(KeyboardInterrupt, ["teardown"])
expected_status = [IterationResult.ABORTED, IterationResult.ABORTED]
expected_signals = [
[
signal.RUN_START.name, signal.RUN_INIT.name,
signal.WORKLOAD_SPEC_START.name, signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name, signal.ITERATION_END.name,
signal.WORKLOAD_SPEC_END.name, signal.RUN_FIN.name,
signal.BEFORE_OVERALL_RESULTS_PROCESSING.name,
signal.SUCCESSFUL_OVERALL_RESULTS_PROCESSING.name,
signal.AFTER_OVERALL_RESULTS_PROCESSING.name,
signal.RUN_END.name
],
[
signal.RUN_START.name, signal.RUN_INIT.name,
signal.WORKLOAD_SPEC_START.name, signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.SUCCESSFUL_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name, signal.ITERATION_END.name,
signal.WORKLOAD_SPEC_END.name, signal.RUN_FIN.name,
signal.BEFORE_OVERALL_RESULTS_PROCESSING.name,
signal.SUCCESSFUL_OVERALL_RESULTS_PROCESSING.name,
signal.AFTER_OVERALL_RESULTS_PROCESSING.name,
signal.RUN_END.name
],
[
signal.RUN_START.name, signal.RUN_INIT.name,
signal.WORKLOAD_SPEC_START.name, signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.SUCCESSFUL_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.SUCCESSFUL_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name, signal.ITERATION_END.name,
signal.WORKLOAD_SPEC_END.name, signal.RUN_FIN.name,
signal.BEFORE_OVERALL_RESULTS_PROCESSING.name,
signal.SUCCESSFUL_OVERALL_RESULTS_PROCESSING.name,
signal.AFTER_OVERALL_RESULTS_PROCESSING.name,
signal.RUN_END.name
],
[
signal.RUN_START.name, signal.RUN_INIT.name,
signal.WORKLOAD_SPEC_START.name, signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.SUCCESSFUL_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE.name,
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name, signal.ITERATION_END.name,
signal.WORKLOAD_SPEC_END.name, signal.RUN_FIN.name,
signal.BEFORE_OVERALL_RESULTS_PROCESSING.name,
signal.SUCCESSFUL_OVERALL_RESULTS_PROCESSING.name,
signal.AFTER_OVERALL_RESULTS_PROCESSING.name,
signal.RUN_END.name
],
]
for i in xrange(0, len(workloads)):
context = Mock()
context.reboot_policy = RebootPolicy("never")
context.config.workload_specs = [workloads[i]]
runner = BySpecRunner(Mock(), context, Mock())
runner.init_queue(context.config.workload_specs)
instrument = _instantiate(SignalCatcher)
instrumentation.install(instrument)
try:
runner.run()
finally:
instrumentation.uninstall(instrument)
#Check queue was handled correctly
assert_equal(len(runner.completed_jobs), 2)
assert_equal(len(runner.job_queue), 0)
#check correct signals were sent
assert_equal(expected_signals[i], instrument.signals_received)
#Check job status'
for j in range(0, len(runner.completed_jobs)):
assert_equal(runner.completed_jobs[j].result.status,
expected_status[j])
def test_bad_workload_status(self):
workloads = [
WorkloadRunSpec(id='1',
number_of_iterations=2,
instrumentation=['Signal Catcher']),
WorkloadRunSpec(id='2',
number_of_iterations=2,
instrumentation=['Signal Catcher']),
WorkloadRunSpec(id='3',
number_of_iterations=2,
instrumentation=['Signal Catcher']),
WorkloadRunSpec(id='4',
number_of_iterations=2,
instrumentation=['Signal Catcher']),
WorkloadRunSpec(id='5',
number_of_iterations=2,
instrumentation=['Signal Catcher'])
]
workloads[0]._workload = BadWorkload(Exception, ["setup"])
workloads[1]._workload = BadWorkload(Exception, ["run"])
workloads[2]._workload = BadWorkload(Exception, ["update_result"])
workloads[3]._workload = BadWorkload(Exception, ["teardown"])
workloads[4]._workload = Mock()
context = Mock()
context.reboot_policy = RebootPolicy("never")
context.config.workload_specs = workloads
runner = BySpecRunner(Mock(), context, Mock())
runner.init_queue(context.config.workload_specs)
instrument = _instantiate(SignalCatcher)
instrumentation.install(instrument)
try:
runner.run()
finally:
instrumentation.uninstall(instrument)
#Check queue was handled correctly
assert_equal(len(runner.completed_jobs), 10)
assert_equal(len(runner.job_queue), 0)
#Check job status'
expected_status = [
IterationResult.FAILED, IterationResult.SKIPPED,
IterationResult.FAILED, IterationResult.FAILED,
IterationResult.PARTIAL, IterationResult.PARTIAL,
IterationResult.NONCRITICAL, IterationResult.NONCRITICAL,
IterationResult.OK, IterationResult.OK
]
for i in range(0, len(runner.completed_jobs)):
assert_equal(runner.completed_jobs[i].result.status,
expected_status[i])
#Check signals were sent correctly
expected_signals = [
signal.RUN_START.name,
signal.RUN_INIT.name,
signal.WORKLOAD_SPEC_START.name, #Fail Setup
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.ITERATION_END.name,
#Skipped iteration
signal.WORKLOAD_SPEC_END.name,
signal.WORKLOAD_SPEC_START.name, #Fail Run
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
#signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE.name, - not sent because run failed
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.SUCCESSFUL_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
#signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE.name, - not sent because run failed
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.SUCCESSFUL_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.WORKLOAD_SPEC_END.name,
signal.WORKLOAD_SPEC_START.name, # Fail Result Update
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.SUCCESSFUL_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.SUCCESSFUL_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.SUCCESSFUL_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.SUCCESSFUL_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.WORKLOAD_SPEC_END.name,
signal.WORKLOAD_SPEC_START.name, # Fail Teardown
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.SUCCESSFUL_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE.name,
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.SUCCESSFUL_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE.name,
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.WORKLOAD_SPEC_END.name,
signal.WORKLOAD_SPEC_START.name, #OK
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.SUCCESSFUL_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE.name,
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.SUCCESSFUL_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.ITERATION_START.name,
signal.BEFORE_WORKLOAD_SETUP.name,
signal.SUCCESSFUL_WORKLOAD_SETUP.name,
signal.AFTER_WORKLOAD_SETUP.name,
signal.BEFORE_WORKLOAD_EXECUTION.name,
signal.SUCCESSFUL_WORKLOAD_EXECUTION.name,
signal.AFTER_WORKLOAD_EXECUTION.name,
signal.BEFORE_WORKLOAD_RESULT_UPDATE.name,
signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE.name,
signal.AFTER_WORKLOAD_RESULT_UPDATE.name,
signal.BEFORE_WORKLOAD_TEARDOWN.name,
signal.SUCCESSFUL_WORKLOAD_TEARDOWN.name,
signal.AFTER_WORKLOAD_TEARDOWN.name,
signal.ITERATION_END.name,
signal.WORKLOAD_SPEC_END.name,
signal.RUN_FIN.name,
signal.BEFORE_OVERALL_RESULTS_PROCESSING.name,
signal.SUCCESSFUL_OVERALL_RESULTS_PROCESSING.name,
signal.AFTER_OVERALL_RESULTS_PROCESSING.name,
signal.RUN_END.name
]
assert_equal(expected_signals, instrument.signals_received)