def test_message_included_in_test_case_finished_as_stacktrace(self):
with TestGtestBinaryRunner.create_harness() as harness:
gtest_output = dedent(
"""
[ RUN ] SbCryptographyTransform/Aes.SunnyDayIdentity/0
this
is
a
message
[ FAILED ] SbCryptographyTransform/Aes.SunnyDayIdentity/0
""")
exec = Mock()
exec.send_line.return_value = gtest_output
with LocalMessageQueue(harness.messagebus, [TEST_CASE_FINISHED]) as queue:
harness.extension._run(message=None, exec=exec, messagebus=harness.messagebus)
test_case_finished_message = queue.get(timeout=1)
self.assertEqual(test_case_finished_message.message_id, TEST_CASE_FINISHED)
self.assertEqual(
test_case_finished_message.data.stacktrace,
dedent(
"""\
this
is
a
message
"""))
def test_connection_reestablished(exec, messagebus):
with LocalMessageQueue(messagebus,
[SERIAL_CONNECTION_LOST, SERIAL_CONNECTED],
[SERIAL_ENDPOINT]) as queue:
exec.send_line_nowait('disconnect')
assert queue.get(timeout=2).message_id == SERIAL_CONNECTION_LOST
assert queue.get(timeout=2).message_id == SERIAL_CONNECTED
def test_running_a_binary_with_mixed_results(self):
success_patterns = ['success']
failure_patterns = ['failure']
with TestConsoleRunner.create_harness(success_patterns,
failure_patterns) as harness:
binary_output = dedent("""
success
some text
some more text
failure
""")
exec = Mock()
exec.send_line.return_value = binary_output
with LocalMessageQueue(harness.messagebus,
[TEST_CASE_FINISHED]) as queue:
harness.extension.run(message=None,
exec=exec,
messagebus=harness.messagebus)
success_message = queue.get(timeout=1)
assert success_message.data.name == 'success'
assert success_message.data.verdict == Verdict.PASSED
failure_message = queue.get(timeout=1)
assert failure_message.data.name == 'failure'
assert failure_message.data.verdict == Verdict.FAILED
def test_triggers_sut_recovery_perform_on_failed_health_check(self):
with TestHealthMonitor.create_harness() as harness:
def handler(message):
raise HealthCheckError('Nope!')
dispatcher = CallbackDispatcher(harness.messagebus, handler)
harness.messagebus.register_dispatcher(dispatcher,
[PERFORM_HEALTH_CHECK],
entities=['mysut'])
with LocalMessageQueue(harness.messagebus,
message_ids=[SUT_RECOVERY_PERFORM],
endpoint_ids=[MOCK_ENDPOINT],
entities=['mysut'
]) as sut_recovery_perform_queue:
data = Mock()
data.verdict = Verdict.FAILED
harness.trigger_event(TEST_CASE_FINISHED,
MOCK_ENDPOINT,
data=data,
entity='mysut')
harness.messagebus.wait_for_not_active()
assert not sut_recovery_perform_queue.empty()
def test_does_not_trigger_perform_health_check_on_passed_test(self):
with TestHealthMonitor.create_harness() as harness:
with LocalMessageQueue(harness.messagebus, message_ids=[PERFORM_HEALTH_CHECK],
endpoint_ids=[HEALTH_CHECK_ENDPOINT],
entities=['mysut']) as perform_health_check_queue, \
LocalMessageQueue(harness.messagebus, message_ids=[SUT_RECOVERY_PERFORM],
endpoint_ids=[MOCK_ENDPOINT],
entities=['mysut']) as sut_recovery_perform_queue:
data = Mock()
data.verdict = Verdict.PASSED
harness.trigger_event(TEST_CASE_FINISHED,
MOCK_ENDPOINT,
data=data,
entity='mysut')
harness.messagebus.wait_for_not_active()
assert perform_health_check_queue.empty()
assert sut_recovery_perform_queue.empty()
def _is_abort_message_triggered_for_verdict(verdict):
with _create_harness() as harness:
with LocalMessageQueue(harness.messagebus, [ABORT]) as q:
harness.trigger_event(TEST_CASE_FINISHED,
RUNNER_ENDPOINT,
data=TestCaseFinished(
ANY, ANY, ANY, verdict))
return not q.empty()
def _handle_start(self, message, messagebus):
with LocalMessageQueue(messagebus, [CONTINUE], [ENDPOINT]) as cont:
assert cont.get(timeout=1).message_id == CONTINUE
assert cont.get(timeout=1).message_id == CONTINUE
messagebus.trigger_event(BLOCK_MESSAGE, ENDPOINT)
assert cont.get(timeout=0)
assert cont.get(timeout=1)
def test_looper_extends_test_queue_when_repeat_set(self):
with create_harness(repeats=3) as harness:
with LocalMessageQueue(harness.messagebus, [ADD_TEST_CASES],
[SCHEDULER_ENDPOINT]) as queue:
harness.trigger_event(RUN_QUEUE_INITIALIZED,
SCHEDULER_ENDPOINT,
data=['test1', 'test2', 'test3'])
result = queue.get(timeout=1)
assert result.data == ['test1', 'test2', 'test3'] * 2
def message_queue(self,
message_ids,
endpoint_ids=None,
entities=None,
match=None,
priority=0):
with LocalMessageQueue(self.messagebus, message_ids, endpoint_ids,
entities, match, priority) as q:
yield q
def exposed_local_message_queue(self, serialized_message_ids,
serialized_endpoint_ids,
serialized_entities):
message_ids = pickle.loads(serialized_message_ids)
endpoint_ids = pickle.loads(serialized_endpoint_ids)
entities = pickle.loads(serialized_entities)
return LocalMessageQueue(self.messagebus, message_ids,
endpoint_ids, entities)
def test_triggers_test_run_started_and_finished_on_test_run(self):
with TestGtestRunner.create_harness() as harness:
events = [TEST_RUN_STARTED, TEST_RUN_FINISHED]
with LocalMessageQueue(harness.messagebus, events) as queue:
harness.messagebus.trigger_event(TEST_RUN,
RUN_COMMAND_ENDPOINT, None)
self.assertEqual(
queue.get(timeout=1).message_id, TEST_RUN_STARTED)
self.assertEqual(
queue.get(timeout=1).message_id, TEST_RUN_FINISHED)
def test_looper_refills_test_queue_if_empty_with_duration_remaining(self):
with create_harness(duration='2') as harness:
harness.trigger_event(TEST_RUN_STARTED, SCHEDULER_ENDPOINT)
harness.trigger_event(RUN_QUEUE_INITIALIZED,
SCHEDULER_ENDPOINT,
data=['test1', 'test2', 'test3'])
with LocalMessageQueue(harness.messagebus, [ADD_TEST_CASES],
[SCHEDULER_ENDPOINT]) as queue:
harness.trigger_event(RUN_QUEUE_EMPTY, SCHEDULER_ENDPOINT)
result = queue.get(timeout=1)
assert result.data == ['test1']
def test_starts_sending_perform_measurement_requests_when_the_test_run_start(
self):
with _create_harness() as harness:
with LocalMessageQueue(harness.messagebus,
message_ids=[PERFORM_MEASUREMENT],
endpoint_ids=[MONITOR_ENDPOINT],
entities=['mysut']) as queue:
initial_timer = harness.extension._timer
harness.messagebus.trigger_event(TEST_RUN_STARTED,
RUNNER_ENDPOINT)
queue.get(timeout=1)
assert harness.extension._timer is not initial_timer
def test_make_triggers_pre_make_event_with_make_arguments_as_data(self):
docker_run = Mock()
scope = Scope('scope')
with create_harness(docker_run) as harness:
application = Mock()
application.config = harness.config
application.messagebus = harness.messagebus
docker_run.docker_config = DockerConfig(harness.config)
with LocalMessageQueue(harness.messagebus, [PRE_MAKE], [MAKE_ENDPOINT]) as queue:
harness.component_factory.call(make, scope, application)
self.assertEqual(queue.get(timeout=1).data, ('arg1', '--opt2'))
def test_critical_extension_exception_sent_if_serial_connection_cant_be_established(
self):
with patch('zserial.serial.find_serial_port',
return_value=('device', False)):
with create_harness() as harness:
with LocalMessageQueue(
harness.messagebus,
[CRITICAL_EXTENSION_ERROR]) as error_queue:
with patch('zserial.serial.start_serial_connection', side_effect=SerialConnectionError),\
patch('time.sleep'):
harness.trigger_event(BEFORE_COMMAND,
APPLICATION_ENDPOINT)
assert error_queue.get(
timeout=1).message_id == CRITICAL_EXTENSION_ERROR
def test_extension_run_all_checks_does_not_trigger_recovery_if_not_configured_to_do_so(
self):
with create_harness(should_recover=False) as harness:
with LocalMessageQueue(harness.messagebus, [SUT_RECOVERY_PERFORM],
entities=['entity']) as queue:
with MockConnectionCheck(harness.messagebus, success=True, required=True, entity='entity'), \
MockConnectionCheck(harness.messagebus, success=False, required=True, entity='entity'), \
MockConnectionCheck(harness.messagebus, success=True, required=True, entity='entity'):
harness.messagebus.send_request(
POST_INITIALIZE_SUT,
RUN_COMMAND_ENDPOINT,
entity='entity').wait(timeout=1)
self.assertRaises(Empty, queue.get_nowait)
def test_extension_run_all_checks_triggers_recovery_when_one_of_multiple_required_checks_fails(
self):
with create_harness() as harness:
with LocalMessageQueue(harness.messagebus, [SUT_RECOVERY_PERFORM],
entities=['entity']) as queue:
with MockConnectionCheck(harness.messagebus, success=True, required=True, entity='entity'), \
MockConnectionCheck(harness.messagebus, success=False, required=True, entity='entity'), \
MockConnectionCheck(harness.messagebus, success=True, required=True, entity='entity'):
harness.messagebus.send_request(POST_INITIALIZE_SUT,
RUN_COMMAND_ENDPOINT,
entity='entity')
assert queue.get(
timeout=1).message_id == SUT_RECOVERY_PERFORM
def test_extension_run_all_checks_does_not_triggers_recovery_when_only_not_required_check_fails(
self):
with create_harness() as harness:
with LocalMessageQueue(harness.messagebus, [SUT_RECOVERY_PERFORM],
entities=['entity']) as queue:
with MockConnectionCheck(harness.messagebus, success=True, required=True, entity='entity'), \
MockConnectionCheck(harness.messagebus, success=False, required=False, entity='entity'), \
MockConnectionCheck(harness.messagebus, success=True, required=True, entity='entity'):
harness.messagebus.send_request(CONNECTIONCHECK_RUN_CHECKS,
CONNECTIONCHECK_ENDPOINT,
entity='entity',
data=False).wait(timeout=1)
self.assertRaises(Empty, queue.get_nowait)
def test_running_a_binary_with_no_patterns(self):
with TestConsoleRunner.create_harness() as harness:
binary_output = dedent("""
some text
some more text
""")
exec = Mock()
exec.send_line.return_value = binary_output
with LocalMessageQueue(harness.messagebus,
[TEST_CASE_FINISHED]) as queue:
harness.extension.run(message=None,
exec=exec,
messagebus=harness.messagebus)
self.assertRaises(Empty, queue.get_nowait)
def test_if_a_matchgroup_called_name_is_provided_it_is_used_as_the_test_case_name(
self):
success_patterns = [r'my long (?P<name>\w+) case name']
with TestConsoleRunner.create_harness(success_patterns) as harness:
binary_output = dedent("""
my long test case name
""")
exec = Mock()
exec.send_line.return_value = binary_output
with LocalMessageQueue(harness.messagebus,
[TEST_CASE_FINISHED]) as queue:
harness.extension.run(message=None,
exec=exec,
messagebus=harness.messagebus)
message = queue.get(timeout=1)
assert message.data.name == 'test'
def test_failed_test_translated_to_test_events(self):
with TestGtestBinaryRunner.create_harness() as harness:
gtest_output = dedent(
"""
[ RUN ] SbCryptographyTransform/Aes.SunnyDayIdentity/0
[ FAILED ] SbCryptographyTransform/Aes.SunnyDayIdentity/0
""")
exec = Mock()
exec.send_line.return_value = gtest_output
events = [TEST_CASE_STARTED, TEST_CASE_FINISHED]
with LocalMessageQueue(harness.messagebus, events) as queue:
harness.extension._run(message=None, exec=exec, messagebus=harness.messagebus)
self.assertEqual(queue.get(timeout=1).message_id, TEST_CASE_STARTED)
test_case_finished_message = queue.get(timeout=1)
self.assertEqual(test_case_finished_message.message_id, TEST_CASE_FINISHED)
self.assertEqual(test_case_finished_message.data.verdict, Verdict.FAILED)
def test_extension_run_all_checks_triggers_recovery_when_specified_and_required_check_fails(
self):
with create_harness() as harness:
with LocalMessageQueue(harness.messagebus, [SUT_RECOVERY_PERFORM],
entities=['entity']) as queue:
with MockConnectionCheck(harness.messagebus,
success=False,
required=True,
entity='entity'):
trigger_recover = True
harness.messagebus.send_request(CONNECTIONCHECK_RUN_CHECKS,
CONNECTIONCHECK_ENDPOINT,
entity='entity',
data=trigger_recover)
assert queue.get(
timeout=1).message_id == SUT_RECOVERY_PERFORM
def test_failure_is_reported_if_both_success_and_failure_patterns_match(
self):
success_patterns = ['pattern']
failure_patterns = ['pattern']
with TestConsoleRunner.create_harness(success_patterns,
failure_patterns) as harness:
binary_output = dedent("""
some text
pattern
some more text
""")
exec = Mock()
exec.send_line.return_value = binary_output
with LocalMessageQueue(harness.messagebus,
[TEST_CASE_FINISHED]) as queue:
harness.extension.run(message=None,
exec=exec,
messagebus=harness.messagebus)
message = queue.get(timeout=1)
assert message.data.name == 'pattern'
assert message.data.verdict == Verdict.FAILED
def await_sut_message(self, message_id, timeout=300):
"""
Context manager that can be used to wait for a SUT event.
The context manager yields a future that will contain the sut message.
Example of how this can be used:
.. code-block:: python
@requires(sut_events='SutEvents')
def test_case(sut_events):
with sut_events.await_sut_message(SUT_RESET_EXPECTED) as future:
do_stuff_that_triggers_reset()
do_stuff_after_reset_expected_has_been_received()
sut_reset_expected_message = future.result()
:param message_id: the ID of the message to wait for
:param timeout: the timeout
:return: future containing the message representing the sut event
"""
logger.debug(
'Await message {id} for sut {entity}'.format(id=message_id, entity=self._sut.entity))
def wait_for_message(queue):
try:
return queue.get(timeout=timeout)
except Empty:
raise SutEventTimeout(
'Timeout when waiting for message {message_id} for sut {sut}'.format(
message_id=message_id, sut=self._sut.entity))
with LocalMessageQueue(self._messagebus, [message_id],
entities=[self._sut.entity]) as queue:
future = Future()
yield future
future.run(wait_for_message, queue)
future.result(timeout=timeout)
def test_blocking_until_stop_blocking_is_sent(self):
config = ConfigManager()
config.set(BLOCKER_ENABLED, True)
with ExtensionTestHarness(Blocker,
endpoints_and_messages={
TEST_BLOCKER_ENDPOINT:
[TEST_BLOCKER_BLOCK],
APPLICATION_ENDPOINT: [AFTER_COMMAND]
},
config=config) as harness:
id_futures = harness.send_request(
START_BLOCKING_ON_MESSAGE,
BLOCKER_ENDPOINT,
data=StartBlockingInfo(message_id=TEST_BLOCKER_BLOCK,
endpoint_id=TEST_BLOCKER_ENDPOINT,
entity=None,
timeout=3))
id = id_futures.wait(timeout=1)[0].result(timeout=1)
with LocalMessageQueue(
harness.messagebus,
[BLOCKING_STARTED, BLOCKING_COMPLETED]) as queue:
thread = threading.Thread(
target=harness.send_request,
args=[TEST_BLOCKER_BLOCK, TEST_BLOCKER_ENDPOINT])
thread.start()
self.assertEqual(queue.get(timeout=1).entity, id)
self.assertTrue(thread.is_alive())
harness.send_request(STOP_BLOCKING_ON_MESSAGE,
BLOCKER_ENDPOINT,
entity=id).wait(timeout=1)
thread.join()
self.assertFalse(thread.is_alive())
self.assertEqual(queue.get(timeout=1).entity, id)
def test_ongoing_blockers_are_stopped_by_destroy(self):
config = ConfigManager()
config.set(BLOCKER_ENABLED, True)
harness = ExtensionTestHarness(Blocker,
endpoints_and_messages={
TEST_BLOCKER_ENDPOINT:
[TEST_BLOCKER_BLOCK],
APPLICATION_ENDPOINT:
[AFTER_COMMAND]
},
config=config)
try:
harness.__enter__()
with LocalMessageQueue(harness.messagebus,
[BLOCKING_COMPLETED]) as queue:
id_futures = harness.send_request(
START_BLOCKING_ON_MESSAGE,
BLOCKER_ENDPOINT,
data=StartBlockingInfo(message_id=TEST_BLOCKER_BLOCK,
endpoint_id=TEST_BLOCKER_ENDPOINT,
entity=None,
timeout=1))
id = id_futures.wait(timeout=1)[0].result(timeout=1)
thread = threading.Thread(
target=harness.send_request,
args=[TEST_BLOCKER_BLOCK, TEST_BLOCKER_ENDPOINT])
thread.start()
harness.extension.destroy()
self.assertEqual(queue.get(timeout=1).entity, id)
thread.join()
except AssertionError:
raise
except Exception:
harness.extension.destroy()
def test_running_a_binary_with_matching_failure_patterns(self):
failure_patterns = ['failure_a', 'failure_b']
with TestConsoleRunner.create_harness(
failure_patterns=failure_patterns) as harness:
binary_output = dedent("""
failure_a
some text
some more text
failure_b
""")
exec = Mock()
exec.send_line.return_value = binary_output
with LocalMessageQueue(harness.messagebus,
[TEST_CASE_FINISHED]) as queue:
harness.extension.run(message=None,
exec=exec,
messagebus=harness.messagebus)
failure_a_message = queue.get(timeout=1)
assert failure_a_message.data.name == 'failure_a'
assert failure_a_message.data.verdict == Verdict.FAILED
failure_b_message = queue.get(timeout=1)
assert failure_b_message.data.name == 'failure_b'
assert failure_b_message.data.verdict == Verdict.FAILED
def test_blocking_timeout_send_timeout_event(self):
config = ConfigManager()
config.set(BLOCKER_ENABLED, True)
with ExtensionTestHarness(Blocker,
endpoints_and_messages={
TEST_BLOCKER_ENDPOINT:
[TEST_BLOCKER_BLOCK],
APPLICATION_ENDPOINT: [AFTER_COMMAND]
},
config=config) as harness:
id_futures = harness.send_request(
START_BLOCKING_ON_MESSAGE,
BLOCKER_ENDPOINT,
data=StartBlockingInfo(message_id=TEST_BLOCKER_BLOCK,
endpoint_id=TEST_BLOCKER_ENDPOINT,
entity=None,
timeout=0))
id = id_futures.wait(timeout=1)[0].result(timeout=1)
with LocalMessageQueue(harness.messagebus,
[BLOCKING_TIMED_OUT]) as queue:
harness.send_request(TEST_BLOCKER_BLOCK, TEST_BLOCKER_ENDPOINT)
self.assertEqual(queue.get(timeout=1).entity, id)
def test_1(messagebus):
with LocalMessageQueue(messagebus, [SUT_RESET_STARTED]) as queue:
logger.info('test 1')
queue.get(timeout=5)
def test_a_generate_metrics_report_event_is_triggered_on_test_run_finished(self):
with TestTriggerMetricsReportGenerationOnTestRunFinished._create_harness() as harness:
with LocalMessageQueue(harness.messagebus, [GENERATE_METRICS_REPORT]) as queue:
harness.messagebus.trigger_event(TEST_RUN_FINISHED, RUNNER_ENDPOINT, data=None)
queue.get(timeout=1)
