def test_message_included_in_test_case_finished_as_stacktrace(self): with TestGtestBinaryRunner.create_harness() as harness: gtest_output = dedent( """ [ RUN ] SbCryptographyTransform/Aes.SunnyDayIdentity/0 this is a message [ FAILED ] SbCryptographyTransform/Aes.SunnyDayIdentity/0 """) exec = Mock() exec.send_line.return_value = gtest_output with LocalMessageQueue(harness.messagebus, [TEST_CASE_FINISHED]) as queue: harness.extension._run(message=None, exec=exec, messagebus=harness.messagebus) test_case_finished_message = queue.get(timeout=1) self.assertEqual(test_case_finished_message.message_id, TEST_CASE_FINISHED) self.assertEqual( test_case_finished_message.data.stacktrace, dedent( """\ this is a message """))
def test_connection_reestablished(exec, messagebus): with LocalMessageQueue(messagebus, [SERIAL_CONNECTION_LOST, SERIAL_CONNECTED], [SERIAL_ENDPOINT]) as queue: exec.send_line_nowait('disconnect') assert queue.get(timeout=2).message_id == SERIAL_CONNECTION_LOST assert queue.get(timeout=2).message_id == SERIAL_CONNECTED
def test_running_a_binary_with_mixed_results(self): success_patterns = ['success'] failure_patterns = ['failure'] with TestConsoleRunner.create_harness(success_patterns, failure_patterns) as harness: binary_output = dedent(""" success some text some more text failure """) exec = Mock() exec.send_line.return_value = binary_output with LocalMessageQueue(harness.messagebus, [TEST_CASE_FINISHED]) as queue: harness.extension.run(message=None, exec=exec, messagebus=harness.messagebus) success_message = queue.get(timeout=1) assert success_message.data.name == 'success' assert success_message.data.verdict == Verdict.PASSED failure_message = queue.get(timeout=1) assert failure_message.data.name == 'failure' assert failure_message.data.verdict == Verdict.FAILED
def test_triggers_sut_recovery_perform_on_failed_health_check(self): with TestHealthMonitor.create_harness() as harness: def handler(message): raise HealthCheckError('Nope!') dispatcher = CallbackDispatcher(harness.messagebus, handler) harness.messagebus.register_dispatcher(dispatcher, [PERFORM_HEALTH_CHECK], entities=['mysut']) with LocalMessageQueue(harness.messagebus, message_ids=[SUT_RECOVERY_PERFORM], endpoint_ids=[MOCK_ENDPOINT], entities=['mysut' ]) as sut_recovery_perform_queue: data = Mock() data.verdict = Verdict.FAILED harness.trigger_event(TEST_CASE_FINISHED, MOCK_ENDPOINT, data=data, entity='mysut') harness.messagebus.wait_for_not_active() assert not sut_recovery_perform_queue.empty()
def test_does_not_trigger_perform_health_check_on_passed_test(self): with TestHealthMonitor.create_harness() as harness: with LocalMessageQueue(harness.messagebus, message_ids=[PERFORM_HEALTH_CHECK], endpoint_ids=[HEALTH_CHECK_ENDPOINT], entities=['mysut']) as perform_health_check_queue, \ LocalMessageQueue(harness.messagebus, message_ids=[SUT_RECOVERY_PERFORM], endpoint_ids=[MOCK_ENDPOINT], entities=['mysut']) as sut_recovery_perform_queue: data = Mock() data.verdict = Verdict.PASSED harness.trigger_event(TEST_CASE_FINISHED, MOCK_ENDPOINT, data=data, entity='mysut') harness.messagebus.wait_for_not_active() assert perform_health_check_queue.empty() assert sut_recovery_perform_queue.empty()
def _is_abort_message_triggered_for_verdict(verdict): with _create_harness() as harness: with LocalMessageQueue(harness.messagebus, [ABORT]) as q: harness.trigger_event(TEST_CASE_FINISHED, RUNNER_ENDPOINT, data=TestCaseFinished( ANY, ANY, ANY, verdict)) return not q.empty()
def _handle_start(self, message, messagebus): with LocalMessageQueue(messagebus, [CONTINUE], [ENDPOINT]) as cont: assert cont.get(timeout=1).message_id == CONTINUE assert cont.get(timeout=1).message_id == CONTINUE messagebus.trigger_event(BLOCK_MESSAGE, ENDPOINT) assert cont.get(timeout=0) assert cont.get(timeout=1)
def test_looper_extends_test_queue_when_repeat_set(self): with create_harness(repeats=3) as harness: with LocalMessageQueue(harness.messagebus, [ADD_TEST_CASES], [SCHEDULER_ENDPOINT]) as queue: harness.trigger_event(RUN_QUEUE_INITIALIZED, SCHEDULER_ENDPOINT, data=['test1', 'test2', 'test3']) result = queue.get(timeout=1) assert result.data == ['test1', 'test2', 'test3'] * 2
def message_queue(self, message_ids, endpoint_ids=None, entities=None, match=None, priority=0): with LocalMessageQueue(self.messagebus, message_ids, endpoint_ids, entities, match, priority) as q: yield q
def exposed_local_message_queue(self, serialized_message_ids, serialized_endpoint_ids, serialized_entities): message_ids = pickle.loads(serialized_message_ids) endpoint_ids = pickle.loads(serialized_endpoint_ids) entities = pickle.loads(serialized_entities) return LocalMessageQueue(self.messagebus, message_ids, endpoint_ids, entities)
def test_triggers_test_run_started_and_finished_on_test_run(self): with TestGtestRunner.create_harness() as harness: events = [TEST_RUN_STARTED, TEST_RUN_FINISHED] with LocalMessageQueue(harness.messagebus, events) as queue: harness.messagebus.trigger_event(TEST_RUN, RUN_COMMAND_ENDPOINT, None) self.assertEqual( queue.get(timeout=1).message_id, TEST_RUN_STARTED) self.assertEqual( queue.get(timeout=1).message_id, TEST_RUN_FINISHED)
def test_looper_refills_test_queue_if_empty_with_duration_remaining(self): with create_harness(duration='2') as harness: harness.trigger_event(TEST_RUN_STARTED, SCHEDULER_ENDPOINT) harness.trigger_event(RUN_QUEUE_INITIALIZED, SCHEDULER_ENDPOINT, data=['test1', 'test2', 'test3']) with LocalMessageQueue(harness.messagebus, [ADD_TEST_CASES], [SCHEDULER_ENDPOINT]) as queue: harness.trigger_event(RUN_QUEUE_EMPTY, SCHEDULER_ENDPOINT) result = queue.get(timeout=1) assert result.data == ['test1']
def test_starts_sending_perform_measurement_requests_when_the_test_run_start( self): with _create_harness() as harness: with LocalMessageQueue(harness.messagebus, message_ids=[PERFORM_MEASUREMENT], endpoint_ids=[MONITOR_ENDPOINT], entities=['mysut']) as queue: initial_timer = harness.extension._timer harness.messagebus.trigger_event(TEST_RUN_STARTED, RUNNER_ENDPOINT) queue.get(timeout=1) assert harness.extension._timer is not initial_timer
def test_make_triggers_pre_make_event_with_make_arguments_as_data(self): docker_run = Mock() scope = Scope('scope') with create_harness(docker_run) as harness: application = Mock() application.config = harness.config application.messagebus = harness.messagebus docker_run.docker_config = DockerConfig(harness.config) with LocalMessageQueue(harness.messagebus, [PRE_MAKE], [MAKE_ENDPOINT]) as queue: harness.component_factory.call(make, scope, application) self.assertEqual(queue.get(timeout=1).data, ('arg1', '--opt2'))
def test_critical_extension_exception_sent_if_serial_connection_cant_be_established( self): with patch('zserial.serial.find_serial_port', return_value=('device', False)): with create_harness() as harness: with LocalMessageQueue( harness.messagebus, [CRITICAL_EXTENSION_ERROR]) as error_queue: with patch('zserial.serial.start_serial_connection', side_effect=SerialConnectionError),\ patch('time.sleep'): harness.trigger_event(BEFORE_COMMAND, APPLICATION_ENDPOINT) assert error_queue.get( timeout=1).message_id == CRITICAL_EXTENSION_ERROR
def test_extension_run_all_checks_does_not_trigger_recovery_if_not_configured_to_do_so( self): with create_harness(should_recover=False) as harness: with LocalMessageQueue(harness.messagebus, [SUT_RECOVERY_PERFORM], entities=['entity']) as queue: with MockConnectionCheck(harness.messagebus, success=True, required=True, entity='entity'), \ MockConnectionCheck(harness.messagebus, success=False, required=True, entity='entity'), \ MockConnectionCheck(harness.messagebus, success=True, required=True, entity='entity'): harness.messagebus.send_request( POST_INITIALIZE_SUT, RUN_COMMAND_ENDPOINT, entity='entity').wait(timeout=1) self.assertRaises(Empty, queue.get_nowait)
def test_extension_run_all_checks_triggers_recovery_when_one_of_multiple_required_checks_fails( self): with create_harness() as harness: with LocalMessageQueue(harness.messagebus, [SUT_RECOVERY_PERFORM], entities=['entity']) as queue: with MockConnectionCheck(harness.messagebus, success=True, required=True, entity='entity'), \ MockConnectionCheck(harness.messagebus, success=False, required=True, entity='entity'), \ MockConnectionCheck(harness.messagebus, success=True, required=True, entity='entity'): harness.messagebus.send_request(POST_INITIALIZE_SUT, RUN_COMMAND_ENDPOINT, entity='entity') assert queue.get( timeout=1).message_id == SUT_RECOVERY_PERFORM
def test_extension_run_all_checks_does_not_triggers_recovery_when_only_not_required_check_fails( self): with create_harness() as harness: with LocalMessageQueue(harness.messagebus, [SUT_RECOVERY_PERFORM], entities=['entity']) as queue: with MockConnectionCheck(harness.messagebus, success=True, required=True, entity='entity'), \ MockConnectionCheck(harness.messagebus, success=False, required=False, entity='entity'), \ MockConnectionCheck(harness.messagebus, success=True, required=True, entity='entity'): harness.messagebus.send_request(CONNECTIONCHECK_RUN_CHECKS, CONNECTIONCHECK_ENDPOINT, entity='entity', data=False).wait(timeout=1) self.assertRaises(Empty, queue.get_nowait)
def test_running_a_binary_with_no_patterns(self): with TestConsoleRunner.create_harness() as harness: binary_output = dedent(""" some text some more text """) exec = Mock() exec.send_line.return_value = binary_output with LocalMessageQueue(harness.messagebus, [TEST_CASE_FINISHED]) as queue: harness.extension.run(message=None, exec=exec, messagebus=harness.messagebus) self.assertRaises(Empty, queue.get_nowait)
def test_if_a_matchgroup_called_name_is_provided_it_is_used_as_the_test_case_name( self): success_patterns = [r'my long (?P<name>\w+) case name'] with TestConsoleRunner.create_harness(success_patterns) as harness: binary_output = dedent(""" my long test case name """) exec = Mock() exec.send_line.return_value = binary_output with LocalMessageQueue(harness.messagebus, [TEST_CASE_FINISHED]) as queue: harness.extension.run(message=None, exec=exec, messagebus=harness.messagebus) message = queue.get(timeout=1) assert message.data.name == 'test'
def test_failed_test_translated_to_test_events(self): with TestGtestBinaryRunner.create_harness() as harness: gtest_output = dedent( """ [ RUN ] SbCryptographyTransform/Aes.SunnyDayIdentity/0 [ FAILED ] SbCryptographyTransform/Aes.SunnyDayIdentity/0 """) exec = Mock() exec.send_line.return_value = gtest_output events = [TEST_CASE_STARTED, TEST_CASE_FINISHED] with LocalMessageQueue(harness.messagebus, events) as queue: harness.extension._run(message=None, exec=exec, messagebus=harness.messagebus) self.assertEqual(queue.get(timeout=1).message_id, TEST_CASE_STARTED) test_case_finished_message = queue.get(timeout=1) self.assertEqual(test_case_finished_message.message_id, TEST_CASE_FINISHED) self.assertEqual(test_case_finished_message.data.verdict, Verdict.FAILED)
def test_extension_run_all_checks_triggers_recovery_when_specified_and_required_check_fails( self): with create_harness() as harness: with LocalMessageQueue(harness.messagebus, [SUT_RECOVERY_PERFORM], entities=['entity']) as queue: with MockConnectionCheck(harness.messagebus, success=False, required=True, entity='entity'): trigger_recover = True harness.messagebus.send_request(CONNECTIONCHECK_RUN_CHECKS, CONNECTIONCHECK_ENDPOINT, entity='entity', data=trigger_recover) assert queue.get( timeout=1).message_id == SUT_RECOVERY_PERFORM
def test_failure_is_reported_if_both_success_and_failure_patterns_match( self): success_patterns = ['pattern'] failure_patterns = ['pattern'] with TestConsoleRunner.create_harness(success_patterns, failure_patterns) as harness: binary_output = dedent(""" some text pattern some more text """) exec = Mock() exec.send_line.return_value = binary_output with LocalMessageQueue(harness.messagebus, [TEST_CASE_FINISHED]) as queue: harness.extension.run(message=None, exec=exec, messagebus=harness.messagebus) message = queue.get(timeout=1) assert message.data.name == 'pattern' assert message.data.verdict == Verdict.FAILED
def await_sut_message(self, message_id, timeout=300): """ Context manager that can be used to wait for a SUT event. The context manager yields a future that will contain the sut message. Example of how this can be used: .. code-block:: python @requires(sut_events='SutEvents') def test_case(sut_events): with sut_events.await_sut_message(SUT_RESET_EXPECTED) as future: do_stuff_that_triggers_reset() do_stuff_after_reset_expected_has_been_received() sut_reset_expected_message = future.result() :param message_id: the ID of the message to wait for :param timeout: the timeout :return: future containing the message representing the sut event """ logger.debug( 'Await message {id} for sut {entity}'.format(id=message_id, entity=self._sut.entity)) def wait_for_message(queue): try: return queue.get(timeout=timeout) except Empty: raise SutEventTimeout( 'Timeout when waiting for message {message_id} for sut {sut}'.format( message_id=message_id, sut=self._sut.entity)) with LocalMessageQueue(self._messagebus, [message_id], entities=[self._sut.entity]) as queue: future = Future() yield future future.run(wait_for_message, queue) future.result(timeout=timeout)
def test_blocking_until_stop_blocking_is_sent(self): config = ConfigManager() config.set(BLOCKER_ENABLED, True) with ExtensionTestHarness(Blocker, endpoints_and_messages={ TEST_BLOCKER_ENDPOINT: [TEST_BLOCKER_BLOCK], APPLICATION_ENDPOINT: [AFTER_COMMAND] }, config=config) as harness: id_futures = harness.send_request( START_BLOCKING_ON_MESSAGE, BLOCKER_ENDPOINT, data=StartBlockingInfo(message_id=TEST_BLOCKER_BLOCK, endpoint_id=TEST_BLOCKER_ENDPOINT, entity=None, timeout=3)) id = id_futures.wait(timeout=1)[0].result(timeout=1) with LocalMessageQueue( harness.messagebus, [BLOCKING_STARTED, BLOCKING_COMPLETED]) as queue: thread = threading.Thread( target=harness.send_request, args=[TEST_BLOCKER_BLOCK, TEST_BLOCKER_ENDPOINT]) thread.start() self.assertEqual(queue.get(timeout=1).entity, id) self.assertTrue(thread.is_alive()) harness.send_request(STOP_BLOCKING_ON_MESSAGE, BLOCKER_ENDPOINT, entity=id).wait(timeout=1) thread.join() self.assertFalse(thread.is_alive()) self.assertEqual(queue.get(timeout=1).entity, id)
def test_ongoing_blockers_are_stopped_by_destroy(self): config = ConfigManager() config.set(BLOCKER_ENABLED, True) harness = ExtensionTestHarness(Blocker, endpoints_and_messages={ TEST_BLOCKER_ENDPOINT: [TEST_BLOCKER_BLOCK], APPLICATION_ENDPOINT: [AFTER_COMMAND] }, config=config) try: harness.__enter__() with LocalMessageQueue(harness.messagebus, [BLOCKING_COMPLETED]) as queue: id_futures = harness.send_request( START_BLOCKING_ON_MESSAGE, BLOCKER_ENDPOINT, data=StartBlockingInfo(message_id=TEST_BLOCKER_BLOCK, endpoint_id=TEST_BLOCKER_ENDPOINT, entity=None, timeout=1)) id = id_futures.wait(timeout=1)[0].result(timeout=1) thread = threading.Thread( target=harness.send_request, args=[TEST_BLOCKER_BLOCK, TEST_BLOCKER_ENDPOINT]) thread.start() harness.extension.destroy() self.assertEqual(queue.get(timeout=1).entity, id) thread.join() except AssertionError: raise except Exception: harness.extension.destroy()
def test_running_a_binary_with_matching_failure_patterns(self): failure_patterns = ['failure_a', 'failure_b'] with TestConsoleRunner.create_harness( failure_patterns=failure_patterns) as harness: binary_output = dedent(""" failure_a some text some more text failure_b """) exec = Mock() exec.send_line.return_value = binary_output with LocalMessageQueue(harness.messagebus, [TEST_CASE_FINISHED]) as queue: harness.extension.run(message=None, exec=exec, messagebus=harness.messagebus) failure_a_message = queue.get(timeout=1) assert failure_a_message.data.name == 'failure_a' assert failure_a_message.data.verdict == Verdict.FAILED failure_b_message = queue.get(timeout=1) assert failure_b_message.data.name == 'failure_b' assert failure_b_message.data.verdict == Verdict.FAILED
def test_blocking_timeout_send_timeout_event(self): config = ConfigManager() config.set(BLOCKER_ENABLED, True) with ExtensionTestHarness(Blocker, endpoints_and_messages={ TEST_BLOCKER_ENDPOINT: [TEST_BLOCKER_BLOCK], APPLICATION_ENDPOINT: [AFTER_COMMAND] }, config=config) as harness: id_futures = harness.send_request( START_BLOCKING_ON_MESSAGE, BLOCKER_ENDPOINT, data=StartBlockingInfo(message_id=TEST_BLOCKER_BLOCK, endpoint_id=TEST_BLOCKER_ENDPOINT, entity=None, timeout=0)) id = id_futures.wait(timeout=1)[0].result(timeout=1) with LocalMessageQueue(harness.messagebus, [BLOCKING_TIMED_OUT]) as queue: harness.send_request(TEST_BLOCKER_BLOCK, TEST_BLOCKER_ENDPOINT) self.assertEqual(queue.get(timeout=1).entity, id)
def test_1(messagebus): with LocalMessageQueue(messagebus, [SUT_RESET_STARTED]) as queue: logger.info('test 1') queue.get(timeout=5)
def test_a_generate_metrics_report_event_is_triggered_on_test_run_finished(self): with TestTriggerMetricsReportGenerationOnTestRunFinished._create_harness() as harness: with LocalMessageQueue(harness.messagebus, [GENERATE_METRICS_REPORT]) as queue: harness.messagebus.trigger_event(TEST_RUN_FINISHED, RUNNER_ENDPOINT, data=None) queue.get(timeout=1)