def main(): # CREATE AN INTERACTIVE TESTPLAN plan = TestplanMock(name="MyPlan", interactive_port=0, interactive_block=False) with open("basic_suite_template.txt") as fobj: template = fobj.read() # WRITE A FAILING TESTCASE with open("basic_suite_with_value.py", "w") as fobj: fobj.write(template.format(VALUE=3)) atexit.register(os.remove, "basic_suite_with_value.py") from basic_suite_with_value import SuiteTemplate # TRIGGER RUN OF INTERACTIVE EXECUTIONER plan.run() # ADD A TEST plan.add(MultiTest(name="Test1", suites=[SuiteTemplate()])) # RUN THE TESTS plan.i.run_all_tests() # EXPECTED 1 != 3 FAILURE serialized = plan.i.test_case_report( test_uid="Test1", suite_uid="SuiteTemplate", case_uid="basic_case", serialized=True, ) assert (compare( serialized["entries"][0]["entries"][0]["entries"][0], FAILED_CASE_REPORT, )[0] is True) assert plan.i.report().passed is False # APPLY A CODE CHANGE - FIX with open("basic_suite_with_value.py", "w") as fobj: fobj.write(template.format(VALUE=1)) # SEND RELOAD CODE plan.i.reload() # RUN TESTS AGAIN plan.i.run_all_tests() # EXPECTED 1 == 1 SUCCESS serialized = plan.i.test_case_report( test_uid="Test1", suite_uid="SuiteTemplate", case_uid="basic_case", serialized=True, ) assert (compare( serialized["entries"][0]["entries"][0]["entries"][0], PASSED_CASE_REPORT, )[0] is True) assert plan.i.report().passed is True
def test_http_operate_tests_async(): with InteractivePlan(name='InteractivePlan', interactive_port=0, interactive_block=False, parse_cmdline=False, logger_level=TEST_INFO) as plan: plan.run() wait(lambda: any(plan.i.http_handler_info), 5, raise_on_timeout=True) addr = 'http://{}:{}'.format(*plan.i.http_handler_info) plan.add(make_multitest(1)) plan.add(make_multitest(2)) # TRIGGER ASYNC RUN OF TESTS -> UID response = post_request('{}/async/run_tests'.format(addr), {}).json() expected = { 'message': 'Async operation performed: run_tests', 'error': False, 'trace': None, 'metadata': {}, 'result': re.compile('[0-9|a-z|-]+') } assert compare(expected, response)[0] is True uid = response['result'] # QUERY UID ASYNC OPERATION UNTIL FINISHED sleeper = get_sleeper(0.6, raise_timeout_with_msg='Async result missing.') while next(sleeper): response = post_request('{}/async_result'.format(addr), {'uid': uid}) json_response = response.json() if json_response['error'] is False: assert response.status_code == 200 expected = { 'result': None, 'trace': None, 'error': False, 'message': re.compile('[0-9|a-z|-]+'), 'metadata': { 'state': 'Finished' } } assert compare(expected, json_response)[0] is True break assert response.status_code == 400 # REPORT VIA HTTP response = post_request('{}/sync/report'.format(addr), { 'serialized': True }).json() expected_response = { 'result': plan.i.report(serialized=True), 'error': False, 'metadata': {}, 'trace': None, 'message': 'Sync operation performed: report' } assert compare(response, expected_response)[0] is True
def test_top_level_tests(): with log_propagation_disabled(TESTPLAN_LOGGER): with InteractivePlan(name='InteractivePlan', interactive=True, interactive_block=False, parse_cmdline=False, logger_level=TEST_INFO) as plan: plan.add(make_multitest(1)) plan.run() wait(lambda: bool(plan.i.http_handler_info), 5, raise_on_timeout=True) plan.add(make_multitest(2)) # Added after plan.run() assert isinstance(plan.i.test('Test1'), MultiTest) assert isinstance(plan.i.test('Test2'), MultiTest) # print_report(plan.i.report(serialized=True)) # TESTS AND ASSIGNED RUNNERS assert list(plan.i.all_tests()) ==\ [('Test1', 'local_runner'), ('Test2', 'local_runner')] # OPERATE TEST DRIVERS (start/stop) resources = [res.uid() for res in plan.i.test('Test2').resources] assert resources == ['server', 'client'] for resource in plan.i.test('Test2').resources: assert resource.status.tag is None plan.i.start_test_resources('Test2') # START for resource in plan.i.test('Test2').resources: assert resource.status.tag is resource.STATUS.STARTED plan.i.stop_test_resources('Test2') # STOP for resource in plan.i.test('Test2').resources: assert resource.status.tag is resource.STATUS.STOPPED # RESET REPORTS plan.i.reset_reports() from .reports.basic_top_level_reset import REPORT as BTLReset assert compare(BTLReset, plan.i.report(serialized=True))[0] is True # RUN ALL TESTS plan.i.run_tests() from .reports.basic_top_level import REPORT as BTLevel assert compare(BTLevel, plan.i.report(serialized=True))[0] is True # RESET REPORTS plan.i.reset_reports() from .reports.basic_top_level_reset import REPORT as BTLReset assert compare(BTLReset, plan.i.report(serialized=True))[0] is True # RUN SINGLE TESTSUITE (CUSTOM NAME) plan.i.run_test_suite('Test2', 'TCPSuite - Custom_1') from .reports.basic_run_suite_test2 import REPORT as BRSTest2 assert compare(BRSTest2, plan.i.test_report('Test2'))[0] is True # RUN SINGLE TESTCASE plan.i.run_test_case('Test1', '*', 'basic_case__arg_1') from .reports.basic_run_case_test1 import REPORT as BRCTest1 assert compare(BRCTest1, plan.i.test_report('Test1'))[0] is True
def evaluate(self): passed, cmp_result = comparison.compare(lhs=self.value, rhs=self.expected, ignore=self.exclude_keys, only=self.include_keys, report_all=self.report_all) self.comparison = flatten_dict_comparison(cmp_result) return passed
def evaluate(self): """Evaluate the dict match.""" passed, cmp_result = comparison.compare( lhs=self.value, rhs=self.expected, ignore=self.exclude_keys, only=self.include_keys, report_mode=self._report_mode, value_cmp_func=self._value_cmp_func) self.comparison = flatten_dict_comparison(cmp_result) return passed
def test_http_operate_tests_sync(): with log_propagation_disabled(TESTPLAN_LOGGER): with InteractivePlan(name='InteractivePlan', interactive=True, interactive_block=False, parse_cmdline=False, logger_level=TEST_INFO) as plan: plan.run() wait(lambda: any(plan.i.http_handler_info), 5, raise_on_timeout=True) addr = 'http://{}:{}'.format(*plan.i.http_handler_info) plan.add(make_multitest(1)) plan.add(make_multitest(2)) # OPERATE TEST DRIVERS (start/stop) for resource in plan.i.test('Test2').resources: assert resource.status.tag is None response = post_request( '{}/sync/start_test_resources'.format(addr), { 'test_uid': 'Test2' }).json() _assert_http_response(response, 'start_test_resources', 'Sync') for resource in plan.i.test('Test2').resources: assert resource.status.tag is resource.STATUS.STARTED response = post_request('{}/sync/stop_test_resources'.format(addr), { 'test_uid': 'Test2' }).json() _assert_http_response(response, 'stop_test_resources', 'Sync') for resource in plan.i.test('Test2').resources: assert resource.status.tag is resource.STATUS.STOPPED # RESET REPORTS response = post_request('{}/sync/reset_reports'.format(addr), {}).json() _assert_http_response(response, 'reset_reports', 'Sync') from .reports.basic_top_level_reset import REPORT as BTLReset result = compare(BTLReset, plan.i.report(serialized=True)) assert result[0] is True # RUN ALL TESTS response = post_request('{}/sync/run_tests'.format(addr), {}).json() _assert_http_response(response, 'run_tests', 'Sync') from .reports.basic_top_level import REPORT as BTLevel result = compare(BTLevel, plan.i.report(serialized=True)) assert result[0] is True # RESET REPORTS response = post_request('{}/sync/reset_reports'.format(addr), {}).json() _assert_http_response(response, 'reset_reports', 'Sync') from .reports.basic_top_level_reset import REPORT as BTLReset assert compare(BTLReset, plan.i.report(serialized=True))[0] is True # REPORT VIA HTTP response = post_request('{}/sync/report'.format(addr), { 'serialized': True }).json() expected_response = { 'result': plan.i.report(serialized=True), 'error': False, 'metadata': {}, 'trace': None, 'message': 'Sync operation performed: report' } assert compare(response, expected_response)[0] is True # RUN SINGLE TESTSUITE (CUSTOM NAME) response = post_request('{}/sync/run_test_suite'.format(addr), { 'test_uid': 'Test2', 'suite_uid': 'TCPSuite - Custom_1' }).json() _assert_http_response(response, 'run_test_suite', 'Sync') from .reports.basic_run_suite_test2 import REPORT as BRSTest2 assert compare(BRSTest2, plan.i.test_report('Test2'))[0] is True # TEST 2 REPORT VIA HTTP response = post_request('{}/sync/test_report'.format(addr), { 'test_uid': 'Test2' }).json() expected_response = { 'result': plan.i.test_report('Test2'), 'error': False, 'metadata': {}, 'trace': None, 'message': 'Sync operation performed: test_report' } assert compare(response, expected_response)[0] is True # RUN SINGLE TESTCASE response = post_request( '{}/sync/run_test_case'.format(addr), { 'test_uid': 'Test1', 'suite_uid': '*', 'case_uid': 'basic_case__arg_1' }).json() _assert_http_response(response, 'run_test_case', 'Sync') from .reports.basic_run_case_test1 import REPORT as BRCTest1 assert compare(BRCTest1, plan.i.test_report('Test1'))[0] is True # TEST 1 REPORT VIA HTTP response = post_request('{}/sync/test_report'.format(addr), { 'test_uid': 'Test1' }).json() expected_response = { 'result': plan.i.test_report('Test1'), 'error': False, 'metadata': {}, 'trace': None, 'message': 'Sync operation performed: test_report' } assert compare(response, expected_response)[0] is True
plan.run() # ADD A TEST plan.add(MultiTest( name='Test1', suites=[SuiteTemplate()])) # RUN THE TESTS plan.i.run_tests() # EXPECTED 1 != 3 FAILURE serialized = plan.i.test_case_report( test_uid='Test1', suite_uid='SuiteTemplate', case_uid='basic_case', serialized=True) assert compare( serialized['entries'][0]['entries'][0]['entries'][0], FAILED_CASE_REPORT)[0] is True assert plan.i.report().passed is False # APPLY A CODE CHANGE - FIX with open('basic_suite_with_value.py', 'w') as fobj: fobj.write(template.format(VALUE=1)) # SEND RELOAD CODE plan.i.reload() # RUN TESTS AGAIN plan.i.run_tests() # EXPECTED 1 == 1 SUCCESS serialized = plan.i.test_case_report(
def test_top_level_tests(): with InteractivePlan( name="InteractivePlan", interactive_port=0, interactive_block=False, parse_cmdline=False, logger_level=TEST_INFO, ) as plan: plan.add(make_multitest("1")) plan.add(make_multitest("2")) plan.run() wait(lambda: bool(plan.i.http_handler_info), 5, raise_on_timeout=True) assert isinstance(plan.i.test("Test1"), MultiTest) assert isinstance(plan.i.test("Test2"), MultiTest) # print_report(plan.i.report(serialized=True)) # TESTS AND ASSIGNED RUNNERS assert list(plan.i.all_tests()) == ["Test1", "Test2"] # OPERATE TEST DRIVERS (start/stop) resources = [res.uid() for res in plan.i.test("Test2").resources] assert resources == ["server", "client"] for resource in plan.i.test("Test2").resources: assert resource.status.tag is None plan.i.start_test_resources("Test2") # START for resource in plan.i.test("Test2").resources: assert resource.status.tag is resource.STATUS.STARTED plan.i.stop_test_resources("Test2") # STOP for resource in plan.i.test("Test2").resources: assert resource.status.tag is resource.STATUS.STOPPED # RESET REPORTS plan.i.reset_all_tests() from .reports.basic_top_level_reset import REPORT as BTLReset assert (compare( BTLReset, plan.i.report.serialize(), ignore=["hash", "information", "line_no"], )[0] is True) # RUN ALL TESTS plan.i.run_all_tests() from .reports.basic_top_level import REPORT as BTLevel assert (compare( BTLevel, plan.i.report.serialize(), ignore=[ "hash", "information", "timer", "machine_time", "utc_time", "line_no", ], )[0] is True) # RESET REPORTS plan.i.reset_all_tests() from .reports.basic_top_level_reset import REPORT as BTLReset assert (compare( BTLReset, plan.i.report.serialize(), ignore=["hash", "information"], )[0] is True) # RUN SINGLE TESTSUITE (CUSTOM NAME) plan.i.run_test_suite("Test2", "TCPSuite - Custom_1") from .reports.basic_run_suite_test2 import REPORT as BRSTest2 assert (compare(BRSTest2, plan.i.test_report("Test2"), ignore=["hash"])[0] is True) # RUN SINGLE TESTCASE plan.i.run_test_case("Test1", "*", "basic_case__arg_1") from .reports.basic_run_case_test1 import REPORT as BRCTest1 assert (compare( BRCTest1, plan.i.test_report("Test1"), ignore=[ "hash", "information", "timer", "machine_time", "utc_time", "line_no", ], )[0] is True)
def test_http_operate_tests_async(): with InteractivePlan( name="InteractivePlan", interactive_port=0, interactive_block=False, parse_cmdline=False, logger_level=TEST_INFO, ) as plan: plan.run() wait(lambda: any(plan.i.http_handler_info), 5, raise_on_timeout=True) addr = "http://{}:{}".format(*plan.i.http_handler_info) plan.add(make_multitest(1)) plan.add(make_multitest(2)) # TRIGGER ASYNC RUN OF TESTS -> UID response = post_request("{}/async/run_tests".format(addr), {}).json() expected = { "message": "Async operation performed: run_tests", "error": False, "trace": None, "metadata": {}, "result": re.compile("[0-9|a-z|-]+"), } assert compare(expected, response)[0] is True uid = response["result"] # QUERY UID ASYNC OPERATION UNTIL FINISHED sleeper = get_sleeper(0.6, raise_timeout_with_msg="Async result missing.") while next(sleeper): response = post_request("{}/async_result".format(addr), {"uid": uid}) json_response = response.json() if json_response["error"] is False: assert response.status_code == 200 expected = { "result": None, "trace": None, "error": False, "message": re.compile("[0-9|a-z|-]+"), "metadata": { "state": "Finished" }, } assert compare(expected, json_response)[0] is True break assert response.status_code == 400 # REPORT VIA HTTP response = post_request("{}/sync/report".format(addr), { "serialized": True }).json() expected_response = { "result": plan.i.report(serialized=True), "error": False, "metadata": {}, "trace": None, "message": "Sync operation performed: report", } assert compare(response, expected_response)[0] is True
def test_http_operate_tests_sync(): with InteractivePlan( name="InteractivePlan", interactive_port=0, interactive_block=False, parse_cmdline=False, logger_level=TEST_INFO, ) as plan: plan.run() wait(lambda: any(plan.i.http_handler_info), 5, raise_on_timeout=True) addr = "http://{}:{}".format(*plan.i.http_handler_info) plan.add(make_multitest(1)) plan.add(make_multitest(2)) # OPERATE TEST DRIVERS (start/stop) for resource in plan.i.test("Test2").resources: assert resource.status.tag is None response = post_request("{}/sync/start_test_resources".format(addr), { "test_uid": "Test2" }).json() _assert_http_response(response, "start_test_resources", "Sync") for resource in plan.i.test("Test2").resources: assert resource.status.tag is resource.STATUS.STARTED response = post_request("{}/sync/stop_test_resources".format(addr), { "test_uid": "Test2" }).json() _assert_http_response(response, "stop_test_resources", "Sync") for resource in plan.i.test("Test2").resources: assert resource.status.tag is resource.STATUS.STOPPED # RESET REPORTS response = post_request("{}/sync/reset_reports".format(addr), {}).json() _assert_http_response(response, "reset_reports", "Sync") from .reports.basic_top_level_reset import REPORT as BTLReset result = compare(BTLReset, plan.i.report(serialized=True)) assert result[0] is True # RUN ALL TESTS response = post_request("{}/sync/run_tests".format(addr), {}).json() _assert_http_response(response, "run_tests", "Sync") from .reports.basic_top_level import REPORT as BTLevel result = compare(BTLevel, plan.i.report(serialized=True)) assert result[0] is True # RESET REPORTS response = post_request("{}/sync/reset_reports".format(addr), {}).json() _assert_http_response(response, "reset_reports", "Sync") from .reports.basic_top_level_reset import REPORT as BTLReset assert compare(BTLReset, plan.i.report(serialized=True))[0] is True # REPORT VIA HTTP response = post_request("{}/sync/report".format(addr), { "serialized": True }).json() expected_response = { "result": plan.i.report(serialized=True), "error": False, "metadata": {}, "trace": None, "message": "Sync operation performed: report", } assert compare(response, expected_response)[0] is True # RUN SINGLE TESTSUITE (CUSTOM NAME) response = post_request( "{}/sync/run_test_suite".format(addr), { "test_uid": "Test2", "suite_uid": "TCPSuite - Custom_1" }, ).json() _assert_http_response(response, "run_test_suite", "Sync") from .reports.basic_run_suite_test2 import REPORT as BRSTest2 assert compare(BRSTest2, plan.i.test_report("Test2"))[0] is True # TEST 2 REPORT VIA HTTP response = post_request("{}/sync/test_report".format(addr), { "test_uid": "Test2" }).json() expected_response = { "result": plan.i.test_report("Test2"), "error": False, "metadata": {}, "trace": None, "message": "Sync operation performed: test_report", } assert compare(response, expected_response)[0] is True # RUN SINGLE TESTCASE response = post_request( "{}/sync/run_test_case".format(addr), { "test_uid": "Test1", "suite_uid": "*", "case_uid": "basic_case__arg_1", }, ).json() _assert_http_response(response, "run_test_case", "Sync") from .reports.basic_run_case_test1 import REPORT as BRCTest1 assert compare(BRCTest1, plan.i.test_report("Test1"))[0] is True # TEST 1 REPORT VIA HTTP response = post_request("{}/sync/test_report".format(addr), { "test_uid": "Test1" }).json() expected_response = { "result": plan.i.test_report("Test1"), "error": False, "metadata": {}, "trace": None, "message": "Sync operation performed: test_report", } assert compare(response, expected_response)[0] is True