def test_parse_tests_variable_not_found(self): tests_mapping = { "project_mapping": { "functions": { "sum_two": sum_two } }, 'testcases': [{ "config": { 'name': '', "base_url": "$host1", 'variables': { "host1": "https://debugtalk.com" } }, "teststeps": [{ 'name': 'testcase1', "base_url": "$host2", "variables": { "host2": "https://apiautotest.org", "num4": "${sum_two($num0, 5)}", "num3": "${sum_two($num2, 4)}", "num2": "${sum_two($num1, 3)}", "num1": "${sum_two(1, 2)}" }, 'request': { 'url': '/api1/?num1=$num1&num2=$num2&num3=$num3&num4=$num4', 'method': 'GET' } }] }] } with self.assertRaises(exceptions.VariableNotFound): parser.parse_tests(tests_mapping)
def test_parse_tests_override_variables(self): tests_mapping = { 'testcases': [{ "config": { 'name': '', 'variables': [{ "password": "******" }, { "creator": "user_test_001" }] }, "teststeps": [{ 'name': 'testcase1', "variables": [{ "creator": "user_test_002" }, { "username": "******" }], 'request': { 'url': '/api1', 'method': 'GET' } }] }] } parsed_testcases = parser.parse_tests(tests_mapping) test_dict1_variables = parsed_testcases[0]["teststeps"][0]["variables"] self.assertEqual(test_dict1_variables["creator"], "user_test_001") self.assertEqual(test_dict1_variables["username"].raw_string, "$creator")
def prepare_locust_tests(path): """ prepare locust testcases Args: path (str): testcase file path. Returns: list: locust tests data [ testcase1_dict, testcase2_dict ] """ tests_mapping = loader.load_tests(path) testcases = parser.parse_tests(tests_mapping) locust_tests = [] for testcase in testcases: testcase_weight = testcase.get("config", {}).pop("weight", 1) for _ in range(testcase_weight): locust_tests.append(testcase) return locust_tests
def test_validate_exception(self): testcases = [{ "config": { 'name': "test validation" }, "teststeps": [{ "name": "test validation", "request": { "url": "http://127.0.0.1:5000/", "method": "GET", }, "variables": { "resp_status_code": 200, "resp_body_success": True }, "validate": [{ "eq": ["$resp_status_code", 201] }, { "check": "$resp_status_code", "expect": 201 }, { "check": "$resp_body_success", "comparator": "eq", "expect": True }] }] }] tests_mapping = {"testcases": testcases} testcases = parser.parse_tests(tests_mapping) parsed_testcase = testcases[0] test_runner = runner.Runner(parsed_testcase["config"]) teststep = parsed_testcase["teststeps"][0] with self.assertRaises(exceptions.ValidationFailure): test_runner.run_test(teststep)
def test_parse_tests_base_url_test_dict(self): tests_mapping = { 'testcases': [{ "config": { 'name': '', "base_url": "$host1", 'variables': { "host1": "https://debugtalk.com" } }, "teststeps": [{ 'name': 'testcase1', "base_url": "$host2", "variables": { "host2": "https://apiautotest.org" }, 'request': { 'url': '/api1', 'method': 'GET' } }] }] } parsed_testcases = parser.parse_tests(tests_mapping) test_dict = parsed_testcases[0]["teststeps"][0] parsed_test_dict = parser.parse_lazy_data(test_dict, test_dict["variables"]) self.assertEqual(parsed_test_dict["base_url"], "https://apiautotest.org")
def test_parse_tests_verify_nested_testcase_unset(self): tests_mapping = { 'testcases': [{ 'config': { 'name': 'inquiry price', 'verify': False }, 'teststeps': [{ 'name': 'login system', 'testcase': 'testcases/deps/login.yml', 'testcase_def': { 'config': { 'name': 'login system' }, 'teststeps': [{ 'name': '/', 'request': { 'method': 'GET', 'url': 'https://httpbin.org/' } }] } }] }] } parsed_testcases = parser.parse_tests(tests_mapping) test_dict = parsed_testcases[0]["teststeps"][0] self.assertEqual(test_dict["teststeps"][0]["request"]["verify"], False)
def test_parse_tests_base_url_priority(self): """ base_url & verify: priority test_dict > config """ tests_mapping = { 'testcases': [{ "config": { 'name': '', "base_url": "$host", 'variables': { "host": "https://debugtalk.com" }, "verify": False }, "teststeps": [{ 'name': 'testcase1', "base_url": "https://apiautotest.org", 'request': { 'url': '/api1', 'method': 'GET', "verify": True } }] }] } parsed_testcases = parser.parse_tests(tests_mapping) test_dict = parsed_testcases[0]["teststeps"][0] self.assertEqual(test_dict["request"]["url"], "/api1") self.assertEqual(test_dict["request"]["verify"], True)
def test_run_testcase_with_hooks(self): start_time = time.time() testcases = [ { "config": { "name": "basic test with httpbin", "base_url": HTTPBIN_SERVER, "setup_hooks": [ "${sleep_N_secs(0.5)}", "${hook_print(setup)}" ], "teardown_hooks": [ "${sleep_N_secs(1)}", "${hook_print(teardown)}" ] }, "teststeps": [ { "name": "get token", "request": { "url": "http://127.0.0.1:5000/api/get-token", "method": "POST", "headers": { "content-type": "application/json", "user_agent": "iOS/10.3", "device_sn": "HZfFBh6tU59EdXJ", "os_platform": "ios", "app_version": "2.8.6" }, "json": { "sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" } }, "validate": [ {"check": "status_code", "expect": 200} ] } ] } ] tests_mapping = { "project_mapping": { "functions": self.debugtalk_functions }, "testcases": testcases } parsed_testcases = parser.parse_tests(tests_mapping) parsed_testcase = parsed_testcases[0] test_runner = runner.Runner(parsed_testcase["config"]) end_time = time.time() # check if testcase setup hook executed self.assertGreater(end_time - start_time, 0.5) start_time = time.time() test_runner.run_test(parsed_testcase["teststeps"][0]) end_time = time.time() # testcase teardown hook has not been executed now self.assertLess(end_time - start_time, 1)
def test_testcase_simple_run_suite(self): testcase_path = "tests/testcases/setup.yml" tests_mapping = loader.load_tests(testcase_path) testcases = parser.parse_tests(tests_mapping) runner = apiautotest() test_suite = runner._add_tests(testcases) tests_results = runner._run_suite(test_suite) self.assertEqual(len(tests_results[0][1].records), 2)
def test_bugfix_type_match(self): testcase_file_path = os.path.join( os.getcwd(), 'tests/data/bugfix_type_match.yml') tests_mapping = loader.load_tests(testcase_file_path) parsed_testcases = parser.parse_tests(tests_mapping) parsed_testcase = parsed_testcases[0] test_runner = runner.Runner(parsed_testcase["config"]) test_runner.run_test(parsed_testcase["teststeps"][0])
def test_testsuite_add_tests(self): testcase_path = "tests/testsuites/create_users.yml" tests_mapping = loader.load_tests(testcase_path) testcases = parser.parse_tests(tests_mapping) runner = apiautotest() test_suite = runner._add_tests(testcases) self.assertEqual(len(test_suite._tests), 2) tests = test_suite._tests[0].teststeps self.assertIn("setup and reset all (override)", tests[0]["config"]["name"].raw_string)
def test_testcase_add_tests(self): testcase_path = "tests/testcases/setup.yml" tests_mapping = loader.load_tests(testcase_path) testcases = parser.parse_tests(tests_mapping) runner = apiautotest() test_suite = runner._add_tests(testcases) self.assertEqual(len(test_suite._tests), 1) teststeps = test_suite._tests[0].teststeps self.assertEqual(teststeps[0]["name"], "get token (setup)") self.assertEqual(teststeps[0]["variables"]["device_sn"], "TESTCASE_SETUP_XXX") self.assertIn("api", teststeps[0])
def test_testcase_complex_run_suite(self): testcase_path = "tests/testcases/create_user.yml" tests_mapping = loader.load_tests(testcase_path) testcases = parser.parse_tests(tests_mapping) runner = apiautotest() test_suite = runner._add_tests(testcases) tests_results = runner._run_suite(test_suite) self.assertEqual(len(tests_results[0][1].records), 2) results = tests_results[0][1] self.assertEqual( results.records[0]["name"], "setup and reset all (override) for TESTCASE_CREATE_XXX.") self.assertEqual(results.records[1]["name"], "create user and check result.")
def test_testsuite_parser(self): testcase_path = "tests/testsuites/create_users.yml" tests_mapping = loader.load_tests(testcase_path) parsed_testcases = parser.parse_tests(tests_mapping) self.assertEqual(len(parsed_testcases), 2) self.assertEqual(len(parsed_testcases[0]["teststeps"]), 2) testcase1 = parsed_testcases[0]["teststeps"][0] self.assertIn("setup and reset all (override)", testcase1["config"]["name"].raw_string) teststeps = testcase1["teststeps"] self.assertNotIn("testcase_def", testcase1) self.assertEqual(len(teststeps), 2) self.assertEqual(teststeps[0]["request"]["url"], "/api/get-token")
def test_run_single_testcase(self): testcase_file_path_list = [ os.path.join( os.getcwd(), 'tests/data/demo_testcase_hardcode.yml'), os.path.join( os.getcwd(), 'tests/data/demo_testcase_hardcode.json') ] for testcase_file_path in testcase_file_path_list: tests_mapping = loader.load_tests(testcase_file_path) parsed_testcases = parser.parse_tests(tests_mapping) parsed_testcase = parsed_testcases[0] test_runner = runner.Runner(parsed_testcase["config"]) test_runner.run_test(parsed_testcase["teststeps"][0]) test_runner.run_test(parsed_testcase["teststeps"][1]) test_runner.run_test(parsed_testcase["teststeps"][2])
def test_testcase_complex_verify(self): testcase_path = "tests/testcases/create_user.yml" tests_mapping = loader.load_tests(testcase_path) testcases = parser.parse_tests(tests_mapping) teststeps = testcases[0]["teststeps"] # testcases/setup.yml teststep0 = teststeps[0] self.assertEqual(teststep0["teststeps"][0]["request"]["verify"], False) self.assertEqual(teststep0["teststeps"][1]["request"]["verify"], False) # testcases/create_user.yml teststep1 = teststeps[1] self.assertEqual(teststep1["teststeps"][0]["request"]["verify"], True) self.assertEqual(teststep1["teststeps"][1]["request"]["verify"], True) self.assertEqual(teststep1["teststeps"][2]["request"]["verify"], True)
def test_parse_tests_variable_with_function(self): tests_mapping = { "project_mapping": { "functions": { "sum_two": sum_two, "gen_random_string": gen_random_string } }, 'testcases': [{ "config": { 'name': '', "base_url": "$host1", 'variables': { "host1": "https://debugtalk.com", "var_a": "${gen_random_string(5)}", "var_b": "$var_a" } }, "teststeps": [{ 'name': 'testcase1', "base_url": "$host2", "variables": { "host2": "https://apiautotest.org", "num3": "${sum_two($num2, 4)}", "num2": "${sum_two($num1, 3)}", "num1": "${sum_two(1, 2)}", "str1": "${gen_random_string(5)}", "str2": "$str1" }, 'request': { 'url': '/api1/?num1=$num1&num2=$num2&num3=$num3', 'method': 'GET' } }] }] } parsed_testcases = parser.parse_tests(tests_mapping) test_dict = parsed_testcases[0]["teststeps"][0] variables = parser.parse_variables_mapping(test_dict["variables"]) self.assertEqual(variables["num3"], 10) self.assertEqual(variables["num2"], 6) parsed_test_dict = parser.parse_lazy_data(test_dict, variables) self.assertEqual(parsed_test_dict["base_url"], "https://apiautotest.org") self.assertEqual(parsed_test_dict["request"]["url"], "/api1/?num1=3&num2=6&num3=10") self.assertEqual(variables["str1"], variables["str2"])
def test_run_testcase_with_teardown_hooks_fail(self): testcases = [ { "config": { "name": "basic test with httpbin" }, "teststeps": [ { "name": "get token", "request": { "url": "http://127.0.0.1:5000/api/get-token2", "method": "POST", "headers": { "content-type": "application/json", "user_agent": "iOS/10.3", "device_sn": "HZfFBh6tU59EdXJ", "os_platform": "ios", "app_version": "2.8.6" }, "json": { "sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" } }, "validate": [ {"check": "status_code", "expect": 404} ], "teardown_hooks": ["${teardown_hook_sleep_N_secs($response, 2)}"] } ] } ] tests_mapping = { "project_mapping": { "functions": self.debugtalk_functions }, "testcases": testcases } parsed_testcases = parser.parse_tests(tests_mapping) parsed_testcase = parsed_testcases[0] test_runner = runner.Runner(parsed_testcase["config"]) start_time = time.time() test_runner.run_test(parsed_testcase["teststeps"][0]) end_time = time.time() # check if teardown function executed self.assertGreater(end_time - start_time, 2)
def test_validate(self): testcases = [{ "config": { 'name': "test validation" }, "teststeps": [{ "name": "test validation", "request": { "url": "http://127.0.0.1:5000/", "method": "GET", }, "variables": { "resp_status_code": 200, "resp_body_success": True }, "validate": [{ "eq": ["$resp_status_code", 200] }, { "check": "$resp_status_code", "comparator": "eq", "expect": 200 }, { "check": "$resp_body_success", "expect": True }, { "check": "${is_status_code_200($resp_status_code)}", "expect": True }] }] }] from tests.debugtalk import is_status_code_200 tests_mapping = { "project_mapping": { "functions": { "is_status_code_200": is_status_code_200 } }, "testcases": testcases } testcases = parser.parse_tests(tests_mapping) parsed_testcase = testcases[0] test_runner = runner.Runner(parsed_testcase["config"]) teststep = parsed_testcase["teststeps"][0] test_runner.run_test(teststep)
def test_parse_tests_testcase(self): testcase_file_path = os.path.join(os.getcwd(), 'tests/data/demo_testcase.yml') tests_mapping = loader.load_tests(testcase_file_path) testcases = tests_mapping["testcases"] self.assertEqual(testcases[0]["config"]["variables"]["var_c"], "${sum_two($var_a, $var_b)}") self.assertEqual(testcases[0]["config"]["variables"]["PROJECT_KEY"], "${ENV(PROJECT_KEY)}") parsed_testcases = parser.parse_tests(tests_mapping) self.assertIsInstance(parsed_testcases, list) test_dict1 = parsed_testcases[0]["teststeps"][0] self.assertEqual(test_dict1["variables"]["var_c"].raw_string, "${sum_two($var_a, $var_b)}") self.assertEqual(test_dict1["variables"]["PROJECT_KEY"].raw_string, "${ENV(PROJECT_KEY)}") self.assertIsInstance(parsed_testcases[0]["config"]["name"], parser.LazyString)
def test_run_testcase_with_hooks_modify_request(self): testcases = [ { "config": { "name": "basic test with httpbin", "base_url": HTTPBIN_SERVER }, "teststeps": [ { "name": "modify request headers", "base_url": HTTPBIN_SERVER, "request": { "url": "/anything", "method": "POST", "headers": { "content-type": "application/json", "user_agent": "iOS/10.3" }, "json": { "os_platform": "ios", "sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" } }, "setup_hooks": [ "${modify_request_json($request, android)}" ], "validate": [ {"check": "status_code", "expect": 200}, {"check": "content.json.os_platform", "expect": "android"} ] } ] } ] tests_mapping = { "project_mapping": { "functions": self.debugtalk_functions }, "testcases": testcases } parsed_testcases = parser.parse_tests(tests_mapping) parsed_testcase = parsed_testcases[0] test_runner = runner.Runner(parsed_testcase["config"]) test_runner.run_test(parsed_testcase["teststeps"][0])
def test_run_testcase_with_hooks_assignment(self): testcases = [ { "config": { "name": "basic test with httpbin", "base_url": HTTPBIN_SERVER }, "teststeps": [ { "name": "modify request headers", "base_url": HTTPBIN_SERVER, "request": { "url": "/anything", "method": "POST", "headers": { "user_agent": "iOS/10.3", "os_platform": "ios" }, "data": "a=1&b=2" }, "setup_hooks": [ {"total": "${sum_two(1, 5)}"} ], "validate": [ {"check": "status_code", "expect": 200} ] } ] } ] tests_mapping = { "project_mapping": { "functions": self.debugtalk_functions }, "testcases": testcases } parsed_testcases = parser.parse_tests(tests_mapping) parsed_testcase = parsed_testcases[0] test_runner = runner.Runner(parsed_testcase["config"]) test_runner.run_test(parsed_testcase["teststeps"][0]) test_variables_mapping = test_runner.session_context.test_variables_mapping self.assertEqual(test_variables_mapping["total"], 6) self.assertEqual(test_variables_mapping["request"]["data"], "a=1&b=2")
def test_run_validate_elapsed(self): testcases = [ { "config": {}, "teststeps": [ { "name": "get token", "request": { "url": "http://127.0.0.1:5000/api/get-token", "method": "POST", "headers": { "content-type": "application/json", "user_agent": "iOS/10.3", "device_sn": "HZfFBh6tU59EdXJ", "os_platform": "ios", "app_version": "2.8.6" }, "json": { "sign": "5188962c489d1a35effa99e9346dd5efd4fdabad" } }, "validate": [ {"check": "status_code", "expect": 200}, {"check": "elapsed.seconds", "comparator": "lt", "expect": 1}, {"check": "elapsed.days", "comparator": "eq", "expect": 0}, {"check": "elapsed.microseconds", "comparator": "gt", "expect": 1000}, {"check": "elapsed.total_seconds", "comparator": "lt", "expect": 1} ] } ] } ] tests_mapping = { "project_mapping": { "functions": self.debugtalk_functions }, "testcases": testcases } parsed_testcases = parser.parse_tests(tests_mapping) parsed_testcase = parsed_testcases[0] test_runner = runner.Runner(parsed_testcase["config"]) test_runner.run_test(parsed_testcase["teststeps"][0])
def test_parse_tests_verify_config_unset(self): """ verify priority: test_dict > config """ tests_mapping = { 'testcases': [{ "config": { 'name': 'bugfix verify', "base_url": "https://httpbin.org/", }, "teststeps": [{ 'name': 'testcase1', 'request': { 'url': '/headers', 'method': 'GET' } }] }] } parsed_testcases = parser.parse_tests(tests_mapping) test_dict = parsed_testcases[0]["teststeps"][0] self.assertEqual(test_dict["request"]["verify"], True)
def test_testcase_parser(self): testcase_path = "tests/testcases/setup.yml" tests_mapping = loader.load_tests(testcase_path) parsed_testcases = parser.parse_tests(tests_mapping) self.assertEqual(len(parsed_testcases), 1) self.assertIn("variables", parsed_testcases[0]["config"]) self.assertEqual(len(parsed_testcases[0]["teststeps"]), 2) test_dict1 = parsed_testcases[0]["teststeps"][0] self.assertEqual(test_dict1["name"], "get token (setup)") self.assertNotIn("api_def", test_dict1) self.assertEqual(test_dict1["variables"]["device_sn"], "TESTCASE_SETUP_XXX") self.assertEqual(test_dict1["request"]["url"], "/api/get-token") self.assertEqual(test_dict1["request"]["verify"], False) test_dict2 = parsed_testcases[0]["teststeps"][1] self.assertEqual(test_dict2["request"]["verify"], False)
def run_tests(self, tests_mapping): """ run testcase/testsuite data """ project_mapping = tests_mapping.get("project_mapping", {}) if self.save_tests: utils.dump_logs(tests_mapping, project_mapping, "loaded") # parse tests self.exception_stage = "parse tests" parsed_testcases = parser.parse_tests(tests_mapping) if self.save_tests: utils.dump_logs(parsed_testcases, project_mapping, "parsed") # add tests to test suite self.exception_stage = "add tests to test suite" test_suite = self._add_tests(parsed_testcases) # run test suite self.exception_stage = "run test suite" results = self._run_suite(test_suite) # aggregate results self.exception_stage = "aggregate results" self._summary = self._aggregate(results) # generate html report self.exception_stage = "generate html report" report.stringify_summary(self._summary) if self.save_tests: utils.dump_logs(self._summary, project_mapping, "summary") report_path = report.render_html_report(self._summary, self.report_template, self.report_dir) return report_path