def test_load_tests_path_not_exist(self): # absolute folder path path = os.path.join(os.getcwd(), 'tests/data_not_exist') with self.assertRaises(exceptions.FileNotFound): loader.load_cases(path) # relative folder path path = 'tests/data_not_exist' with self.assertRaises(exceptions.FileNotFound): loader.load_cases(path)
def test_load_tests_folder_path(self): # absolute folder path path = os.path.join(os.getcwd(), 'tests/data') tests_mapping = loader.load_cases(path) testcase_list_1 = tests_mapping["testcases"] self.assertGreater(len(testcase_list_1), 4) # relative folder path path = 'tests/data/' tests_mapping = loader.load_cases(path) testcase_list_2 = tests_mapping["testcases"] self.assertEqual(len(testcase_list_1), len(testcase_list_2))
def test_testsuite_loader(self): testcase_path = "tests/testsuites/create_users.yml" tests_mapping = loader.load_cases(testcase_path) project_mapping = tests_mapping["project_mapping"] self.assertIsInstance(project_mapping, dict) self.assertIn("PWD", project_mapping) self.assertIn("functions", project_mapping) self.assertIn("env", project_mapping) testsuites = tests_mapping["testsuites"] self.assertIsInstance(testsuites, list) self.assertEqual(len(testsuites), 1) self.assertIn("path", testsuites[0]) testsuite_config = testsuites[0]["config"] self.assertEqual(testsuite_config["name"], "create users with uid") testcases = testsuites[0]["testcases"] self.assertEqual(len(testcases), 2) self.assertIn("create user 1000 and check result.", testcases) testcase_tests = testcases["create user 1000 and check result."] self.assertIn("testcase_def", testcase_tests) self.assertEqual(testcase_tests["name"], "create user 1000 and check result.") self.assertIsInstance(testcase_tests["testcase_def"], dict) self.assertEqual(testcase_tests["testcase_def"]["config"]["name"], "create user and check result.") self.assertEqual(len(testcase_tests["testcase_def"]["teststeps"]), 2) self.assertEqual( testcase_tests["testcase_def"]["teststeps"][0]["name"], "setup and reset all (override) for $device_sn." )
def test_load_tests_api_file(self): path = os.path.join(os.getcwd(), 'tests/api/create_user.yml') tests_mapping = loader.load_cases(path) project_mapping = tests_mapping["project_mapping"] api_list = tests_mapping["apis"] self.assertEqual(len(api_list), 1) self.assertEqual(api_list[0]["request"]["url"], "/api/users/$uid")
def prepare_locust_tests(path): """ prepare locust testcases Args: path (str): testcase file path. Returns: list: locust tests data [ testcase1_dict, testcase2_dict ] """ tests_mapping = loader.load_cases(path) testcases = parser.parse_tests(tests_mapping) locust_tests = [] for testcase in testcases: testcase_weight = testcase.get("config", {}).pop("weight", 1) for _ in range(testcase_weight): locust_tests.append(testcase) return locust_tests
def test_testcase_simple_run_suite(self): testcase_path = "tests/testcases/setup.yml" tests_mapping = loader.load_cases(testcase_path) testcases = parser.parse_tests(tests_mapping) runner = HttpRunner() test_suite = runner._add_tests(testcases) tests_results = runner._run_suite(test_suite) self.assertEqual(len(tests_results[0][1].records), 2)
def test_load_tests_testcase_file(self): # absolute file path path = os.path.join(os.getcwd(), 'tests/data/demo_testcase_hardcode.json') tests_mapping = loader.load_cases(path) project_mapping = tests_mapping["project_mapping"] testcases_list = tests_mapping["testcases"] self.assertEqual(len(testcases_list), 1) self.assertEqual(len(testcases_list[0]["teststeps"]), 3) self.assertIn("get_sign", project_mapping["functions"]) # relative file path path = 'tests/data/demo_testcase_hardcode.yml' tests_mapping = loader.load_cases(path) project_mapping = tests_mapping["project_mapping"] testcases_list = tests_mapping["testcases"] self.assertEqual(len(testcases_list), 1) self.assertEqual(len(testcases_list[0]["teststeps"]), 3) self.assertIn("get_sign", project_mapping["functions"])
def test_testsuite_add_tests(self): testcase_path = "tests/testsuites/create_users.yml" tests_mapping = loader.load_cases(testcase_path) testcases = parser.parse_tests(tests_mapping) runner = HttpRunner() test_suite = runner._add_tests(testcases) self.assertEqual(len(test_suite._tests), 2) tests = test_suite._tests[0].teststeps self.assertIn("setup and reset all (override)", tests[0]["config"]["name"].raw_string)
def test_load_tests_testcase_file_2(self): testcase_file_path = os.path.join(os.getcwd(), 'tests/data/demo_testcase.yml') tests_mapping = loader.load_cases(testcase_file_path) testcases = tests_mapping["testcases"] self.assertIsInstance(testcases, list) self.assertEqual(testcases[0]["config"]["name"], '123t$var_a') self.assertIn("sum_two", tests_mapping["project_mapping"]["functions"]) self.assertEqual(testcases[0]["config"]["variables"]["var_c"], "${sum_two($var_a, $var_b)}") self.assertEqual(testcases[0]["config"]["variables"]["PROJECT_KEY"], "${ENV(PROJECT_KEY)}")
def test_testcase_add_tests(self): testcase_path = "tests/testcases/setup.yml" tests_mapping = loader.load_cases(testcase_path) testcases = parser.parse_tests(tests_mapping) runner = HttpRunner() test_suite = runner._add_tests(testcases) self.assertEqual(len(test_suite._tests), 1) teststeps = test_suite._tests[0].teststeps self.assertEqual(teststeps[0]["name"], "get token (setup)") self.assertEqual(teststeps[0]["variables"]["device_sn"], "TESTCASE_SETUP_XXX") self.assertIn("api", teststeps[0])
def test_load_tests_testcase_file_with_api_ref(self): path = os.path.join(os.getcwd(), 'tests/data/demo_testcase_layer.yml') tests_mapping = loader.load_cases(path) project_mapping = tests_mapping["project_mapping"] testcases_list = tests_mapping["testcases"] self.assertIn('device_sn', testcases_list[0]["config"]["variables"]) self.assertIn("gen_md5", project_mapping["functions"]) self.assertIn("base_url", testcases_list[0]["config"]) test_dict0 = testcases_list[0]["teststeps"][0] self.assertEqual("get token with $user_agent, $app_version", test_dict0["name"]) self.assertIn("/api/get-token", test_dict0["api_def"]["request"]["url"]) self.assertIn({'eq': ['status_code', 200]}, test_dict0["validate"])
def test_testcase_complex_verify(self): testcase_path = "tests/testcases/create_user.yml" tests_mapping = loader.load_cases(testcase_path) testcases = parser.parse_tests(tests_mapping) teststeps = testcases[0]["teststeps"] # testcases/setup.yml teststep0 = teststeps[0] self.assertEqual(teststep0["teststeps"][0]["request"]["verify"], False) self.assertEqual(teststep0["teststeps"][1]["request"]["verify"], False) # testcases/create_user.yml teststep1 = teststeps[1] self.assertEqual(teststep1["teststeps"][0]["request"]["verify"], True) self.assertEqual(teststep1["teststeps"][1]["request"]["verify"], True) self.assertEqual(teststep1["teststeps"][2]["request"]["verify"], True)
def test_testsuite_parser(self): testcase_path = "tests/testsuites/create_users.yml" tests_mapping = loader.load_cases(testcase_path) parsed_testcases = parser.parse_tests(tests_mapping) self.assertEqual(len(parsed_testcases), 2) self.assertEqual(len(parsed_testcases[0]["teststeps"]), 2) testcase1 = parsed_testcases[0]["teststeps"][0] self.assertIn("setup and reset all (override)", testcase1["config"]["name"].raw_string) teststeps = testcase1["teststeps"] self.assertNotIn("testcase_def", testcase1) self.assertEqual(len(teststeps), 2) self.assertEqual( teststeps[0]["request"]["url"], "/api/get-token" )
def test_load_tests_testsuite_file_with_testcase_ref(self): path = os.path.join(os.getcwd(), 'tests/testsuites/create_users.yml') tests_mapping = loader.load_cases(path) project_mapping = tests_mapping["project_mapping"] testsuites_list = tests_mapping["testsuites"] self.assertEqual("create users with uid", testsuites_list[0]["config"]["name"]) self.assertEqual( '${gen_random_string(15)}', testsuites_list[0]["config"]["variables"]['device_sn']) self.assertIn("create user 1000 and check result.", testsuites_list[0]["testcases"]) self.assertEqual( testsuites_list[0]["testcases"] ["create user 1000 and check result."]["testcase_def"]["config"] ["name"], "create user and check result.")
def test_testcase_parser(self): testcase_path = "tests/testcases/setup.yml" tests_mapping = loader.load_cases(testcase_path) parsed_testcases = parser.parse_tests(tests_mapping) self.assertEqual(len(parsed_testcases), 1) self.assertIn("variables", parsed_testcases[0]["config"]) self.assertEqual(len(parsed_testcases[0]["teststeps"]), 2) test_dict1 = parsed_testcases[0]["teststeps"][0] self.assertEqual(test_dict1["name"], "get token (setup)") self.assertNotIn("api_def", test_dict1) self.assertEqual(test_dict1["variables"]["device_sn"], "TESTCASE_SETUP_XXX") self.assertEqual(test_dict1["request"]["url"], "/api/get-token") self.assertEqual(test_dict1["request"]["verify"], False) test_dict2 = parsed_testcases[0]["teststeps"][1] self.assertEqual(test_dict2["request"]["verify"], False)
def run_path(self, path, dot_env_path=None, mapping=None): """ run testcase/testsuite file or folder. Args: path (str): testcase/testsuite file/foler path. dot_env_path (str): specified .env file path. mapping (dict): if mapping is specified, it will override variables in config block. Returns: dict: result summary """ # load tests self.exception_stage = "load tests" tests_mapping = loader.load_cases(path, dot_env_path) if mapping: tests_mapping["project_mapping"]["variables"] = mapping return self.run_tests(tests_mapping)
def test_testsuite_run_suite(self): testcase_path = "tests/testsuites/create_users.yml" tests_mapping = loader.load_cases(testcase_path) testcases = parser.parse_tests(tests_mapping) runner = HttpRunner() test_suite = runner._add_tests(testcases) tests_results = runner._run_suite(test_suite) self.assertEqual(len(tests_results[0][1].records), 2) results = tests_results[0][1] self.assertIn( "setup and reset all (override)", results.records[0]["name"] ) self.assertEqual( results.records[1]["name"], "create user and check result." )
def test_testcase_complex_run_suite(self): for testcase_path in [ "tests/testcases/create_user.yml", "tests/testcases/create_user.v2.yml", "tests/testcases/create_user.json", "tests/testcases/create_user.v2.json" ]: tests_mapping = loader.load_cases(testcase_path) testcases = parser.parse_tests(tests_mapping) runner = HttpRunner() test_suite = runner._add_tests(testcases) tests_results = runner._run_suite(test_suite) self.assertEqual(len(tests_results[0][1].records), 2) results = tests_results[0][1] self.assertEqual( results.records[0]["name"], "setup and reset all (override) for TESTCASE_CREATE_XXX." ) self.assertEqual( results.records[1]["name"], "create user and check result." )
def test_testcase_loader(self): testcase_path = "tests/testcases/setup.yml" tests_mapping = loader.load_cases(testcase_path) project_mapping = tests_mapping["project_mapping"] self.assertIsInstance(project_mapping, dict) self.assertIn("PWD", project_mapping) self.assertIn("functions", project_mapping) self.assertIn("env", project_mapping) testcases = tests_mapping["testcases"] self.assertIsInstance(testcases, list) self.assertEqual(len(testcases), 1) testcase_config = testcases[0]["config"] self.assertEqual(testcase_config["name"], "setup and reset all.") self.assertIn("path", testcases[0]) testcase_tests = testcases[0]["teststeps"] self.assertEqual(len(testcase_tests), 2) self.assertIn("api", testcase_tests[0]) self.assertEqual(testcase_tests[0]["name"], "get token (setup)") self.assertIsInstance(testcase_tests[0]["variables"], dict) self.assertIn("api_def", testcase_tests[0]) self.assertEqual(testcase_tests[0]["api_def"]["request"]["url"], "/api/get-token")
def main(): """ API test: parse command line options and run commands. """ if is_py2: color_print(get_python2_retire_msg(), "YELLOW") parser = argparse.ArgumentParser(description=__description__) parser.add_argument( '-V', '--version', dest='version', action='store_true', help="show version") parser.add_argument( 'testfile_paths', nargs='*', help="Specify api/testcase/testsuite file paths to run.") parser.add_argument( '--log-level', default='INFO', help="Specify logging level, default is INFO.") parser.add_argument( '--log-file', help="Write logs to specified file path.") parser.add_argument( '--dot-env-path', help="Specify .env file path, which is useful for keeping sensitive data.") parser.add_argument( '--report-template', help="Specify report template path.") parser.add_argument( '--report-dir', help="Specify report save directory.") parser.add_argument( '--report-file', help="Specify report file path, this has higher priority than specifying report dir.") parser.add_argument( '--save-tests', action='store_true', default=False, help="Save loaded/parsed/summary json data to JSON files.") parser.add_argument( '--failfast', action='store_true', default=False, help="Stop the test run on the first error or failure.") parser.add_argument( '--startproject', help="Specify new project name.") parser.add_argument( '--validate', nargs='*', help="Validate YAML/JSON api/testcase/testsuite format.") parser.add_argument( '--prettify', nargs='*', help="Prettify JSON testcase format.") args = parser.parse_args() if len(sys.argv) == 1: # no argument passed parser.print_help() sys.exit(0) if args.version: color_print("{}".format(__version__), "GREEN") sys.exit(0) if args.validate: for validate_path in args.validate: try: color_print("validate test file: {}".format(validate_path), "GREEN") load_cases(validate_path, args.dot_env_path) except exceptions.MyBaseError as ex: log_error(str(ex)) continue color_print("done!", "BLUE") sys.exit(0) if args.prettify: prettify_json_file(args.prettify) sys.exit(0) project_name = args.startproject if project_name: create_scaffold(project_name) sys.exit(0) runner = HttpRunner( failfast=args.failfast, save_tests=args.save_tests, log_level=args.log_level, log_file=args.log_file ) err_code = 0 try: for path in args.testfile_paths: summary = runner.run(path, dot_env_path=args.dot_env_path) report_dir = args.report_dir or os.path.join(runner.project_working_directory, "reports") gen_html_report( summary, report_template=args.report_template, report_dir=report_dir, report_file=args.report_file ) err_code |= (0 if summary and summary["success"] else 1) except Exception as ex: color_print("!!!!!!!!!! exception stage: {} !!!!!!!!!!".format(runner.exception_stage), "YELLOW") color_print(str(ex), "RED") sentry_sdk.capture_exception(ex) err_code = 1 sys.exit(err_code)