def test_load_tests_path_not_exist(self):
# absolute folder path
path = os.path.join(os.getcwd(), 'tests/data_not_exist')
with self.assertRaises(exceptions.FileNotFound):
loader.load_tests(path)
# relative folder path
path = 'tests/data_not_exist'
with self.assertRaises(exceptions.FileNotFound):
loader.load_tests(path)
def test_load_tests_folder_path(self):
# absolute folder path
path = os.path.join(os.getcwd(), 'tests/data')
tests_mapping = loader.load_tests(path)
testcase_list_1 = tests_mapping["testcases"]
self.assertGreater(len(testcase_list_1), 4)
# relative folder path
path = 'tests/data/'
tests_mapping = loader.load_tests(path)
testcase_list_2 = tests_mapping["testcases"]
self.assertEqual(len(testcase_list_1), len(testcase_list_2))
def test_load_testcases_by_path_folder(self):
# absolute folder path
path = os.path.join(os.getcwd(), 'tests/data')
testcase_list_1 = loader.load_tests(path)
self.assertGreater(len(testcase_list_1), 4)
# relative folder path
path = 'tests/data/'
testcase_list_2 = loader.load_tests(path)
self.assertEqual(len(testcase_list_1), len(testcase_list_2))
# list/set container with file(s)
path = [os.path.join(os.getcwd(), 'tests/data'), 'tests/data/']
testcase_list_3 = loader.load_tests(path)
self.assertEqual(len(testcase_list_3), 2 * len(testcase_list_1))
def test_testsuite_loader(self):
testcase_path = "tests/testsuites/create_users.yml"
tests_mapping = loader.load_tests(testcase_path)
project_mapping = tests_mapping["project_mapping"]
self.assertIsInstance(project_mapping, dict)
self.assertIn("PWD", project_mapping)
self.assertIn("functions", project_mapping)
self.assertIn("env", project_mapping)
testsuites = tests_mapping["testsuites"]
self.assertIsInstance(testsuites, list)
self.assertEqual(len(testsuites), 1)
self.assertIn("path", testsuites[0])
testsuite_config = testsuites[0]["config"]
self.assertEqual(testsuite_config["name"], "create users with uid")
testcases = testsuites[0]["testcases"]
self.assertEqual(len(testcases), 2)
self.assertIn("create user 1000 and check result.", testcases)
testcase_tests = testcases["create user 1000 and check result."]
self.assertIn("testcase_def", testcase_tests)
self.assertEqual(testcase_tests["name"], "create user 1000 and check result.")
self.assertIsInstance(testcase_tests["testcase_def"], dict)
self.assertEqual(testcase_tests["testcase_def"]["config"]["name"], "create user and check result.")
self.assertEqual(len(testcase_tests["testcase_def"]["teststeps"]), 4)
self.assertEqual(
testcase_tests["testcase_def"]["teststeps"][0]["name"],
"setup and reset all (override) for $device_sn."
)
def prepare_locust_tests(path):
""" prepare locust testcases
Args:
path (str): testcase file path.
Returns:
dict: locust tests data
{
"functions": {},
"tests": []
}
"""
tests_mapping = loader.load_tests(path)
parsed_tests_mapping = parser.parse_tests(tests_mapping)
functions = parsed_tests_mapping.get("project_mapping",
{}).get("functions", {})
tests = []
for testcase in parsed_tests_mapping["testcases"]:
testcase_weight = testcase.get("config", {}).pop("weight", 1)
for _ in range(testcase_weight):
tests.append(testcase)
return {"functions": functions, "tests": tests}
def test_load_tests_api_file(self):
path = os.path.join(os.getcwd(), 'tests/api/create_user.yml')
tests_mapping = loader.load_tests(path)
project_mapping = tests_mapping["project_mapping"]
api_list = tests_mapping["apis"]
self.assertEqual(len(api_list), 1)
self.assertEqual(api_list[0]["request"]["url"], "/api/users/$uid")
def test_testcase_loader(self):
testcase_path = "tests/testcases/setup.yml"
tests_mapping = loader.load_tests(testcase_path)
project_mapping = tests_mapping["project_mapping"]
self.assertIsInstance(project_mapping, dict)
self.assertIn("PWD", project_mapping)
self.assertIn("functions", project_mapping)
self.assertIn("env", project_mapping)
testcases = tests_mapping["testcases"]
self.assertIsInstance(testcases, list)
self.assertEqual(len(testcases), 1)
testcase_config = testcases[0]["config"]
self.assertEqual(testcase_config["name"], "setup and reset all.")
self.assertIn("path", testcases[0])
testcase_tests = testcases[0]["teststeps"]
self.assertEqual(len(testcase_tests), 2)
self.assertIn("api", testcase_tests[0])
self.assertEqual(testcase_tests[0]["name"], "get token (setup)")
self.assertIsInstance(testcase_tests[0]["variables"], dict)
self.assertIn("api_def", testcase_tests[0])
self.assertEqual(testcase_tests[0]["api_def"]["request"]["url"],
"/api/get-token")
def run(self, path_or_testcases, dot_env_path=None, mapping=None):
""" main interface, run testcases with variables mapping.
Args:
path_or_testcases (str/list/dict): testcase file/foler path, or valid testcases.
dot_env_path (str): specified .env file path.
mapping (dict): if mapping is specified, it will override variables in config block.
Returns:
instance: HttpRunner() instance
"""
self.exception_stage = "load tests"
if validator.is_testcases(path_or_testcases):
if isinstance(path_or_testcases, dict):
testcases = [path_or_testcases]
else:
testcases = path_or_testcases
elif validator.is_testcase_path(path_or_testcases):
testcases = loader.load_tests(path_or_testcases, dot_env_path)
else:
raise exceptions.ParamsError("invalid testcase path or testcases.")
return self._run_tests(testcases, mapping)
def test_testsuite_parser(self):
testcase_path = "tests/testsuites/create_users.yml"
tests_mapping = loader.load_tests(testcase_path)
parsed_tests_mapping = parser.parse_tests(tests_mapping)
parsed_testcases = parsed_tests_mapping["testcases"]
self.assertEqual(len(parsed_testcases), 2)
self.assertEqual(len(parsed_testcases[0]["teststeps"]), 4)
testcase1 = parsed_testcases[0]["teststeps"][0]
self.assertIn("setup and reset all (override)",
testcase1["config"]["name"])
self.assertEqual(testcase1["teststeps"][0]["variables"]["var_c"],
testcase1["teststeps"][0]["variables"]["var_d"])
self.assertEqual(testcase1["teststeps"][0]["variables"]["var_a"],
testcase1["teststeps"][0]["variables"]["var_b"])
self.assertNotEqual(testcase1["teststeps"][0]["variables"]["var_a"],
testcase1["teststeps"][0]["variables"]["var_c"])
self.assertNotIn("testcase_def", testcase1)
self.assertEqual(len(testcase1["teststeps"]), 2)
self.assertEqual(testcase1["teststeps"][0]["request"]["url"],
"http://127.0.0.1:5000/api/get-token")
self.assertEqual(
len(testcase1["teststeps"][0]["variables"]["device_sn"]), 15)
def prepare_locust_tests(path):
""" prepare locust testcases
Args:
path (str): testcase file path.
Returns:
list: locust tests data
[
testcase1_dict,
testcase2_dict
]
"""
tests_mapping = loader.load_tests(path)
testcases = parser.parse_tests(tests_mapping)
locust_tests = []
for testcase in testcases:
testcase_weight = testcase.get("config", {}).pop("weight", 1)
for _ in range(testcase_weight):
locust_tests.append(testcase)
return locust_tests
def test_load_testcases_by_path_files(self):
testcases_list = []
# absolute file path
path = os.path.join(os.getcwd(),
'tests/data/demo_testcase_hardcode.json')
testcases_list = loader.load_tests(path)
self.assertEqual(len(testcases_list), 1)
self.assertEqual(len(testcases_list[0]["teststeps"]), 3)
self.assertEqual(
testcases_list[0]["config"]["refs"]["debugtalk"]["variables"]
["SECRET_KEY"], "DebugTalk")
self.assertIn(
"get_sign",
testcases_list[0]["config"]["refs"]["debugtalk"]["functions"])
# relative file path
path = 'tests/data/demo_testcase_hardcode.yml'
testcases_list = loader.load_tests(path)
self.assertEqual(len(testcases_list), 1)
self.assertEqual(len(testcases_list[0]["teststeps"]), 3)
self.assertEqual(
testcases_list[0]["config"]["refs"]["debugtalk"]["variables"]
["SECRET_KEY"], "DebugTalk")
self.assertIn(
"get_sign",
testcases_list[0]["config"]["refs"]["debugtalk"]["functions"])
# list/set container with file(s)
path = [
os.path.join(os.getcwd(),
'tests/data/demo_testcase_hardcode.json'),
'tests/data/demo_testcase_hardcode.yml'
]
testcases_list = loader.load_tests(path)
self.assertEqual(len(testcases_list), 2)
self.assertEqual(len(testcases_list[0]["teststeps"]), 3)
self.assertEqual(len(testcases_list[1]["teststeps"]), 3)
testcases_list.extend(testcases_list)
self.assertEqual(len(testcases_list), 4)
for testcase in testcases_list:
for teststep in testcase["teststeps"]:
self.assertIn('name', teststep)
self.assertIn('request', teststep)
self.assertIn('url', teststep['request'])
self.assertIn('method', teststep['request'])
def test_testcase_simple_run_suite(self):
testcase_path = "tests/testcases/setup.yml"
tests_mapping = loader.load_tests(testcase_path)
parsed_tests_mapping = parser.parse_tests(tests_mapping)
runner = HttpRunner()
test_suite = runner._add_tests(parsed_tests_mapping)
tests_results = runner._run_suite(test_suite)
self.assertEqual(len(tests_results[0][1].records), 2)
def test_bugfix_type_match(self):
testcase_file_path = os.path.join(os.getcwd(),
'tests/data/bugfix_type_match.yml')
tests_mapping = loader.load_tests(testcase_file_path)
parsed_testcases = parser.parse_tests(tests_mapping)
parsed_testcase = parsed_testcases[0]
test_runner = runner.Runner(parsed_testcase["config"])
test_runner.run_test(parsed_testcase["teststeps"][0])
def test_load_testcases_by_path_layered(self):
path = os.path.join(os.getcwd(), 'tests/data/demo_testcase_layer.yml')
testcases_list = loader.load_tests(path)
self.assertIn("variables", testcases_list[0]["config"])
self.assertIn("request", testcases_list[0]["config"])
self.assertIn("request", testcases_list[0]["teststeps"][0])
self.assertIn("url", testcases_list[0]["teststeps"][0]["request"])
self.assertIn("validate", testcases_list[0]["teststeps"][0])
def test_run_testcase_with_empty_header(self):
testcase_file_path = os.path.join(os.getcwd(),
'tests/data/test_bugfix.yml')
tests_mapping = loader.load_tests(testcase_file_path)
testcase = tests_mapping["testcases"][0]
config_dict_headers = testcase["config"]["request"]["headers"]
test_dict_headers = testcase["teststeps"][0]["request"]["headers"]
headers = deep_update_dict(config_dict_headers, test_dict_headers)
self.assertEqual(headers["Content-Type"], "application/json")
def test_load_tests_testcase_file(self):
# absolute file path
path = os.path.join(os.getcwd(),
'tests/data/demo_testcase_hardcode.json')
tests_mapping = loader.load_tests(path)
project_mapping = tests_mapping["project_mapping"]
testcases_list = tests_mapping["testcases"]
self.assertEqual(len(testcases_list), 1)
self.assertEqual(len(testcases_list[0]["teststeps"]), 3)
self.assertIn("get_sign", project_mapping["functions"])
# relative file path
path = 'tests/data/demo_testcase_hardcode.yml'
tests_mapping = loader.load_tests(path)
project_mapping = tests_mapping["project_mapping"]
testcases_list = tests_mapping["testcases"]
self.assertEqual(len(testcases_list), 1)
self.assertEqual(len(testcases_list[0]["teststeps"]), 3)
self.assertIn("get_sign", project_mapping["functions"])
def test_load_tests(self):
testcase_file_path = os.path.join(os.getcwd(),
'tests/data/demo_testcase.yml')
testcases = loader.load_tests(testcase_file_path)
self.assertIsInstance(testcases, list)
self.assertEqual(testcases[0]["config"]["request"],
'$demo_default_request')
self.assertEqual(testcases[0]["config"]["name"], '123$var_a')
self.assertIn("sum_two",
testcases[0]["config"]["refs"]["debugtalk"]["functions"])
def test_testsuite_add_tests(self):
testcase_path = "tests/testsuites/create_users.yml"
tests_mapping = loader.load_tests(testcase_path)
parsed_tests_mapping = parser.parse_tests(tests_mapping)
runner = HttpRunner()
test_suite = runner._add_tests(parsed_tests_mapping)
self.assertEqual(len(test_suite._tests), 2)
tests = test_suite._tests[0].teststeps
self.assertIn("setup and reset all (override)", tests[0]["config"]["name"])
def hookFunction(testcase_path, variables):
testcase_path = os.path.join(os.getcwd(), testcase_path)
runner = HttpRunner(failfast=True)
tests_mapping = loader.load_tests(testcase_path)
# to add variables into tests_mapping.teststeps
tests_mapping['apis'][0]['variables'] = variables
parsed_tests_mapping = parser.parse_tests(tests_mapping)
runner.run(parsed_tests_mapping)
def test_load_tests_testcase_file_2(self):
testcase_file_path = os.path.join(os.getcwd(),
'tests/data/demo_testcase.yml')
tests_mapping = loader.load_tests(testcase_file_path)
testcases = tests_mapping["testcases"]
self.assertIsInstance(testcases, list)
self.assertEqual(testcases[0]["config"]["name"], '123$var_a')
self.assertIn("sum_two", tests_mapping["project_mapping"]["functions"])
self.assertEqual(testcases[0]["config"]["variables"]["var_c"],
"${sum_two(1, 2)}")
self.assertEqual(testcases[0]["config"]["variables"]["PROJECT_KEY"],
"${ENV(PROJECT_KEY)}")
def test_testcase_add_tests(self):
testcase_path = "tests/testcases/setup.yml"
tests_mapping = loader.load_tests(testcase_path)
parsed_tests_mapping = parser.parse_tests(tests_mapping)
runner = HttpRunner()
test_suite = runner._add_tests(parsed_tests_mapping)
self.assertEqual(len(test_suite._tests), 1)
teststeps = test_suite._tests[0].teststeps
self.assertEqual(teststeps[0]["name"], "get token (setup)")
self.assertEqual(teststeps[0]["variables"]["device_sn"], "TESTCASE_SETUP_XXX")
self.assertIn("api", teststeps[0])
def test_run_single_testcase(self):
testcase_file_path_list = [
os.path.join(os.getcwd(), 'tests/data/demo_testcase_hardcode.yml'),
os.path.join(os.getcwd(), 'tests/data/demo_testcase_hardcode.json')
]
for testcase_file_path in testcase_file_path_list:
tests_mapping = loader.load_tests(testcase_file_path)
parsed_testcases = parser.parse_tests(tests_mapping)
parsed_testcase = parsed_testcases[0]
test_runner = runner.Runner(parsed_testcase["config"])
test_runner.run_test(parsed_testcase["teststeps"][0])
test_runner.run_test(parsed_testcase["teststeps"][1])
test_runner.run_test(parsed_testcase["teststeps"][2])
def test_load_tests_testcase_file_with_api_ref(self):
path = os.path.join(os.getcwd(), 'tests/data/demo_testcase_layer.yml')
tests_mapping = loader.load_tests(path)
project_mapping = tests_mapping["project_mapping"]
testcases_list = tests_mapping["testcases"]
self.assertIn('device_sn', testcases_list[0]["config"]["variables"])
self.assertIn("gen_md5", project_mapping["functions"])
self.assertIn("base_url", testcases_list[0]["config"])
test_dict0 = testcases_list[0]["teststeps"][0]
self.assertEqual("get token with $user_agent, $app_version",
test_dict0["name"])
self.assertIn("/api/get-token",
test_dict0["api_def"]["request"]["url"])
self.assertIn({'eq': ['status_code', 200]}, test_dict0["validate"])
def test_testsuite_parser(self):
testcase_path = "tests/testsuites/create_users.yml"
tests_mapping = loader.load_tests(testcase_path)
parsed_testcases = parser.parse_tests(tests_mapping)
self.assertEqual(len(parsed_testcases), 2)
self.assertEqual(len(parsed_testcases[0]["teststeps"]), 2)
testcase1 = parsed_testcases[0]["teststeps"][0]
self.assertIn("setup and reset all (override)",
testcase1["config"]["name"].raw_string)
teststeps = testcase1["teststeps"]
self.assertNotIn("testcase_def", testcase1)
self.assertEqual(len(teststeps), 2)
self.assertEqual(teststeps[0]["request"]["url"], "/api/get-token")
def test_testcase_complex_run_suite(self):
testcase_path = "tests/testcases/create_and_check.yml"
tests_mapping = loader.load_tests(testcase_path)
parsed_tests_mapping = parser.parse_tests(tests_mapping)
runner = HttpRunner()
test_suite = runner._add_tests(parsed_tests_mapping)
tests_results = runner._run_suite(test_suite)
self.assertEqual(len(tests_results[0][1].records), 4)
results = tests_results[0][1]
self.assertEqual(
results.records[0]["name"],
"setup and reset all (override) for TESTCASE_CREATE_XXX.")
self.assertEqual(results.records[1]["name"],
"make sure user 9001 does not exist")
def test_testcase_complex_verify(self):
testcase_path = "tests/testcases/create_and_check.yml"
tests_mapping = loader.load_tests(testcase_path)
parsed_tests_mapping = parser.parse_tests(tests_mapping)
teststeps = parsed_tests_mapping["testcases"][0]["teststeps"]
# testcases/setup.yml
teststep1 = teststeps[0]
self.assertEqual(teststep1["teststeps"][0]["request"]["verify"], False)
self.assertEqual(teststep1["teststeps"][1]["request"]["verify"], False)
# testcases/create_and_check.yml teststep 2/3/4
self.assertEqual(teststeps[1]["request"]["verify"], True)
self.assertEqual(teststeps[2]["request"]["verify"], True)
self.assertEqual(teststeps[3]["request"]["verify"], True)
def test_testcase_complex_run_suite(self):
testcase_path = "tests/testcases/create_user.yml"
tests_mapping = loader.load_tests(testcase_path)
testcases = parser.parse_tests(tests_mapping)
runner = HttpRunner()
test_suite = runner._add_tests(testcases)
tests_results = runner._run_suite(test_suite)
self.assertEqual(len(tests_results[0][1].records), 2)
results = tests_results[0][1]
self.assertEqual(
results.records[0]["name"],
"setup and reset all (override) for TESTCASE_CREATE_XXX.")
self.assertEqual(results.records[1]["name"],
"create user and check result.")
def test_parse_tests(self):
testcase_file_path = os.path.join(os.getcwd(),
'tests/data/demo_testcase.yml')
testcases = loader.load_tests(testcase_file_path)
parsed_testcases = parser.parse_tests(testcases)
self.assertEqual(parsed_testcases[0]["config"]["variables"]["var_c"],
3)
self.assertEqual(len(parsed_testcases), 2 * 2)
self.assertEqual(parsed_testcases[0]["config"]["request"]["base_url"],
'$BASE_URL')
self.assertEqual(
parsed_testcases[0]["config"]["variables"]["BASE_URL"],
'http://127.0.0.1:5000')
self.assertIsInstance(parsed_testcases, list)
self.assertEqual(parsed_testcases[0]["config"]["name"], '12311')
def test_testcase_complex_verify(self):
testcase_path = "tests/testcases/create_user.yml"
tests_mapping = loader.load_tests(testcase_path)
testcases = parser.parse_tests(tests_mapping)
teststeps = testcases[0]["teststeps"]
# testcases/setup.yml
teststep0 = teststeps[0]
self.assertEqual(teststep0["teststeps"][0]["request"]["verify"], False)
self.assertEqual(teststep0["teststeps"][1]["request"]["verify"], False)
# testcases/create_user.yml
teststep1 = teststeps[1]
self.assertEqual(teststep1["teststeps"][0]["request"]["verify"], True)
self.assertEqual(teststep1["teststeps"][1]["request"]["verify"], True)
self.assertEqual(teststep1["teststeps"][2]["request"]["verify"], True)
def test_load_tests_testsuite_file_with_testcase_ref(self):
path = os.path.join(os.getcwd(), 'tests/testsuites/create_users.yml')
tests_mapping = loader.load_tests(path)
project_mapping = tests_mapping["project_mapping"]
testsuites_list = tests_mapping["testsuites"]
self.assertEqual("create users with uid",
testsuites_list[0]["config"]["name"])
self.assertEqual({'device_sn': '${gen_random_string(15)}'},
testsuites_list[0]["config"]["variables"])
self.assertIn("create user 1000 and check result.",
testsuites_list[0]["testcases"])
self.assertEqual(
testsuites_list[0]["testcases"]
["create user 1000 and check result."]["testcase_def"]["config"]
["name"], "create user and check result.")