def test_parse_tests_variable_not_found(self):
tests_mapping = {
"project_mapping": {
"functions": {
"sum_two": sum_two
}
},
'testcases': [{
"config": {
'name': '',
"base_url": "$host1",
'variables': {
"host1": "https://debugtalk.com"
}
},
"teststeps": [{
'name': 'testcase1',
"base_url": "$host2",
"variables": {
"host2": "https://apiautotest.org",
"num4": "${sum_two($num0, 5)}",
"num3": "${sum_two($num2, 4)}",
"num2": "${sum_two($num1, 3)}",
"num1": "${sum_two(1, 2)}"
},
'request': {
'url':
'/api1/?num1=$num1&num2=$num2&num3=$num3&num4=$num4',
'method': 'GET'
}
}]
}]
}
with self.assertRaises(exceptions.VariableNotFound):
parser.parse_tests(tests_mapping)
def test_parse_tests_override_variables(self):
tests_mapping = {
'testcases': [{
"config": {
'name':
'',
'variables': [{
"password": "******"
}, {
"creator": "user_test_001"
}]
},
"teststeps": [{
'name':
'testcase1',
"variables": [{
"creator": "user_test_002"
}, {
"username": "******"
}],
'request': {
'url': '/api1',
'method': 'GET'
}
}]
}]
}
parsed_testcases = parser.parse_tests(tests_mapping)
test_dict1_variables = parsed_testcases[0]["teststeps"][0]["variables"]
self.assertEqual(test_dict1_variables["creator"], "user_test_001")
self.assertEqual(test_dict1_variables["username"].raw_string,
"$creator")
def prepare_locust_tests(path):
""" prepare locust testcases
Args:
path (str): testcase file path.
Returns:
list: locust tests data
[
testcase1_dict,
testcase2_dict
]
"""
tests_mapping = loader.load_tests(path)
testcases = parser.parse_tests(tests_mapping)
locust_tests = []
for testcase in testcases:
testcase_weight = testcase.get("config", {}).pop("weight", 1)
for _ in range(testcase_weight):
locust_tests.append(testcase)
return locust_tests
def test_validate_exception(self):
testcases = [{
"config": {
'name': "test validation"
},
"teststeps": [{
"name":
"test validation",
"request": {
"url": "http://127.0.0.1:5000/",
"method": "GET",
},
"variables": {
"resp_status_code": 200,
"resp_body_success": True
},
"validate": [{
"eq": ["$resp_status_code", 201]
}, {
"check": "$resp_status_code",
"expect": 201
}, {
"check": "$resp_body_success",
"comparator": "eq",
"expect": True
}]
}]
}]
tests_mapping = {"testcases": testcases}
testcases = parser.parse_tests(tests_mapping)
parsed_testcase = testcases[0]
test_runner = runner.Runner(parsed_testcase["config"])
teststep = parsed_testcase["teststeps"][0]
with self.assertRaises(exceptions.ValidationFailure):
test_runner.run_test(teststep)
def test_parse_tests_base_url_test_dict(self):
tests_mapping = {
'testcases': [{
"config": {
'name': '',
"base_url": "$host1",
'variables': {
"host1": "https://debugtalk.com"
}
},
"teststeps": [{
'name': 'testcase1',
"base_url": "$host2",
"variables": {
"host2": "https://apiautotest.org"
},
'request': {
'url': '/api1',
'method': 'GET'
}
}]
}]
}
parsed_testcases = parser.parse_tests(tests_mapping)
test_dict = parsed_testcases[0]["teststeps"][0]
parsed_test_dict = parser.parse_lazy_data(test_dict,
test_dict["variables"])
self.assertEqual(parsed_test_dict["base_url"],
"https://apiautotest.org")
def test_parse_tests_verify_nested_testcase_unset(self):
tests_mapping = {
'testcases': [{
'config': {
'name': 'inquiry price',
'verify': False
},
'teststeps': [{
'name': 'login system',
'testcase': 'testcases/deps/login.yml',
'testcase_def': {
'config': {
'name': 'login system'
},
'teststeps': [{
'name': '/',
'request': {
'method': 'GET',
'url': 'https://httpbin.org/'
}
}]
}
}]
}]
}
parsed_testcases = parser.parse_tests(tests_mapping)
test_dict = parsed_testcases[0]["teststeps"][0]
self.assertEqual(test_dict["teststeps"][0]["request"]["verify"], False)
def test_parse_tests_base_url_priority(self):
""" base_url & verify: priority test_dict > config
"""
tests_mapping = {
'testcases': [{
"config": {
'name': '',
"base_url": "$host",
'variables': {
"host": "https://debugtalk.com"
},
"verify": False
},
"teststeps": [{
'name': 'testcase1',
"base_url": "https://apiautotest.org",
'request': {
'url': '/api1',
'method': 'GET',
"verify": True
}
}]
}]
}
parsed_testcases = parser.parse_tests(tests_mapping)
test_dict = parsed_testcases[0]["teststeps"][0]
self.assertEqual(test_dict["request"]["url"], "/api1")
self.assertEqual(test_dict["request"]["verify"], True)
def test_run_testcase_with_hooks(self):
start_time = time.time()
testcases = [
{
"config": {
"name": "basic test with httpbin",
"base_url": HTTPBIN_SERVER,
"setup_hooks": [
"${sleep_N_secs(0.5)}",
"${hook_print(setup)}"
],
"teardown_hooks": [
"${sleep_N_secs(1)}",
"${hook_print(teardown)}"
]
},
"teststeps": [
{
"name": "get token",
"request": {
"url": "http://127.0.0.1:5000/api/get-token",
"method": "POST",
"headers": {
"content-type": "application/json",
"user_agent": "iOS/10.3",
"device_sn": "HZfFBh6tU59EdXJ",
"os_platform": "ios",
"app_version": "2.8.6"
},
"json": {
"sign": "5188962c489d1a35effa99e9346dd5efd4fdabad"
}
},
"validate": [
{"check": "status_code", "expect": 200}
]
}
]
}
]
tests_mapping = {
"project_mapping": {
"functions": self.debugtalk_functions
},
"testcases": testcases
}
parsed_testcases = parser.parse_tests(tests_mapping)
parsed_testcase = parsed_testcases[0]
test_runner = runner.Runner(parsed_testcase["config"])
end_time = time.time()
# check if testcase setup hook executed
self.assertGreater(end_time - start_time, 0.5)
start_time = time.time()
test_runner.run_test(parsed_testcase["teststeps"][0])
end_time = time.time()
# testcase teardown hook has not been executed now
self.assertLess(end_time - start_time, 1)
def test_testcase_simple_run_suite(self):
testcase_path = "tests/testcases/setup.yml"
tests_mapping = loader.load_tests(testcase_path)
testcases = parser.parse_tests(tests_mapping)
runner = apiautotest()
test_suite = runner._add_tests(testcases)
tests_results = runner._run_suite(test_suite)
self.assertEqual(len(tests_results[0][1].records), 2)
def test_bugfix_type_match(self):
testcase_file_path = os.path.join(
os.getcwd(), 'tests/data/bugfix_type_match.yml')
tests_mapping = loader.load_tests(testcase_file_path)
parsed_testcases = parser.parse_tests(tests_mapping)
parsed_testcase = parsed_testcases[0]
test_runner = runner.Runner(parsed_testcase["config"])
test_runner.run_test(parsed_testcase["teststeps"][0])
def test_testsuite_add_tests(self):
testcase_path = "tests/testsuites/create_users.yml"
tests_mapping = loader.load_tests(testcase_path)
testcases = parser.parse_tests(tests_mapping)
runner = apiautotest()
test_suite = runner._add_tests(testcases)
self.assertEqual(len(test_suite._tests), 2)
tests = test_suite._tests[0].teststeps
self.assertIn("setup and reset all (override)",
tests[0]["config"]["name"].raw_string)
def test_testcase_add_tests(self):
testcase_path = "tests/testcases/setup.yml"
tests_mapping = loader.load_tests(testcase_path)
testcases = parser.parse_tests(tests_mapping)
runner = apiautotest()
test_suite = runner._add_tests(testcases)
self.assertEqual(len(test_suite._tests), 1)
teststeps = test_suite._tests[0].teststeps
self.assertEqual(teststeps[0]["name"], "get token (setup)")
self.assertEqual(teststeps[0]["variables"]["device_sn"],
"TESTCASE_SETUP_XXX")
self.assertIn("api", teststeps[0])
def test_testcase_complex_run_suite(self):
testcase_path = "tests/testcases/create_user.yml"
tests_mapping = loader.load_tests(testcase_path)
testcases = parser.parse_tests(tests_mapping)
runner = apiautotest()
test_suite = runner._add_tests(testcases)
tests_results = runner._run_suite(test_suite)
self.assertEqual(len(tests_results[0][1].records), 2)
results = tests_results[0][1]
self.assertEqual(
results.records[0]["name"],
"setup and reset all (override) for TESTCASE_CREATE_XXX.")
self.assertEqual(results.records[1]["name"],
"create user and check result.")
def test_testsuite_parser(self):
testcase_path = "tests/testsuites/create_users.yml"
tests_mapping = loader.load_tests(testcase_path)
parsed_testcases = parser.parse_tests(tests_mapping)
self.assertEqual(len(parsed_testcases), 2)
self.assertEqual(len(parsed_testcases[0]["teststeps"]), 2)
testcase1 = parsed_testcases[0]["teststeps"][0]
self.assertIn("setup and reset all (override)",
testcase1["config"]["name"].raw_string)
teststeps = testcase1["teststeps"]
self.assertNotIn("testcase_def", testcase1)
self.assertEqual(len(teststeps), 2)
self.assertEqual(teststeps[0]["request"]["url"], "/api/get-token")
def test_run_single_testcase(self):
testcase_file_path_list = [
os.path.join(
os.getcwd(), 'tests/data/demo_testcase_hardcode.yml'),
os.path.join(
os.getcwd(), 'tests/data/demo_testcase_hardcode.json')
]
for testcase_file_path in testcase_file_path_list:
tests_mapping = loader.load_tests(testcase_file_path)
parsed_testcases = parser.parse_tests(tests_mapping)
parsed_testcase = parsed_testcases[0]
test_runner = runner.Runner(parsed_testcase["config"])
test_runner.run_test(parsed_testcase["teststeps"][0])
test_runner.run_test(parsed_testcase["teststeps"][1])
test_runner.run_test(parsed_testcase["teststeps"][2])
def test_testcase_complex_verify(self):
testcase_path = "tests/testcases/create_user.yml"
tests_mapping = loader.load_tests(testcase_path)
testcases = parser.parse_tests(tests_mapping)
teststeps = testcases[0]["teststeps"]
# testcases/setup.yml
teststep0 = teststeps[0]
self.assertEqual(teststep0["teststeps"][0]["request"]["verify"], False)
self.assertEqual(teststep0["teststeps"][1]["request"]["verify"], False)
# testcases/create_user.yml
teststep1 = teststeps[1]
self.assertEqual(teststep1["teststeps"][0]["request"]["verify"], True)
self.assertEqual(teststep1["teststeps"][1]["request"]["verify"], True)
self.assertEqual(teststep1["teststeps"][2]["request"]["verify"], True)
def test_parse_tests_variable_with_function(self):
tests_mapping = {
"project_mapping": {
"functions": {
"sum_two": sum_two,
"gen_random_string": gen_random_string
}
},
'testcases': [{
"config": {
'name': '',
"base_url": "$host1",
'variables': {
"host1": "https://debugtalk.com",
"var_a": "${gen_random_string(5)}",
"var_b": "$var_a"
}
},
"teststeps": [{
'name': 'testcase1',
"base_url": "$host2",
"variables": {
"host2": "https://apiautotest.org",
"num3": "${sum_two($num2, 4)}",
"num2": "${sum_two($num1, 3)}",
"num1": "${sum_two(1, 2)}",
"str1": "${gen_random_string(5)}",
"str2": "$str1"
},
'request': {
'url': '/api1/?num1=$num1&num2=$num2&num3=$num3',
'method': 'GET'
}
}]
}]
}
parsed_testcases = parser.parse_tests(tests_mapping)
test_dict = parsed_testcases[0]["teststeps"][0]
variables = parser.parse_variables_mapping(test_dict["variables"])
self.assertEqual(variables["num3"], 10)
self.assertEqual(variables["num2"], 6)
parsed_test_dict = parser.parse_lazy_data(test_dict, variables)
self.assertEqual(parsed_test_dict["base_url"],
"https://apiautotest.org")
self.assertEqual(parsed_test_dict["request"]["url"],
"/api1/?num1=3&num2=6&num3=10")
self.assertEqual(variables["str1"], variables["str2"])
def test_run_testcase_with_teardown_hooks_fail(self):
testcases = [
{
"config": {
"name": "basic test with httpbin"
},
"teststeps": [
{
"name": "get token",
"request": {
"url": "http://127.0.0.1:5000/api/get-token2",
"method": "POST",
"headers": {
"content-type": "application/json",
"user_agent": "iOS/10.3",
"device_sn": "HZfFBh6tU59EdXJ",
"os_platform": "ios",
"app_version": "2.8.6"
},
"json": {
"sign": "5188962c489d1a35effa99e9346dd5efd4fdabad"
}
},
"validate": [
{"check": "status_code", "expect": 404}
],
"teardown_hooks": ["${teardown_hook_sleep_N_secs($response, 2)}"]
}
]
}
]
tests_mapping = {
"project_mapping": {
"functions": self.debugtalk_functions
},
"testcases": testcases
}
parsed_testcases = parser.parse_tests(tests_mapping)
parsed_testcase = parsed_testcases[0]
test_runner = runner.Runner(parsed_testcase["config"])
start_time = time.time()
test_runner.run_test(parsed_testcase["teststeps"][0])
end_time = time.time()
# check if teardown function executed
self.assertGreater(end_time - start_time, 2)
def test_validate(self):
testcases = [{
"config": {
'name': "test validation"
},
"teststeps": [{
"name":
"test validation",
"request": {
"url": "http://127.0.0.1:5000/",
"method": "GET",
},
"variables": {
"resp_status_code": 200,
"resp_body_success": True
},
"validate": [{
"eq": ["$resp_status_code", 200]
}, {
"check": "$resp_status_code",
"comparator": "eq",
"expect": 200
}, {
"check": "$resp_body_success",
"expect": True
}, {
"check": "${is_status_code_200($resp_status_code)}",
"expect": True
}]
}]
}]
from tests.debugtalk import is_status_code_200
tests_mapping = {
"project_mapping": {
"functions": {
"is_status_code_200": is_status_code_200
}
},
"testcases": testcases
}
testcases = parser.parse_tests(tests_mapping)
parsed_testcase = testcases[0]
test_runner = runner.Runner(parsed_testcase["config"])
teststep = parsed_testcase["teststeps"][0]
test_runner.run_test(teststep)
def test_parse_tests_testcase(self):
testcase_file_path = os.path.join(os.getcwd(),
'tests/data/demo_testcase.yml')
tests_mapping = loader.load_tests(testcase_file_path)
testcases = tests_mapping["testcases"]
self.assertEqual(testcases[0]["config"]["variables"]["var_c"],
"${sum_two($var_a, $var_b)}")
self.assertEqual(testcases[0]["config"]["variables"]["PROJECT_KEY"],
"${ENV(PROJECT_KEY)}")
parsed_testcases = parser.parse_tests(tests_mapping)
self.assertIsInstance(parsed_testcases, list)
test_dict1 = parsed_testcases[0]["teststeps"][0]
self.assertEqual(test_dict1["variables"]["var_c"].raw_string,
"${sum_two($var_a, $var_b)}")
self.assertEqual(test_dict1["variables"]["PROJECT_KEY"].raw_string,
"${ENV(PROJECT_KEY)}")
self.assertIsInstance(parsed_testcases[0]["config"]["name"],
parser.LazyString)
def test_run_testcase_with_hooks_modify_request(self):
testcases = [
{
"config": {
"name": "basic test with httpbin",
"base_url": HTTPBIN_SERVER
},
"teststeps": [
{
"name": "modify request headers",
"base_url": HTTPBIN_SERVER,
"request": {
"url": "/anything",
"method": "POST",
"headers": {
"content-type": "application/json",
"user_agent": "iOS/10.3"
},
"json": {
"os_platform": "ios",
"sign": "5188962c489d1a35effa99e9346dd5efd4fdabad"
}
},
"setup_hooks": [
"${modify_request_json($request, android)}"
],
"validate": [
{"check": "status_code", "expect": 200},
{"check": "content.json.os_platform", "expect": "android"}
]
}
]
}
]
tests_mapping = {
"project_mapping": {
"functions": self.debugtalk_functions
},
"testcases": testcases
}
parsed_testcases = parser.parse_tests(tests_mapping)
parsed_testcase = parsed_testcases[0]
test_runner = runner.Runner(parsed_testcase["config"])
test_runner.run_test(parsed_testcase["teststeps"][0])
def test_run_testcase_with_hooks_assignment(self):
testcases = [
{
"config": {
"name": "basic test with httpbin",
"base_url": HTTPBIN_SERVER
},
"teststeps": [
{
"name": "modify request headers",
"base_url": HTTPBIN_SERVER,
"request": {
"url": "/anything",
"method": "POST",
"headers": {
"user_agent": "iOS/10.3",
"os_platform": "ios"
},
"data": "a=1&b=2"
},
"setup_hooks": [
{"total": "${sum_two(1, 5)}"}
],
"validate": [
{"check": "status_code", "expect": 200}
]
}
]
}
]
tests_mapping = {
"project_mapping": {
"functions": self.debugtalk_functions
},
"testcases": testcases
}
parsed_testcases = parser.parse_tests(tests_mapping)
parsed_testcase = parsed_testcases[0]
test_runner = runner.Runner(parsed_testcase["config"])
test_runner.run_test(parsed_testcase["teststeps"][0])
test_variables_mapping = test_runner.session_context.test_variables_mapping
self.assertEqual(test_variables_mapping["total"], 6)
self.assertEqual(test_variables_mapping["request"]["data"], "a=1&b=2")
def test_run_validate_elapsed(self):
testcases = [
{
"config": {},
"teststeps": [
{
"name": "get token",
"request": {
"url": "http://127.0.0.1:5000/api/get-token",
"method": "POST",
"headers": {
"content-type": "application/json",
"user_agent": "iOS/10.3",
"device_sn": "HZfFBh6tU59EdXJ",
"os_platform": "ios",
"app_version": "2.8.6"
},
"json": {
"sign": "5188962c489d1a35effa99e9346dd5efd4fdabad"
}
},
"validate": [
{"check": "status_code", "expect": 200},
{"check": "elapsed.seconds", "comparator": "lt", "expect": 1},
{"check": "elapsed.days", "comparator": "eq", "expect": 0},
{"check": "elapsed.microseconds", "comparator": "gt", "expect": 1000},
{"check": "elapsed.total_seconds", "comparator": "lt", "expect": 1}
]
}
]
}
]
tests_mapping = {
"project_mapping": {
"functions": self.debugtalk_functions
},
"testcases": testcases
}
parsed_testcases = parser.parse_tests(tests_mapping)
parsed_testcase = parsed_testcases[0]
test_runner = runner.Runner(parsed_testcase["config"])
test_runner.run_test(parsed_testcase["teststeps"][0])
def test_parse_tests_verify_config_unset(self):
""" verify priority: test_dict > config
"""
tests_mapping = {
'testcases': [{
"config": {
'name': 'bugfix verify',
"base_url": "https://httpbin.org/",
},
"teststeps": [{
'name': 'testcase1',
'request': {
'url': '/headers',
'method': 'GET'
}
}]
}]
}
parsed_testcases = parser.parse_tests(tests_mapping)
test_dict = parsed_testcases[0]["teststeps"][0]
self.assertEqual(test_dict["request"]["verify"], True)
def test_testcase_parser(self):
testcase_path = "tests/testcases/setup.yml"
tests_mapping = loader.load_tests(testcase_path)
parsed_testcases = parser.parse_tests(tests_mapping)
self.assertEqual(len(parsed_testcases), 1)
self.assertIn("variables", parsed_testcases[0]["config"])
self.assertEqual(len(parsed_testcases[0]["teststeps"]), 2)
test_dict1 = parsed_testcases[0]["teststeps"][0]
self.assertEqual(test_dict1["name"], "get token (setup)")
self.assertNotIn("api_def", test_dict1)
self.assertEqual(test_dict1["variables"]["device_sn"],
"TESTCASE_SETUP_XXX")
self.assertEqual(test_dict1["request"]["url"], "/api/get-token")
self.assertEqual(test_dict1["request"]["verify"], False)
test_dict2 = parsed_testcases[0]["teststeps"][1]
self.assertEqual(test_dict2["request"]["verify"], False)
def run_tests(self, tests_mapping):
""" run testcase/testsuite data
"""
project_mapping = tests_mapping.get("project_mapping", {})
if self.save_tests:
utils.dump_logs(tests_mapping, project_mapping, "loaded")
# parse tests
self.exception_stage = "parse tests"
parsed_testcases = parser.parse_tests(tests_mapping)
if self.save_tests:
utils.dump_logs(parsed_testcases, project_mapping, "parsed")
# add tests to test suite
self.exception_stage = "add tests to test suite"
test_suite = self._add_tests(parsed_testcases)
# run test suite
self.exception_stage = "run test suite"
results = self._run_suite(test_suite)
# aggregate results
self.exception_stage = "aggregate results"
self._summary = self._aggregate(results)
# generate html report
self.exception_stage = "generate html report"
report.stringify_summary(self._summary)
if self.save_tests:
utils.dump_logs(self._summary, project_mapping, "summary")
report_path = report.render_html_report(self._summary,
self.report_template,
self.report_dir)
return report_path
