def __init__(self, config_dict=None, http_client_session=None):
self.http_client_session = http_client_session
self.context = Context()
testcase.load_test_dependencies()
config_dict = config_dict or {}
self.init_config(config_dict, "testset")
def __init__(self, config_dict=None, http_client_session=None):
self.http_client_session = http_client_session
self.context = Context()
config_dict = config_dict or {}
self.init_config(config_dict, "testset")
# testset setup hooks
testset_setup_hooks = config_dict.pop("setup_hooks", [])
if testset_setup_hooks:
self.do_hook_actions(testset_setup_hooks)
# testset teardown hooks
self.testset_teardown_hooks = config_dict.pop("teardown_hooks", [])
def __init__(self, config_dict=None, http_client_session=None):
"""
"""
self.http_client_session = http_client_session
config_dict = config_dict or {}
self.evaluated_validators = []
# testcase variables
config_variables = config_dict.get("variables", {})
# testcase functions
config_functions = config_dict.get("functions", {})
# testcase setup hooks
testcase_setup_hooks = config_dict.pop("setup_hooks", [])
# testcase teardown hooks
self.testcase_teardown_hooks = config_dict.pop("teardown_hooks", [])
self.context = Context(config_variables, config_functions)
self.init_config(config_dict, "testcase")
if testcase_setup_hooks:
self.do_hook_actions(testcase_setup_hooks)
def __init__(self, config_dict=None, http_client_session=None):
self.http_client_session = http_client_session
self.context = Context()
config_dict = config_dict or {}
self.init_config(config_dict, "testset")
# testset setup hooks
testset_setup_hooks = config_dict.pop("setup_hooks", [])
if testset_setup_hooks:
self.do_hook_actions(testset_setup_hooks)
# testset teardown hooks
self.testset_teardown_hooks = config_dict.pop("teardown_hooks", [])
def setUp(self):
self.context = Context()
testcase_file_path = os.path.join(os.getcwd(),
'tests/data/demo_binds.yml')
self.testcases = testcase.load_file(testcase_file_path)
def setUp(self):
self.context = Context()
testcase_file_path = os.path.join(os.getcwd(), 'tests/data/demo_binds.yml')
self.testcases = FileUtils.load_file(testcase_file_path)
class VariableBindsUnittest(ApiServerUnittest):
def setUp(self):
self.context = Context()
testcase_file_path = os.path.join(os.getcwd(), 'tests/data/demo_binds.yml')
self.testcases = FileUtils.load_file(testcase_file_path)
def test_context_init_functions(self):
self.assertIn("get_timestamp", self.context.testset_functions_config)
self.assertIn("gen_random_string", self.context.testset_functions_config)
variables = [
{"random": "${gen_random_string(5)}"},
{"timestamp10": "${get_timestamp(10)}"}
]
self.context.bind_variables(variables)
context_variables = self.context.testcase_variables_mapping
self.assertEqual(len(context_variables["random"]), 5)
self.assertEqual(len(context_variables["timestamp10"]), 10)
def test_context_bind_testset_variables(self):
# testcase in JSON format
testcase1 = {
"variables": [
{"GLOBAL_TOKEN": "debugtalk"},
{"token": "$GLOBAL_TOKEN"}
]
}
# testcase in YAML format
testcase2 = self.testcases["bind_variables"]
for testcase in [testcase1, testcase2]:
variables = testcase['variables']
self.context.bind_variables(variables, level="testset")
testset_variables = self.context.testset_shared_variables_mapping
testcase_variables = self.context.testcase_variables_mapping
self.assertIn("GLOBAL_TOKEN", testset_variables)
self.assertIn("GLOBAL_TOKEN", testcase_variables)
self.assertEqual(testset_variables["GLOBAL_TOKEN"], "debugtalk")
self.assertIn("token", testset_variables)
self.assertIn("token", testcase_variables)
self.assertEqual(testset_variables["token"], "debugtalk")
def test_context_bind_testcase_variables(self):
testcase1 = {
"variables": [
{"GLOBAL_TOKEN": "debugtalk"},
{"token": "$GLOBAL_TOKEN"}
]
}
testcase2 = self.testcases["bind_variables"]
for testcase in [testcase1, testcase2]:
variables = testcase['variables']
self.context.bind_variables(variables)
testset_variables = self.context.testset_shared_variables_mapping
testcase_variables = self.context.testcase_variables_mapping
self.assertNotIn("GLOBAL_TOKEN", testset_variables)
self.assertIn("GLOBAL_TOKEN", testcase_variables)
self.assertEqual(testcase_variables["GLOBAL_TOKEN"], "debugtalk")
self.assertNotIn("token", testset_variables)
self.assertIn("token", testcase_variables)
self.assertEqual(testcase_variables["token"], "debugtalk")
def test_context_bind_lambda_functions(self):
testcase1 = {
"function_binds": {
"add_one": lambda x: x + 1,
"add_two_nums": lambda x, y: x + y
},
"variables": [
{"add1": "${add_one(2)}"},
{"sum2nums": "${add_two_nums(2,3)}"}
]
}
testcase2 = self.testcases["bind_lambda_functions"]
for testcase in [testcase1, testcase2]:
function_binds = testcase.get('function_binds', {})
self.context.bind_functions(function_binds)
variables = testcase['variables']
self.context.bind_variables(variables)
context_variables = self.context.testcase_variables_mapping
self.assertIn("add1", context_variables)
self.assertEqual(context_variables["add1"], 3)
self.assertIn("sum2nums", context_variables)
self.assertEqual(context_variables["sum2nums"], 5)
def test_call_builtin_functions(self):
testcase1 = {
"variables": [
{"length": "${len(debugtalk)}"},
{"smallest": "${min(2, 3, 8)}"},
{"largest": "${max(2, 3, 8)}"}
]
}
testcase2 = self.testcases["builtin_functions"]
for testcase in [testcase1, testcase2]:
variables = testcase['variables']
self.context.bind_variables(variables)
context_variables = self.context.testcase_variables_mapping
self.assertEqual(context_variables["length"], 9)
self.assertEqual(context_variables["smallest"], 2)
self.assertEqual(context_variables["largest"], 8)
def test_context_bind_lambda_functions_with_import(self):
testcase1 = {
"requires": ["random", "string", "hashlib"],
"function_binds": {
"gen_random_string": "lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(str_len))",
"gen_md5": "lambda *str_args: hashlib.md5(''.join(str_args).encode('utf-8')).hexdigest()"
},
"variables": [
{"TOKEN": "debugtalk"},
{"random": "${gen_random_string(5)}"},
{"data": '{"name": "user", "password": "******"}'},
{"authorization": "${gen_md5($TOKEN, $data, $random)}"}
]
}
testcase2 = self.testcases["bind_lambda_functions_with_import"]
for testcase in [testcase1, testcase2]:
requires = testcase.get('requires', [])
self.context.import_requires(requires)
function_binds = testcase.get('function_binds', {})
self.context.bind_functions(function_binds)
variables = testcase['variables']
self.context.bind_variables(variables)
context_variables = self.context.testcase_variables_mapping
self.assertIn("TOKEN", context_variables)
TOKEN = context_variables["TOKEN"]
self.assertEqual(TOKEN, "debugtalk")
self.assertIn("random", context_variables)
self.assertIsInstance(context_variables["random"], str)
self.assertEqual(len(context_variables["random"]), 5)
random = context_variables["random"]
self.assertIn("data", context_variables)
data = context_variables["data"]
self.assertIn("authorization", context_variables)
self.assertEqual(len(context_variables["authorization"]), 32)
authorization = context_variables["authorization"]
self.assertEqual(gen_md5(TOKEN, data, random), authorization)
def test_import_module_items(self):
testcase1 = {
"import_module_items": ["tests.debugtalk"],
"variables": [
{"TOKEN": "debugtalk"},
{"random": "${gen_random_string(5)}"},
{"data": '{"name": "user", "password": "******"}'},
{"authorization": "${gen_md5($TOKEN, $data, $random)}"}
]
}
testcase2 = self.testcases["bind_module_functions"]
for testcase in [testcase1, testcase2]:
module_items = testcase.get('import_module_items', [])
self.context.import_module_items(module_items)
variables = testcase['variables']
self.context.bind_variables(variables)
context_variables = self.context.testcase_variables_mapping
self.assertIn("TOKEN", context_variables)
TOKEN = context_variables["TOKEN"]
self.assertEqual(TOKEN, "debugtalk")
self.assertIn("random", context_variables)
self.assertIsInstance(context_variables["random"], str)
self.assertEqual(len(context_variables["random"]), 5)
random = context_variables["random"]
self.assertIn("data", context_variables)
data = context_variables["data"]
self.assertIn("authorization", context_variables)
self.assertEqual(len(context_variables["authorization"]), 32)
authorization = context_variables["authorization"]
self.assertEqual(gen_md5(TOKEN, data, random), authorization)
self.assertIn("SECRET_KEY", context_variables)
SECRET_KEY = context_variables["SECRET_KEY"]
self.assertEqual(SECRET_KEY, "DebugTalk")
def test_get_parsed_request(self):
test_runner = runner.Runner()
testcase = {
"import_module_items": ["tests.debugtalk"],
"variables": [
{"TOKEN": "debugtalk"},
{"random": "${gen_random_string(5)}"},
{"data": '{"name": "user", "password": "******"}'},
{"authorization": "${gen_md5($TOKEN, $data, $random)}"}
],
"request": {
"url": "http://127.0.0.1:5000/api/users/1000",
"METHOD": "POST",
"Headers": {
"Content-Type": "application/json",
"authorization": "$authorization",
"random": "$random",
"SECRET_KEY": "$SECRET_KEY"
},
"Data": "$data"
}
}
parsed_request = test_runner.init_config(testcase, level="testcase")
self.assertIn("authorization", parsed_request["headers"])
self.assertEqual(len(parsed_request["headers"]["authorization"]), 32)
self.assertIn("random", parsed_request["headers"])
self.assertEqual(len(parsed_request["headers"]["random"]), 5)
self.assertIn("data", parsed_request)
self.assertEqual(parsed_request["data"], testcase["variables"][2]["data"])
self.assertEqual(parsed_request["headers"]["secret_key"], "DebugTalk")
def test_exec_content_functions(self):
test_runner = runner.Runner()
content = "${sleep_N_secs(1)}"
start_time = time.time()
test_runner.context.eval_content(content)
end_time = time.time()
elapsed_time = end_time - start_time
self.assertGreater(elapsed_time, 1)
def test_do_validation(self):
self.context.do_validation(
{"check": "check", "check_value": 1, "expect": 1, "comparator": "eq"}
)
self.context.do_validation(
{"check": "check", "check_value": "abc", "expect": "abc", "comparator": "=="}
)
config_dict = {
"path": 'tests/data/demo_testset_hardcode.yml'
}
self.context.config_context(config_dict, "testset")
self.context.do_validation(
{"check": "status_code", "check_value": "201", "expect": 3, "comparator": "sum_status_code"}
)
def test_validate(self):
url = "http://127.0.0.1:5000/"
resp = requests.get(url)
resp_obj = response.ResponseObject(resp)
validators = [
{"eq": ["$resp_status_code", 201]},
{"check": "$resp_status_code", "comparator": "eq", "expect": 201},
{"check": "$resp_body_success", "comparator": "eq", "expect": True},
{"check": "${is_status_code_200($resp_status_code)}", "comparator": "eq", "expect": False}
]
variables = [
{"resp_status_code": 200},
{"resp_body_success": True}
]
self.context.bind_variables(variables)
with self.assertRaises(exception.ValidationError):
self.context.validate(validators, resp_obj)
variables = [
{"resp_status_code": 201},
{"resp_body_success": True}
]
self.context.bind_variables(variables)
from tests.debugtalk import is_status_code_200
functions = {
"is_status_code_200": is_status_code_200
}
self.context.bind_functions(functions)
self.assertTrue(self.context.validate(validators, resp_obj))
def test_validate_exception(self):
url = "http://127.0.0.1:5000/"
resp = requests.get(url)
resp_obj = response.ResponseObject(resp)
# expected value missed in validators
validators = [
{"eq": ["$resp_status_code", 201]},
{"check": "$resp_status_code", "comparator": "eq", "expect": 201},
{"check": "$resp_body_success", "comparator": "eq", "expect": True}
]
variables = []
self.context.bind_variables(variables)
with self.assertRaises(exception.ParamsError):
self.context.validate(validators, resp_obj)
# expected value missed in variables mapping
variables = [
{"resp_status_code": 200}
]
self.context.bind_variables(variables)
with self.assertRaises(exception.ValidationError):
self.context.validate(validators, resp_obj)
def __init__(self, http_client_session=None, request_failure_hook=None):
self.http_client_session = http_client_session
self.context = Context()
# testcase.load_test_dependencies()
self.request_failure_hook = request_failure_hook
class Runner(object):
def __init__(self, http_client_session=None, request_failure_hook=None):
self.http_client_session = http_client_session
self.context = Context()
# testcase.load_test_dependencies()
self.request_failure_hook = request_failure_hook
def init_config(self, config_dict, level):
""" create/update context variables binds
@param (dict) config_dict
@param (str) level, "testset" or "testcase"
testset:
{
"name": "smoke testset",
"path": "tests/data/demo_testset_variables.yml",
"requires": [], # optional
"function_binds": {}, # optional
"import_module_items": [], # optional
"variables": [], # optional
"request": {
"base_url": "http://127.0.0.1:5000",
"headers": {
"User-Agent": "iOS/2.8.3"
}
}
}
testcase:
{
"name": "testcase description",
"requires": [], # optional
"function_binds": {}, # optional
"import_module_items": [], # optional
"variables": [], # optional
"request": {
"url": "/api/get-token",
"method": "POST",
"headers": {
"Content-Type": "application/json"
}
},
"json": {
"sign": "f1219719911caae89ccc301679857ebfda115ca2"
}
}
@param (str) context level, testcase or testset
"""
# convert keys in request headers to lowercase
config_dict = utils.lower_config_dict_key(config_dict)
self.context.init_context(level)
self.context.config_context(config_dict, level)
request_config = config_dict.get('request', {})
parsed_request = self.context.get_parsed_request(request_config, level)
base_url = parsed_request.pop("base_url", None)
self.http_client_session = self.http_client_session or HttpSession(
base_url)
return parsed_request
def _run_test(self, testcase_dict):
""" run single testcase.
@param (dict) testcase_dict
{
"name": "testcase description",
"times": 3,
"requires": [], # optional, override
"function_binds": {}, # optional, override
"variables": [], # optional, override
"request": {
"url": "http://127.0.0.1:5000/api/users/1000",
"method": "POST",
"headers": {
"Content-Type": "application/json",
"authorization": "$authorization",
"random": "$random"
},
"body": '{"name": "user", "password": "******"}'
},
"extract": [], # optional
"validate": [], # optional
"setup": [], # optional
"teardown": [] # optional
}
@return True or raise exception during test
"""
parsed_request = self.init_config(testcase_dict, level="testcase")
try:
url = parsed_request.pop('url')
method = parsed_request.pop('method')
group_name = parsed_request.pop("group", None)
except KeyError:
raise exception.ParamsError("URL or METHOD missed!")
run_times = int(testcase_dict.get("times", 1))
extractors = testcase_dict.get("extract") \
or testcase_dict.get("extractors") \
or testcase_dict.get("extract_binds", [])
validators = testcase_dict.get("validate") \
or testcase_dict.get("validators", [])
setup_actions = testcase_dict.get("setup", [])
teardown_actions = testcase_dict.get("teardown", [])
def setup_teardown(actions):
for action in actions:
self.context.exec_content_functions(action)
for _ in range(run_times):
setup_teardown(setup_actions)
resp = self.http_client_session.request(method,
url,
name=group_name,
**parsed_request)
resp_obj = response.ResponseObject(resp)
extracted_variables_mapping = resp_obj.extract_response(extractors)
self.context.bind_extracted_variables(extracted_variables_mapping)
try:
self.context.validate(validators, resp_obj)
except (exception.ParamsError, exception.ResponseError,
exception.ValidationError):
err_msg = u"Exception occured.\n"
err_msg += u"HTTP request url: {}\n".format(url)
err_msg += u"HTTP request kwargs: {}\n".format(parsed_request)
err_msg += u"HTTP response status_code: {}\n".format(
resp.status_code)
err_msg += u"HTTP response content: \n{}".format(resp.text)
logging.error(err_msg)
raise
finally:
setup_teardown(teardown_actions)
return True
def _run_testset(self, testset, variables_mapping=None):
""" run single testset, including one or several testcases.
@param
(dict) testset
{
"name": "testset description",
"config": {
"name": "testset description",
"requires": [],
"function_binds": {},
"variables": [],
"request": {}
},
"testcases": [
{
"name": "testcase description",
"variables": [], # optional, override
"request": {},
"extract": {}, # optional
"validate": {} # optional
},
testcase12
]
}
(dict) variables_mapping:
passed in variables mapping, it will override variables in config block
@return (dict) test result of testset
{
"success": True,
"output": {} # variables mapping
}
"""
success = True
config_dict = testset.get("config", {})
variables = config_dict.get("variables", [])
variables_mapping = variables_mapping or {}
config_dict["variables"] = utils.override_variables_binds(
variables, variables_mapping)
self.init_config(config_dict, level="testset")
testcases = testset.get("testcases", [])
for testcase_dict in testcases:
try:
self._run_test(testcase_dict)
except exception.MyBaseError as ex:
success = False
if self.request_failure_hook:
self.request_failure_hook.fire(
request_type=testcase_dict.get("request",
{}).get("method"),
name=testcase_dict.get("request", {}).get("url"),
response_time=0,
exception=ex)
else:
logging.exception(
"Exception occured in testcase: {}".format(
testcase_dict.get("name")))
break
output_variables_list = config_dict.get("output", [])
return {
"success": success,
"output": self.generate_output(output_variables_list)
}
def run(self, path, mapping=None):
""" run specified testset path or folder path.
@param
path: path could be in several type
- absolute/relative file path
- absolute/relative folder path
- list/set container with file(s) and/or folder(s)
(dict) mapping:
passed in variables mapping, it will override variables in config block
"""
success = True
mapping = mapping or {}
output = {}
testsets = testcase.load_testcases_by_path(path)
for testset in testsets:
try:
result = self._run_testset(testset, mapping)
assert result["success"]
output.update(result["output"])
except AssertionError:
success = False
return {"success": success, "output": output}
def generate_output(self, output_variables_list):
""" generate and print output
"""
variables_mapping = self.context.get_testcase_variables_mapping()
output = {
variable: variables_mapping[variable]
for variable in output_variables_list
}
utils.print_output(output)
return output
def __init__(self, http_client_session=None, request_failure_hook=None):
self.http_client_session = http_client_session
self.context = Context()
testcase.load_test_dependencies()
self.request_failure_hook = request_failure_hook
class VariableBindsUnittest(unittest.TestCase):
def setUp(self):
self.context = Context()
testcase_file_path = os.path.join(os.getcwd(), 'tests/data/demo_binds.yml')
self.testcases = testcase._load_file(testcase_file_path)
def test_context_init_functions(self):
self.assertIn("get_timestamp", self.context.testset_functions_config)
self.assertIn("gen_random_string", self.context.testset_functions_config)
variables = [
{"random": "${gen_random_string(5)}"},
{"timestamp10": "${get_timestamp(10)}"}
]
self.context.bind_variables(variables)
context_variables = self.context.get_testcase_variables_mapping()
self.assertEqual(len(context_variables["random"]), 5)
self.assertEqual(len(context_variables["timestamp10"]), 10)
def test_context_bind_testset_variables(self):
# testcase in JSON format
testcase1 = {
"variables": [
{"GLOBAL_TOKEN": "debugtalk"},
{"token": "$GLOBAL_TOKEN"}
]
}
# testcase in YAML format
testcase2 = self.testcases["bind_variables"]
for testcase in [testcase1, testcase2]:
variables = testcase['variables']
self.context.bind_variables(variables, level="testset")
testset_variables = self.context.testset_shared_variables_mapping
testcase_variables = self.context.get_testcase_variables_mapping()
self.assertIn("GLOBAL_TOKEN", testset_variables)
self.assertIn("GLOBAL_TOKEN", testcase_variables)
self.assertEqual(testset_variables["GLOBAL_TOKEN"], "debugtalk")
self.assertIn("token", testset_variables)
self.assertIn("token", testcase_variables)
self.assertEqual(testset_variables["token"], "debugtalk")
def test_context_bind_testcase_variables(self):
testcase1 = {
"variables": [
{"GLOBAL_TOKEN": "debugtalk"},
{"token": "$GLOBAL_TOKEN"}
]
}
testcase2 = self.testcases["bind_variables"]
for testcase in [testcase1, testcase2]:
variables = testcase['variables']
self.context.bind_variables(variables)
testset_variables = self.context.testset_shared_variables_mapping
testcase_variables = self.context.get_testcase_variables_mapping()
self.assertNotIn("GLOBAL_TOKEN", testset_variables)
self.assertIn("GLOBAL_TOKEN", testcase_variables)
self.assertEqual(testcase_variables["GLOBAL_TOKEN"], "debugtalk")
self.assertNotIn("token", testset_variables)
self.assertIn("token", testcase_variables)
self.assertEqual(testcase_variables["token"], "debugtalk")
def test_context_bind_lambda_functions(self):
testcase1 = {
"function_binds": {
"add_one": lambda x: x + 1,
"add_two_nums": lambda x, y: x + y
},
"variables": [
{"add1": "${add_one(2)}"},
{"sum2nums": "${add_two_nums(2,3)}"}
]
}
testcase2 = self.testcases["bind_lambda_functions"]
for testcase in [testcase1, testcase2]:
function_binds = testcase.get('function_binds', {})
self.context.bind_functions(function_binds)
variables = testcase['variables']
self.context.bind_variables(variables)
context_variables = self.context.get_testcase_variables_mapping()
self.assertIn("add1", context_variables)
self.assertEqual(context_variables["add1"], 3)
self.assertIn("sum2nums", context_variables)
self.assertEqual(context_variables["sum2nums"], 5)
def test_context_bind_lambda_functions_with_import(self):
testcase1 = {
"requires": ["random", "string", "hashlib"],
"function_binds": {
"gen_random_string": "lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(str_len))",
"gen_md5": "lambda *str_args: hashlib.md5(''.join(str_args).encode('utf-8')).hexdigest()"
},
"variables": [
{"TOKEN": "debugtalk"},
{"random": "${gen_random_string(5)}"},
{"data": '{"name": "user", "password": "******"}'},
{"authorization": "${gen_md5($TOKEN, $data, $random)}"}
]
}
testcase2 = self.testcases["bind_lambda_functions_with_import"]
for testcase in [testcase1, testcase2]:
requires = testcase.get('requires', [])
self.context.import_requires(requires)
function_binds = testcase.get('function_binds', {})
self.context.bind_functions(function_binds)
variables = testcase['variables']
self.context.bind_variables(variables)
context_variables = self.context.get_testcase_variables_mapping()
self.assertIn("TOKEN", context_variables)
TOKEN = context_variables["TOKEN"]
self.assertEqual(TOKEN, "debugtalk")
self.assertIn("random", context_variables)
self.assertIsInstance(context_variables["random"], str)
self.assertEqual(len(context_variables["random"]), 5)
random = context_variables["random"]
self.assertIn("data", context_variables)
data = context_variables["data"]
self.assertIn("authorization", context_variables)
self.assertEqual(len(context_variables["authorization"]), 32)
authorization = context_variables["authorization"]
self.assertEqual(utils.gen_md5(TOKEN, data, random), authorization)
def test_import_module_items(self):
testcase1 = {
"import_module_items": ["tests.data.debugtalk"],
"variables": [
{"TOKEN": "debugtalk"},
{"random": "${gen_random_string(5)}"},
{"data": '{"name": "user", "password": "******"}'},
{"authorization": "${gen_md5($TOKEN, $data, $random)}"}
]
}
testcase2 = self.testcases["bind_module_functions"]
for testcase in [testcase1, testcase2]:
module_items = testcase.get('import_module_items', [])
self.context.import_module_items(module_items)
variables = testcase['variables']
self.context.bind_variables(variables)
context_variables = self.context.get_testcase_variables_mapping()
self.assertIn("TOKEN", context_variables)
TOKEN = context_variables["TOKEN"]
self.assertEqual(TOKEN, "debugtalk")
self.assertIn("random", context_variables)
self.assertIsInstance(context_variables["random"], str)
self.assertEqual(len(context_variables["random"]), 5)
random = context_variables["random"]
self.assertIn("data", context_variables)
data = context_variables["data"]
self.assertIn("authorization", context_variables)
self.assertEqual(len(context_variables["authorization"]), 32)
authorization = context_variables["authorization"]
self.assertEqual(utils.gen_md5(TOKEN, data, random), authorization)
self.assertIn("SECRET_KEY", context_variables)
SECRET_KEY = context_variables["SECRET_KEY"]
self.assertEqual(SECRET_KEY, "DebugTalk")
def test_get_parsed_request(self):
test_runner = runner.Runner()
testcase = {
"import_module_items": ["tests.data.debugtalk"],
"variables": [
{"TOKEN": "debugtalk"},
{"random": "${gen_random_string(5)}"},
{"data": '{"name": "user", "password": "******"}'},
{"authorization": "${gen_md5($TOKEN, $data, $random)}"}
],
"request": {
"url": "http://127.0.0.1:5000/api/users/1000",
"METHOD": "POST",
"Headers": {
"Content-Type": "application/json",
"authorization": "$authorization",
"random": "$random",
"SECRET_KEY": "$SECRET_KEY"
},
"Data": "$data"
}
}
parsed_request = test_runner.init_config(testcase, level="testcase")
self.assertIn("authorization", parsed_request["headers"])
self.assertEqual(len(parsed_request["headers"]["authorization"]), 32)
self.assertIn("random", parsed_request["headers"])
self.assertEqual(len(parsed_request["headers"]["random"]), 5)
self.assertIn("data", parsed_request)
self.assertEqual(parsed_request["data"], testcase["variables"][2]["data"])
self.assertEqual(parsed_request["headers"]["SECRET_KEY"], "DebugTalk")
def test_exec_content_functions(self):
test_runner = runner.Runner()
content = "${sleep(1)}"
start_time = time.time()
test_runner.context.exec_content_functions(content)
end_time = time.time()
elapsed_time = end_time - start_time
self.assertGreater(elapsed_time, 1)
def __init__(self, config_dict=None, http_client_session=None):
self.http_client_session = http_client_session
self.context = Context()
config_dict = config_dict or {}
self.init_config(config_dict, "testset")
class Runner(object):
def __init__(self, config_dict=None, http_client_session=None):
self.http_client_session = http_client_session
self.context = Context()
config_dict = config_dict or {}
self.init_config(config_dict, "testset")
def init_config(self, config_dict, level):
""" create/update context variables binds
@param (dict) config_dict
@param (str) level, "testset" or "testcase"
testset:
{
"name": "smoke testset",
"path": "tests/data/demo_testset_variables.yml",
"requires": [], # optional
"function_binds": {}, # optional
"import_module_items": [], # optional
"variables": [], # optional
"request": {
"base_url": "http://127.0.0.1:5000",
"headers": {
"User-Agent": "iOS/2.8.3"
}
}
}
testcase:
{
"name": "testcase description",
"requires": [], # optional
"function_binds": {}, # optional
"import_module_items": [], # optional
"variables": [], # optional
"request": {
"url": "/api/get-token",
"method": "POST",
"headers": {
"Content-Type": "application/json"
}
},
"json": {
"sign": "f1219719911caae89ccc301679857ebfda115ca2"
}
}
@param (str) context level, testcase or testset
"""
# convert keys in request headers to lowercase
config_dict = utils.lower_config_dict_key(config_dict)
self.context.init_context(level)
self.context.config_context(config_dict, level)
request_config = config_dict.get('request', {})
parsed_request = self.context.get_parsed_request(request_config, level)
base_url = parsed_request.pop("base_url", None)
self.http_client_session = self.http_client_session or HttpSession(
base_url)
return parsed_request
def _handle_skip_feature(self, testcase_dict):
""" handle skip feature for testcase
- skip: skip current test unconditionally
- skipIf: skip current test if condition is true
- skipUnless: skip current test unless condition is true
"""
skip_reason = None
if "skip" in testcase_dict:
skip_reason = testcase_dict["skip"]
elif "skipIf" in testcase_dict:
skip_if_condition = testcase_dict["skipIf"]
if self.context.eval_content(skip_if_condition):
skip_reason = "{} evaluate to True".format(skip_if_condition)
elif "skipUnless" in testcase_dict:
skip_unless_condition = testcase_dict["skipUnless"]
if not self.context.eval_content(skip_unless_condition):
skip_reason = "{} evaluate to False".format(
skip_unless_condition)
if skip_reason:
raise SkipTest(skip_reason)
def _prepare_hooks_event(self, hooks):
if not hooks:
return None
event = EventHook()
for hook in hooks:
func = self.context.testcase_parser.get_bind_function(hook)
event += func
return event
def _call_setup_hooks(self, hooks, method, url, kwargs):
""" call hook functions before request
Listeners should take the following arguments:
* *method*: request method type, e.g. GET, POST, PUT
* *url*: URL that was called (or override name if it was used in the call to the client)
* *kwargs*: kwargs of request
"""
hooks.insert(0, "setup_hook_prepare_kwargs")
event = self._prepare_hooks_event(hooks)
if not event:
return
event.fire(method=method, url=url, kwargs=kwargs)
def _call_teardown_hooks(self, hooks, resp_obj):
""" call hook functions after request
Listeners should take the following arguments:
* *resp_obj*: response object
"""
event = self._prepare_hooks_event(hooks)
if not event:
return
event.fire(resp_obj=resp_obj)
def run_test(self, testcase_dict):
""" run single testcase.
@param (dict) testcase_dict
{
"name": "testcase description",
"skip": "skip this test unconditionally",
"times": 3,
"requires": [], # optional, override
"function_binds": {}, # optional, override
"variables": [], # optional, override
"request": {
"url": "http://127.0.0.1:5000/api/users/1000",
"method": "POST",
"headers": {
"Content-Type": "application/json",
"authorization": "$authorization",
"random": "$random"
},
"body": '{"name": "user", "password": "******"}'
},
"extract": [], # optional
"validate": [], # optional
"setup_hooks": [], # optional
"teardown_hooks": [] # optional
}
@return True or raise exception during test
"""
parsed_request = self.init_config(testcase_dict, level="testcase")
try:
url = parsed_request.pop('url')
method = parsed_request.pop('method')
group_name = parsed_request.pop("group", None)
except KeyError:
raise exception.ParamsError("URL or METHOD missed!")
self._handle_skip_feature(testcase_dict)
extractors = testcase_dict.get("extract", []) or testcase_dict.get(
"extractors", [])
validators = testcase_dict.get("validate", []) or testcase_dict.get(
"validators", [])
setup_hooks = testcase_dict.get("setup_hooks", [])
teardown_hooks = testcase_dict.get("teardown_hooks", [])
logger.log_info("{method} {url}".format(method=method, url=url))
logger.log_debug(
"request kwargs(raw): {kwargs}".format(kwargs=parsed_request))
self._call_setup_hooks(setup_hooks, method, url, parsed_request)
resp = self.http_client_session.request(method,
url,
name=group_name,
**parsed_request)
self._call_teardown_hooks(teardown_hooks, resp)
resp_obj = response.ResponseObject(resp)
extracted_variables_mapping = resp_obj.extract_response(extractors)
self.context.bind_extracted_variables(extracted_variables_mapping)
try:
self.context.validate(validators, resp_obj)
except (exception.ParamsError, exception.ResponseError, \
exception.ValidationError, exception.ParseResponseError):
# log request
err_req_msg = "request: \n"
err_req_msg += "headers: {}\n".format(
parsed_request.pop("headers", {}))
for k, v in parsed_request.items():
err_req_msg += "{}: {}\n".format(k, v)
logger.log_error(err_req_msg)
# log response
err_resp_msg = "response: \n"
err_resp_msg += "status_code: {}\n".format(resp.status_code)
err_resp_msg += "headers: {}\n".format(resp.headers)
err_resp_msg += "body: {}\n".format(resp.text)
logger.log_error(err_resp_msg)
raise
def extract_output(self, output_variables_list):
""" extract output variables
"""
variables_mapping = self.context.testcase_variables_mapping
output = {}
for variable in output_variables_list:
if variable not in variables_mapping:
logger.log_warning(
"variable '{}' can not be found in variables mapping, failed to ouput!"\
.format(variable)
)
continue
output[variable] = variables_mapping[variable]
return output
class VariableBindsUnittest(ApiServerUnittest):
def setUp(self):
self.context = Context()
testcase_file_path = os.path.join(os.getcwd(),
'tests/data/demo_binds.yml')
self.testcases = FileUtils.load_file(testcase_file_path)
def test_context_init_functions(self):
self.assertIn("get_timestamp", self.context.testset_functions_config)
self.assertIn("gen_random_string",
self.context.testset_functions_config)
variables = [{
"random": "${gen_random_string(5)}"
}, {
"timestamp10": "${get_timestamp(10)}"
}]
self.context.bind_variables(variables)
context_variables = self.context.testcase_variables_mapping
self.assertEqual(len(context_variables["random"]), 5)
self.assertEqual(len(context_variables["timestamp10"]), 10)
def test_context_bind_testset_variables(self):
# testcase in JSON format
testcase1 = {
"variables": [{
"GLOBAL_TOKEN": "debugtalk"
}, {
"token": "$GLOBAL_TOKEN"
}]
}
# testcase in YAML format
testcase2 = self.testcases["bind_variables"]
for testcase in [testcase1, testcase2]:
variables = testcase['variables']
self.context.bind_variables(variables, level="testset")
testset_variables = self.context.testset_shared_variables_mapping
testcase_variables = self.context.testcase_variables_mapping
self.assertIn("GLOBAL_TOKEN", testset_variables)
self.assertIn("GLOBAL_TOKEN", testcase_variables)
self.assertEqual(testset_variables["GLOBAL_TOKEN"], "debugtalk")
self.assertIn("token", testset_variables)
self.assertIn("token", testcase_variables)
self.assertEqual(testset_variables["token"], "debugtalk")
def test_context_bind_testcase_variables(self):
testcase1 = {
"variables": [{
"GLOBAL_TOKEN": "debugtalk"
}, {
"token": "$GLOBAL_TOKEN"
}]
}
testcase2 = self.testcases["bind_variables"]
for testcase in [testcase1, testcase2]:
variables = testcase['variables']
self.context.bind_variables(variables)
testset_variables = self.context.testset_shared_variables_mapping
testcase_variables = self.context.testcase_variables_mapping
self.assertNotIn("GLOBAL_TOKEN", testset_variables)
self.assertIn("GLOBAL_TOKEN", testcase_variables)
self.assertEqual(testcase_variables["GLOBAL_TOKEN"], "debugtalk")
self.assertNotIn("token", testset_variables)
self.assertIn("token", testcase_variables)
self.assertEqual(testcase_variables["token"], "debugtalk")
def test_context_bind_lambda_functions(self):
function_binds = {
"add_one": lambda x: x + 1,
"add_two_nums": lambda x, y: x + y
}
variables = [{
"add1": "${add_one(2)}"
}, {
"sum2nums": "${add_two_nums(2,3)}"
}]
self.context.bind_functions(function_binds)
self.context.bind_variables(variables)
context_variables = self.context.testcase_variables_mapping
self.assertIn("add1", context_variables)
self.assertEqual(context_variables["add1"], 3)
self.assertIn("sum2nums", context_variables)
self.assertEqual(context_variables["sum2nums"], 5)
def test_call_builtin_functions(self):
testcase1 = {
"variables": [{
"length": "${len(debugtalk)}"
}, {
"smallest": "${min(2, 3, 8)}"
}, {
"largest": "${max(2, 3, 8)}"
}]
}
testcase2 = self.testcases["builtin_functions"]
for testcase in [testcase1, testcase2]:
variables = testcase['variables']
self.context.bind_variables(variables)
context_variables = self.context.testcase_variables_mapping
self.assertEqual(context_variables["length"], 9)
self.assertEqual(context_variables["smallest"], 2)
self.assertEqual(context_variables["largest"], 8)
def test_import_module_items(self):
variables = [{
"TOKEN": "debugtalk"
}, {
"random": "${gen_random_string(5)}"
}, {
"data": '{"name": "user", "password": "******"}'
}, {
"authorization": "${gen_md5($TOKEN, $data, $random)}"
}]
from tests import debugtalk
self.context.import_module_items(debugtalk)
self.context.bind_variables(variables)
context_variables = self.context.testcase_variables_mapping
self.assertIn("TOKEN", context_variables)
TOKEN = context_variables["TOKEN"]
self.assertEqual(TOKEN, "debugtalk")
self.assertIn("random", context_variables)
self.assertIsInstance(context_variables["random"], str)
self.assertEqual(len(context_variables["random"]), 5)
random = context_variables["random"]
self.assertIn("data", context_variables)
data = context_variables["data"]
self.assertIn("authorization", context_variables)
self.assertEqual(len(context_variables["authorization"]), 32)
authorization = context_variables["authorization"]
self.assertEqual(gen_md5(TOKEN, data, random), authorization)
self.assertIn("SECRET_KEY", context_variables)
SECRET_KEY = context_variables["SECRET_KEY"]
self.assertEqual(SECRET_KEY, "DebugTalk")
def test_get_parsed_request(self):
test_runner = runner.Runner()
testcase = {
"variables": [{
"TOKEN": "debugtalk"
}, {
"random": "${gen_random_string(5)}"
}, {
"data": '{"name": "user", "password": "******"}'
}, {
"authorization": "${gen_md5($TOKEN, $data, $random)}"
}],
"request": {
"url": "http://127.0.0.1:5000/api/users/1000",
"method": "POST",
"headers": {
"Content-Type": "application/json",
"authorization": "$authorization",
"random": "$random",
"secret_key": "$SECRET_KEY"
},
"data": "$data"
}
}
from tests import debugtalk
self.context.import_module_items(debugtalk)
self.context.bind_variables(testcase["variables"])
parsed_request = self.context.get_parsed_request(testcase["request"])
self.assertIn("authorization", parsed_request["headers"])
self.assertEqual(len(parsed_request["headers"]["authorization"]), 32)
self.assertIn("random", parsed_request["headers"])
self.assertEqual(len(parsed_request["headers"]["random"]), 5)
self.assertIn("data", parsed_request)
self.assertEqual(parsed_request["data"],
testcase["variables"][2]["data"])
self.assertEqual(parsed_request["headers"]["secret_key"], "DebugTalk")
def test_exec_content_functions(self):
test_runner = runner.Runner()
content = "${sleep_N_secs(1)}"
start_time = time.time()
test_runner.context.eval_content(content)
end_time = time.time()
elapsed_time = end_time - start_time
self.assertGreater(elapsed_time, 1)
def test_do_validation(self):
self.context.do_validation({
"check": "check",
"check_value": 1,
"expect": 1,
"comparator": "eq"
})
self.context.do_validation({
"check": "check",
"check_value": "abc",
"expect": "abc",
"comparator": "=="
})
config_dict = {"path": 'tests/data/demo_testset_hardcode.yml'}
self.context.config_context(config_dict, "testset")
self.context.do_validation({
"check": "status_code",
"check_value": "201",
"expect": 3,
"comparator": "sum_status_code"
})
def test_validate(self):
url = "http://127.0.0.1:5000/"
resp = requests.get(url)
resp_obj = response.ResponseObject(resp)
validators = [{
"eq": ["$resp_status_code", 201]
}, {
"check": "$resp_status_code",
"comparator": "eq",
"expect": 201
}, {
"check": "$resp_body_success",
"comparator": "eq",
"expect": True
}]
variables = [{"resp_status_code": 200}, {"resp_body_success": True}]
self.context.bind_variables(variables)
with self.assertRaises(exceptions.ValidationFailure):
self.context.validate(validators, resp_obj)
validators = [{
"eq": ["$resp_status_code", 201]
}, {
"check": "$resp_status_code",
"comparator": "eq",
"expect": 201
}, {
"check": "$resp_body_success",
"comparator": "eq",
"expect": True
}, {
"check": "${is_status_code_200($resp_status_code)}",
"comparator": "eq",
"expect": False
}]
variables = [{"resp_status_code": 201}, {"resp_body_success": True}]
self.context.bind_variables(variables)
from tests.debugtalk import is_status_code_200
functions = {"is_status_code_200": is_status_code_200}
self.context.bind_functions(functions)
self.context.validate(validators, resp_obj)
def test_validate_exception(self):
url = "http://127.0.0.1:5000/"
resp = requests.get(url)
resp_obj = response.ResponseObject(resp)
# expected value missed in validators
validators = [{
"eq": ["$resp_status_code", 201]
}, {
"check": "$resp_status_code",
"comparator": "eq",
"expect": 201
}]
variables = []
self.context.bind_variables(variables)
with self.assertRaises(exceptions.ParamsError):
self.context.validate(validators, resp_obj)
# expected value missed in variables mapping
variables = [{"resp_status_code": 200}]
self.context.bind_variables(variables)
with self.assertRaises(exceptions.ValidationFailure):
self.context.validate(validators, resp_obj)
class Runner(object):
def __init__(self, config_dict=None, http_client_session=None):
self.http_client_session = http_client_session
self.context = Context()
config_dict = config_dict or {}
self.init_config(config_dict, "testset")
# testset setup hooks
testset_setup_hooks = config_dict.pop("setup_hooks", [])
if testset_setup_hooks:
self.do_hook_actions(testset_setup_hooks)
# testset teardown hooks
self.testset_teardown_hooks = config_dict.pop("teardown_hooks", [])
def __del__(self):
if self.testset_teardown_hooks:
self.do_hook_actions(self.testset_teardown_hooks)
def init_config(self, config_dict, level):
""" create/update context variables binds
@param (dict) config_dict
@param (str) level, "testset" or "testcase"
testset:
{
"name": "smoke testset",
"path": "tests/data/demo_testset_variables.yml",
"requires": [], # optional
"function_binds": {}, # optional
"import_module_items": [], # optional
"variables": [], # optional
"request": {
"base_url": "http://127.0.0.1:5000",
"headers": {
"User-Agent": "iOS/2.8.3"
}
}
}
testcase:
{
"name": "testcase description",
"requires": [], # optional
"function_binds": {}, # optional
"import_module_items": [], # optional
"variables": [], # optional
"request": {
"url": "/api/get-token",
"method": "POST",
"headers": {
"Content-Type": "application/json"
}
},
"json": {
"sign": "f1219719911caae89ccc301679857ebfda115ca2"
}
}
@param (str) context level, testcase or testset
"""
# convert keys in request headers to lowercase
config_dict = utils.lower_config_dict_key(config_dict)
self.context.init_context(level)
self.context.config_context(config_dict, level)
request_config = config_dict.get('request', {})
parsed_request = self.context.get_parsed_request(request_config, level)
base_url = parsed_request.pop("base_url", None)
self.http_client_session = self.http_client_session or HttpSession(base_url)
return parsed_request
def _handle_skip_feature(self, testcase_dict):
""" handle skip feature for testcase
- skip: skip current test unconditionally
- skipIf: skip current test if condition is true
- skipUnless: skip current test unless condition is true
"""
skip_reason = None
if "skip" in testcase_dict:
skip_reason = testcase_dict["skip"]
elif "skipIf" in testcase_dict:
skip_if_condition = testcase_dict["skipIf"]
if self.context.eval_content(skip_if_condition):
skip_reason = "{} evaluate to True".format(skip_if_condition)
elif "skipUnless" in testcase_dict:
skip_unless_condition = testcase_dict["skipUnless"]
if not self.context.eval_content(skip_unless_condition):
skip_reason = "{} evaluate to False".format(skip_unless_condition)
if skip_reason:
raise SkipTest(skip_reason)
def do_hook_actions(self, actions):
for action in actions:
logger.log_debug("call hook: {}".format(action))
self.context.eval_content(action)
def run_test(self, testcase_dict):
""" run single testcase.
@param (dict) testcase_dict
{
"name": "testcase description",
"skip": "skip this test unconditionally",
"times": 3,
"requires": [], # optional, override
"function_binds": {}, # optional, override
"variables": [], # optional, override
"request": {
"url": "http://127.0.0.1:5000/api/users/1000",
"method": "POST",
"headers": {
"Content-Type": "application/json",
"authorization": "$authorization",
"random": "$random"
},
"body": '{"name": "user", "password": "******"}'
},
"extract": [], # optional
"validate": [], # optional
"setup_hooks": [], # optional
"teardown_hooks": [] # optional
}
@return True or raise exception during test
"""
# check skip
self._handle_skip_feature(testcase_dict)
# prepare
parsed_request = self.init_config(testcase_dict, level="testcase")
self.context.bind_testcase_variable("request", parsed_request)
# setup hooks
setup_hooks = testcase_dict.get("setup_hooks", [])
setup_hooks.insert(0, "${setup_hook_prepare_kwargs($request)}")
self.do_hook_actions(setup_hooks)
try:
url = parsed_request.pop('url')
method = parsed_request.pop('method')
group_name = parsed_request.pop("group", None)
except KeyError:
raise exception.ParamsError("URL or METHOD missed!")
logger.log_info("{method} {url}".format(method=method, url=url))
logger.log_debug("request kwargs(raw): {kwargs}".format(kwargs=parsed_request))
# request
resp = self.http_client_session.request(
method,
url,
name=group_name,
**parsed_request
)
resp_obj = response.ResponseObject(resp)
# teardown hooks
teardown_hooks = testcase_dict.get("teardown_hooks", [])
if teardown_hooks:
self.context.bind_testcase_variable("response", resp_obj)
self.do_hook_actions(teardown_hooks)
# extract
extractors = testcase_dict.get("extract", []) or testcase_dict.get("extractors", [])
extracted_variables_mapping = resp_obj.extract_response(extractors)
self.context.bind_extracted_variables(extracted_variables_mapping)
# validate
validators = testcase_dict.get("validate", []) or testcase_dict.get("validators", [])
try:
self.context.validate(validators, resp_obj)
except (exception.ParamsError, exception.ResponseError, \
exception.ValidationError, exception.ParseResponseError):
# log request
err_req_msg = "request: \n"
err_req_msg += "headers: {}\n".format(parsed_request.pop("headers", {}))
for k, v in parsed_request.items():
err_req_msg += "{}: {}\n".format(k, v)
logger.log_error(err_req_msg)
# log response
err_resp_msg = "response: \n"
err_resp_msg += "status_code: {}\n".format(resp_obj.status_code)
err_resp_msg += "headers: {}\n".format(resp_obj.headers)
err_resp_msg += "content: {}\n".format(resp_obj.content)
logger.log_error(err_resp_msg)
raise
def extract_output(self, output_variables_list):
""" extract output variables
"""
variables_mapping = self.context.testcase_variables_mapping
output = {}
for variable in output_variables_list:
if variable not in variables_mapping:
logger.log_warning(
"variable '{}' can not be found in variables mapping, failed to output!"\
.format(variable)
)
continue
output[variable] = variables_mapping[variable]
return output
class VariableBindsUnittest(ApiServerUnittest):
def setUp(self):
self.context = Context()
testcase_file_path = os.path.join(os.getcwd(),
'tests/data/demo_binds.yml')
self.testcases = testcase.load_file(testcase_file_path)
def test_context_init_functions(self):
self.assertIn("get_timestamp", self.context.testset_functions_config)
self.assertIn("gen_random_string",
self.context.testset_functions_config)
variables = [{
"random": "${gen_random_string(5)}"
}, {
"timestamp10": "${get_timestamp(10)}"
}]
self.context.bind_variables(variables)
context_variables = self.context.testcase_variables_mapping
self.assertEqual(len(context_variables["random"]), 5)
self.assertEqual(len(context_variables["timestamp10"]), 10)
def test_context_bind_testset_variables(self):
# testcase in JSON format
testcase1 = {
"variables": [{
"GLOBAL_TOKEN": "debugtalk"
}, {
"token": "$GLOBAL_TOKEN"
}]
}
# testcase in YAML format
testcase2 = self.testcases["bind_variables"]
for testcase in [testcase1, testcase2]:
variables = testcase['variables']
self.context.bind_variables(variables, level="testset")
testset_variables = self.context.testset_shared_variables_mapping
testcase_variables = self.context.testcase_variables_mapping
self.assertIn("GLOBAL_TOKEN", testset_variables)
self.assertIn("GLOBAL_TOKEN", testcase_variables)
self.assertEqual(testset_variables["GLOBAL_TOKEN"], "debugtalk")
self.assertIn("token", testset_variables)
self.assertIn("token", testcase_variables)
self.assertEqual(testset_variables["token"], "debugtalk")
def test_context_bind_testcase_variables(self):
testcase1 = {
"variables": [{
"GLOBAL_TOKEN": "debugtalk"
}, {
"token": "$GLOBAL_TOKEN"
}]
}
testcase2 = self.testcases["bind_variables"]
for testcase in [testcase1, testcase2]:
variables = testcase['variables']
self.context.bind_variables(variables)
testset_variables = self.context.testset_shared_variables_mapping
testcase_variables = self.context.testcase_variables_mapping
self.assertNotIn("GLOBAL_TOKEN", testset_variables)
self.assertIn("GLOBAL_TOKEN", testcase_variables)
self.assertEqual(testcase_variables["GLOBAL_TOKEN"], "debugtalk")
self.assertNotIn("token", testset_variables)
self.assertIn("token", testcase_variables)
self.assertEqual(testcase_variables["token"], "debugtalk")
def test_context_bind_lambda_functions(self):
testcase1 = {
"function_binds": {
"add_one": lambda x: x + 1,
"add_two_nums": lambda x, y: x + y
},
"variables": [{
"add1": "${add_one(2)}"
}, {
"sum2nums": "${add_two_nums(2,3)}"
}]
}
testcase2 = self.testcases["bind_lambda_functions"]
for testcase in [testcase1, testcase2]:
function_binds = testcase.get('function_binds', {})
self.context.bind_functions(function_binds)
variables = testcase['variables']
self.context.bind_variables(variables)
context_variables = self.context.testcase_variables_mapping
self.assertIn("add1", context_variables)
self.assertEqual(context_variables["add1"], 3)
self.assertIn("sum2nums", context_variables)
self.assertEqual(context_variables["sum2nums"], 5)
def test_context_bind_lambda_functions_with_import(self):
testcase1 = {
"requires": ["random", "string", "hashlib"],
"function_binds": {
"gen_random_string":
"lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(str_len))",
"gen_md5":
"lambda *str_args: hashlib.md5(''.join(str_args).encode('utf-8')).hexdigest()"
},
"variables": [{
"TOKEN": "debugtalk"
}, {
"random": "${gen_random_string(5)}"
}, {
"data": '{"name": "user", "password": "******"}'
}, {
"authorization": "${gen_md5($TOKEN, $data, $random)}"
}]
}
testcase2 = self.testcases["bind_lambda_functions_with_import"]
for testcase in [testcase1, testcase2]:
requires = testcase.get('requires', [])
self.context.import_requires(requires)
function_binds = testcase.get('function_binds', {})
self.context.bind_functions(function_binds)
variables = testcase['variables']
self.context.bind_variables(variables)
context_variables = self.context.testcase_variables_mapping
self.assertIn("TOKEN", context_variables)
TOKEN = context_variables["TOKEN"]
self.assertEqual(TOKEN, "debugtalk")
self.assertIn("random", context_variables)
self.assertIsInstance(context_variables["random"], str)
self.assertEqual(len(context_variables["random"]), 5)
random = context_variables["random"]
self.assertIn("data", context_variables)
data = context_variables["data"]
self.assertIn("authorization", context_variables)
self.assertEqual(len(context_variables["authorization"]), 32)
authorization = context_variables["authorization"]
self.assertEqual(utils.gen_md5(TOKEN, data, random), authorization)
def test_import_module_items(self):
testcase1 = {
"import_module_items": ["tests.data.debugtalk"],
"variables": [{
"TOKEN": "debugtalk"
}, {
"random": "${gen_random_string(5)}"
}, {
"data": '{"name": "user", "password": "******"}'
}, {
"authorization": "${gen_md5($TOKEN, $data, $random)}"
}]
}
testcase2 = self.testcases["bind_module_functions"]
for testcase in [testcase1, testcase2]:
module_items = testcase.get('import_module_items', [])
self.context.import_module_items(module_items)
variables = testcase['variables']
self.context.bind_variables(variables)
context_variables = self.context.testcase_variables_mapping
self.assertIn("TOKEN", context_variables)
TOKEN = context_variables["TOKEN"]
self.assertEqual(TOKEN, "debugtalk")
self.assertIn("random", context_variables)
self.assertIsInstance(context_variables["random"], str)
self.assertEqual(len(context_variables["random"]), 5)
random = context_variables["random"]
self.assertIn("data", context_variables)
data = context_variables["data"]
self.assertIn("authorization", context_variables)
self.assertEqual(len(context_variables["authorization"]), 32)
authorization = context_variables["authorization"]
self.assertEqual(utils.gen_md5(TOKEN, data, random), authorization)
self.assertIn("SECRET_KEY", context_variables)
SECRET_KEY = context_variables["SECRET_KEY"]
self.assertEqual(SECRET_KEY, "DebugTalk")
def test_get_parsed_request(self):
test_runner = runner.Runner()
testcase = {
"import_module_items": ["tests.data.debugtalk"],
"variables": [{
"TOKEN": "debugtalk"
}, {
"random": "${gen_random_string(5)}"
}, {
"data": '{"name": "user", "password": "******"}'
}, {
"authorization": "${gen_md5($TOKEN, $data, $random)}"
}],
"request": {
"url": "http://127.0.0.1:5000/api/users/1000",
"METHOD": "POST",
"Headers": {
"Content-Type": "application/json",
"authorization": "$authorization",
"random": "$random",
"SECRET_KEY": "$SECRET_KEY"
},
"Data": "$data"
}
}
parsed_request = test_runner.init_config(testcase, level="testcase")
self.assertIn("authorization", parsed_request["headers"])
self.assertEqual(len(parsed_request["headers"]["authorization"]), 32)
self.assertIn("random", parsed_request["headers"])
self.assertEqual(len(parsed_request["headers"]["random"]), 5)
self.assertIn("data", parsed_request)
self.assertEqual(parsed_request["data"],
testcase["variables"][2]["data"])
self.assertEqual(parsed_request["headers"]["secret_key"], "DebugTalk")
def test_exec_content_functions(self):
test_runner = runner.Runner()
content = "${sleep(1)}"
start_time = time.time()
test_runner.context.eval_content(content)
end_time = time.time()
elapsed_time = end_time - start_time
self.assertGreater(elapsed_time, 1)
def test_do_validation(self):
self.context.do_validation({
"check": "check",
"check_value": 1,
"expect": 1,
"comparator": "eq"
})
self.context.do_validation({
"check": "check",
"check_value": "abc",
"expect": "abc",
"comparator": "=="
})
config_dict = {"path": 'tests/data/demo_testset_hardcode.yml'}
self.context.config_context(config_dict, "testset")
self.context.do_validation({
"check": "status_code",
"check_value": "201",
"expect": 3,
"comparator": "sum_status_code"
})
def test_validate(self):
url = "http://127.0.0.1:5000/"
resp = requests.get(url)
resp_obj = response.ResponseObject(resp)
validators = [{
"eq": ["$resp_status_code", 201]
}, {
"check": "$resp_status_code",
"comparator": "eq",
"expect": 201
}, {
"check": "$resp_body_success",
"comparator": "eq",
"expect": True
}]
variables = [{"resp_status_code": 200}, {"resp_body_success": True}]
self.context.bind_variables(variables)
with self.assertRaises(exception.ValidationError):
self.context.validate(validators, resp_obj)
variables = [{"resp_status_code": 201}, {"resp_body_success": True}]
self.context.bind_variables(variables)
self.assertTrue(self.context.validate(validators, resp_obj))
def test_validate_exception(self):
url = "http://127.0.0.1:5000/"
resp = requests.get(url)
resp_obj = response.ResponseObject(resp)
# expected value missed in validators
validators = [{
"eq": ["$resp_status_code", 201]
}, {
"check": "$resp_status_code",
"comparator": "eq",
"expect": 201
}, {
"check": "$resp_body_success",
"comparator": "eq",
"expect": True
}]
variables = []
self.context.bind_variables(variables)
with self.assertRaises(exception.ParamsError):
self.context.validate(validators, resp_obj)
# expected value missed in variables mapping
variables = [{"resp_status_code": 200}]
self.context.bind_variables(variables)
with self.assertRaises(exception.ValidationError):
self.context.validate(validators, resp_obj)
def __init__(self, testgroup=None):
self.TestGroup = testgroup
self.context = Context()
class Runner(object):
def __init__(self, config_dict=None, http_client_session=None):
self.http_client_session = http_client_session
self.context = Context()
testcase.load_test_dependencies()
config_dict = config_dict or {}
self.init_config(config_dict, "testset")
def init_config(self, config_dict, level):
""" create/update context variables binds
@param (dict) config_dict
@param (str) level, "testset" or "testcase"
testset:
{
"name": "smoke testset",
"path": "tests/data/demo_testset_variables.yml",
"requires": [], # optional
"function_binds": {}, # optional
"import_module_items": [], # optional
"variables": [], # optional
"request": {
"base_url": "http://127.0.0.1:5000",
"headers": {
"User-Agent": "iOS/2.8.3"
}
}
}
testcase:
{
"name": "testcase description",
"requires": [], # optional
"function_binds": {}, # optional
"import_module_items": [], # optional
"variables": [], # optional
"request": {
"url": "/api/get-token",
"method": "POST",
"headers": {
"Content-Type": "application/json"
}
},
"json": {
"sign": "f1219719911caae89ccc301679857ebfda115ca2"
}
}
@param (str) context level, testcase or testset
"""
# convert keys in request headers to lowercase
config_dict = utils.lower_config_dict_key(config_dict)
self.context.init_context(level)
self.context.config_context(config_dict, level)
request_config = config_dict.get('request', {})
parsed_request = self.context.get_parsed_request(request_config, level)
base_url = parsed_request.pop("base_url", None)
self.http_client_session = self.http_client_session or HttpSession(base_url)
return parsed_request
def _handle_skip_feature(self, testcase_dict):
""" handle skip feature for testcase
- skip: skip current test unconditionally
- skipIf: skip current test if condition is true
- skipUnless: skip current test unless condition is true
"""
skip_reason = None
if "skip" in testcase_dict:
skip_reason = testcase_dict["skip"]
elif "skipIf" in testcase_dict:
skip_if_condition = testcase_dict["skipIf"]
if self.context.eval_content(skip_if_condition):
skip_reason = "{} evaluate to True".format(skip_if_condition)
elif "skipUnless" in testcase_dict:
skip_unless_condition = testcase_dict["skipUnless"]
if not self.context.eval_content(skip_unless_condition):
skip_reason = "{} evaluate to False".format(skip_unless_condition)
if skip_reason:
raise SkipTest(skip_reason)
def _run_test(self, testcase_dict):
""" run single testcase.
@param (dict) testcase_dict
{
"name": "testcase description",
"skip": "skip this test unconditionally",
"times": 3,
"requires": [], # optional, override
"function_binds": {}, # optional, override
"variables": [], # optional, override
"request": {
"url": "http://127.0.0.1:5000/api/users/1000",
"method": "POST",
"headers": {
"Content-Type": "application/json",
"authorization": "$authorization",
"random": "$random"
},
"body": '{"name": "user", "password": "******"}'
},
"extract": [], # optional
"validate": [], # optional
"setup": [], # optional
"teardown": [] # optional
}
@return True or raise exception during test
"""
parsed_request = self.init_config(testcase_dict, level="testcase")
try:
url = parsed_request.pop('url')
method = parsed_request.pop('method')
group_name = parsed_request.pop("group", None)
except KeyError:
raise exception.ParamsError("URL or METHOD missed!")
extractors = testcase_dict.get("extract", [])
validators = testcase_dict.get("validate", [])
setup_actions = testcase_dict.get("setup", [])
teardown_actions = testcase_dict.get("teardown", [])
self._handle_skip_feature(testcase_dict)
def setup_teardown(actions):
for action in actions:
self.context.eval_content(action)
setup_teardown(setup_actions)
resp = self.http_client_session.request(
method,
url,
name=group_name,
**parsed_request
)
resp_obj = response.ResponseObject(resp)
extracted_variables_mapping = resp_obj.extract_response(extractors)
self.context.bind_extracted_variables(extracted_variables_mapping)
try:
self.context.validate(validators, resp_obj)
except (exception.ParamsError, exception.ResponseError, exception.ValidationError):
err_msg = u"Exception occured.\n"
err_msg += u"HTTP request url: {}\n".format(url)
err_msg += u"HTTP request kwargs: {}\n".format(parsed_request)
err_msg += u"HTTP response status_code: {}\n".format(resp.status_code)
err_msg += u"HTTP response content: \n{}".format(resp.text)
logging.error(err_msg)
raise
finally:
setup_teardown(teardown_actions)
return True
def extract_output(self, output_variables_list):
""" extract output variables
"""
variables_mapping = self.context.testcase_variables_mapping
output = {}
for variable in output_variables_list:
if variable not in variables_mapping:
logging.warning(
"variable '{}' can not be found in variables mapping, failed to ouput!"\
.format(variable)
)
continue
output[variable] = variables_mapping[variable]
return output
class Runner(object):
def __init__(self, config_dict=None, http_client_session=None):
self.http_client_session = http_client_session
self.context = Context()
config_dict = config_dict or {}
self.init_config(config_dict, "testset")
# testset setup hooks
testset_setup_hooks = config_dict.pop("setup_hooks", [])
if testset_setup_hooks:
self.do_hook_actions(testset_setup_hooks)
# testset teardown hooks
self.testset_teardown_hooks = config_dict.pop("teardown_hooks", [])
def __del__(self):
if self.testset_teardown_hooks:
self.do_hook_actions(self.testset_teardown_hooks)
def init_config(self, config_dict, level):
""" create/update context variables binds
@param (dict) config_dict
@param (str) level, "testset" or "testcase"
testset:
{
"name": "smoke testset",
"path": "tests/data/demo_testset_variables.yml",
"variables": [], # optional
"request": {
"base_url": "http://127.0.0.1:5000",
"headers": {
"User-Agent": "iOS/2.8.3"
}
}
}
testcase:
{
"name": "testcase description",
"variables": [], # optional
"request": {
"url": "/api/get-token",
"method": "POST",
"headers": {
"Content-Type": "application/json"
}
},
"json": {
"sign": "f1219719911caae89ccc301679857ebfda115ca2"
}
}
@param (str) context level, testcase or testset
"""
# convert keys in request headers to lowercase
config_dict = utils.lower_config_dict_key(config_dict)
self.context.init_context(level)
self.context.config_context(config_dict, level)
request_config = config_dict.get('request', {})
parsed_request = self.context.get_parsed_request(request_config, level)
base_url = parsed_request.pop("base_url", None)
self.http_client_session = self.http_client_session or HttpSession(base_url)
return parsed_request
def _handle_skip_feature(self, testcase_dict):
""" handle skip feature for testcase
- skip: skip current test unconditionally
- skipIf: skip current test if condition is true
- skipUnless: skip current test unless condition is true
"""
skip_reason = None
if "skip" in testcase_dict:
skip_reason = testcase_dict["skip"]
elif "skipIf" in testcase_dict:
skip_if_condition = testcase_dict["skipIf"]
if self.context.eval_content(skip_if_condition):
skip_reason = "{} evaluate to True".format(skip_if_condition)
elif "skipUnless" in testcase_dict:
skip_unless_condition = testcase_dict["skipUnless"]
if not self.context.eval_content(skip_unless_condition):
skip_reason = "{} evaluate to False".format(skip_unless_condition)
if skip_reason:
raise SkipTest(skip_reason)
def do_hook_actions(self, actions):
for action in actions:
logger.log_debug("call hook: {}".format(action))
self.context.eval_content(action)
def run_test(self, testcase_dict):
""" run single testcase.
@param (dict) testcase_dict
{
"name": "testcase description",
"skip": "skip this test unconditionally",
"times": 3,
"variables": [], # optional, override
"request": {
"url": "http://127.0.0.1:5000/api/users/1000",
"method": "POST",
"headers": {
"Content-Type": "application/json",
"authorization": "$authorization",
"random": "$random"
},
"body": '{"name": "user", "password": "******"}'
},
"extract": [], # optional
"validate": [], # optional
"setup_hooks": [], # optional
"teardown_hooks": [] # optional
}
@return True or raise exception during test
"""
# check skip
self._handle_skip_feature(testcase_dict)
# prepare
parsed_request = self.init_config(testcase_dict, level="testcase")
self.context.bind_testcase_variable("request", parsed_request)
# setup hooks
setup_hooks = testcase_dict.get("setup_hooks", [])
setup_hooks.insert(0, "${setup_hook_prepare_kwargs($request)}")
self.do_hook_actions(setup_hooks)
try:
url = parsed_request.pop('url')
method = parsed_request.pop('method')
group_name = parsed_request.pop("group", None)
except KeyError:
raise exceptions.ParamsError("URL or METHOD missed!")
# TODO: move method validation to json schema
valid_methods = ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"]
if method.upper() not in valid_methods:
err_msg = u"Invalid HTTP method! => {}\n".format(method)
err_msg += "Available HTTP methods: {}".format("/".join(valid_methods))
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
logger.log_info("{method} {url}".format(method=method, url=url))
logger.log_debug("request kwargs(raw): {kwargs}".format(kwargs=parsed_request))
# request
resp = self.http_client_session.request(
method,
url,
name=group_name,
**parsed_request
)
resp_obj = response.ResponseObject(resp)
# teardown hooks
teardown_hooks = testcase_dict.get("teardown_hooks", [])
if teardown_hooks:
self.context.bind_testcase_variable("response", resp_obj)
self.do_hook_actions(teardown_hooks)
# extract
extractors = testcase_dict.get("extract", []) or testcase_dict.get("extractors", [])
extracted_variables_mapping = resp_obj.extract_response(extractors,context=self.context)
self.context.bind_extracted_variables(extracted_variables_mapping)
# validate
validators = testcase_dict.get("validate", []) or testcase_dict.get("validators", [])
try:
self.context.validate(validators, resp_obj)
except (exceptions.ParamsError, \
exceptions.ValidationFailure, exceptions.ExtractFailure):
# log request
err_req_msg = "request: \n"
err_req_msg += "headers: {}\n".format(parsed_request.pop("headers", {}))
for k, v in parsed_request.items():
err_req_msg += "{}: {}\n".format(k, repr(v))
logger.log_error(err_req_msg)
# log response
err_resp_msg = "response: \n"
err_resp_msg += "status_code: {}\n".format(resp_obj.status_code)
err_resp_msg += "headers: {}\n".format(resp_obj.headers)
err_resp_msg += "body: {}\n".format(repr(resp_obj.text))
logger.log_error(err_resp_msg)
raise
def extract_output(self, output_variables_list):
""" extract output variables
"""
variables_mapping = self.context.testcase_variables_mapping
output = {}
for variable in output_variables_list:
if variable not in variables_mapping:
logger.log_warning(
"variable '{}' can not be found in variables mapping, failed to output!"\
.format(variable)
)
continue
output[variable] = variables_mapping[variable]
return output
class Runner(object):
def __init__(self, http_client_session=None, request_failure_hook=None):
self.http_client_session = http_client_session
self.context = Context()
testcase.load_test_dependencies()
self.request_failure_hook = request_failure_hook
def init_config(self, config_dict, level):
""" create/update context variables binds
@param (dict) config_dict
@param (str) level, "testset" or "testcase"
testset:
{
"name": "smoke testset",
"path": "tests/data/demo_testset_variables.yml",
"requires": [], # optional
"function_binds": {}, # optional
"import_module_items": [], # optional
"variables": [], # optional
"request": {
"base_url": "http://127.0.0.1:5000",
"headers": {
"User-Agent": "iOS/2.8.3"
}
}
}
testcase:
{
"name": "testcase description",
"requires": [], # optional
"function_binds": {}, # optional
"import_module_items": [], # optional
"variables": [], # optional
"request": {
"url": "/api/get-token",
"method": "POST",
"headers": {
"Content-Type": "application/json"
}
},
"json": {
"sign": "f1219719911caae89ccc301679857ebfda115ca2"
}
}
@param (str) context level, testcase or testset
"""
# convert keys in request headers to lowercase
config_dict = utils.lower_config_dict_key(config_dict)
self.context.init_context(level)
self.context.config_context(config_dict, level)
request_config = config_dict.get('request', {})
parsed_request = self.context.get_parsed_request(request_config, level)
base_url = parsed_request.pop("base_url", None)
self.http_client_session = self.http_client_session or HttpSession(base_url)
return parsed_request
def _run_test(self, testcase_dict):
""" run single testcase.
@param (dict) testcase_dict
{
"name": "testcase description",
"times": 3,
"requires": [], # optional, override
"function_binds": {}, # optional, override
"variables": [], # optional, override
"request": {
"url": "http://127.0.0.1:5000/api/users/1000",
"method": "POST",
"headers": {
"Content-Type": "application/json",
"authorization": "$authorization",
"random": "$random"
},
"body": '{"name": "user", "password": "******"}'
},
"extract": [], # optional
"validate": [], # optional
"setup": [], # optional
"teardown": [] # optional
}
@return True or raise exception during test
"""
parsed_request = self.init_config(testcase_dict, level="testcase")
try:
url = parsed_request.pop('url')
method = parsed_request.pop('method')
group_name = parsed_request.pop("group", None)
except KeyError:
raise exception.ParamsError("URL or METHOD missed!")
run_times = int(testcase_dict.get("times", 1))
extractors = testcase_dict.get("extract") \
or testcase_dict.get("extractors") \
or testcase_dict.get("extract_binds", [])
validators = testcase_dict.get("validate") \
or testcase_dict.get("validators", [])
setup_actions = testcase_dict.get("setup", [])
teardown_actions = testcase_dict.get("teardown", [])
def setup_teardown(actions):
for action in actions:
self.context.exec_content_functions(action)
for _ in range(run_times):
setup_teardown(setup_actions)
resp = self.http_client_session.request(
method,
url,
name=group_name,
**parsed_request
)
resp_obj = response.ResponseObject(resp)
extracted_variables_mapping = resp_obj.extract_response(extractors)
self.context.bind_extracted_variables(extracted_variables_mapping)
try:
self.context.validate(validators, resp_obj)
except (exception.ParamsError, exception.ResponseError, exception.ValidationError):
err_msg = u"Exception occured.\n"
err_msg += u"HTTP request url: {}\n".format(url)
err_msg += u"HTTP request kwargs: {}\n".format(parsed_request)
err_msg += u"HTTP response status_code: {}\n".format(resp.status_code)
err_msg += u"HTTP response content: \n{}".format(resp.text)
logging.error(err_msg)
raise
finally:
setup_teardown(teardown_actions)
return True
def _run_testset(self, testset, variables_mapping=None):
""" run single testset, including one or several testcases.
@param
(dict) testset
{
"name": "testset description",
"config": {
"name": "testset description",
"requires": [],
"function_binds": {},
"variables": [],
"request": {}
},
"testcases": [
{
"name": "testcase description",
"variables": [], # optional, override
"request": {},
"extract": {}, # optional
"validate": {} # optional
},
testcase12
]
}
(dict) variables_mapping:
passed in variables mapping, it will override variables in config block
@return (dict) test result of testset
{
"success": True,
"output": {} # variables mapping
}
"""
success = True
config_dict = testset.get("config", {})
variables = config_dict.get("variables", [])
variables_mapping = variables_mapping or {}
config_dict["variables"] = utils.override_variables_binds(variables, variables_mapping)
self.init_config(config_dict, level="testset")
testcases = testset.get("testcases", [])
for testcase_dict in testcases:
try:
self._run_test(testcase_dict)
except exception.MyBaseError as ex:
success = False
if self.request_failure_hook:
self.request_failure_hook.fire(
request_type=testcase_dict.get("request", {}).get("method"),
name=testcase_dict.get("request", {}).get("url"),
response_time=0,
exception=ex
)
else:
logging.exception(
"Exception occured in testcase: {}".format(testcase_dict.get("name")))
break
output_variables_list = config_dict.get("output", [])
return {
"success": success,
"output": self.generate_output(output_variables_list)
}
def run(self, path, mapping=None):
""" run specified testset path or folder path.
@param
path: path could be in several type
- absolute/relative file path
- absolute/relative folder path
- list/set container with file(s) and/or folder(s)
(dict) mapping:
passed in variables mapping, it will override variables in config block
"""
success = True
mapping = mapping or {}
output = {}
testsets = testcase.load_testcases_by_path(path)
for testset in testsets:
try:
result = self._run_testset(testset, mapping)
assert result["success"]
output.update(result["output"])
except AssertionError:
success = False
return {
"success": success,
"output": output
}
def generate_output(self, output_variables_list):
""" generate and print output
"""
variables_mapping = self.context.get_testcase_variables_mapping()
output = {
variable: variables_mapping[variable]
for variable in output_variables_list
}
utils.print_output(output)
return output
