def __init__(self, config_dict=None, http_client_session=None): self.http_client_session = http_client_session self.context = Context() testcase.load_test_dependencies() config_dict = config_dict or {} self.init_config(config_dict, "testset")
def __init__(self, config_dict=None, http_client_session=None): self.http_client_session = http_client_session self.context = Context() config_dict = config_dict or {} self.init_config(config_dict, "testset") # testset setup hooks testset_setup_hooks = config_dict.pop("setup_hooks", []) if testset_setup_hooks: self.do_hook_actions(testset_setup_hooks) # testset teardown hooks self.testset_teardown_hooks = config_dict.pop("teardown_hooks", [])
def __init__(self, config_dict=None, http_client_session=None): """ """ self.http_client_session = http_client_session config_dict = config_dict or {} self.evaluated_validators = [] # testcase variables config_variables = config_dict.get("variables", {}) # testcase functions config_functions = config_dict.get("functions", {}) # testcase setup hooks testcase_setup_hooks = config_dict.pop("setup_hooks", []) # testcase teardown hooks self.testcase_teardown_hooks = config_dict.pop("teardown_hooks", []) self.context = Context(config_variables, config_functions) self.init_config(config_dict, "testcase") if testcase_setup_hooks: self.do_hook_actions(testcase_setup_hooks)
def setUp(self): self.context = Context() testcase_file_path = os.path.join(os.getcwd(), 'tests/data/demo_binds.yml') self.testcases = testcase.load_file(testcase_file_path)
def setUp(self): self.context = Context() testcase_file_path = os.path.join(os.getcwd(), 'tests/data/demo_binds.yml') self.testcases = FileUtils.load_file(testcase_file_path)
class VariableBindsUnittest(ApiServerUnittest): def setUp(self): self.context = Context() testcase_file_path = os.path.join(os.getcwd(), 'tests/data/demo_binds.yml') self.testcases = FileUtils.load_file(testcase_file_path) def test_context_init_functions(self): self.assertIn("get_timestamp", self.context.testset_functions_config) self.assertIn("gen_random_string", self.context.testset_functions_config) variables = [ {"random": "${gen_random_string(5)}"}, {"timestamp10": "${get_timestamp(10)}"} ] self.context.bind_variables(variables) context_variables = self.context.testcase_variables_mapping self.assertEqual(len(context_variables["random"]), 5) self.assertEqual(len(context_variables["timestamp10"]), 10) def test_context_bind_testset_variables(self): # testcase in JSON format testcase1 = { "variables": [ {"GLOBAL_TOKEN": "debugtalk"}, {"token": "$GLOBAL_TOKEN"} ] } # testcase in YAML format testcase2 = self.testcases["bind_variables"] for testcase in [testcase1, testcase2]: variables = testcase['variables'] self.context.bind_variables(variables, level="testset") testset_variables = self.context.testset_shared_variables_mapping testcase_variables = self.context.testcase_variables_mapping self.assertIn("GLOBAL_TOKEN", testset_variables) self.assertIn("GLOBAL_TOKEN", testcase_variables) self.assertEqual(testset_variables["GLOBAL_TOKEN"], "debugtalk") self.assertIn("token", testset_variables) self.assertIn("token", testcase_variables) self.assertEqual(testset_variables["token"], "debugtalk") def test_context_bind_testcase_variables(self): testcase1 = { "variables": [ {"GLOBAL_TOKEN": "debugtalk"}, {"token": "$GLOBAL_TOKEN"} ] } testcase2 = self.testcases["bind_variables"] for testcase in [testcase1, testcase2]: variables = testcase['variables'] self.context.bind_variables(variables) testset_variables = self.context.testset_shared_variables_mapping testcase_variables = self.context.testcase_variables_mapping self.assertNotIn("GLOBAL_TOKEN", testset_variables) self.assertIn("GLOBAL_TOKEN", testcase_variables) self.assertEqual(testcase_variables["GLOBAL_TOKEN"], "debugtalk") self.assertNotIn("token", testset_variables) self.assertIn("token", testcase_variables) self.assertEqual(testcase_variables["token"], "debugtalk") def test_context_bind_lambda_functions(self): testcase1 = { "function_binds": { "add_one": lambda x: x + 1, "add_two_nums": lambda x, y: x + y }, "variables": [ {"add1": "${add_one(2)}"}, {"sum2nums": "${add_two_nums(2,3)}"} ] } testcase2 = self.testcases["bind_lambda_functions"] for testcase in [testcase1, testcase2]: function_binds = testcase.get('function_binds', {}) self.context.bind_functions(function_binds) variables = testcase['variables'] self.context.bind_variables(variables) context_variables = self.context.testcase_variables_mapping self.assertIn("add1", context_variables) self.assertEqual(context_variables["add1"], 3) self.assertIn("sum2nums", context_variables) self.assertEqual(context_variables["sum2nums"], 5) def test_call_builtin_functions(self): testcase1 = { "variables": [ {"length": "${len(debugtalk)}"}, {"smallest": "${min(2, 3, 8)}"}, {"largest": "${max(2, 3, 8)}"} ] } testcase2 = self.testcases["builtin_functions"] for testcase in [testcase1, testcase2]: variables = testcase['variables'] self.context.bind_variables(variables) context_variables = self.context.testcase_variables_mapping self.assertEqual(context_variables["length"], 9) self.assertEqual(context_variables["smallest"], 2) self.assertEqual(context_variables["largest"], 8) def test_context_bind_lambda_functions_with_import(self): testcase1 = { "requires": ["random", "string", "hashlib"], "function_binds": { "gen_random_string": "lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(str_len))", "gen_md5": "lambda *str_args: hashlib.md5(''.join(str_args).encode('utf-8')).hexdigest()" }, "variables": [ {"TOKEN": "debugtalk"}, {"random": "${gen_random_string(5)}"}, {"data": '{"name": "user", "password": "******"}'}, {"authorization": "${gen_md5($TOKEN, $data, $random)}"} ] } testcase2 = self.testcases["bind_lambda_functions_with_import"] for testcase in [testcase1, testcase2]: requires = testcase.get('requires', []) self.context.import_requires(requires) function_binds = testcase.get('function_binds', {}) self.context.bind_functions(function_binds) variables = testcase['variables'] self.context.bind_variables(variables) context_variables = self.context.testcase_variables_mapping self.assertIn("TOKEN", context_variables) TOKEN = context_variables["TOKEN"] self.assertEqual(TOKEN, "debugtalk") self.assertIn("random", context_variables) self.assertIsInstance(context_variables["random"], str) self.assertEqual(len(context_variables["random"]), 5) random = context_variables["random"] self.assertIn("data", context_variables) data = context_variables["data"] self.assertIn("authorization", context_variables) self.assertEqual(len(context_variables["authorization"]), 32) authorization = context_variables["authorization"] self.assertEqual(gen_md5(TOKEN, data, random), authorization) def test_import_module_items(self): testcase1 = { "import_module_items": ["tests.debugtalk"], "variables": [ {"TOKEN": "debugtalk"}, {"random": "${gen_random_string(5)}"}, {"data": '{"name": "user", "password": "******"}'}, {"authorization": "${gen_md5($TOKEN, $data, $random)}"} ] } testcase2 = self.testcases["bind_module_functions"] for testcase in [testcase1, testcase2]: module_items = testcase.get('import_module_items', []) self.context.import_module_items(module_items) variables = testcase['variables'] self.context.bind_variables(variables) context_variables = self.context.testcase_variables_mapping self.assertIn("TOKEN", context_variables) TOKEN = context_variables["TOKEN"] self.assertEqual(TOKEN, "debugtalk") self.assertIn("random", context_variables) self.assertIsInstance(context_variables["random"], str) self.assertEqual(len(context_variables["random"]), 5) random = context_variables["random"] self.assertIn("data", context_variables) data = context_variables["data"] self.assertIn("authorization", context_variables) self.assertEqual(len(context_variables["authorization"]), 32) authorization = context_variables["authorization"] self.assertEqual(gen_md5(TOKEN, data, random), authorization) self.assertIn("SECRET_KEY", context_variables) SECRET_KEY = context_variables["SECRET_KEY"] self.assertEqual(SECRET_KEY, "DebugTalk") def test_get_parsed_request(self): test_runner = runner.Runner() testcase = { "import_module_items": ["tests.debugtalk"], "variables": [ {"TOKEN": "debugtalk"}, {"random": "${gen_random_string(5)}"}, {"data": '{"name": "user", "password": "******"}'}, {"authorization": "${gen_md5($TOKEN, $data, $random)}"} ], "request": { "url": "http://127.0.0.1:5000/api/users/1000", "METHOD": "POST", "Headers": { "Content-Type": "application/json", "authorization": "$authorization", "random": "$random", "SECRET_KEY": "$SECRET_KEY" }, "Data": "$data" } } parsed_request = test_runner.init_config(testcase, level="testcase") self.assertIn("authorization", parsed_request["headers"]) self.assertEqual(len(parsed_request["headers"]["authorization"]), 32) self.assertIn("random", parsed_request["headers"]) self.assertEqual(len(parsed_request["headers"]["random"]), 5) self.assertIn("data", parsed_request) self.assertEqual(parsed_request["data"], testcase["variables"][2]["data"]) self.assertEqual(parsed_request["headers"]["secret_key"], "DebugTalk") def test_exec_content_functions(self): test_runner = runner.Runner() content = "${sleep_N_secs(1)}" start_time = time.time() test_runner.context.eval_content(content) end_time = time.time() elapsed_time = end_time - start_time self.assertGreater(elapsed_time, 1) def test_do_validation(self): self.context.do_validation( {"check": "check", "check_value": 1, "expect": 1, "comparator": "eq"} ) self.context.do_validation( {"check": "check", "check_value": "abc", "expect": "abc", "comparator": "=="} ) config_dict = { "path": 'tests/data/demo_testset_hardcode.yml' } self.context.config_context(config_dict, "testset") self.context.do_validation( {"check": "status_code", "check_value": "201", "expect": 3, "comparator": "sum_status_code"} ) def test_validate(self): url = "http://127.0.0.1:5000/" resp = requests.get(url) resp_obj = response.ResponseObject(resp) validators = [ {"eq": ["$resp_status_code", 201]}, {"check": "$resp_status_code", "comparator": "eq", "expect": 201}, {"check": "$resp_body_success", "comparator": "eq", "expect": True}, {"check": "${is_status_code_200($resp_status_code)}", "comparator": "eq", "expect": False} ] variables = [ {"resp_status_code": 200}, {"resp_body_success": True} ] self.context.bind_variables(variables) with self.assertRaises(exception.ValidationError): self.context.validate(validators, resp_obj) variables = [ {"resp_status_code": 201}, {"resp_body_success": True} ] self.context.bind_variables(variables) from tests.debugtalk import is_status_code_200 functions = { "is_status_code_200": is_status_code_200 } self.context.bind_functions(functions) self.assertTrue(self.context.validate(validators, resp_obj)) def test_validate_exception(self): url = "http://127.0.0.1:5000/" resp = requests.get(url) resp_obj = response.ResponseObject(resp) # expected value missed in validators validators = [ {"eq": ["$resp_status_code", 201]}, {"check": "$resp_status_code", "comparator": "eq", "expect": 201}, {"check": "$resp_body_success", "comparator": "eq", "expect": True} ] variables = [] self.context.bind_variables(variables) with self.assertRaises(exception.ParamsError): self.context.validate(validators, resp_obj) # expected value missed in variables mapping variables = [ {"resp_status_code": 200} ] self.context.bind_variables(variables) with self.assertRaises(exception.ValidationError): self.context.validate(validators, resp_obj)
def __init__(self, http_client_session=None, request_failure_hook=None): self.http_client_session = http_client_session self.context = Context() # testcase.load_test_dependencies() self.request_failure_hook = request_failure_hook
class Runner(object): def __init__(self, http_client_session=None, request_failure_hook=None): self.http_client_session = http_client_session self.context = Context() # testcase.load_test_dependencies() self.request_failure_hook = request_failure_hook def init_config(self, config_dict, level): """ create/update context variables binds @param (dict) config_dict @param (str) level, "testset" or "testcase" testset: { "name": "smoke testset", "path": "tests/data/demo_testset_variables.yml", "requires": [], # optional "function_binds": {}, # optional "import_module_items": [], # optional "variables": [], # optional "request": { "base_url": "http://127.0.0.1:5000", "headers": { "User-Agent": "iOS/2.8.3" } } } testcase: { "name": "testcase description", "requires": [], # optional "function_binds": {}, # optional "import_module_items": [], # optional "variables": [], # optional "request": { "url": "/api/get-token", "method": "POST", "headers": { "Content-Type": "application/json" } }, "json": { "sign": "f1219719911caae89ccc301679857ebfda115ca2" } } @param (str) context level, testcase or testset """ # convert keys in request headers to lowercase config_dict = utils.lower_config_dict_key(config_dict) self.context.init_context(level) self.context.config_context(config_dict, level) request_config = config_dict.get('request', {}) parsed_request = self.context.get_parsed_request(request_config, level) base_url = parsed_request.pop("base_url", None) self.http_client_session = self.http_client_session or HttpSession( base_url) return parsed_request def _run_test(self, testcase_dict): """ run single testcase. @param (dict) testcase_dict { "name": "testcase description", "times": 3, "requires": [], # optional, override "function_binds": {}, # optional, override "variables": [], # optional, override "request": { "url": "http://127.0.0.1:5000/api/users/1000", "method": "POST", "headers": { "Content-Type": "application/json", "authorization": "$authorization", "random": "$random" }, "body": '{"name": "user", "password": "******"}' }, "extract": [], # optional "validate": [], # optional "setup": [], # optional "teardown": [] # optional } @return True or raise exception during test """ parsed_request = self.init_config(testcase_dict, level="testcase") try: url = parsed_request.pop('url') method = parsed_request.pop('method') group_name = parsed_request.pop("group", None) except KeyError: raise exception.ParamsError("URL or METHOD missed!") run_times = int(testcase_dict.get("times", 1)) extractors = testcase_dict.get("extract") \ or testcase_dict.get("extractors") \ or testcase_dict.get("extract_binds", []) validators = testcase_dict.get("validate") \ or testcase_dict.get("validators", []) setup_actions = testcase_dict.get("setup", []) teardown_actions = testcase_dict.get("teardown", []) def setup_teardown(actions): for action in actions: self.context.exec_content_functions(action) for _ in range(run_times): setup_teardown(setup_actions) resp = self.http_client_session.request(method, url, name=group_name, **parsed_request) resp_obj = response.ResponseObject(resp) extracted_variables_mapping = resp_obj.extract_response(extractors) self.context.bind_extracted_variables(extracted_variables_mapping) try: self.context.validate(validators, resp_obj) except (exception.ParamsError, exception.ResponseError, exception.ValidationError): err_msg = u"Exception occured.\n" err_msg += u"HTTP request url: {}\n".format(url) err_msg += u"HTTP request kwargs: {}\n".format(parsed_request) err_msg += u"HTTP response status_code: {}\n".format( resp.status_code) err_msg += u"HTTP response content: \n{}".format(resp.text) logging.error(err_msg) raise finally: setup_teardown(teardown_actions) return True def _run_testset(self, testset, variables_mapping=None): """ run single testset, including one or several testcases. @param (dict) testset { "name": "testset description", "config": { "name": "testset description", "requires": [], "function_binds": {}, "variables": [], "request": {} }, "testcases": [ { "name": "testcase description", "variables": [], # optional, override "request": {}, "extract": {}, # optional "validate": {} # optional }, testcase12 ] } (dict) variables_mapping: passed in variables mapping, it will override variables in config block @return (dict) test result of testset { "success": True, "output": {} # variables mapping } """ success = True config_dict = testset.get("config", {}) variables = config_dict.get("variables", []) variables_mapping = variables_mapping or {} config_dict["variables"] = utils.override_variables_binds( variables, variables_mapping) self.init_config(config_dict, level="testset") testcases = testset.get("testcases", []) for testcase_dict in testcases: try: self._run_test(testcase_dict) except exception.MyBaseError as ex: success = False if self.request_failure_hook: self.request_failure_hook.fire( request_type=testcase_dict.get("request", {}).get("method"), name=testcase_dict.get("request", {}).get("url"), response_time=0, exception=ex) else: logging.exception( "Exception occured in testcase: {}".format( testcase_dict.get("name"))) break output_variables_list = config_dict.get("output", []) return { "success": success, "output": self.generate_output(output_variables_list) } def run(self, path, mapping=None): """ run specified testset path or folder path. @param path: path could be in several type - absolute/relative file path - absolute/relative folder path - list/set container with file(s) and/or folder(s) (dict) mapping: passed in variables mapping, it will override variables in config block """ success = True mapping = mapping or {} output = {} testsets = testcase.load_testcases_by_path(path) for testset in testsets: try: result = self._run_testset(testset, mapping) assert result["success"] output.update(result["output"]) except AssertionError: success = False return {"success": success, "output": output} def generate_output(self, output_variables_list): """ generate and print output """ variables_mapping = self.context.get_testcase_variables_mapping() output = { variable: variables_mapping[variable] for variable in output_variables_list } utils.print_output(output) return output
def __init__(self, http_client_session=None, request_failure_hook=None): self.http_client_session = http_client_session self.context = Context() testcase.load_test_dependencies() self.request_failure_hook = request_failure_hook
class VariableBindsUnittest(unittest.TestCase): def setUp(self): self.context = Context() testcase_file_path = os.path.join(os.getcwd(), 'tests/data/demo_binds.yml') self.testcases = testcase._load_file(testcase_file_path) def test_context_init_functions(self): self.assertIn("get_timestamp", self.context.testset_functions_config) self.assertIn("gen_random_string", self.context.testset_functions_config) variables = [ {"random": "${gen_random_string(5)}"}, {"timestamp10": "${get_timestamp(10)}"} ] self.context.bind_variables(variables) context_variables = self.context.get_testcase_variables_mapping() self.assertEqual(len(context_variables["random"]), 5) self.assertEqual(len(context_variables["timestamp10"]), 10) def test_context_bind_testset_variables(self): # testcase in JSON format testcase1 = { "variables": [ {"GLOBAL_TOKEN": "debugtalk"}, {"token": "$GLOBAL_TOKEN"} ] } # testcase in YAML format testcase2 = self.testcases["bind_variables"] for testcase in [testcase1, testcase2]: variables = testcase['variables'] self.context.bind_variables(variables, level="testset") testset_variables = self.context.testset_shared_variables_mapping testcase_variables = self.context.get_testcase_variables_mapping() self.assertIn("GLOBAL_TOKEN", testset_variables) self.assertIn("GLOBAL_TOKEN", testcase_variables) self.assertEqual(testset_variables["GLOBAL_TOKEN"], "debugtalk") self.assertIn("token", testset_variables) self.assertIn("token", testcase_variables) self.assertEqual(testset_variables["token"], "debugtalk") def test_context_bind_testcase_variables(self): testcase1 = { "variables": [ {"GLOBAL_TOKEN": "debugtalk"}, {"token": "$GLOBAL_TOKEN"} ] } testcase2 = self.testcases["bind_variables"] for testcase in [testcase1, testcase2]: variables = testcase['variables'] self.context.bind_variables(variables) testset_variables = self.context.testset_shared_variables_mapping testcase_variables = self.context.get_testcase_variables_mapping() self.assertNotIn("GLOBAL_TOKEN", testset_variables) self.assertIn("GLOBAL_TOKEN", testcase_variables) self.assertEqual(testcase_variables["GLOBAL_TOKEN"], "debugtalk") self.assertNotIn("token", testset_variables) self.assertIn("token", testcase_variables) self.assertEqual(testcase_variables["token"], "debugtalk") def test_context_bind_lambda_functions(self): testcase1 = { "function_binds": { "add_one": lambda x: x + 1, "add_two_nums": lambda x, y: x + y }, "variables": [ {"add1": "${add_one(2)}"}, {"sum2nums": "${add_two_nums(2,3)}"} ] } testcase2 = self.testcases["bind_lambda_functions"] for testcase in [testcase1, testcase2]: function_binds = testcase.get('function_binds', {}) self.context.bind_functions(function_binds) variables = testcase['variables'] self.context.bind_variables(variables) context_variables = self.context.get_testcase_variables_mapping() self.assertIn("add1", context_variables) self.assertEqual(context_variables["add1"], 3) self.assertIn("sum2nums", context_variables) self.assertEqual(context_variables["sum2nums"], 5) def test_context_bind_lambda_functions_with_import(self): testcase1 = { "requires": ["random", "string", "hashlib"], "function_binds": { "gen_random_string": "lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(str_len))", "gen_md5": "lambda *str_args: hashlib.md5(''.join(str_args).encode('utf-8')).hexdigest()" }, "variables": [ {"TOKEN": "debugtalk"}, {"random": "${gen_random_string(5)}"}, {"data": '{"name": "user", "password": "******"}'}, {"authorization": "${gen_md5($TOKEN, $data, $random)}"} ] } testcase2 = self.testcases["bind_lambda_functions_with_import"] for testcase in [testcase1, testcase2]: requires = testcase.get('requires', []) self.context.import_requires(requires) function_binds = testcase.get('function_binds', {}) self.context.bind_functions(function_binds) variables = testcase['variables'] self.context.bind_variables(variables) context_variables = self.context.get_testcase_variables_mapping() self.assertIn("TOKEN", context_variables) TOKEN = context_variables["TOKEN"] self.assertEqual(TOKEN, "debugtalk") self.assertIn("random", context_variables) self.assertIsInstance(context_variables["random"], str) self.assertEqual(len(context_variables["random"]), 5) random = context_variables["random"] self.assertIn("data", context_variables) data = context_variables["data"] self.assertIn("authorization", context_variables) self.assertEqual(len(context_variables["authorization"]), 32) authorization = context_variables["authorization"] self.assertEqual(utils.gen_md5(TOKEN, data, random), authorization) def test_import_module_items(self): testcase1 = { "import_module_items": ["tests.data.debugtalk"], "variables": [ {"TOKEN": "debugtalk"}, {"random": "${gen_random_string(5)}"}, {"data": '{"name": "user", "password": "******"}'}, {"authorization": "${gen_md5($TOKEN, $data, $random)}"} ] } testcase2 = self.testcases["bind_module_functions"] for testcase in [testcase1, testcase2]: module_items = testcase.get('import_module_items', []) self.context.import_module_items(module_items) variables = testcase['variables'] self.context.bind_variables(variables) context_variables = self.context.get_testcase_variables_mapping() self.assertIn("TOKEN", context_variables) TOKEN = context_variables["TOKEN"] self.assertEqual(TOKEN, "debugtalk") self.assertIn("random", context_variables) self.assertIsInstance(context_variables["random"], str) self.assertEqual(len(context_variables["random"]), 5) random = context_variables["random"] self.assertIn("data", context_variables) data = context_variables["data"] self.assertIn("authorization", context_variables) self.assertEqual(len(context_variables["authorization"]), 32) authorization = context_variables["authorization"] self.assertEqual(utils.gen_md5(TOKEN, data, random), authorization) self.assertIn("SECRET_KEY", context_variables) SECRET_KEY = context_variables["SECRET_KEY"] self.assertEqual(SECRET_KEY, "DebugTalk") def test_get_parsed_request(self): test_runner = runner.Runner() testcase = { "import_module_items": ["tests.data.debugtalk"], "variables": [ {"TOKEN": "debugtalk"}, {"random": "${gen_random_string(5)}"}, {"data": '{"name": "user", "password": "******"}'}, {"authorization": "${gen_md5($TOKEN, $data, $random)}"} ], "request": { "url": "http://127.0.0.1:5000/api/users/1000", "METHOD": "POST", "Headers": { "Content-Type": "application/json", "authorization": "$authorization", "random": "$random", "SECRET_KEY": "$SECRET_KEY" }, "Data": "$data" } } parsed_request = test_runner.init_config(testcase, level="testcase") self.assertIn("authorization", parsed_request["headers"]) self.assertEqual(len(parsed_request["headers"]["authorization"]), 32) self.assertIn("random", parsed_request["headers"]) self.assertEqual(len(parsed_request["headers"]["random"]), 5) self.assertIn("data", parsed_request) self.assertEqual(parsed_request["data"], testcase["variables"][2]["data"]) self.assertEqual(parsed_request["headers"]["SECRET_KEY"], "DebugTalk") def test_exec_content_functions(self): test_runner = runner.Runner() content = "${sleep(1)}" start_time = time.time() test_runner.context.exec_content_functions(content) end_time = time.time() elapsed_time = end_time - start_time self.assertGreater(elapsed_time, 1)
def __init__(self, config_dict=None, http_client_session=None): self.http_client_session = http_client_session self.context = Context() config_dict = config_dict or {} self.init_config(config_dict, "testset")
class Runner(object): def __init__(self, config_dict=None, http_client_session=None): self.http_client_session = http_client_session self.context = Context() config_dict = config_dict or {} self.init_config(config_dict, "testset") def init_config(self, config_dict, level): """ create/update context variables binds @param (dict) config_dict @param (str) level, "testset" or "testcase" testset: { "name": "smoke testset", "path": "tests/data/demo_testset_variables.yml", "requires": [], # optional "function_binds": {}, # optional "import_module_items": [], # optional "variables": [], # optional "request": { "base_url": "http://127.0.0.1:5000", "headers": { "User-Agent": "iOS/2.8.3" } } } testcase: { "name": "testcase description", "requires": [], # optional "function_binds": {}, # optional "import_module_items": [], # optional "variables": [], # optional "request": { "url": "/api/get-token", "method": "POST", "headers": { "Content-Type": "application/json" } }, "json": { "sign": "f1219719911caae89ccc301679857ebfda115ca2" } } @param (str) context level, testcase or testset """ # convert keys in request headers to lowercase config_dict = utils.lower_config_dict_key(config_dict) self.context.init_context(level) self.context.config_context(config_dict, level) request_config = config_dict.get('request', {}) parsed_request = self.context.get_parsed_request(request_config, level) base_url = parsed_request.pop("base_url", None) self.http_client_session = self.http_client_session or HttpSession( base_url) return parsed_request def _handle_skip_feature(self, testcase_dict): """ handle skip feature for testcase - skip: skip current test unconditionally - skipIf: skip current test if condition is true - skipUnless: skip current test unless condition is true """ skip_reason = None if "skip" in testcase_dict: skip_reason = testcase_dict["skip"] elif "skipIf" in testcase_dict: skip_if_condition = testcase_dict["skipIf"] if self.context.eval_content(skip_if_condition): skip_reason = "{} evaluate to True".format(skip_if_condition) elif "skipUnless" in testcase_dict: skip_unless_condition = testcase_dict["skipUnless"] if not self.context.eval_content(skip_unless_condition): skip_reason = "{} evaluate to False".format( skip_unless_condition) if skip_reason: raise SkipTest(skip_reason) def _prepare_hooks_event(self, hooks): if not hooks: return None event = EventHook() for hook in hooks: func = self.context.testcase_parser.get_bind_function(hook) event += func return event def _call_setup_hooks(self, hooks, method, url, kwargs): """ call hook functions before request Listeners should take the following arguments: * *method*: request method type, e.g. GET, POST, PUT * *url*: URL that was called (or override name if it was used in the call to the client) * *kwargs*: kwargs of request """ hooks.insert(0, "setup_hook_prepare_kwargs") event = self._prepare_hooks_event(hooks) if not event: return event.fire(method=method, url=url, kwargs=kwargs) def _call_teardown_hooks(self, hooks, resp_obj): """ call hook functions after request Listeners should take the following arguments: * *resp_obj*: response object """ event = self._prepare_hooks_event(hooks) if not event: return event.fire(resp_obj=resp_obj) def run_test(self, testcase_dict): """ run single testcase. @param (dict) testcase_dict { "name": "testcase description", "skip": "skip this test unconditionally", "times": 3, "requires": [], # optional, override "function_binds": {}, # optional, override "variables": [], # optional, override "request": { "url": "http://127.0.0.1:5000/api/users/1000", "method": "POST", "headers": { "Content-Type": "application/json", "authorization": "$authorization", "random": "$random" }, "body": '{"name": "user", "password": "******"}' }, "extract": [], # optional "validate": [], # optional "setup_hooks": [], # optional "teardown_hooks": [] # optional } @return True or raise exception during test """ parsed_request = self.init_config(testcase_dict, level="testcase") try: url = parsed_request.pop('url') method = parsed_request.pop('method') group_name = parsed_request.pop("group", None) except KeyError: raise exception.ParamsError("URL or METHOD missed!") self._handle_skip_feature(testcase_dict) extractors = testcase_dict.get("extract", []) or testcase_dict.get( "extractors", []) validators = testcase_dict.get("validate", []) or testcase_dict.get( "validators", []) setup_hooks = testcase_dict.get("setup_hooks", []) teardown_hooks = testcase_dict.get("teardown_hooks", []) logger.log_info("{method} {url}".format(method=method, url=url)) logger.log_debug( "request kwargs(raw): {kwargs}".format(kwargs=parsed_request)) self._call_setup_hooks(setup_hooks, method, url, parsed_request) resp = self.http_client_session.request(method, url, name=group_name, **parsed_request) self._call_teardown_hooks(teardown_hooks, resp) resp_obj = response.ResponseObject(resp) extracted_variables_mapping = resp_obj.extract_response(extractors) self.context.bind_extracted_variables(extracted_variables_mapping) try: self.context.validate(validators, resp_obj) except (exception.ParamsError, exception.ResponseError, \ exception.ValidationError, exception.ParseResponseError): # log request err_req_msg = "request: \n" err_req_msg += "headers: {}\n".format( parsed_request.pop("headers", {})) for k, v in parsed_request.items(): err_req_msg += "{}: {}\n".format(k, v) logger.log_error(err_req_msg) # log response err_resp_msg = "response: \n" err_resp_msg += "status_code: {}\n".format(resp.status_code) err_resp_msg += "headers: {}\n".format(resp.headers) err_resp_msg += "body: {}\n".format(resp.text) logger.log_error(err_resp_msg) raise def extract_output(self, output_variables_list): """ extract output variables """ variables_mapping = self.context.testcase_variables_mapping output = {} for variable in output_variables_list: if variable not in variables_mapping: logger.log_warning( "variable '{}' can not be found in variables mapping, failed to ouput!"\ .format(variable) ) continue output[variable] = variables_mapping[variable] return output
class VariableBindsUnittest(ApiServerUnittest): def setUp(self): self.context = Context() testcase_file_path = os.path.join(os.getcwd(), 'tests/data/demo_binds.yml') self.testcases = FileUtils.load_file(testcase_file_path) def test_context_init_functions(self): self.assertIn("get_timestamp", self.context.testset_functions_config) self.assertIn("gen_random_string", self.context.testset_functions_config) variables = [{ "random": "${gen_random_string(5)}" }, { "timestamp10": "${get_timestamp(10)}" }] self.context.bind_variables(variables) context_variables = self.context.testcase_variables_mapping self.assertEqual(len(context_variables["random"]), 5) self.assertEqual(len(context_variables["timestamp10"]), 10) def test_context_bind_testset_variables(self): # testcase in JSON format testcase1 = { "variables": [{ "GLOBAL_TOKEN": "debugtalk" }, { "token": "$GLOBAL_TOKEN" }] } # testcase in YAML format testcase2 = self.testcases["bind_variables"] for testcase in [testcase1, testcase2]: variables = testcase['variables'] self.context.bind_variables(variables, level="testset") testset_variables = self.context.testset_shared_variables_mapping testcase_variables = self.context.testcase_variables_mapping self.assertIn("GLOBAL_TOKEN", testset_variables) self.assertIn("GLOBAL_TOKEN", testcase_variables) self.assertEqual(testset_variables["GLOBAL_TOKEN"], "debugtalk") self.assertIn("token", testset_variables) self.assertIn("token", testcase_variables) self.assertEqual(testset_variables["token"], "debugtalk") def test_context_bind_testcase_variables(self): testcase1 = { "variables": [{ "GLOBAL_TOKEN": "debugtalk" }, { "token": "$GLOBAL_TOKEN" }] } testcase2 = self.testcases["bind_variables"] for testcase in [testcase1, testcase2]: variables = testcase['variables'] self.context.bind_variables(variables) testset_variables = self.context.testset_shared_variables_mapping testcase_variables = self.context.testcase_variables_mapping self.assertNotIn("GLOBAL_TOKEN", testset_variables) self.assertIn("GLOBAL_TOKEN", testcase_variables) self.assertEqual(testcase_variables["GLOBAL_TOKEN"], "debugtalk") self.assertNotIn("token", testset_variables) self.assertIn("token", testcase_variables) self.assertEqual(testcase_variables["token"], "debugtalk") def test_context_bind_lambda_functions(self): function_binds = { "add_one": lambda x: x + 1, "add_two_nums": lambda x, y: x + y } variables = [{ "add1": "${add_one(2)}" }, { "sum2nums": "${add_two_nums(2,3)}" }] self.context.bind_functions(function_binds) self.context.bind_variables(variables) context_variables = self.context.testcase_variables_mapping self.assertIn("add1", context_variables) self.assertEqual(context_variables["add1"], 3) self.assertIn("sum2nums", context_variables) self.assertEqual(context_variables["sum2nums"], 5) def test_call_builtin_functions(self): testcase1 = { "variables": [{ "length": "${len(debugtalk)}" }, { "smallest": "${min(2, 3, 8)}" }, { "largest": "${max(2, 3, 8)}" }] } testcase2 = self.testcases["builtin_functions"] for testcase in [testcase1, testcase2]: variables = testcase['variables'] self.context.bind_variables(variables) context_variables = self.context.testcase_variables_mapping self.assertEqual(context_variables["length"], 9) self.assertEqual(context_variables["smallest"], 2) self.assertEqual(context_variables["largest"], 8) def test_import_module_items(self): variables = [{ "TOKEN": "debugtalk" }, { "random": "${gen_random_string(5)}" }, { "data": '{"name": "user", "password": "******"}' }, { "authorization": "${gen_md5($TOKEN, $data, $random)}" }] from tests import debugtalk self.context.import_module_items(debugtalk) self.context.bind_variables(variables) context_variables = self.context.testcase_variables_mapping self.assertIn("TOKEN", context_variables) TOKEN = context_variables["TOKEN"] self.assertEqual(TOKEN, "debugtalk") self.assertIn("random", context_variables) self.assertIsInstance(context_variables["random"], str) self.assertEqual(len(context_variables["random"]), 5) random = context_variables["random"] self.assertIn("data", context_variables) data = context_variables["data"] self.assertIn("authorization", context_variables) self.assertEqual(len(context_variables["authorization"]), 32) authorization = context_variables["authorization"] self.assertEqual(gen_md5(TOKEN, data, random), authorization) self.assertIn("SECRET_KEY", context_variables) SECRET_KEY = context_variables["SECRET_KEY"] self.assertEqual(SECRET_KEY, "DebugTalk") def test_get_parsed_request(self): test_runner = runner.Runner() testcase = { "variables": [{ "TOKEN": "debugtalk" }, { "random": "${gen_random_string(5)}" }, { "data": '{"name": "user", "password": "******"}' }, { "authorization": "${gen_md5($TOKEN, $data, $random)}" }], "request": { "url": "http://127.0.0.1:5000/api/users/1000", "method": "POST", "headers": { "Content-Type": "application/json", "authorization": "$authorization", "random": "$random", "secret_key": "$SECRET_KEY" }, "data": "$data" } } from tests import debugtalk self.context.import_module_items(debugtalk) self.context.bind_variables(testcase["variables"]) parsed_request = self.context.get_parsed_request(testcase["request"]) self.assertIn("authorization", parsed_request["headers"]) self.assertEqual(len(parsed_request["headers"]["authorization"]), 32) self.assertIn("random", parsed_request["headers"]) self.assertEqual(len(parsed_request["headers"]["random"]), 5) self.assertIn("data", parsed_request) self.assertEqual(parsed_request["data"], testcase["variables"][2]["data"]) self.assertEqual(parsed_request["headers"]["secret_key"], "DebugTalk") def test_exec_content_functions(self): test_runner = runner.Runner() content = "${sleep_N_secs(1)}" start_time = time.time() test_runner.context.eval_content(content) end_time = time.time() elapsed_time = end_time - start_time self.assertGreater(elapsed_time, 1) def test_do_validation(self): self.context.do_validation({ "check": "check", "check_value": 1, "expect": 1, "comparator": "eq" }) self.context.do_validation({ "check": "check", "check_value": "abc", "expect": "abc", "comparator": "==" }) config_dict = {"path": 'tests/data/demo_testset_hardcode.yml'} self.context.config_context(config_dict, "testset") self.context.do_validation({ "check": "status_code", "check_value": "201", "expect": 3, "comparator": "sum_status_code" }) def test_validate(self): url = "http://127.0.0.1:5000/" resp = requests.get(url) resp_obj = response.ResponseObject(resp) validators = [{ "eq": ["$resp_status_code", 201] }, { "check": "$resp_status_code", "comparator": "eq", "expect": 201 }, { "check": "$resp_body_success", "comparator": "eq", "expect": True }] variables = [{"resp_status_code": 200}, {"resp_body_success": True}] self.context.bind_variables(variables) with self.assertRaises(exceptions.ValidationFailure): self.context.validate(validators, resp_obj) validators = [{ "eq": ["$resp_status_code", 201] }, { "check": "$resp_status_code", "comparator": "eq", "expect": 201 }, { "check": "$resp_body_success", "comparator": "eq", "expect": True }, { "check": "${is_status_code_200($resp_status_code)}", "comparator": "eq", "expect": False }] variables = [{"resp_status_code": 201}, {"resp_body_success": True}] self.context.bind_variables(variables) from tests.debugtalk import is_status_code_200 functions = {"is_status_code_200": is_status_code_200} self.context.bind_functions(functions) self.context.validate(validators, resp_obj) def test_validate_exception(self): url = "http://127.0.0.1:5000/" resp = requests.get(url) resp_obj = response.ResponseObject(resp) # expected value missed in validators validators = [{ "eq": ["$resp_status_code", 201] }, { "check": "$resp_status_code", "comparator": "eq", "expect": 201 }] variables = [] self.context.bind_variables(variables) with self.assertRaises(exceptions.ParamsError): self.context.validate(validators, resp_obj) # expected value missed in variables mapping variables = [{"resp_status_code": 200}] self.context.bind_variables(variables) with self.assertRaises(exceptions.ValidationFailure): self.context.validate(validators, resp_obj)
class Runner(object): def __init__(self, config_dict=None, http_client_session=None): self.http_client_session = http_client_session self.context = Context() config_dict = config_dict or {} self.init_config(config_dict, "testset") # testset setup hooks testset_setup_hooks = config_dict.pop("setup_hooks", []) if testset_setup_hooks: self.do_hook_actions(testset_setup_hooks) # testset teardown hooks self.testset_teardown_hooks = config_dict.pop("teardown_hooks", []) def __del__(self): if self.testset_teardown_hooks: self.do_hook_actions(self.testset_teardown_hooks) def init_config(self, config_dict, level): """ create/update context variables binds @param (dict) config_dict @param (str) level, "testset" or "testcase" testset: { "name": "smoke testset", "path": "tests/data/demo_testset_variables.yml", "requires": [], # optional "function_binds": {}, # optional "import_module_items": [], # optional "variables": [], # optional "request": { "base_url": "http://127.0.0.1:5000", "headers": { "User-Agent": "iOS/2.8.3" } } } testcase: { "name": "testcase description", "requires": [], # optional "function_binds": {}, # optional "import_module_items": [], # optional "variables": [], # optional "request": { "url": "/api/get-token", "method": "POST", "headers": { "Content-Type": "application/json" } }, "json": { "sign": "f1219719911caae89ccc301679857ebfda115ca2" } } @param (str) context level, testcase or testset """ # convert keys in request headers to lowercase config_dict = utils.lower_config_dict_key(config_dict) self.context.init_context(level) self.context.config_context(config_dict, level) request_config = config_dict.get('request', {}) parsed_request = self.context.get_parsed_request(request_config, level) base_url = parsed_request.pop("base_url", None) self.http_client_session = self.http_client_session or HttpSession(base_url) return parsed_request def _handle_skip_feature(self, testcase_dict): """ handle skip feature for testcase - skip: skip current test unconditionally - skipIf: skip current test if condition is true - skipUnless: skip current test unless condition is true """ skip_reason = None if "skip" in testcase_dict: skip_reason = testcase_dict["skip"] elif "skipIf" in testcase_dict: skip_if_condition = testcase_dict["skipIf"] if self.context.eval_content(skip_if_condition): skip_reason = "{} evaluate to True".format(skip_if_condition) elif "skipUnless" in testcase_dict: skip_unless_condition = testcase_dict["skipUnless"] if not self.context.eval_content(skip_unless_condition): skip_reason = "{} evaluate to False".format(skip_unless_condition) if skip_reason: raise SkipTest(skip_reason) def do_hook_actions(self, actions): for action in actions: logger.log_debug("call hook: {}".format(action)) self.context.eval_content(action) def run_test(self, testcase_dict): """ run single testcase. @param (dict) testcase_dict { "name": "testcase description", "skip": "skip this test unconditionally", "times": 3, "requires": [], # optional, override "function_binds": {}, # optional, override "variables": [], # optional, override "request": { "url": "http://127.0.0.1:5000/api/users/1000", "method": "POST", "headers": { "Content-Type": "application/json", "authorization": "$authorization", "random": "$random" }, "body": '{"name": "user", "password": "******"}' }, "extract": [], # optional "validate": [], # optional "setup_hooks": [], # optional "teardown_hooks": [] # optional } @return True or raise exception during test """ # check skip self._handle_skip_feature(testcase_dict) # prepare parsed_request = self.init_config(testcase_dict, level="testcase") self.context.bind_testcase_variable("request", parsed_request) # setup hooks setup_hooks = testcase_dict.get("setup_hooks", []) setup_hooks.insert(0, "${setup_hook_prepare_kwargs($request)}") self.do_hook_actions(setup_hooks) try: url = parsed_request.pop('url') method = parsed_request.pop('method') group_name = parsed_request.pop("group", None) except KeyError: raise exception.ParamsError("URL or METHOD missed!") logger.log_info("{method} {url}".format(method=method, url=url)) logger.log_debug("request kwargs(raw): {kwargs}".format(kwargs=parsed_request)) # request resp = self.http_client_session.request( method, url, name=group_name, **parsed_request ) resp_obj = response.ResponseObject(resp) # teardown hooks teardown_hooks = testcase_dict.get("teardown_hooks", []) if teardown_hooks: self.context.bind_testcase_variable("response", resp_obj) self.do_hook_actions(teardown_hooks) # extract extractors = testcase_dict.get("extract", []) or testcase_dict.get("extractors", []) extracted_variables_mapping = resp_obj.extract_response(extractors) self.context.bind_extracted_variables(extracted_variables_mapping) # validate validators = testcase_dict.get("validate", []) or testcase_dict.get("validators", []) try: self.context.validate(validators, resp_obj) except (exception.ParamsError, exception.ResponseError, \ exception.ValidationError, exception.ParseResponseError): # log request err_req_msg = "request: \n" err_req_msg += "headers: {}\n".format(parsed_request.pop("headers", {})) for k, v in parsed_request.items(): err_req_msg += "{}: {}\n".format(k, v) logger.log_error(err_req_msg) # log response err_resp_msg = "response: \n" err_resp_msg += "status_code: {}\n".format(resp_obj.status_code) err_resp_msg += "headers: {}\n".format(resp_obj.headers) err_resp_msg += "content: {}\n".format(resp_obj.content) logger.log_error(err_resp_msg) raise def extract_output(self, output_variables_list): """ extract output variables """ variables_mapping = self.context.testcase_variables_mapping output = {} for variable in output_variables_list: if variable not in variables_mapping: logger.log_warning( "variable '{}' can not be found in variables mapping, failed to output!"\ .format(variable) ) continue output[variable] = variables_mapping[variable] return output
class VariableBindsUnittest(ApiServerUnittest): def setUp(self): self.context = Context() testcase_file_path = os.path.join(os.getcwd(), 'tests/data/demo_binds.yml') self.testcases = testcase.load_file(testcase_file_path) def test_context_init_functions(self): self.assertIn("get_timestamp", self.context.testset_functions_config) self.assertIn("gen_random_string", self.context.testset_functions_config) variables = [{ "random": "${gen_random_string(5)}" }, { "timestamp10": "${get_timestamp(10)}" }] self.context.bind_variables(variables) context_variables = self.context.testcase_variables_mapping self.assertEqual(len(context_variables["random"]), 5) self.assertEqual(len(context_variables["timestamp10"]), 10) def test_context_bind_testset_variables(self): # testcase in JSON format testcase1 = { "variables": [{ "GLOBAL_TOKEN": "debugtalk" }, { "token": "$GLOBAL_TOKEN" }] } # testcase in YAML format testcase2 = self.testcases["bind_variables"] for testcase in [testcase1, testcase2]: variables = testcase['variables'] self.context.bind_variables(variables, level="testset") testset_variables = self.context.testset_shared_variables_mapping testcase_variables = self.context.testcase_variables_mapping self.assertIn("GLOBAL_TOKEN", testset_variables) self.assertIn("GLOBAL_TOKEN", testcase_variables) self.assertEqual(testset_variables["GLOBAL_TOKEN"], "debugtalk") self.assertIn("token", testset_variables) self.assertIn("token", testcase_variables) self.assertEqual(testset_variables["token"], "debugtalk") def test_context_bind_testcase_variables(self): testcase1 = { "variables": [{ "GLOBAL_TOKEN": "debugtalk" }, { "token": "$GLOBAL_TOKEN" }] } testcase2 = self.testcases["bind_variables"] for testcase in [testcase1, testcase2]: variables = testcase['variables'] self.context.bind_variables(variables) testset_variables = self.context.testset_shared_variables_mapping testcase_variables = self.context.testcase_variables_mapping self.assertNotIn("GLOBAL_TOKEN", testset_variables) self.assertIn("GLOBAL_TOKEN", testcase_variables) self.assertEqual(testcase_variables["GLOBAL_TOKEN"], "debugtalk") self.assertNotIn("token", testset_variables) self.assertIn("token", testcase_variables) self.assertEqual(testcase_variables["token"], "debugtalk") def test_context_bind_lambda_functions(self): testcase1 = { "function_binds": { "add_one": lambda x: x + 1, "add_two_nums": lambda x, y: x + y }, "variables": [{ "add1": "${add_one(2)}" }, { "sum2nums": "${add_two_nums(2,3)}" }] } testcase2 = self.testcases["bind_lambda_functions"] for testcase in [testcase1, testcase2]: function_binds = testcase.get('function_binds', {}) self.context.bind_functions(function_binds) variables = testcase['variables'] self.context.bind_variables(variables) context_variables = self.context.testcase_variables_mapping self.assertIn("add1", context_variables) self.assertEqual(context_variables["add1"], 3) self.assertIn("sum2nums", context_variables) self.assertEqual(context_variables["sum2nums"], 5) def test_context_bind_lambda_functions_with_import(self): testcase1 = { "requires": ["random", "string", "hashlib"], "function_binds": { "gen_random_string": "lambda str_len: ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(str_len))", "gen_md5": "lambda *str_args: hashlib.md5(''.join(str_args).encode('utf-8')).hexdigest()" }, "variables": [{ "TOKEN": "debugtalk" }, { "random": "${gen_random_string(5)}" }, { "data": '{"name": "user", "password": "******"}' }, { "authorization": "${gen_md5($TOKEN, $data, $random)}" }] } testcase2 = self.testcases["bind_lambda_functions_with_import"] for testcase in [testcase1, testcase2]: requires = testcase.get('requires', []) self.context.import_requires(requires) function_binds = testcase.get('function_binds', {}) self.context.bind_functions(function_binds) variables = testcase['variables'] self.context.bind_variables(variables) context_variables = self.context.testcase_variables_mapping self.assertIn("TOKEN", context_variables) TOKEN = context_variables["TOKEN"] self.assertEqual(TOKEN, "debugtalk") self.assertIn("random", context_variables) self.assertIsInstance(context_variables["random"], str) self.assertEqual(len(context_variables["random"]), 5) random = context_variables["random"] self.assertIn("data", context_variables) data = context_variables["data"] self.assertIn("authorization", context_variables) self.assertEqual(len(context_variables["authorization"]), 32) authorization = context_variables["authorization"] self.assertEqual(utils.gen_md5(TOKEN, data, random), authorization) def test_import_module_items(self): testcase1 = { "import_module_items": ["tests.data.debugtalk"], "variables": [{ "TOKEN": "debugtalk" }, { "random": "${gen_random_string(5)}" }, { "data": '{"name": "user", "password": "******"}' }, { "authorization": "${gen_md5($TOKEN, $data, $random)}" }] } testcase2 = self.testcases["bind_module_functions"] for testcase in [testcase1, testcase2]: module_items = testcase.get('import_module_items', []) self.context.import_module_items(module_items) variables = testcase['variables'] self.context.bind_variables(variables) context_variables = self.context.testcase_variables_mapping self.assertIn("TOKEN", context_variables) TOKEN = context_variables["TOKEN"] self.assertEqual(TOKEN, "debugtalk") self.assertIn("random", context_variables) self.assertIsInstance(context_variables["random"], str) self.assertEqual(len(context_variables["random"]), 5) random = context_variables["random"] self.assertIn("data", context_variables) data = context_variables["data"] self.assertIn("authorization", context_variables) self.assertEqual(len(context_variables["authorization"]), 32) authorization = context_variables["authorization"] self.assertEqual(utils.gen_md5(TOKEN, data, random), authorization) self.assertIn("SECRET_KEY", context_variables) SECRET_KEY = context_variables["SECRET_KEY"] self.assertEqual(SECRET_KEY, "DebugTalk") def test_get_parsed_request(self): test_runner = runner.Runner() testcase = { "import_module_items": ["tests.data.debugtalk"], "variables": [{ "TOKEN": "debugtalk" }, { "random": "${gen_random_string(5)}" }, { "data": '{"name": "user", "password": "******"}' }, { "authorization": "${gen_md5($TOKEN, $data, $random)}" }], "request": { "url": "http://127.0.0.1:5000/api/users/1000", "METHOD": "POST", "Headers": { "Content-Type": "application/json", "authorization": "$authorization", "random": "$random", "SECRET_KEY": "$SECRET_KEY" }, "Data": "$data" } } parsed_request = test_runner.init_config(testcase, level="testcase") self.assertIn("authorization", parsed_request["headers"]) self.assertEqual(len(parsed_request["headers"]["authorization"]), 32) self.assertIn("random", parsed_request["headers"]) self.assertEqual(len(parsed_request["headers"]["random"]), 5) self.assertIn("data", parsed_request) self.assertEqual(parsed_request["data"], testcase["variables"][2]["data"]) self.assertEqual(parsed_request["headers"]["secret_key"], "DebugTalk") def test_exec_content_functions(self): test_runner = runner.Runner() content = "${sleep(1)}" start_time = time.time() test_runner.context.eval_content(content) end_time = time.time() elapsed_time = end_time - start_time self.assertGreater(elapsed_time, 1) def test_do_validation(self): self.context.do_validation({ "check": "check", "check_value": 1, "expect": 1, "comparator": "eq" }) self.context.do_validation({ "check": "check", "check_value": "abc", "expect": "abc", "comparator": "==" }) config_dict = {"path": 'tests/data/demo_testset_hardcode.yml'} self.context.config_context(config_dict, "testset") self.context.do_validation({ "check": "status_code", "check_value": "201", "expect": 3, "comparator": "sum_status_code" }) def test_validate(self): url = "http://127.0.0.1:5000/" resp = requests.get(url) resp_obj = response.ResponseObject(resp) validators = [{ "eq": ["$resp_status_code", 201] }, { "check": "$resp_status_code", "comparator": "eq", "expect": 201 }, { "check": "$resp_body_success", "comparator": "eq", "expect": True }] variables = [{"resp_status_code": 200}, {"resp_body_success": True}] self.context.bind_variables(variables) with self.assertRaises(exception.ValidationError): self.context.validate(validators, resp_obj) variables = [{"resp_status_code": 201}, {"resp_body_success": True}] self.context.bind_variables(variables) self.assertTrue(self.context.validate(validators, resp_obj)) def test_validate_exception(self): url = "http://127.0.0.1:5000/" resp = requests.get(url) resp_obj = response.ResponseObject(resp) # expected value missed in validators validators = [{ "eq": ["$resp_status_code", 201] }, { "check": "$resp_status_code", "comparator": "eq", "expect": 201 }, { "check": "$resp_body_success", "comparator": "eq", "expect": True }] variables = [] self.context.bind_variables(variables) with self.assertRaises(exception.ParamsError): self.context.validate(validators, resp_obj) # expected value missed in variables mapping variables = [{"resp_status_code": 200}] self.context.bind_variables(variables) with self.assertRaises(exception.ValidationError): self.context.validate(validators, resp_obj)
def __init__(self, testgroup=None): self.TestGroup = testgroup self.context = Context()
class Runner(object): def __init__(self, config_dict=None, http_client_session=None): self.http_client_session = http_client_session self.context = Context() testcase.load_test_dependencies() config_dict = config_dict or {} self.init_config(config_dict, "testset") def init_config(self, config_dict, level): """ create/update context variables binds @param (dict) config_dict @param (str) level, "testset" or "testcase" testset: { "name": "smoke testset", "path": "tests/data/demo_testset_variables.yml", "requires": [], # optional "function_binds": {}, # optional "import_module_items": [], # optional "variables": [], # optional "request": { "base_url": "http://127.0.0.1:5000", "headers": { "User-Agent": "iOS/2.8.3" } } } testcase: { "name": "testcase description", "requires": [], # optional "function_binds": {}, # optional "import_module_items": [], # optional "variables": [], # optional "request": { "url": "/api/get-token", "method": "POST", "headers": { "Content-Type": "application/json" } }, "json": { "sign": "f1219719911caae89ccc301679857ebfda115ca2" } } @param (str) context level, testcase or testset """ # convert keys in request headers to lowercase config_dict = utils.lower_config_dict_key(config_dict) self.context.init_context(level) self.context.config_context(config_dict, level) request_config = config_dict.get('request', {}) parsed_request = self.context.get_parsed_request(request_config, level) base_url = parsed_request.pop("base_url", None) self.http_client_session = self.http_client_session or HttpSession(base_url) return parsed_request def _handle_skip_feature(self, testcase_dict): """ handle skip feature for testcase - skip: skip current test unconditionally - skipIf: skip current test if condition is true - skipUnless: skip current test unless condition is true """ skip_reason = None if "skip" in testcase_dict: skip_reason = testcase_dict["skip"] elif "skipIf" in testcase_dict: skip_if_condition = testcase_dict["skipIf"] if self.context.eval_content(skip_if_condition): skip_reason = "{} evaluate to True".format(skip_if_condition) elif "skipUnless" in testcase_dict: skip_unless_condition = testcase_dict["skipUnless"] if not self.context.eval_content(skip_unless_condition): skip_reason = "{} evaluate to False".format(skip_unless_condition) if skip_reason: raise SkipTest(skip_reason) def _run_test(self, testcase_dict): """ run single testcase. @param (dict) testcase_dict { "name": "testcase description", "skip": "skip this test unconditionally", "times": 3, "requires": [], # optional, override "function_binds": {}, # optional, override "variables": [], # optional, override "request": { "url": "http://127.0.0.1:5000/api/users/1000", "method": "POST", "headers": { "Content-Type": "application/json", "authorization": "$authorization", "random": "$random" }, "body": '{"name": "user", "password": "******"}' }, "extract": [], # optional "validate": [], # optional "setup": [], # optional "teardown": [] # optional } @return True or raise exception during test """ parsed_request = self.init_config(testcase_dict, level="testcase") try: url = parsed_request.pop('url') method = parsed_request.pop('method') group_name = parsed_request.pop("group", None) except KeyError: raise exception.ParamsError("URL or METHOD missed!") extractors = testcase_dict.get("extract", []) validators = testcase_dict.get("validate", []) setup_actions = testcase_dict.get("setup", []) teardown_actions = testcase_dict.get("teardown", []) self._handle_skip_feature(testcase_dict) def setup_teardown(actions): for action in actions: self.context.eval_content(action) setup_teardown(setup_actions) resp = self.http_client_session.request( method, url, name=group_name, **parsed_request ) resp_obj = response.ResponseObject(resp) extracted_variables_mapping = resp_obj.extract_response(extractors) self.context.bind_extracted_variables(extracted_variables_mapping) try: self.context.validate(validators, resp_obj) except (exception.ParamsError, exception.ResponseError, exception.ValidationError): err_msg = u"Exception occured.\n" err_msg += u"HTTP request url: {}\n".format(url) err_msg += u"HTTP request kwargs: {}\n".format(parsed_request) err_msg += u"HTTP response status_code: {}\n".format(resp.status_code) err_msg += u"HTTP response content: \n{}".format(resp.text) logging.error(err_msg) raise finally: setup_teardown(teardown_actions) return True def extract_output(self, output_variables_list): """ extract output variables """ variables_mapping = self.context.testcase_variables_mapping output = {} for variable in output_variables_list: if variable not in variables_mapping: logging.warning( "variable '{}' can not be found in variables mapping, failed to ouput!"\ .format(variable) ) continue output[variable] = variables_mapping[variable] return output
class Runner(object): def __init__(self, config_dict=None, http_client_session=None): self.http_client_session = http_client_session self.context = Context() config_dict = config_dict or {} self.init_config(config_dict, "testset") # testset setup hooks testset_setup_hooks = config_dict.pop("setup_hooks", []) if testset_setup_hooks: self.do_hook_actions(testset_setup_hooks) # testset teardown hooks self.testset_teardown_hooks = config_dict.pop("teardown_hooks", []) def __del__(self): if self.testset_teardown_hooks: self.do_hook_actions(self.testset_teardown_hooks) def init_config(self, config_dict, level): """ create/update context variables binds @param (dict) config_dict @param (str) level, "testset" or "testcase" testset: { "name": "smoke testset", "path": "tests/data/demo_testset_variables.yml", "variables": [], # optional "request": { "base_url": "http://127.0.0.1:5000", "headers": { "User-Agent": "iOS/2.8.3" } } } testcase: { "name": "testcase description", "variables": [], # optional "request": { "url": "/api/get-token", "method": "POST", "headers": { "Content-Type": "application/json" } }, "json": { "sign": "f1219719911caae89ccc301679857ebfda115ca2" } } @param (str) context level, testcase or testset """ # convert keys in request headers to lowercase config_dict = utils.lower_config_dict_key(config_dict) self.context.init_context(level) self.context.config_context(config_dict, level) request_config = config_dict.get('request', {}) parsed_request = self.context.get_parsed_request(request_config, level) base_url = parsed_request.pop("base_url", None) self.http_client_session = self.http_client_session or HttpSession(base_url) return parsed_request def _handle_skip_feature(self, testcase_dict): """ handle skip feature for testcase - skip: skip current test unconditionally - skipIf: skip current test if condition is true - skipUnless: skip current test unless condition is true """ skip_reason = None if "skip" in testcase_dict: skip_reason = testcase_dict["skip"] elif "skipIf" in testcase_dict: skip_if_condition = testcase_dict["skipIf"] if self.context.eval_content(skip_if_condition): skip_reason = "{} evaluate to True".format(skip_if_condition) elif "skipUnless" in testcase_dict: skip_unless_condition = testcase_dict["skipUnless"] if not self.context.eval_content(skip_unless_condition): skip_reason = "{} evaluate to False".format(skip_unless_condition) if skip_reason: raise SkipTest(skip_reason) def do_hook_actions(self, actions): for action in actions: logger.log_debug("call hook: {}".format(action)) self.context.eval_content(action) def run_test(self, testcase_dict): """ run single testcase. @param (dict) testcase_dict { "name": "testcase description", "skip": "skip this test unconditionally", "times": 3, "variables": [], # optional, override "request": { "url": "http://127.0.0.1:5000/api/users/1000", "method": "POST", "headers": { "Content-Type": "application/json", "authorization": "$authorization", "random": "$random" }, "body": '{"name": "user", "password": "******"}' }, "extract": [], # optional "validate": [], # optional "setup_hooks": [], # optional "teardown_hooks": [] # optional } @return True or raise exception during test """ # check skip self._handle_skip_feature(testcase_dict) # prepare parsed_request = self.init_config(testcase_dict, level="testcase") self.context.bind_testcase_variable("request", parsed_request) # setup hooks setup_hooks = testcase_dict.get("setup_hooks", []) setup_hooks.insert(0, "${setup_hook_prepare_kwargs($request)}") self.do_hook_actions(setup_hooks) try: url = parsed_request.pop('url') method = parsed_request.pop('method') group_name = parsed_request.pop("group", None) except KeyError: raise exceptions.ParamsError("URL or METHOD missed!") # TODO: move method validation to json schema valid_methods = ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"] if method.upper() not in valid_methods: err_msg = u"Invalid HTTP method! => {}\n".format(method) err_msg += "Available HTTP methods: {}".format("/".join(valid_methods)) logger.log_error(err_msg) raise exceptions.ParamsError(err_msg) logger.log_info("{method} {url}".format(method=method, url=url)) logger.log_debug("request kwargs(raw): {kwargs}".format(kwargs=parsed_request)) # request resp = self.http_client_session.request( method, url, name=group_name, **parsed_request ) resp_obj = response.ResponseObject(resp) # teardown hooks teardown_hooks = testcase_dict.get("teardown_hooks", []) if teardown_hooks: self.context.bind_testcase_variable("response", resp_obj) self.do_hook_actions(teardown_hooks) # extract extractors = testcase_dict.get("extract", []) or testcase_dict.get("extractors", []) extracted_variables_mapping = resp_obj.extract_response(extractors,context=self.context) self.context.bind_extracted_variables(extracted_variables_mapping) # validate validators = testcase_dict.get("validate", []) or testcase_dict.get("validators", []) try: self.context.validate(validators, resp_obj) except (exceptions.ParamsError, \ exceptions.ValidationFailure, exceptions.ExtractFailure): # log request err_req_msg = "request: \n" err_req_msg += "headers: {}\n".format(parsed_request.pop("headers", {})) for k, v in parsed_request.items(): err_req_msg += "{}: {}\n".format(k, repr(v)) logger.log_error(err_req_msg) # log response err_resp_msg = "response: \n" err_resp_msg += "status_code: {}\n".format(resp_obj.status_code) err_resp_msg += "headers: {}\n".format(resp_obj.headers) err_resp_msg += "body: {}\n".format(repr(resp_obj.text)) logger.log_error(err_resp_msg) raise def extract_output(self, output_variables_list): """ extract output variables """ variables_mapping = self.context.testcase_variables_mapping output = {} for variable in output_variables_list: if variable not in variables_mapping: logger.log_warning( "variable '{}' can not be found in variables mapping, failed to output!"\ .format(variable) ) continue output[variable] = variables_mapping[variable] return output
class Runner(object): def __init__(self, http_client_session=None, request_failure_hook=None): self.http_client_session = http_client_session self.context = Context() testcase.load_test_dependencies() self.request_failure_hook = request_failure_hook def init_config(self, config_dict, level): """ create/update context variables binds @param (dict) config_dict @param (str) level, "testset" or "testcase" testset: { "name": "smoke testset", "path": "tests/data/demo_testset_variables.yml", "requires": [], # optional "function_binds": {}, # optional "import_module_items": [], # optional "variables": [], # optional "request": { "base_url": "http://127.0.0.1:5000", "headers": { "User-Agent": "iOS/2.8.3" } } } testcase: { "name": "testcase description", "requires": [], # optional "function_binds": {}, # optional "import_module_items": [], # optional "variables": [], # optional "request": { "url": "/api/get-token", "method": "POST", "headers": { "Content-Type": "application/json" } }, "json": { "sign": "f1219719911caae89ccc301679857ebfda115ca2" } } @param (str) context level, testcase or testset """ # convert keys in request headers to lowercase config_dict = utils.lower_config_dict_key(config_dict) self.context.init_context(level) self.context.config_context(config_dict, level) request_config = config_dict.get('request', {}) parsed_request = self.context.get_parsed_request(request_config, level) base_url = parsed_request.pop("base_url", None) self.http_client_session = self.http_client_session or HttpSession(base_url) return parsed_request def _run_test(self, testcase_dict): """ run single testcase. @param (dict) testcase_dict { "name": "testcase description", "times": 3, "requires": [], # optional, override "function_binds": {}, # optional, override "variables": [], # optional, override "request": { "url": "http://127.0.0.1:5000/api/users/1000", "method": "POST", "headers": { "Content-Type": "application/json", "authorization": "$authorization", "random": "$random" }, "body": '{"name": "user", "password": "******"}' }, "extract": [], # optional "validate": [], # optional "setup": [], # optional "teardown": [] # optional } @return True or raise exception during test """ parsed_request = self.init_config(testcase_dict, level="testcase") try: url = parsed_request.pop('url') method = parsed_request.pop('method') group_name = parsed_request.pop("group", None) except KeyError: raise exception.ParamsError("URL or METHOD missed!") run_times = int(testcase_dict.get("times", 1)) extractors = testcase_dict.get("extract") \ or testcase_dict.get("extractors") \ or testcase_dict.get("extract_binds", []) validators = testcase_dict.get("validate") \ or testcase_dict.get("validators", []) setup_actions = testcase_dict.get("setup", []) teardown_actions = testcase_dict.get("teardown", []) def setup_teardown(actions): for action in actions: self.context.exec_content_functions(action) for _ in range(run_times): setup_teardown(setup_actions) resp = self.http_client_session.request( method, url, name=group_name, **parsed_request ) resp_obj = response.ResponseObject(resp) extracted_variables_mapping = resp_obj.extract_response(extractors) self.context.bind_extracted_variables(extracted_variables_mapping) try: self.context.validate(validators, resp_obj) except (exception.ParamsError, exception.ResponseError, exception.ValidationError): err_msg = u"Exception occured.\n" err_msg += u"HTTP request url: {}\n".format(url) err_msg += u"HTTP request kwargs: {}\n".format(parsed_request) err_msg += u"HTTP response status_code: {}\n".format(resp.status_code) err_msg += u"HTTP response content: \n{}".format(resp.text) logging.error(err_msg) raise finally: setup_teardown(teardown_actions) return True def _run_testset(self, testset, variables_mapping=None): """ run single testset, including one or several testcases. @param (dict) testset { "name": "testset description", "config": { "name": "testset description", "requires": [], "function_binds": {}, "variables": [], "request": {} }, "testcases": [ { "name": "testcase description", "variables": [], # optional, override "request": {}, "extract": {}, # optional "validate": {} # optional }, testcase12 ] } (dict) variables_mapping: passed in variables mapping, it will override variables in config block @return (dict) test result of testset { "success": True, "output": {} # variables mapping } """ success = True config_dict = testset.get("config", {}) variables = config_dict.get("variables", []) variables_mapping = variables_mapping or {} config_dict["variables"] = utils.override_variables_binds(variables, variables_mapping) self.init_config(config_dict, level="testset") testcases = testset.get("testcases", []) for testcase_dict in testcases: try: self._run_test(testcase_dict) except exception.MyBaseError as ex: success = False if self.request_failure_hook: self.request_failure_hook.fire( request_type=testcase_dict.get("request", {}).get("method"), name=testcase_dict.get("request", {}).get("url"), response_time=0, exception=ex ) else: logging.exception( "Exception occured in testcase: {}".format(testcase_dict.get("name"))) break output_variables_list = config_dict.get("output", []) return { "success": success, "output": self.generate_output(output_variables_list) } def run(self, path, mapping=None): """ run specified testset path or folder path. @param path: path could be in several type - absolute/relative file path - absolute/relative folder path - list/set container with file(s) and/or folder(s) (dict) mapping: passed in variables mapping, it will override variables in config block """ success = True mapping = mapping or {} output = {} testsets = testcase.load_testcases_by_path(path) for testset in testsets: try: result = self._run_testset(testset, mapping) assert result["success"] output.update(result["output"]) except AssertionError: success = False return { "success": success, "output": output } def generate_output(self, output_variables_list): """ generate and print output """ variables_mapping = self.context.get_testcase_variables_mapping() output = { variable: variables_mapping[variable] for variable in output_variables_list } utils.print_output(output) return output