def __getattr__(self, key): if key in ["json", "content", "body"]: try: value = self.resp_obj.json() except ValueError: value = self.resp_obj.content elif key == "cookies": value = self.resp_obj.cookies.get_dict() else: try: value = getattr(self.resp_obj, key) except AttributeError: err_msg = "ResponseObject does not have attribute: {}".format( key) logger.error(err_msg) raise exceptions.ParamsError(err_msg) self.__dict__[key] = value return value
def extract_field(self, field,context=""): """ extract value from requests.Response. """ if not isinstance(field, basestring): err_msg = u"Invalid extractor! => {}\n".format(field) logger.log_error(err_msg) raise exceptions.ParamsError(err_msg) msg = "extract: {}".format(field) if text_extractor_regexp_compile.match(field): value = self._extract_field_with_regex(field,context=context) else: value = self._extract_field_with_delimiter(field,context=context) msg += "\t=> {}".format(value) logger.log_debug(msg) return value
def run(self, path_or_tests, dot_env_path=None, mapping=None): """ main interface. Args: path_or_tests: str: testcase/testsuite file/foler path dict: valid testcase/testsuite data Returns: dict: result summary """ logger.log_info("HttpRunner version: {}".format(__version__)) if loader.is_testcase_path(path_or_tests): return self.run_path(path_or_tests, dot_env_path, mapping) elif loader.is_testcases(path_or_tests): return self.run_tests(path_or_tests) else: raise exceptions.ParamsError("Invalid testcase path or testcases: {}".format(path_or_tests))
def do_validation(self, validator_dict): """ validate with functions """ # TODO: move comparator uniform to init_test_suites comparator = utils.get_uniform_comparator(validator_dict["comparator"]) validate_func = self.testcase_parser.get_bind_function(comparator) if not validate_func: raise exceptions.FunctionNotFound("comparator not found: {}".format(comparator)) check_item = validator_dict["check"] check_value = validator_dict["check_value"] expect_value = validator_dict["expect"] if (check_value is None or expect_value is None) \ and comparator not in ["is", "eq", "equals", "=="]: raise exceptions.ParamsError("Null value can only be compared with comparator: eq/equals/==") validate_msg = "validate: {} {} {}({})".format( check_item, comparator, expect_value, type(expect_value).__name__ ) try: validator_dict["check_result"] = "pass" validate_func(check_value, expect_value) validate_msg += "\t==> pass" logger.log_debug(validate_msg) except (AssertionError, TypeError): validate_msg += "\t==> fail" validate_msg += "\n{}({}) {} {}({})".format( check_value, type(check_value).__name__, comparator, expect_value, type(expect_value).__name__ ) logger.log_error(validate_msg) validator_dict["check_result"] = "fail" raise exceptions.ValidationFailure(validate_msg)
def override_mapping_list(variables, new_mapping): """ override variables with new mapping. Args: variables (list): variables list [ {"var_a": 1}, {"var_b": "world"} ] new_mapping (dict): overrided variables mapping { "var_a": "hello" } Returns: OrderedDict: overrided variables mapping. Examples: >>> variables = [ {"var_a": 1}, {"var_b": "world"} ] >>> new_mapping = { "var_a": "hello" } >>> override_mapping_list(variables, new_mapping) OrderedDict( { "var_a": "hello", "var_b": "world" } ) """ if isinstance(variables, list): variables_ordered_dict = convert_mappinglist_to_orderdict(variables) elif isinstance(variables, (OrderedDict, dict)): variables_ordered_dict = variables else: raise exceptions.ParamsError("variables error!") return update_ordered_dict(variables_ordered_dict, new_mapping)
def convert_testcase_path(testcase_path: Text) -> Tuple[Text, Text]: """convert single YAML/JSON testcase path to python file""" if os.path.isdir(testcase_path): # folder does not need to convert return testcase_path, "" raw_file_name, file_suffix = os.path.splitext(os.path.basename(testcase_path)) file_suffix = file_suffix.lower() if file_suffix not in [".json", ".yml", ".yaml"]: raise exceptions.ParamsError("") file_name = raw_file_name.replace(" ", "_").replace(".", "_").replace("-", "_") testcase_dir = os.path.dirname(testcase_path) testcase_python_path = os.path.join(testcase_dir, f"{file_name}_test.py") # convert title case, e.g. request_with_variables => RequestWithVariables name_in_title_case = file_name.title().replace("_", "") return testcase_python_path, name_in_title_case
def _get_block_by_name(ref_call, ref_type, project_mapping): """ get test content by reference name. Args: ref_call (str): call function. e.g. api_v1_Account_Login_POST($UserName, $Password) ref_type (enum): "def-api" or "def-testcase" project_mapping (dict): project_mapping Returns: dict: api/testcase definition. Raises: exceptions.ParamsError: call args number is not equal to defined args number. """ function_meta = parser.parse_function(ref_call) func_name = function_meta["func_name"] call_args = function_meta["args"] block = _get_test_definition(func_name, ref_type, project_mapping) def_args = block.get("function_meta", {}).get("args", []) if len(call_args) != len(def_args): err_msg = "{}: call args number is not equal to defined args number!\n".format( func_name) err_msg += "defined args: {}\n".format(def_args) err_msg += "reference args: {}".format(call_args) logger.log_error(err_msg) raise exceptions.ParamsError(err_msg) args_mapping = {} for index, item in enumerate(def_args): if call_args[index] == item: continue args_mapping[item] = call_args[index] if args_mapping: block = parser.substitute_variables(block, args_mapping) return block
def main_run(extra_args): tests_path_list = [] for index, item in enumerate(extra_args): if not os.path.exists(item): # item is not file/folder path continue elif os.path.isfile(item): # replace YAML/JSON file path with generated python file extra_args[index] = convert_testcase_path(item) tests_path_list.append(item) if len(tests_path_list) == 0: # has not specified any testcase path raise exceptions.ParamsError("Missed testcase path") main_make(tests_path_list) if "-s" not in extra_args: extra_args.insert(0, "-s") pytest.main(extra_args)
def __ensure_absolute(path: Text) -> Text: if path.startswith("./"): # Linux/Darwin, hrun ./test.yml path = path[len("./"):] elif path.startswith(".\\"): # Windows, hrun .\\test.yml path = path[len(".\\"):] path = ensure_path_sep(path) project_meta = load_project_meta(path) if os.path.isabs(path): absolute_path = path else: absolute_path = os.path.join(project_meta.RootDir, path) if not os.path.isfile(absolute_path): raise exceptions.ParamsError( f"Invalid testcase file path: {absolute_path}") return absolute_path
def run(self, path_or_tests, dot_env_path=None, mapping=None): """ main interface. Args: path_or_tests: str: testcase/testsuite file/foler path dict: valid testcase/testsuite data dot_env_path (str): specified .env file path. mapping (dict): if mapping is specified, it will override variables in config block. Returns: dict: result summary """ logger.log_info("HttpRunner version: {}".format(__version__)) if loader.is_test_path(path_or_tests): return self.run_path(path_or_tests, dot_env_path, mapping) elif loader.is_test_content(path_or_tests): return self.run_tests(path_or_tests) else: raise exceptions.ParamsError("Invalid testcase path or testcases: {}".format(path_or_tests))
def extract_field(self, field): """ extract value from requests.Response. """ if not isinstance(field, str): err_msg = f"Invalid extractor! => {field}\n" logger.error(err_msg) raise exceptions.ParamsError(err_msg) msg = f"extract: {field}" if field.startswith("$"): value = self._extract_field_with_jsonpath(field) elif text_extractor_regexp_compile.match(field): value = self._extract_field_with_regex(field) else: value = self._extract_field_with_delimiter(field) msg += f"\t=> {value}" logger.debug(msg) return value
def load_test_dependencies(): """ load all api and suite definitions. default api folder is "$CWD/tests/api/". default suite folder is "$CWD/tests/suite/". """ # TODO: cache api and suite loading # load api definitions api_def_folder = os.path.join(os.getcwd(), "tests", "api") for test_file in load_folder_files(api_def_folder): load_api_file(test_file) # load suite definitions suite_def_folder = os.path.join(os.getcwd(), "tests", "suite") for suite_file in load_folder_files(suite_def_folder): suite = load_test_file(suite_file) if "def" not in suite["config"]: raise exceptions.ParamsError("def missed in suite file: {}!".format(suite_file)) call_func = suite["config"]["def"] function_meta = parser.parse_function(call_func) suite["function_meta"] = function_meta overall_def_dict["suite"][function_meta["func_name"]] = suite
def __run_step_testcase(self, step: TStep) -> StepData: """run teststep: referenced testcase""" step_data = StepData(name=step.name) step_variables = step.variables step_export = step.export if hasattr(step.testcase, "config") and hasattr( step.testcase, "teststeps"): testcase_cls = step.testcase case_result = (testcase_cls().with_session( self.__session).with_case_id(self.__case_id).with_variables( step_variables).with_export(step_export).run()) elif isinstance(step.testcase, Text): if os.path.isabs(step.testcase): ref_testcase_path = step.testcase else: ref_testcase_path = os.path.join(self.__project_meta.RootDir, step.testcase) case_result = (HttpRunner().with_session( self.__session).with_case_id( self.__case_id).with_variables(step_variables).with_export( step_export).run_path(ref_testcase_path)) else: raise exceptions.ParamsError( f"Invalid teststep referenced testcase: {step.dict()}") step_data.data = case_result.get_step_datas() # list of step data step_data.export_vars = case_result.get_export_variables() step_data.success = case_result.success self.success = case_result.success if step_data.export_vars: logger.info(f"export variables: {step_data.export_vars}") return step_data
def run(self, path_or_tests, dot_env_path=None, mapping=None): """ main interface. Args: path_or_tests: str: testcase/testsuite file/foler path dict: valid testcase/testsuite data dot_env_path (str): specified .env file path. mapping (dict): if mapping is specified, it will override variables in config block. Returns: dict: result summary """ logger.info(f"HttpRunner version: {__version__}") if loader.is_test_path(path_or_tests): return self.run_path(path_or_tests, dot_env_path, mapping) elif loader.is_test_content(path_or_tests): project_working_directory = path_or_tests.get("project_mapping", {}).get("PWD", os.getcwd()) loader.init_pwd(project_working_directory) return self.run_tests(path_or_tests) else: raise exceptions.ParamsError(f"Invalid testcase path or testcases: {path_or_tests}")
def extract_field(self, field): """ extract value from requests.Response. """ if not isinstance(field, basestring): err_msg = u"Invalid extractor! => {}\n".format(field) logger.log_error(err_msg) raise exceptions.ParamsError(err_msg) msg = "extract: {}".format(field) if field.startswith("$"): value = self._extract_field_with_jsonpath(field) elif text_extractor_regexp_compile.match(field): value = self._extract_field_with_regex(field) else: value = self._extract_field_with_delimiter(field) if is_py2 and isinstance(value, unicode): value = value.encode("utf-8") msg += "\t=> {}".format(value) logger.log_debug(msg) return value
def _extract_field_with_delimiter(self, field): """ response content could be json or html text. Args: field (str): string joined by delimiter. e.g. "status_code" "headers" "cookies" "content" "headers.content-type" "content.person.name.first_name" """ # string.split(sep=None, maxsplit=1) -> list of strings # e.g. "content.person.name" => ["content", "person.name"] try: top_query, sub_query = field.split('.', 1) except ValueError: top_query = field sub_query = None # status_code if top_query in ["status_code", "encoding", "ok", "reason", "url"]: if sub_query: # status_code.XX err_msg = u"Failed to extract: {}\n".format(field) logger.log_error(err_msg) raise exceptions.ParamsError(err_msg) return getattr(self, top_query) # cookies elif top_query == "cookies": cookies = self.cookies if not sub_query: # extract cookies return cookies try: return cookies[sub_query] except KeyError: err_msg = u"Failed to extract cookie! => {}\n".format(field) err_msg += u"response cookies: {}\n".format(cookies) logger.log_error(err_msg) raise exceptions.ExtractFailure(err_msg) # elapsed elif top_query == "elapsed": available_attributes = u"available attributes: days, seconds, microseconds, total_seconds" if not sub_query: err_msg = u"elapsed is datetime.timedelta instance, attribute should also be specified!\n" err_msg += available_attributes logger.log_error(err_msg) raise exceptions.ParamsError(err_msg) elif sub_query in ["days", "seconds", "microseconds"]: return getattr(self.elapsed, sub_query) elif sub_query == "total_seconds": return self.elapsed.total_seconds() else: err_msg = "{} is not valid datetime.timedelta attribute.\n".format( sub_query) err_msg += available_attributes logger.log_error(err_msg) raise exceptions.ParamsError(err_msg) # req_headers elif top_query == "req_headers": headers = self.resp_obj.request.headers if not sub_query: # extract headers return headers try: return headers[sub_query] except KeyError: err_msg = u"Failed to extract req_header! => {}\n".format( field) err_msg += u"response headers: {}\n".format(headers) logger.log_error(err_msg) raise exceptions.ExtractFailure(err_msg) # headers elif top_query == "headers": headers = self.headers if not sub_query: # extract headers return headers try: return headers[sub_query] except KeyError: err_msg = u"Failed to extract header! => {}\n".format(field) err_msg += u"response headers: {}\n".format(headers) logger.log_error(err_msg) raise exceptions.ExtractFailure(err_msg) # response body elif top_query in ["body", "content", "text", "json"]: try: body = self.json except exceptions.JSONDecodeError: body = self.text if not sub_query: # extract response body return self.content # return body if isinstance(body, (dict, list)): # content = {"xxx": 123}, content.xxx return utils.query_json(body, sub_query) elif sub_query.isdigit(): # content = "abcdefg", content.3 => d return utils.query_json(body, sub_query) else: # content = "<html>abcdefg</html>", content.xxx err_msg = u"Failed to extract attribute from response body! => {}\n".format( field) err_msg += u"response body: {}\n".format(body) logger.log_error(err_msg) raise exceptions.ExtractFailure(err_msg) # new set response attributes in teardown_hooks elif top_query in self.__dict__: attributes = self.__dict__[top_query] if not sub_query: # extract response attributes return attributes if isinstance(attributes, (dict, list)): # attributes = {"xxx": 123}, content.xxx return utils.query_json(attributes, sub_query) elif sub_query.isdigit(): # attributes = "abcdefg", attributes.3 => d return utils.query_json(attributes, sub_query) else: # content = "attributes.new_attribute_not_exist" err_msg = u"Failed to extract cumstom set attribute from teardown hooks! => {}\n".format( field) err_msg += u"response set attributes: {}\n".format(attributes) logger.log_error(err_msg) raise exceptions.TeardownHooksFailure(err_msg) # others else: err_msg = u"Failed to extract attribute from response! => {}\n".format( field) err_msg += u"available response attributes: status_code, cookies, elapsed, headers, content, " \ u"text, json, encoding, ok, reason, url.\n\n" err_msg += u"If you want to set attribute in teardown_hooks, take the following example as reference:\n" err_msg += u"response.new_attribute = 'new_attribute_value'\n" logger.log_error(err_msg) raise exceptions.ParamsError(err_msg)
def parse_parameters(parameters: Dict, ) -> List[Dict]: """ parse parameters and generate cartesian product. Args: parameters (Dict) parameters: parameter name and value mapping parameter value may be in three types: (1) data list, e.g. ["iOS/10.1", "iOS/10.2", "iOS/10.3"] (2) call built-in parameterize function, "${parameterize(account.csv)}" (3) call custom function in debugtalk.py, "${gen_app_version()}" Returns: list: cartesian product list Examples: >>> parameters = { "user_agent": ["iOS/10.1", "iOS/10.2", "iOS/10.3"], "username-password": "******", "app_version": "${gen_app_version()}", } >>> parse_parameters(parameters) """ parsed_parameters_list: List[List[Dict]] = [] # load project_meta functions project_meta = loader.load_project_meta(os.getcwd()) functions_mapping = project_meta.functions for parameter_name, parameter_content in parameters.items(): parameter_name_list = parameter_name.split("-") if isinstance(parameter_content, List): # (1) data list # e.g. {"app_version": ["2.8.5", "2.8.6"]} # => [{"app_version": "2.8.5", "app_version": "2.8.6"}] # e.g. {"username-password": [["user1", "111111"], ["test2", "222222"]} # => [{"username": "******", "password": "******"}, {"username": "******", "password": "******"}] parameter_content_list: List[Dict] = [] for parameter_item in parameter_content: if not isinstance(parameter_item, (list, tuple)): # "2.8.5" => ["2.8.5"] parameter_item = [parameter_item] # ["app_version"], ["2.8.5"] => {"app_version": "2.8.5"} # ["username", "password"], ["user1", "111111"] => {"username": "******", "password": "******"} parameter_content_dict = dict(zip(parameter_name_list, parameter_item)) parameter_content_list.append(parameter_content_dict) elif isinstance(parameter_content, Text): # (2) & (3) parsed_parameter_content: List = parse_data( parameter_content, {}, functions_mapping ) if not isinstance(parsed_parameter_content, List): raise exceptions.ParamsError( f"parameters content should be in List type, got {parsed_parameter_content} for {parameter_content}" ) parameter_content_list: List[Dict] = [] for parameter_item in parsed_parameter_content: if isinstance(parameter_item, Dict): # get subset by parameter name # {"app_version": "${gen_app_version()}"} # gen_app_version() => [{'app_version': '2.8.5'}, {'app_version': '2.8.6'}] # {"username-password": "******"} # get_account() => [ # {"username": "******", "password": "******"}, # {"username": "******", "password": "******"} # ] parameter_dict: Dict = { key: parameter_item[key] for key in parameter_name_list } elif isinstance(parameter_item, (List, tuple)): if len(parameter_name_list) == len(parameter_item): # {"username-password": "******"} # get_account() => [("user1", "111111"), ("user2", "222222")] parameter_dict = dict(zip(parameter_name_list, parameter_item)) else: raise exceptions.ParamsError( f"parameter names length are not equal to value length.\n" f"parameter names: {parameter_name_list}\n" f"parameter values: {parameter_item}" ) elif len(parameter_name_list) == 1: # {"user_agent": "${get_user_agent()}"} # get_user_agent() => ["iOS/10.1", "iOS/10.2"] # parameter_dict will get: {"user_agent": "iOS/10.1", "user_agent": "iOS/10.2"} parameter_dict = {parameter_name_list[0]: parameter_item} else: raise exceptions.ParamsError( f"Invalid parameter names and values:\n" f"parameter names: {parameter_name_list}\n" f"parameter values: {parameter_item}" ) parameter_content_list.append(parameter_dict) else: raise exceptions.ParamsError( f"parameter content should be List or Text(variables or functions call), got {parameter_content}" ) parsed_parameters_list.append(parameter_content_list) return utils.gen_cartesian_product(*parsed_parameters_list)
def _add_tests(self, testcases): """ initialize testcase with Runner() and add to test suite. Args: testcases (list): testcases list. Returns: unittest.TestSuite() """ def _add_test(test_runner, test_dict): """ add test to testcase. """ def test(self): try: test_runner.run_test(test_dict) except exceptions.MyBaseFailure as ex: self.fail(str(ex)) finally: self.meta_datas = test_runner.meta_datas if "config" in test_dict: # run nested testcase test.__doc__ = test_dict["config"].get("name") variables = test_dict["config"].get("variables", {}) else: # run api test test.__doc__ = test_dict.get("name") variables = test_dict.get("variables", {}) if isinstance(test.__doc__, parser.LazyString): try: parsed_variables = parser.parse_variables_mapping( variables) test.__doc__ = parser.parse_lazy_data( test.__doc__, parsed_variables) except exceptions.VariableNotFound: test.__doc__ = str(test.__doc__) return test test_suite = unittest.TestSuite() for testcase in testcases: config = testcase.get("config", {}) test_runner = runner.Runner(config) TestSequense = type('TestSequense', (unittest.TestCase, ), {}) tests = testcase.get("teststeps", []) for index, test_dict in enumerate(tests): times = test_dict.get("times", 1) try: times = int(times) except ValueError: raise exceptions.ParamsError( "times should be digit, given: {}".format(times)) for times_index in range(times): # suppose one testcase should not have more than 9999 steps, # and one step should not run more than 999 times. test_method_name = 'test_{:04}_{:03}'.format( index, times_index) test_method = _add_test(test_runner, test_dict) setattr(TestSequense, test_method_name, test_method) loaded_testcase = self.test_loader.loadTestsFromTestCase( TestSequense) setattr(loaded_testcase, "config", config) setattr(loaded_testcase, "teststeps", tests) setattr(loaded_testcase, "runner", test_runner) test_suite.addTest(loaded_testcase) return test_suite
def parse_parameters(parameters, variables_mapping=None, functions_mapping=None): """ parse parameters and generate cartesian product. Args: parameters (list) parameters: parameter name and value in list parameter value may be in three types: (1) data list, e.g. ["iOS/10.1", "iOS/10.2", "iOS/10.3"] (2) call built-in parameterize function, "${parameterize(account.csv)}" (3) call custom function in debugtalk.py, "${gen_app_version()}" variables_mapping (dict): variables mapping loaded from testcase config functions_mapping (dict): functions mapping loaded from debugtalk.py Returns: list: cartesian product list Examples: >>> parameters = [ {"user_agent": ["iOS/10.1", "iOS/10.2", "iOS/10.3"]}, {"username-password": "******"}, {"app_version": "${gen_app_version()}"} ] >>> parse_parameters(parameters) """ variables_mapping = variables_mapping or {} functions_mapping = functions_mapping or {} parsed_parameters_list = [] parameters = utils.ensure_mapping_format(parameters) for parameter_name, parameter_content in parameters.items(): parameter_name_list = parameter_name.split("-") if isinstance(parameter_content, list): # (1) data list # e.g. {"app_version": ["2.8.5", "2.8.6"]} # => [{"app_version": "2.8.5", "app_version": "2.8.6"}] # e.g. {"username-password": [["user1", "111111"], ["test2", "222222"]} # => [{"username": "******", "password": "******"}, {"username": "******", "password": "******"}] parameter_content_list = [] for parameter_item in parameter_content: if not isinstance(parameter_item, (list, tuple)): # "2.8.5" => ["2.8.5"] parameter_item = [parameter_item] # ["app_version"], ["2.8.5"] => {"app_version": "2.8.5"} # ["username", "password"], ["user1", "111111"] => {"username": "******", "password": "******"} parameter_content_dict = dict( zip(parameter_name_list, parameter_item)) parameter_content_list.append(parameter_content_dict) else: # (2) & (3) parsed_variables_mapping = parse_variables_mapping( variables_mapping) parsed_parameter_content = eval_lazy_data( parameter_content, parsed_variables_mapping, functions_mapping) if not isinstance(parsed_parameter_content, list): raise exceptions.ParamsError("parameters syntax error!") parameter_content_list = [] for parameter_item in parsed_parameter_content: if isinstance(parameter_item, dict): # get subset by parameter name # {"app_version": "${gen_app_version()}"} # gen_app_version() => [{'app_version': '2.8.5'}, {'app_version': '2.8.6'}] # {"username-password": "******"} # get_account() => [ # {"username": "******", "password": "******"}, # {"username": "******", "password": "******"} # ] parameter_dict = { key: parameter_item[key] for key in parameter_name_list } elif isinstance(parameter_item, (list, tuple)): # {"username-password": "******"} # get_account() => [("user1", "111111"), ("user2", "222222")] parameter_dict = dict( zip(parameter_name_list, parameter_item)) elif len(parameter_name_list) == 1: # {"user_agent": "${get_user_agent()}"} # get_user_agent() => ["iOS/10.1", "iOS/10.2"] parameter_dict = {parameter_name_list[0]: parameter_item} parameter_content_list.append(parameter_dict) parsed_parameters_list.append(parameter_content_list) return utils.gen_cartesian_product(*parsed_parameters_list)
def _run_dubbo_test(self, test_dict): # clear meta data first to ensure independence for each test self.__clear_test_data() # check skip self._handle_skip_feature(test_dict) # prepare test_dict = utils.lower_test_dict_keys(test_dict) test_variables = test_dict.get("variables", {}) self.session_context.init_test_variables(test_variables) # parse test request raw_request = test_dict.get('request', {}) parsed_test_request = self.session_context.eval_content(raw_request) self.session_context.update_test_variables("request", parsed_test_request) # prepend url with base_url unless it's already an absolute URL host = parsed_test_request.pop('host', '') port = parsed_test_request.pop('port', '') # setup hooks setup_hooks = test_dict.get("setup_hooks", []) if setup_hooks: self.do_hook_actions(setup_hooks, HookTypeEnum.SETUP) try: service = parsed_test_request.pop('service') method = parsed_test_request.pop('method') params = parsed_test_request.pop("params", "") parsed_test_request.setdefault("verify", self.verify) group_name = parsed_test_request.pop("group", None) except KeyError: raise exceptions.ParamsError("SERVICE or METHOD missed!") logger.log_info("{host}:{port} {service}.{method}".format( host=host, port=port, service=service, method=method)) logger.log_debug( "request kwargs(raw): {kwargs}".format(kwargs=parsed_test_request)) # request resp = self.dubbo_client_session.request( host, port, service, method, params, name=(group_name or self.session_context.eval_content( test_dict.get("name", ""))), **parsed_test_request) # TODO: responseObject resp_obj = DubboResponseObject(resp) def log_req_resp_details(): err_msg = "{} DETAILED REQUEST & RESPONSE {}\n".format( "*" * 32, "*" * 32) # log request err_msg += "====== request details ======\n" err_msg += "host: {}\n".format(host) err_msg += "port: {}\n".format(port) err_msg += "service: {}\n".format(service) err_msg += "method: {}\n".format(method) err_msg += "params: {}\n".format(params) for k, v in parsed_test_request.items(): v = utils.omit_long_data(v) err_msg += "{}: {}\n".format(k, repr(v)) err_msg += "\n" # log response err_msg += "====== response details ======\n" err_msg += "body: {}\n".format(repr(resp_obj.json)) logger.log_error(err_msg) # teardown hooks teardown_hooks = test_dict.get("teardown_hooks", []) if teardown_hooks: self.session_context.update_test_variables("response", resp_obj) self.do_hook_actions(teardown_hooks, HookTypeEnum.TEARDOWN) self.dubbo_client_session.update_last_req_resp_record(resp_obj) # extract extractors = test_dict.get("extract", {}) try: extracted_variables_mapping = resp_obj.extract_response(extractors) self.session_context.update_session_variables( extracted_variables_mapping) except (exceptions.ParamsError, exceptions.ExtractFailure): log_req_resp_details() raise # validate validators = test_dict.get("validate") or test_dict.get( "validators") or [] validate_script = test_dict.get("validate_script", []) if validate_script: validators.append({ "type": "python_script", "script": validate_script }) validator = Validator(self.session_context, resp_obj) try: validator.validate(validators) except exceptions.ValidationFailure: log_req_resp_details() raise return validator.validation_results
def _run_test(self, test_dict): """ run single teststep. Args: test_dict (dict): teststep info { "name": "teststep description", "skip": "skip this test unconditionally", "times": 3, "variables": [], # optional, override "request": { "url": "http://127.0.0.1:5000/api/users/1000", "method": "POST", "headers": { "Content-Type": "application/json", "authorization": "$authorization", "random": "$random" }, "json": {"name": "user", "password": "******"} }, "extract": {}, # optional "validate": [], # optional "setup_hooks": [], # optional "teardown_hooks": [] # optional } Raises: exceptions.ParamsError exceptions.ValidationFailure exceptions.ExtractFailure """ # clear meta data first to ensure independence for each test self.__clear_test_data() # check skip self._handle_skip_feature(test_dict) # prepare test_dict = utils.lower_test_dict_keys(test_dict) test_variables = test_dict.get("variables", {}) self.session_context.init_test_variables(test_variables) # teststep name test_name = self.session_context.eval_content(test_dict.get( "name", "")) # parse test request raw_request = test_dict.get('request', {}) parsed_test_request = self.session_context.eval_content(raw_request) self.session_context.update_test_variables("request", parsed_test_request) # setup hooks setup_hooks = test_dict.get("setup_hooks", []) if setup_hooks: self.do_hook_actions(setup_hooks, HookTypeEnum.SETUP) # prepend url with base_url unless it's already an absolute URL url = parsed_test_request.pop('url') base_url = self.session_context.eval_content( test_dict.get("base_url", "")) parsed_url = utils.build_url(base_url, url) request_headers = parsed_test_request.setdefault("headers", {}) if "HRUN-Request-ID" not in request_headers: parsed_test_request["headers"]["HRUN-Request-ID"] = \ self.session_context.session_variables_mapping["HRUN-Request-ID"] try: method = parsed_test_request.pop('method') parsed_test_request.setdefault("verify", self.verify) group_name = parsed_test_request.pop("group", None) except KeyError: raise exceptions.ParamsError("URL or METHOD missed!") logger.info(f"{method} {parsed_url}") logger.debug(f"request kwargs(raw): {parsed_test_request}") # request resp = self.http_client_session.request(method, parsed_url, name=(group_name or test_name), **parsed_test_request) resp_obj = response.ResponseObject(resp) def log_req_resp_details(): err_msg = "{} DETAILED REQUEST & RESPONSE {}\n".format( "*" * 32, "*" * 32) # log request err_msg += "====== request details ======\n" err_msg += f"url: {parsed_url}\n" err_msg += f"method: {method}\n" headers = parsed_test_request.pop("headers", {}) err_msg += f"headers: {headers}\n" for k, v in parsed_test_request.items(): v = utils.omit_long_data(v) err_msg += f"{k}: {repr(v)}\n" err_msg += "\n" # log response err_msg += "====== response details ======\n" err_msg += f"status_code: {resp_obj.status_code}\n" err_msg += f"headers: {resp_obj.headers}\n" err_msg += f"body: {repr(resp_obj.text)}\n" logger.error(err_msg) # teardown hooks teardown_hooks = test_dict.get("teardown_hooks", []) if teardown_hooks: self.session_context.update_test_variables("response", resp_obj) self.do_hook_actions(teardown_hooks, HookTypeEnum.TEARDOWN) self.http_client_session.update_last_req_resp_record(resp_obj) # extract extractors = test_dict.get("extract", {}) try: extracted_variables_mapping = resp_obj.extract_response(extractors) self.session_context.update_session_variables( extracted_variables_mapping) except (exceptions.ParamsError, exceptions.ExtractFailure): log_req_resp_details() raise # validate validators = test_dict.get("validate") or test_dict.get( "validators") or [] validate_script = test_dict.get("validate_script", []) if validate_script: validators.append({ "type": "python_script", "script": validate_script }) validator = Validator(self.session_context, resp_obj) try: validator.validate(validators) except exceptions.ValidationFailure: log_req_resp_details() raise finally: self.validation_results = validator.validation_results
def _extract_field_with_delimiter(self, field): """ response content could be json or html text. @param (str) field should be string joined by delimiter. e.g. "status_code" "headers" "cookies" "content" "headers.content-type" "content.person.name.first_name" """ # string.split(sep=None, maxsplit=-1) -> list of strings # e.g. "content.person.name" => ["content", "person.name"] try: top_query, sub_query = field.split('.', 1) except ValueError: top_query = field sub_query = None # status_code if top_query in ["status_code", "encoding", "ok", "reason", "url"]: if sub_query: # status_code.XX err_msg = u"Failed to extract: {}\n".format(field) logger.log_error(err_msg) raise exceptions.ParamsError(err_msg) return getattr(self, top_query) # cookies elif top_query == "cookies": cookies = self.cookies.get_dict() if not sub_query: # extract cookies return cookies try: return cookies[sub_query] except KeyError: err_msg = u"Failed to extract cookie! => {}\n".format(field) err_msg += u"response cookies: {}\n".format(cookies) logger.log_error(err_msg) raise exceptions.ExtractFailure(err_msg) # elapsed elif top_query == "elapsed": available_attributes = u"available attributes: days, seconds, microseconds, total_seconds" if not sub_query: err_msg = u"elapsed is datetime.timedelta instance, attribute should also be specified!\n" err_msg += available_attributes logger.log_error(err_msg) raise exceptions.ParamsError(err_msg) elif sub_query in ["days", "seconds", "microseconds"]: return getattr(self.elapsed, sub_query) elif sub_query == "total_seconds": return self.elapsed.total_seconds() else: err_msg = "{} is not valid datetime.timedelta attribute.\n".format( sub_query) err_msg += available_attributes logger.log_error(err_msg) raise exceptions.ParamsError(err_msg) # headers elif top_query == "headers": headers = self.headers if not sub_query: # extract headers return headers try: return headers[sub_query] except KeyError: err_msg = u"Failed to extract header! => {}\n".format(field) err_msg += u"response headers: {}\n".format(headers) logger.log_error(err_msg) raise exceptions.ExtractFailure(err_msg) # response body elif top_query in ["content", "text", "json"]: try: body = self.json except exceptions.JSONDecodeError: body = self.text if not sub_query: # extract response body return body if isinstance(body, (dict, list)): # content = {"xxx": 123}, content.xxx return utils.query_json(body, sub_query) elif sub_query.isdigit(): # content = "abcdefg", content.3 => d return utils.query_json(body, sub_query) else: # content = "<html>abcdefg</html>", content.xxx err_msg = u"Failed to extract attribute from response body! => {}\n".format( field) err_msg += u"response body: {}\n".format(body) logger.log_error(err_msg) raise exceptions.ExtractFailure(err_msg) # response jsonpath elif top_query == 'jsonpath': try: body = self.json except exceptions.JSONDecodeError: body = self.text if not sub_query: # extract response body return body re = jsonpath.jsonpath(body, expr=sub_query) if re: # return first value of result(list) return re[0] else: return 'null' # others else: err_msg = u"Failed to extract attribute from response! => {}\n".format( field) err_msg += u"available response attributes: status_code, cookies, elapsed, headers, content, text, json, encoding, ok, reason, url." logger.log_error(err_msg) raise exceptions.ParamsError(err_msg)
def parse_parameters(parameters, testset_path=None): """ parse parameters and generate cartesian product @params (list) parameters: parameter name and value in list parameter value may be in three types: (1) data list (2) call built-in parameterize function (3) call custom function in debugtalk.py e.g. [ {"user_agent": ["iOS/10.1", "iOS/10.2", "iOS/10.3"]}, {"username-password": "******"}, {"app_version": "${gen_app_version()}"} ] (str) testset_path: testset file path, used for locating csv file and debugtalk.py @return cartesian product in list """ testcase_parser = TestcaseParser(file_path=testset_path) parsed_parameters_list = [] for parameter in parameters: parameter_name, parameter_content = list(parameter.items())[0] parameter_name_list = parameter_name.split("-") if isinstance(parameter_content, list): # (1) data list # e.g. {"app_version": ["2.8.5", "2.8.6"]} # => [{"app_version": "2.8.5", "app_version": "2.8.6"}] # e.g. {"username-password": [["user1", "111111"], ["test2", "222222"]} # => [{"username": "******", "password": "******"}, {"username": "******", "password": "******"}] parameter_content_list = [] for parameter_item in parameter_content: if not isinstance(parameter_item, (list, tuple)): # "2.8.5" => ["2.8.5"] parameter_item = [parameter_item] # ["app_version"], ["2.8.5"] => {"app_version": "2.8.5"} # ["username", "password"], ["user1", "111111"] => {"username": "******", "password": "******"} parameter_content_dict = dict( zip(parameter_name_list, parameter_item)) parameter_content_list.append(parameter_content_dict) else: # (2) & (3) parsed_parameter_content = testcase_parser.eval_content_with_bindings( parameter_content) # e.g. [{'app_version': '2.8.5'}, {'app_version': '2.8.6'}] # e.g. [{"username": "******", "password": "******"}, {"username": "******", "password": "******"}] if not isinstance(parsed_parameter_content, list): raise exceptions.ParamsError("parameters syntax error!") parameter_content_list = [ # get subset by parameter name {key: parameter_item[key] for key in parameter_name_list} for parameter_item in parsed_parameter_content ] parsed_parameters_list.append(parameter_content_list) return gen_cartesian_product(*parsed_parameters_list)
def _run_test(self, test_dict): """ run single teststep. Args: test_dict (dict): teststep info { "name": "teststep description", "skip": "skip this test unconditionally", "times": 3, "variables": [], # optional, override "request": { "url": "http://127.0.0.1:5000/api/users/1000", "method": "POST", "headers": { "Content-Type": "application/json", "authorization": "$authorization", "random": "$random" }, "json": {"name": "user", "password": "******"} }, "extract": {}, # optional "validate": [], # optional "setup_hooks": [], # optional "teardown_hooks": [], # optional "wait":{} # optional } Raises: exceptions.ParamsError exceptions.ValidationFailure exceptions.ExtractFailure """ # get wait parameter, if this step result needs to waif for a right one wait_params = test_dict.get("wait", {}) validate_pass = None failures = [] if wait_params: wait_total_time = wait_params["Total_Time"] if wait_params.has_key( 'Total_Time') else 300 wait_interval = wait_params["Interval"] if wait_params.has_key( 'Interval') else 5 # validate for wait function wait_validators = wait_params.get("validate", []) for item in [wait_total_time, wait_interval]: if not isinstance(item, int): err_msg = u"Invalid wait function parameters! => {}\n".format( item) logger.log_error(err_msg) raise exceptions.ParamsError(err_msg) start_time = time.time() current_time = time.time() try: while current_time - start_time < wait_total_time: validate_pass, failures = self._run_test_once( test_dict=test_dict, wait_validators=wait_validators) # to break while when validate is OK if validate_pass: break current_time = time.time() time.sleep(wait_interval) # to raise alarm when failures except (exceptions.ParamsError, exceptions.ValidationFailure, exceptions.ExtractFailure): raise else: validate_pass, failures = self._run_test_once(test_dict=test_dict) if not validate_pass: failures_string = "\n".join([failure for failure in failures]) raise exceptions.ValidationFailure(failures_string)
def _add_tests(self, testcases): """ initialize testcase with Runner() and add to test suite. 把测试用例加载到测试套件中 Args: testcases (list): testcases list. Returns: unittest.TestSuite() """ def _add_test(test_runner, test_dict): """ add test to testcase. 把步骤添加到用例,返回一个测试步骤 test_dict = parsed_testcases[0][teststeps][0] test_dict = { "name": "/mgmt/store/checkBusinessAddressIsExist", "request": { "headers": {"Authorization": "LazyString(${token_type} ${access_token})"}, "method": "GET", "params": { "provinceName": "LazyString(${provinceName})", "cityName": "LazyString(${cityName})", "areaName": LazyString(${areaName}), "streetName": LazyString(${streetName}), "detailAddress": LazyString(${detailAddress})}, "url": LazyString(${base_url}/mgmt/store/checkBusinessAddressIsExist), "verify": True }, "variables": { "provinceName": "广东省", "cityName": "广州市", "areaName": "海珠区", "streetName": "南州街道", "detailAddress": "广州市海珠区南洲街道新滘中路88号唯品同创汇6区东三街17号自编23号", "access_token": LazyString(${ENV(access_token)}), "token_type": LazyString(${ENV(token_type)}), "base_url": LazyString(${ENV(base_url)}) }, "validate": [LazyFunction(equals(status_code, 200))] } """ def test(self): try: test_runner.run_test(test_dict) pprint(test_runner) except exceptions.MyBaseFailure as ex: self.fail(str(ex)) finally: self.meta_datas = test_runner.meta_datas if "config" in test_dict: # run nested testcase 嵌套testcase运行:testcase引用了testcase test.__doc__ = test_dict["config"].get("name") variables = test_dict["config"].get("variables", {}) else: # run api test 运行api测试 testcase引用了api test.__doc__ = test_dict.get("name") variables = test_dict.get("variables", {}) if isinstance(test.__doc__, parser.LazyString): #懒惰的字符串:名字中有引用的变量 try: parsed_variables = parser.parse_variables_mapping( variables) # 所有的变量字典 test.__doc__ = parser.parse_lazy_data( # 找到引用变量对应的值 test.__doc__, parsed_variables) except exceptions.VariableNotFound: test.__doc__ = str(test.__doc__) # 返回函数<function HttpRunner._add_tests.<locals>._add_test.<locals>.test at 0x000002D69C38A550> return test test_suite = unittest.TestSuite() # 用例集的子类 for testcase in testcases: #遍历用例中引用的每一个用例 config = testcase.get("config", {}) # <httprunner.runner.Runner object at 0x000002629251F520> test_runner = runner.Runner(config) TestSequense = type('TestSequense', (unittest.TestCase, ), {}) #创建测试用例:unittest.TestCase子类 tests = testcase.get("teststeps", []) # 测试用例 for index, test_dict in enumerate(tests): # 遍历每一个测试步骤 times = test_dict.get("times", 1) try: times = int(times) except ValueError: raise exceptions.ParamsError( "times should be digit, given: {}".format(times)) for times_index in range(times): #根据times设置,运行一个测试步骤N次 # suppose one testcase should not have more than 9999 steps, # 假设一个测试用例不应该有超过9999个步骤 # and one step should not run more than 999 times. # 一个步骤不应该运行超过999次 test_method_name = 'test_{:04}_{:03}'.format( index, times_index) # 测试方法名 test_method = _add_test(test_runner, test_dict) # 测试方法 setattr(TestSequense, test_method_name, test_method) #加载测试方法到测试用例中:unittest.TestCase子类 loaded_testcase = self.test_loader.loadTestsFromTestCase( TestSequense) # 加载测试用例到用例集(小集:测试用例) setattr(loaded_testcase, "config", config) # 加载属性到用例集(小集:测试用例) setattr(loaded_testcase, "teststeps", tests) # 加载属性到用例集(小集:测试用例) setattr(loaded_testcase, "runner", test_runner) # 加载运行方法到用例集(小集:测试用例) test_suite.addTest(loaded_testcase) # 加载用例集(小集:测试用例)到大用例集 return test_suite
def run_path(self, path: Text) -> "HttpRunner": if not os.path.isfile(path): raise exceptions.ParamsError(f"Invalid testcase path: {path}") _, testcase_obj = load_testcase_file(path) return self.run(testcase_obj)
def load_api_folder(api_folder_path): """ load api definitions from api folder. Args: api_folder_path (str): api files folder. api file should be in the following format: [ { "api": { "def": "api_login", "request": {}, "validate": [] } }, { "api": { "def": "api_logout", "request": {}, "validate": [] } } ] Returns: dict: api definition mapping. { "api_login": { "function_meta": {"func_name": "api_login", "args": [], "kwargs": {}} "request": {} }, "api_logout": { "function_meta": {"func_name": "api_logout", "args": [], "kwargs": {}} "request": {} } } """ api_definition_mapping = {} api_items_mapping = load_folder_content(api_folder_path) for api_file_path, api_items in api_items_mapping.items(): # TODO: add JSON schema validation if isinstance(api_items, list): for api_item in api_items: key, api_dict = api_item.popitem() api_id = api_dict.get("id") or api_dict.get("def") \ or api_dict.get("name") if key != "api" or not api_id: raise exceptions.ParamsError( "Invalid API defined in {}".format(api_file_path)) if api_id in api_definition_mapping: raise exceptions.ParamsError( "Duplicated API ({}) defined in {}".format( api_id, api_file_path)) else: api_definition_mapping[api_id] = api_dict elif isinstance(api_items, dict): if api_file_path in api_definition_mapping: raise exceptions.ParamsError( "Duplicated API defined: {}".format(api_file_path)) else: api_definition_mapping[api_file_path] = api_items return api_definition_mapping
def _run_test_once(self, test_dict, wait_validators=None): # clear meta data first to ensure independence for each test self.__clear_test_data() # check skip self._handle_skip_feature(test_dict) # prepare test_dict = utils.lower_test_dict_keys(test_dict) test_variables = test_dict.get("variables", {}) self.session_context.init_test_variables(test_variables) # teststep name test_name = test_dict.get("name", "") # parse test request raw_request = test_dict.get('request', {}) parsed_test_request = self.session_context.eval_content(raw_request) self.session_context.update_test_variables("request", parsed_test_request) # setup hooks setup_hooks = test_dict.get("setup_hooks", []) if setup_hooks: self.do_hook_actions(setup_hooks, "setup") try: url = parsed_test_request.pop('url') method = parsed_test_request.pop('method') parsed_test_request.setdefault("verify", self.verify) group_name = parsed_test_request.pop("group", None) except KeyError: raise exceptions.ParamsError("URL or METHOD missed!") # TODO: move method validation to json schema valid_methods = [ "GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS" ] if method.upper() not in valid_methods: err_msg = u"Invalid HTTP method! => {}\n".format(method) err_msg += "Available HTTP methods: {}".format( "/".join(valid_methods)) logger.log_error(err_msg) raise exceptions.ParamsError(err_msg) logger.log_info("{method} {url}".format(method=method, url=url)) logger.log_debug( "request kwargs(raw): {kwargs}".format(kwargs=parsed_test_request)) # request resp = self.http_client_session.request(method, url, name=(group_name or test_name), **parsed_test_request) resp_obj = response.ResponseObject(resp) # teardown hooks teardown_hooks = test_dict.get("teardown_hooks", []) if teardown_hooks: self.session_context.update_test_variables("response", resp_obj) self.do_hook_actions(teardown_hooks, "teardown") # extract extractors = test_dict.get("extract", {}) extracted_variables_mapping = resp_obj.extract_response(extractors) self.session_context.update_session_variables( extracted_variables_mapping) # validate if wait_validators: wait_validate_pass, wait_failures = self.session_context.validate( wait_validators, resp_obj) if not wait_validate_pass: # err_msg = "{} DETAILED REQUEST & RESPONSE {}\n".format("*" * 32, "*" * 32) # # # log request # err_msg += "====== request details ======\n" # err_msg += "url: {}\n".format(url) # err_msg += "method: {}\n".format(method) # err_msg += "headers: {}\n".format(parsed_test_request.pop("headers", {})) # for k, v in parsed_test_request.items(): # v = utils.omit_long_data(v) # err_msg += "{}: {}\n".format(k, repr(v)) # err_msg += "\n" # log response # err_msg += "====== response details ======\n" # err_msg += "status_code: {}\n".format(resp_obj.status_code) # err_msg += "headers: {}\n".format(resp_obj.headers) # err_msg += "body: {}\n".format(repr(resp_obj.text)) # logger.log_error("During the wait: \n" + err_msg) logger.log_error( "====== During the wait: the expect conditions can not Meet the requirements! ======" ) failures_string = "\n".join( [failure for failure in wait_failures]) self.validation_results = self.session_context.validation_results raise exceptions.ValidationFailure(failures_string) validators = test_dict.get("validate", []) validate_pass, failures = self.session_context.validate( validators, resp_obj) self.validation_results = self.session_context.validation_results # try: # validate_pass, failures = self.session_context.validate(validators, resp_obj) # except (exceptions.ParamsError, exceptions.ValidationFailure, exceptions.ExtractFailure): # err_msg = "{} DETAILED REQUEST & RESPONSE {}\n".format("*" * 32, "*" * 32) # # # log request # err_msg += "====== request details ======\n" # err_msg += "url: {}\n".format(url) # err_msg += "method: {}\n".format(method) # err_msg += "headers: {}\n".format(parsed_test_request.pop("headers", {})) # for k, v in parsed_test_request.items(): # v = utils.omit_long_data(v) # err_msg += "{}: {}\n".format(k, repr(v)) # # err_msg += "\n" # # # log response # err_msg += "====== response details ======\n" # err_msg += "status_code: {}\n".format(resp_obj.status_code) # err_msg += "headers: {}\n".format(resp_obj.headers) # err_msg += "body: {}\n".format(repr(resp_obj.text)) # logger.log_error(err_msg) # raise # finally: # self.validation_results = self.session_context.validation_results return validate_pass, failures
def run_test(self, testcase_dict): """ run single testcase. @param (dict) testcase_dict { "name": "testcase description", "skip": "skip this test unconditionally", "times": 3, "variables": [], # optional, override "request": { "url": "http://127.0.0.1:5000/api/users/1000", "method": "POST", "headers": { "Content-Type": "application/json", "authorization": "$authorization", "random": "$random" }, "body": '{"name": "user", "password": "******"}' }, "extract": [], # optional "validate": [], # optional "setup_hooks": [], # optional "teardown_hooks": [] # optional } @return True or raise exception during test """ # check skip self._handle_skip_feature(testcase_dict) # prepare parsed_request = self.init_config(testcase_dict, level="testcase") self.context.bind_testcase_variable("request", parsed_request) # setup hooks setup_hooks = testcase_dict.get("setup_hooks", []) setup_hooks.insert(0, "${setup_hook_prepare_kwargs($request)}") self.do_hook_actions(setup_hooks) try: url = parsed_request.pop('url') method = parsed_request.pop('method') group_name = parsed_request.pop("group", None) except KeyError: raise exceptions.ParamsError("URL or METHOD missed!") # TODO: move method validation to json schema valid_methods = ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"] if method.upper() not in valid_methods: err_msg = u"Invalid HTTP method! => {}\n".format(method) err_msg += "Available HTTP methods: {}".format("/".join(valid_methods)) logger.log_error(err_msg) raise exceptions.ParamsError(err_msg) logger.log_info("{method} {url}".format(method=method, url=url)) logger.log_debug("request kwargs(raw): {kwargs}".format(kwargs=parsed_request)) # request resp = self.http_client_session.request( method, url, name=group_name, **parsed_request ) resp_obj = response.ResponseObject(resp) # teardown hooks teardown_hooks = testcase_dict.get("teardown_hooks", []) if teardown_hooks: self.context.bind_testcase_variable("response", resp_obj) self.do_hook_actions(teardown_hooks) # extract extractors = testcase_dict.get("extract", []) or testcase_dict.get("extractors", []) extracted_variables_mapping = resp_obj.extract_response(extractors,context=self.context) self.context.bind_extracted_variables(extracted_variables_mapping) # validate validators = testcase_dict.get("validate", []) or testcase_dict.get("validators", []) try: self.context.validate(validators, resp_obj) except (exceptions.ParamsError, \ exceptions.ValidationFailure, exceptions.ExtractFailure): # log request err_req_msg = "request: \n" err_req_msg += "headers: {}\n".format(parsed_request.pop("headers", {})) for k, v in parsed_request.items(): err_req_msg += "{}: {}\n".format(k, repr(v)) logger.log_error(err_req_msg) # log response err_resp_msg = "response: \n" err_resp_msg += "status_code: {}\n".format(resp_obj.status_code) err_resp_msg += "headers: {}\n".format(resp_obj.headers) err_resp_msg += "body: {}\n".format(repr(resp_obj.text)) logger.log_error(err_resp_msg) raise
def _run_test(self, test_dict): """ run single teststep. Args: test_dict (dict): teststep info { "name": "teststep description", "skip": "skip this test unconditionally", "times": 3, "variables": [], # optional, override "request": { "url": "http://127.0.0.1:5000/api/users/1000", "method": "POST", "headers": { "Content-Type": "application/json", "authorization": "$authorization", "random": "$random" }, "json": {"name": "user", "password": "******"} }, "extract": {}, # optional "validate": [], # optional "setup_hooks": [], # optional "teardown_hooks": [] # optional } Raises: exceptions.ParamsError exceptions.ValidationFailure exceptions.ExtractFailure """ # clear meta data first to ensure independence for each test self.__clear_test_data() # check skip self._handle_skip_feature(test_dict) # prepare test_dict = utils.lower_test_dict_keys(test_dict) test_variables = test_dict.get("variables", {}) self.session_context.init_test_variables(test_variables) # teststep name test_name = test_dict.get("name", "") # parse test request raw_request = test_dict.get('request', {}) parsed_test_request = self.session_context.eval_content(raw_request) self.session_context.update_test_variables("request", parsed_test_request) # setup hooks setup_hooks = test_dict.get("setup_hooks", []) if setup_hooks: self.do_hook_actions(setup_hooks, "setup") try: url = parsed_test_request.pop('url') method = parsed_test_request.pop('method') parsed_test_request.setdefault("verify", self.verify) group_name = parsed_test_request.pop("group", None) except KeyError: raise exceptions.ParamsError("URL or METHOD missed!") # TODO: move method validation to json schema valid_methods = [ "GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS" ] if method.upper() not in valid_methods: err_msg = u"Invalid HTTP method! => {}\n".format(method) err_msg += "Available HTTP methods: {}".format( "/".join(valid_methods)) logger.log_error(err_msg) raise exceptions.ParamsError(err_msg) logger.log_info("{method} {url}".format(method=method, url=url)) logger.log_debug( "request kwargs(raw): {kwargs}".format(kwargs=parsed_test_request)) # request resp = self.http_client_session.request(method, url, name=(group_name or test_name), **parsed_test_request) resp_obj = response.ResponseObject(resp) # teardown hooks teardown_hooks = test_dict.get("teardown_hooks", []) if teardown_hooks: self.session_context.update_test_variables("response", resp_obj) self.do_hook_actions(teardown_hooks, "teardown") # extract extractors = test_dict.get("extract", {}) extracted_variables_mapping = resp_obj.extract_response(extractors) self.session_context.update_session_variables( extracted_variables_mapping) # validate validators = test_dict.get("validate", []) try: self.session_context.validate(validators, resp_obj) except (exceptions.ParamsError, exceptions.ValidationFailure, exceptions.ExtractFailure): err_msg = "{} DETAILED REQUEST & RESPONSE {}\n".format( "*" * 32, "*" * 32) # log request err_msg += "====== request details ======\n" err_msg += "url: {}\n".format(url) err_msg += "method: {}\n".format(method) err_msg += "headers: {}\n".format( parsed_test_request.pop("headers", {})) for k, v in parsed_test_request.items(): v = utils.omit_long_data(v) err_msg += "{}: {}\n".format(k, repr(v)) err_msg += "\n" # log response err_msg += "====== response details ======\n" err_msg += "status_code: {}\n".format(resp_obj.status_code) err_msg += "headers: {}\n".format(resp_obj.headers) err_msg += "body: {}\n".format(repr(resp_obj.text)) logger.log_error(err_msg) raise finally: self.validation_results = self.session_context.validation_results