def parse_validator(validator): """ parse validator Args: validator (dict): validator maybe in two formats: format1: this is kept for compatiblity with the previous versions. {"check": "status_code", "comparator": "eq", "expect": 201} {"check": "$resp_body_success", "comparator": "eq", "expect": True} format2: recommended new version {'eq': ['status_code', 201]} {'eq': ['$resp_body_success', True]} Returns dict: validator info { "check": "status_code", "expect": 201, "comparator": "eq" } """ if not isinstance(validator, dict): raise exceptions.ParamsError("invalid validator: {}".format(validator)) if "check" in validator and len(validator) > 1: # format1 check_item = validator.get("check") if "expect" in validator: expect_value = validator.get("expect") elif "expected" in validator: expect_value = validator.get("expected") else: raise exceptions.ParamsError( "invalid validator: {}".format(validator)) comparator = validator.get("comparator", "eq") elif len(validator) == 1: # format2 comparator = list(validator.keys())[0] compare_values = validator[comparator] if not isinstance(compare_values, list) or len(compare_values) != 2: raise exceptions.ParamsError( "invalid validator: {}".format(validator)) check_item, expect_value = compare_values else: raise exceptions.ParamsError("invalid validator: {}".format(validator)) return { "check": check_item, "expect": expect_value, "comparator": comparator }
def ensure_mapping_format(variables): """ ensure variables are in mapping format. Args: variables (list/dict): original variables Returns: dict: ensured variables in dict format Examples: >>> variables = [ {"a": 1}, {"b": 2} ] >>> print(ensure_mapping_format(variables)) { "a": 1, "b": 2 } """ if isinstance(variables, list): variables_dict = {} for map_dict in variables: variables_dict.update(map_dict) return variables_dict elif isinstance(variables, dict): return variables else: raise exceptions.ParamsError("variables format error!")
def __getattr__(self, key): try: if key == "json": value = self.resp_obj.json() elif key == "cookies": value = self.resp_obj.cookies.get_dict() else: value = getattr(self.resp_obj, key) self.__dict__[key] = value return value except AttributeError: err_msg = "ResponseObject does not have attribute: {}".format(key) logger.log_error(err_msg) raise exceptions.ParamsError(err_msg)
def run(self, path_or_tests): """ main interface. Args: path_or_tests: str: testcase/testsuite file/foler path dict: valid testcase/testsuite data """ if validator.is_testcase_path(path_or_tests): self.run_path(path_or_tests) elif validator.is_testcases(path_or_tests): self.run_tests(path_or_tests) else: raise exceptions.ParamsError("invalid testcase path or testcases.")
def _do_validation(self, validator_dict): """ validate with functions Args: validator_dict (dict): validator dict { "check": "status_code", "check_value": 200, "expect": 201, comparator: "eq" } """ # TODO: move comp uniform to init_test_suites comp = comparator.get_uniform_comparator(validator_dict["comparator"]) validate_func = parse.get_mapping_function(comp, self.FUNCTIONS_MAPPING) check_item = validator_dict["check"] check_value = validator_dict["check_value"] expect_value = validator_dict["expect"] if (check_value is None or expect_value is None) \ and comp not in ["is", "eq", "equals", "=="]: raise exceptions.ParamsError( "Null value can only be compared with comparator: eq/equals/==" ) validate_msg = "validate: {} {} {}({})".format( check_item, comp, expect_value, type(expect_value).__name__) try: validator_dict["check_result"] = "pass" validate_func(check_value, expect_value) validate_msg += "\t==> pass" logger.debug(validate_msg) except (AssertionError, TypeError): validate_msg += "\t==> fail" validate_msg += "\n{}({}) {} {}({})".format( check_value, type(check_value).__name__, comp, expect_value, type(expect_value).__name__) logger.error(validate_msg) validator_dict["check_result"] = "fail" raise exceptions.ValidationFailure(validate_msg)
def extract_field(self, field): """ extract value from requests.Response. """ if not isinstance(field, str): err_msg = u"Invalid extractor! => {}\n".format(field) logger.log_error(err_msg) raise exceptions.ParamsError(err_msg) msg = "extract: {}".format(field) if text_extractor_regexp_compile.match(field): value = self._extract_field_with_regex(field) else: value = self._extract_field_with_delimiter(field) msg += "\t=> {}".format(value) logger.log_debug(msg) return value
def _extract_field_with_delimiter(self, field): """ response content could be json or html text. Args: field (str): string joined by delimiter. e.g. "status_code" "headers" "cookies" "content" "headers.content-type" "content.person.name.first_name" """ # string.split(sep=None, maxsplit=-1) -> list of strings # e.g. "content.person.name" => ["content", "person.name"] try: top_query, sub_query = field.split('.', 1) except ValueError: top_query = field sub_query = None # status_code if top_query in ["status_code", "encoding", "reason", "url"]: if sub_query: # status_code.XX err_msg = u"Failed to extract: {}\n".format(field) logger.log_error(err_msg) raise exceptions.ParamsError(err_msg) return getattr(self, top_query) # cookies elif top_query == "cookies": cookies = self.cookies if not sub_query: # extract cookies return cookies try: return cookies[sub_query] except KeyError: err_msg = u"Failed to extract cookie! => {}\n".format(field) err_msg += u"response cookies: {}\n".format(cookies) logger.log_error(err_msg) raise exceptions.ExtractFailure(err_msg) # headers elif top_query == "headers": headers = self.headers if not sub_query: # extract headers return headers try: return headers[sub_query] except KeyError: err_msg = u"Failed to extract header! => {}\n".format(field) err_msg += u"response headers: {}\n".format(headers) logger.log_error(err_msg) raise exceptions.ExtractFailure(err_msg) # response body elif top_query in ["content", "text", "json"]: try: body = self.json except exceptions.JSONDecodeError: body = self.text if not sub_query: # extract response body return body if isinstance(body, (dict, list)): # content = {"xxx": 123}, content.xxx return utils.query_json(body, sub_query) elif sub_query.isdigit(): # content = "abcdefg", content.3 => d return utils.query_json(body, sub_query) else: # content = "<html>abcdefg</html>", content.xxx err_msg = u"Failed to extract attribute from response body! => {}\n".format( field) err_msg += u"response body: {}\n".format(body) logger.log_error(err_msg) raise exceptions.ExtractFailure(err_msg) else: err_msg = u"Failed to extract attribute from response! => {}\n".format( field) err_msg += u"available response attributes: status_code, cookies, headers, content, text, json, encoding, reason, url.\n\n" err_msg += u"response.new_attribute = 'new_attribute_value'\n" logger.log_error(err_msg) raise exceptions.ParamsError(err_msg)
def _run_test(self, test_dict): """ run single teststep. Args: test_dict (dict): teststep info { "name": "teststep description", "skip": "skip this test unconditionally", "times": 3, "variables": [], # optional, override "request": { "url": "http://127.0.0.1:5000/api/users/1000", "method": "POST", "headers": { "Content-Type": "application/json", "authorization": "$authorization", "random": "$random" }, "json": {"name": "user", "password": "******"} }, "extract": {}, # optional "validate": [], # optional "setup_hooks": [], # optional "teardown_hooks": [] # optional } Raises: exceptions.ParamsError exceptions.ValidationFailure exceptions.ExtractFailure """ # clear meta data first to ensure independence for each test self.__clear_test_data() # check skip self._handle_skip_feature(test_dict) # prepare test_dict = lower_test_dict_keys(test_dict) test_variables = test_dict.get("variables", {}) self.session_context.init_test_variables(test_variables) # teststep name test_name = test_dict.get("name", "") # parse test request raw_request = test_dict.get('request', {}) parsed_test_request = self.session_context.eval_content(raw_request) self.session_context.update_test_variables("request", parsed_test_request) # setup hooks setup_hooks = test_dict.get("setup_hooks", []) if setup_hooks: self.do_hook_actions(setup_hooks, "setup") try: url = parsed_test_request.pop('url') method = parsed_test_request.pop('method') parsed_test_request.setdefault("verify", self.verify) group_name = parsed_test_request.pop("group", None) except KeyError: raise exceptions.ParamsError("URL or METHOD missed!") # TODO: move method validation to json schema valid_methods = [ "GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS" ] if method.upper() not in valid_methods: err_msg = u"Invalid HTTP method! => {}\n".format(method) err_msg += "Available HTTP methods: {}".format( "/".join(valid_methods)) logger.log_error(err_msg) raise exceptions.ParamsError(err_msg) logger.log_info("{method} {url}".format(method=method, url=url)) logger.debug( "request kwargs(raw): {kwargs}".format(kwargs=parsed_test_request)) # request resp = self.http_client_session.request(method, url, name=(group_name or test_name), **parsed_test_request) resp_obj = response.ResponseObject(resp) # teardown hooks teardown_hooks = test_dict.get("teardown_hooks", []) if teardown_hooks: self.session_context.update_test_variables("response", resp_obj) self.do_hook_actions(teardown_hooks, "teardown") # extract extractors = test_dict.get("extract", {}) extracted_variables_mapping = resp_obj.extract_response(extractors) self.session_context.update_session_variables( extracted_variables_mapping) # validate validators = test_dict.get("validate", []) try: self.session_context.validate(validators, resp_obj) except (exceptions.ParamsError, exceptions.ValidationFailure, exceptions.ExtractFailure): err_msg = "{} DETAILED REQUEST & RESPONSE {}\n".format( "*" * 32, "*" * 32) # log request err_msg += "====== request details ======\n" err_msg += "url: {}\n".format(url) err_msg += "method: {}\n".format(method) err_msg += "headers: {}\n".format( parsed_test_request.pop("headers", {})) for k, v in parsed_test_request.items(): v = omit_long_data(v) err_msg += "{}: {}\n".format(k, repr(v)) err_msg += "\n" # log response err_msg += "====== response details ======\n" err_msg += "status_code: {}\n".format(resp_obj.status_code) err_msg += "headers: {}\n".format(resp_obj.headers) err_msg += "body: {}\n".format(repr(resp_obj.text)) logger.log_error(err_msg) raise finally: self.validation_results = self.session_context.validation_results