def load_test_dependencies(): """ load all api and suite definitions. default api folder is "$CWD/tests/api/". default suite folder is "$CWD/tests/suite/". """ test_def_overall_dict["loaded"] = True test_def_overall_dict["api"] = {} test_def_overall_dict["suite"] = {} # load api definitions api_def_folder = os.path.join(os.getcwd(), "tests", "api") api_files = utils.load_folder_files(api_def_folder) for test_file in api_files: testset = load_test_file(test_file) test_def_overall_dict["api"].update(testset["api"]) # load suite definitions suite_def_folder = os.path.join(os.getcwd(), "tests", "suite") suite_files = utils.load_folder_files(suite_def_folder) for suite_file in suite_files: suite = load_test_file(suite_file) if "def" not in suite["config"]: raise exception.ParamsError( "def missed in suite file: {}!".format(suite_file)) call_func = suite["config"]["def"] function_meta = parse_function(call_func) suite["function_meta"] = function_meta test_def_overall_dict["suite"][function_meta["func_name"]] = suite
def get_testinfo_by_reference(ref_name, ref_type): """ get test content by reference name @params: ref_name: reference name, e.g. api_v1_Account_Login_POST($UserName, $Password) ref_type: "api" or "suite" """ function_meta = parse_function(ref_name) func_name = function_meta["func_name"] call_args = function_meta["args"] test_info = get_test_definition(func_name, ref_type) def_args = test_info.get("function_meta").get("args", []) if len(call_args) != len(def_args): raise exception.ParamsError("call args mismatch defined args!") args_mapping = {} for index, item in enumerate(def_args): if call_args[index] == item: continue args_mapping[item] = call_args[index] if args_mapping: test_info = substitute_variables_with_mapping(test_info, args_mapping) return test_info
def extract_field(self, field, delimiter='.'): """ extract field from requests.Response @param (str) field of requests.Response object, and may be joined by delimiter "status_code" "content" "headers.content-type" "content.person.name.first_name" """ try: field += "." # string.split(sep=None, maxsplit=-1) -> list of strings # e.g. "content.person.name" => ["content", "person.name"] top_query, sub_query = field.split(delimiter, 1) if top_query in ["body", "content", "text"]: json_content = self.parsed_body() else: json_content = getattr(self.resp_obj, top_query) if sub_query: # e.g. key: resp_headers_content_type, sub_query = "content-type" return utils.query_json(json_content, sub_query) else: # e.g. key: resp_status_code, resp_content return json_content except AttributeError: raise exception.ParamsError("invalid extract_binds!")
def eval_content_variables(content, variable_mapping): """ replace all variables of string content with mapping value. @param (str) content @return (str) parsed content e.g. variable_mapping = { "var_1": "abc", "var_2": "def" } $var_1 => "abc" $var_1#XYZ => "abc#XYZ" /$var_1/$var_2/var3 => "/abc/def/var3" ${func($var_1, $var_2, xyz)} => "${func(abc, def, xyz)}" """ variables_list = extract_variables(content) for variable_name in variables_list: if variable_name not in variable_mapping: raise exception.ParamsError( "%s is not defined in bind variables!" % variable_name) variable_value = variable_mapping.get(variable_name) if "${}".format(variable_name) == content: # content is a variable content = variable_value else: # content contains one or many variables content = content.replace( "${}".format(variable_name), str(variable_value), 1 ) return content
def run_test(self, testcase): """ run single testcase. @param (dict) testcase { "name": "testcase description", "requires": [], # optional, override "function_binds": {}, # optional, override "variable_binds": {}, # optional, override "request": {}, "response": {} } @return (tuple) test result of single testcase (success, diff_content) """ testcase = self.parse_testcase(testcase) req_kwargs = testcase['request'] try: url = req_kwargs.pop('url') method = req_kwargs.pop('method') except KeyError: raise exception.ParamsError("URL or METHOD missed!") resp_obj = self.client.request(url=url, method=method, **req_kwargs) response.extract_response(resp_obj, self.context) diff_content = response.diff_response(resp_obj, testcase['response']) success = False if diff_content else True return success, diff_content
def validate(self, validators, variables_mapping): """ Bind named validators to value within the context. @param (list) validators [ {"check": "status_code", "comparator": "eq", "expected": 201}, {"check": "resp_body_success", "comparator": "eq", "expected": True} ] @param (dict) variables_mapping { "resp_body_success": True } @return (list) content differences [ { "check": "status_code", "comparator": "eq", "expected": 201, "value": 200 } ] """ for validator_dict in validators: check_item = validator_dict.get("check") if not check_item: raise exception.ParamsError("invalid check item in testcase validators!") if "expected" not in validator_dict: raise exception.ParamsError("expected item missed in testcase validators!") expected = validator_dict.get("expected") comparator = validator_dict.get("comparator", "eq") if check_item in variables_mapping: validator_dict["actual_value"] = variables_mapping[check_item] else: try: validator_dict["actual_value"] = self.extract_field(check_item) except exception.ParseResponseError: raise exception.ParseResponseError("failed to extract check item in response!") utils.match_expected( validator_dict["actual_value"], expected, comparator, check_item ) return True
def run_test(self, testcase): """ run single testcase. @param (dict) testcase { "name": "testcase description", "times": 3, "requires": [], # optional, override "function_binds": {}, # optional, override "variable_binds": {}, # optional, override "request": { "url": "http://127.0.0.1:5000/api/users/1000", "method": "POST", "headers": { "Content-Type": "application/json", "authorization": "$authorization", "random": "$random" }, "body": '{"name": "user", "password": "******"}' }, "extract_binds": [], # optional "validators": [], # optional "setup": [], # optional "teardown": [] # optional } @return True or raise exception during test """ self.init_config(testcase, level="testcase") parsed_request = self.context.get_parsed_request() try: url = parsed_request.pop('url') method = parsed_request.pop('method') except KeyError: raise exception.ParamsError("URL or METHOD missed!") run_times = int(testcase.get("times", 1)) extract_binds = testcase.get("extract_binds", []) validators = testcase.get("validators", []) setup_actions = testcase.get("setup", []) teardown_actions = testcase.get("teardown", []) def setup_teardown(actions): for action in actions: self.context.exec_content_functions(action) for _ in range(run_times): setup_teardown(setup_actions) resp = self.http_client_session.request(url=url, method=method, **parsed_request) resp_obj = response.ResponseObject(resp) extracted_variables_mapping_list = resp_obj.extract_response(extract_binds) self.context.bind_variables(extracted_variables_mapping_list, level="testset") resp_obj.validate(validators, self.context.get_testcase_variables_mapping()) setup_teardown(teardown_actions) return True
def get_bind_item(self, item_type, item_name): if item_type == "function": if item_name in self.functions: return self.functions[item_name] elif item_type == "variable": if item_name in self.variables: return self.variables[item_name] else: raise exception.ParamsError( "bind item should only be function or variable.") try: assert self.file_path is not None return utils.search_conf_item(self.file_path, item_type, item_name) except (AssertionError, exception.FunctionNotFound): raise exception.ParamsError( "{} is not defined in bind {}s!".format(item_name, item_type))
def _extract_field_with_delimiter(self, field): """ response content could be json or html text. @param (str) field should be string joined by delimiter. e.g. "status_code" "content" "headers.content-type" "content.person.name.first_name" """ try: # string.split(sep=None, maxsplit=-1) -> list of strings # e.g. "content.person.name" => ["content", "person.name"] try: top_query, sub_query = field.split('.', 1) except ValueError: top_query = field sub_query = None if top_query in ["body", "content", "text"]: top_query_content = self.parsed_body() else: top_query_content = getattr(self.resp_obj, top_query) if sub_query: if not isinstance(top_query_content, (dict, CaseInsensitiveDict, list)): err_msg = u"Extractor error: failed to extract data with regex!\n" err_msg += u"response: {}\n".format(self.parsed_dict()) err_msg += u"regex: {}\n".format(field) logging.error(err_msg) raise exception.ParamsError(err_msg) # e.g. key: resp_headers_content_type, sub_query = "content-type" return utils.query_json(top_query_content, sub_query) else: # e.g. key: resp_status_code, resp_content return top_query_content except AttributeError: err_msg = u"Failed to extract value from response!\n" err_msg += u"response: {}\n".format(self.parsed_dict()) err_msg += u"extract field: {}\n".format(field) logging.error(err_msg) raise exception.ParamsError(err_msg)
def override_variables_binds(variables, new_mapping): """ convert variables in testcase to ordered mapping, with new_mapping overrided """ if isinstance(variables, list): variables_ordered_dict = convert_to_order_dict(variables) elif isinstance(variables, OrderedDict): variables_ordered_dict = variables else: raise exception.ParamsError("variables error!") return update_ordered_dict(variables_ordered_dict, new_mapping)
def run_test(self, testcase): """ run single testcase. @param (dict) testcase { "name": "testcase description", "requires": [], # optional, override "function_binds": {}, # optional, override "variable_binds": {}, # optional, override "request": { "url": "http://127.0.0.1:5000/api/users/1000", "method": "POST", "headers": { "Content-Type": "application/json", "authorization": "${authorization}", "random": "${random}" }, "body": '{"name": "user", "password": "******"}' }, "extract_binds": {}, "validators": [] } @return (tuple) test result of single testcase (success, diff_content_list) """ self.update_context(testcase) # each testcase shall inherit from testset request configs, # but can not override testset configs, # that's why we use copy.deepcopy here. testcase_request = copy.deepcopy(self.testset_req_overall_configs) testcase_request.update(testcase["request"]) parsed_request = parse_template(testcase_request, self.context.variables) try: url = parsed_request.pop('url') method = parsed_request.pop('method') except KeyError: raise exception.ParamsError("URL or METHOD missed!") resp = self.client.request(url=url, method=method, **parsed_request) resp_obj = response.ResponseObject(resp) extract_binds = testcase.get("extract_binds", {}) extracted_variables_mapping = resp_obj.extract_response(extract_binds) self.context.update_variables(extracted_variables_mapping) validators = testcase.get("validators", []) diff_content_list = resp_obj.validate(validators, self.context.variables) return resp_obj.success, diff_content_list
def extract_response(self, extract_binds, delimiter='.'): """ extract content from requests.Response @param (dict) extract_binds { "resp_status_code": "status_code", "resp_headers_content_type": "headers.content-type", "resp_content": "content", "resp_content_person_first_name": "content.person.name.first_name" } """ extract_binds_dict = {} for key, value in extract_binds.items(): if not isinstance(value, utils.string_type): raise exception.ParamsError("invalid extract_binds!") try: value += "." # string.split(sep=None, maxsplit=-1) -> list of strings # e.g. "content.person.name" => ["content", "person.name"] top_query, sub_query = value.split(delimiter, 1) if top_query in ["body", "content", "text"]: json_content = self.parsed_body() else: json_content = getattr(self.resp_obj, top_query) if sub_query: # e.g. key: resp_headers_content_type, sub_query = "content-type" answer = utils.query_json(json_content, sub_query) extract_binds_dict[key] = answer else: # e.g. key: resp_status_code, resp_content extract_binds_dict[key] = json_content except AttributeError: raise exception.ParamsError("invalid extract_binds!") return extract_binds_dict
def run_single_testcase(self, testcase): req_kwargs = testcase['request'] try: url = req_kwargs.pop('url') method = req_kwargs.pop('method') except KeyError: raise exception.ParamsError("URL or METHOD missed!") resp_obj = self.client.request(url=url, method=method, **req_kwargs) diff_content = utils.diff_response(resp_obj, testcase['response']) success = False if diff_content else True return success, diff_content
def run_test(self, testcase): """ run single testcase. @param (dict) testcase { "name": "testcase description", "requires": [], # optional, override "function_binds": {}, # optional, override "variable_binds": {}, # optional, override "request": { "url": "http://127.0.0.1:5000/api/users/1000", "method": "POST", "headers": { "Content-Type": "application/json", "authorization": "$authorization", "random": "$random" }, "body": '{"name": "user", "password": "******"}' }, "extract_binds": {}, # optional "validators": [] # optional } @return (tuple) test result of single testcase (success, diff_content_list) """ self.init_config(testcase, level="testcase") parsed_request = self.context.get_parsed_request() try: url = parsed_request.pop('url') method = parsed_request.pop('method') except KeyError: raise exception.ParamsError("URL or METHOD missed!") resp = self.http_client_session.request(url=url, method=method, **parsed_request) resp_obj = response.ResponseObject(resp) extract_binds = testcase.get("extract_binds", {}) extracted_variables_mapping_list = resp_obj.extract_response( extract_binds) self.context.bind_variables(extracted_variables_mapping_list, level="testset") validators = testcase.get("validators", []) diff_content_list = resp_obj.validate( validators, self.context.get_testcase_variables_mapping()) return resp_obj.success, diff_content_list
def _extract_field_with_regex(self, field): """ extract field from response content with regex. requests.Response body could be json or html text. @param (str) field should only be regex string that matched r".*\(.*\).*" e.g. self.resp_text: "LB123abcRB789" field: "LB[\d]*(.*)RB[\d]*" return: abc """ matched = re.search(field, self.resp_text) if not matched: err_msg = u"Extractor error: failed to extract data with regex!\n" err_msg += u"response body: {}\n".format(self.resp_text) err_msg += u"regex: {}\n".format(field) logging.error(err_msg) raise exception.ParamsError(err_msg) return matched.group(1)
def extract_response(self, extract_binds): """ extract content from requests.Response @param (dict) extract_binds { "resp_status_code": "status_code", "resp_headers_content_type": "headers.content-type", "resp_content": "content", "resp_content_person_first_name": "content.person.name.first_name" } """ extracted_variables_mapping = {} for key, field in extract_binds.items(): if not isinstance(field, utils.string_type): raise exception.ParamsError("invalid extract_binds!") extracted_variables_mapping[key] = self.extract_field(field) return extracted_variables_mapping
def validate(self, validators, variables_mapping): """ Bind named validators to value within the context. @param (list) validators [ {"check": "status_code", "comparator": "eq", "expected": 201}, {"check": "resp_body_success", "comparator": "eq", "expected": True} ] @param (dict) variables_mapping { "resp_body_success": True } @return (list) content differences [ { "check": "status_code", "comparator": "eq", "expected": 201, "value": 200 } ] """ diff_content_list = [] for validator_dict in validators: if "expected" not in validator_dict or "check" not in validator_dict: raise exception.ParamsError( "expected not specified in validator") validator_key = validator_dict["check"] try: validator_dict["value"] = variables_mapping[validator_key] except KeyError: validator_dict["value"] = self.extract_field(validator_key) match_expected = utils.match_expected( validator_dict["value"], validator_dict["expected"], validator_dict.get("comparator", "eq")) if not match_expected: diff_content_list.append(validator_dict) self.success = False if diff_content_list else True return diff_content_list
def extract_response(self, extractors): """ extract content from requests.Response @param (list) extractors [ {"resp_status_code": "status_code"}, {"resp_headers_content_type": "headers.content-type"}, {"resp_content": "content"}, {"resp_content_person_first_name": "content.person.name.first_name"} ] @return (OrderDict) variable binds ordered dict """ extracted_variables_mapping = OrderedDict() extract_binds_order_dict = utils.convert_to_order_dict(extractors) for key, field in extract_binds_order_dict.items(): if not isinstance(field, utils.string_type): raise exception.ParamsError("invalid extractors in testcase!") extracted_variables_mapping[key] = self.extract_field(field) return extracted_variables_mapping
def validate(self, validators, variables_mapping): """ Bind named validators to value within the context. @param (dict) validators { "resp_status_code": {"comparator": "eq", "expected": 201}, "resp_body_success": {"comparator": "eq", "expected": True} } @param (dict) variables_mapping { "resp_status_code": 200, "resp_body_success": True } @return (dict) content differences { "resp_status_code": { "comparator": "eq", "expected": 201, "value": 200 } } """ diff_content_dict = {} for validator_key, validator_dict in validators.items(): try: value = variables_mapping[validator_key] validator_dict["value"] = value except KeyError: raise exception.ParamsError("invalid validator %s" % validator_key) match_expected = utils.match_expected( value, validator_dict["expected"], validator_dict["comparator"] ) if not match_expected: diff_content_dict[validator_key] = validator_dict self.success = False if diff_content_dict else True return diff_content_dict
def extract_response(self, extract_binds): """ extract content from requests.Response @param (list) extract_binds [ {"resp_status_code": "status_code"}, {"resp_headers_content_type": "headers.content-type"}, {"resp_content": "content"}, {"resp_content_person_first_name": "content.person.name.first_name"} ] @return (list) variable binds list """ extracted_variables_mapping_list = [] for extract_bind in extract_binds: for key, field in extract_bind.items(): if not isinstance(field, utils.string_type): raise exception.ParamsError("invalid extract_binds!") extracted_variables_mapping_list.append( {key: self.extract_field(field)}) return extracted_variables_mapping_list
def get_eval_value(self, data): """ evaluate data recursively, each variable in data will be evaluated. """ if isinstance(data, (list, tuple)): return [self.get_eval_value(item) for item in data] if isinstance(data, dict): evaluated_data = {} for key, value in data.items(): evaluated_data[key] = self.get_eval_value(value) return evaluated_data if isinstance(data, (int, float)): return data # data is in string format here data = "" if data is None else data.strip() if utils.is_variable(data): # variable marker: $var variable_name = utils.parse_variable(data) value = self.testcase_variables_mapping.get(variable_name) if value is None: raise exception.ParamsError( "%s is not defined in bind variables!" % variable_name) return value elif utils.is_functon(data): # function marker: ${func(1, 2, a=3, b=4)} fuction_meta = utils.parse_function(data) func_name = fuction_meta['func_name'] args = fuction_meta.get('args', []) kwargs = fuction_meta.get('kwargs', {}) args = self.get_eval_value(args) kwargs = self.get_eval_value(kwargs) return self.testcase_config["functions"][func_name](*args, **kwargs) else: return data
def get_test_definition(name, ref_type): """ get expected api or suite. @params: name: api or suite name ref_type: "api" or "suite" @return expected api info if found, otherwise raise ApiNotFound exception """ if not test_def_overall_dict.get("loaded", False): load_test_dependencies() test_info = test_def_overall_dict.get(ref_type, {}).get(name) if not test_info: err_msg = "{} {} not found!".format(ref_type, name) if ref_type == "api": raise exception.ApiNotFound(err_msg) elif ref_type == "suite": raise exception.SuiteNotFound(err_msg) else: raise exception.ParamsError("ref_type can only be api or suite!") return test_info
def extract_response(resp_obj, context, delimiter='.'): """ extract content from requests.Response, and bind extracted value to context.extractors @param (requests.Response instance) resp_obj @param (ate.context.Context instance) context context.extractors: { "resp_status_code": "status_code", "resp_headers_content_type": "headers.content-type", "resp_content": "content", "resp_content_person_first_name": "content.person.name.first_name" } """ for key, value in context.extractors.items(): try: if isinstance(value, str): value += "." # string.split(sep=None, maxsplit=-1) -> list of strings # e.g. "content.person.name" => ["content", "person.name"] top_query, sub_query = value.split(delimiter, 1) if top_query in ["body", "content", "text"]: json_content = parse_response_body(resp_obj) else: json_content = getattr(resp_obj, top_query) if sub_query: # e.g. key: resp_headers_content_type, sub_query = "content-type" answer = utils.query_json(json_content, sub_query) context.extractors[key] = answer else: # e.g. key: resp_status_code, resp_content context.extractors[key] = json_content else: raise NotImplementedError("TODO: support template.") except AttributeError: raise exception.ParamsError("invalid extract_binds!")
def match_expected(value, expected, comparator="eq", check_item=""): """ check if value matches expected value. @param value: actual value that get from response. @param expected: expected result described in testcase @param comparator: compare method @param check_item: check item name """ try: if value is None or expected is None: assert comparator in ["is", "eq", "equals", "=="] assert value is None assert expected is None if comparator in ["eq", "equals", "=="]: assert value == expected elif comparator in ["lt", "less_than"]: assert value < expected elif comparator in ["le", "less_than_or_equals"]: assert value <= expected elif comparator in ["gt", "greater_than"]: assert value > expected elif comparator in ["ge", "greater_than_or_equals"]: assert value >= expected elif comparator in ["ne", "not_equals"]: assert value != expected elif comparator in ["str_eq", "string_equals"]: assert str(value) == str(expected) elif comparator in ["len_eq", "length_equals", "count_eq"]: assert isinstance(expected, int) assert len(value) == expected elif comparator in ["len_gt", "count_gt", "length_greater_than", "count_greater_than"]: assert isinstance(expected, int) assert len(value) > expected elif comparator in ["len_ge", "count_ge", "length_greater_than_or_equals", \ "count_greater_than_or_equals"]: assert isinstance(expected, int) assert len(value) >= expected elif comparator in ["len_lt", "count_lt", "length_less_than", "count_less_than"]: assert isinstance(expected, int) assert len(value) < expected elif comparator in ["len_le", "count_le", "length_less_than_or_equals", \ "count_less_than_or_equals"]: assert isinstance(expected, int) assert len(value) <= expected elif comparator in ["contains"]: assert isinstance(value, (list,tuple,dict,string_type)) assert expected in value elif comparator in ["contained_by"]: assert isinstance(expected, (list,tuple,dict,string_type)) assert value in expected elif comparator in ["type"]: assert isinstance(value, expected) elif comparator in ["regex"]: assert isinstance(expected, string_type) assert isinstance(value, string_type) assert re.match(expected, value) elif comparator in ["startswith"]: assert str(value).startswith(str(expected)) elif comparator in ["endswith"]: assert str(value).endswith(str(expected)) else: raise exception.ParamsError("comparator not supported!") return True except (AssertionError, TypeError): err_msg = "\n".join([ "check item name: %s;" % check_item, "check item value: %s (%s);" % (value, type(value).__name__), "comparator: %s;" % comparator, "expected value: %s (%s)." % (expected, type(expected).__name__) ]) raise exception.ValidationError(err_msg)
def _run_test(self, testcase): """ run single testcase. @param (dict) testcase { "name": "testcase description", "times": 3, "requires": [], # optional, override "function_binds": {}, # optional, override "variables": [], # optional, override "request": { "url": "http://127.0.0.1:5000/api/users/1000", "method": "POST", "headers": { "Content-Type": "application/json", "authorization": "$authorization", "random": "$random" }, "body": '{"name": "user", "password": "******"}' }, "extract": [], # optional "validate": [], # optional "setup": [], # optional "teardown": [] # optional } @return True or raise exception during test """ parsed_request = self.init_config(testcase, level="testcase") try: url = parsed_request.pop('url') method = parsed_request.pop('method') group_name = parsed_request.pop("group", None) except KeyError: raise exception.ParamsError("URL or METHOD missed!") run_times = int(testcase.get("times", 1)) extractors = testcase.get("extract") \ or testcase.get("extractors") \ or testcase.get("extract_binds", []) validators = testcase.get("validate") \ or testcase.get("validators", []) setup_actions = testcase.get("setup", []) teardown_actions = testcase.get("teardown", []) def setup_teardown(actions): for action in actions: self.context.exec_content_functions(action) for _ in range(run_times): setup_teardown(setup_actions) resp = self.http_client_session.request(method, url, name=group_name, **parsed_request) resp_obj = response.ResponseObject(resp) extracted_variables_mapping = resp_obj.extract_response(extractors) self.context.bind_extracted_variables(extracted_variables_mapping) try: resp_obj.validate( validators, self.context.get_testcase_variables_mapping()) except (exception.ParamsError, exception.ResponseError, exception.ValidationError): logging.error("Exception occured.") logging.error( "HTTP request kwargs: \n{}".format(parsed_request)) logging.error("HTTP response content: \n{}".format(resp.text)) raise setup_teardown(teardown_actions) return True