def run(self, path_or_testsets, mapping=None): """ start to run test with varaibles mapping @param path_or_testsets: YAML/JSON testset file path or testset list path: path could be in several type - absolute/relative file path - absolute/relative folder path - list/set container with file(s) and/or folder(s) testsets: testset or list of testset - (dict) testset_dict - (list) list of testset_dict [ testset_dict_1, testset_dict_2 ] @param (dict) mapping: if mapping specified, it will override variables in config block """ try: test_suite_list = init_test_suites(path_or_testsets, mapping) except exceptions.TestcaseNotFound: logger.log_error( "Testcases not found in {}".format(path_or_testsets)) sys.exit(1) self.summary = {"success": True, "stat": {}, "time": {}, "details": []} def accumulate_stat(origin_stat, new_stat): """ accumulate new_stat to origin_stat """ for key in new_stat: if key not in origin_stat: origin_stat[key] = new_stat[key] elif key == "start_at": # start datetime origin_stat[key] = min(origin_stat[key], new_stat[key]) else: origin_stat[key] += new_stat[key] for test_suite in test_suite_list: result = self.runner.run(test_suite) # test_suite_summary = get_summary(result) # # self.summary["success"] &= test_suite_summary["success"] # test_suite_summary["name"] = test_suite.config.get("name") # test_suite_summary["base_url"] = test_suite.config.get("request", {}).get("base_url", "") # test_suite_summary["output"] = test_suite.output # print_output(test_suite_summary["output"]) # # accumulate_stat(self.summary["stat"], test_suite_summary["stat"]) # accumulate_stat(self.summary["time"], test_suite_summary["time"]) # # self.summary["details"].append(test_suite_summary) return self
def _load_json_file(json_file): """ load json file and check file content format """ with io.open(json_file, encoding='utf-8') as data_file: try: json_content = json.load(data_file) except exceptions.JSONDecodeError: err_msg = u"JSONDecodeError: JSON file format error: {}".format( json_file) logger.log_error(err_msg) raise exceptions.FileFormatError(err_msg) FileUtils._check_format(json_file, json_content) return json_content
def load_testsets_by_path(path): """ load testcases from file path @param path: path could be in several type - absolute/relative file path - absolute/relative folder path - list/set container with file(s) and/or folder(s) @return testcase sets list, each testset is corresponding to a file [ testset_dict_1, testset_dict_2 ] """ if isinstance(path, (list, set)): testsets = [] for file_path in set(path): testset = TestcaseLoader.load_testsets_by_path(file_path) if not testset: continue testsets.extend(testset) return testsets if not os.path.isabs(path): path = os.path.join(os.getcwd(), path) if path in TestcaseLoader.testcases_cache_mapping: return TestcaseLoader.testcases_cache_mapping[path] if os.path.isdir(path): files_list = FileUtils.load_folder_files(path) testcases_list = TestcaseLoader.load_testsets_by_path(files_list) elif os.path.isfile(path): try: testset = TestcaseLoader.load_test_file(path) if testset["testcases"] or testset["api"]: testcases_list = [testset] else: testcases_list = [] except exceptions.FileFormatError: testcases_list = [] else: logger.log_error(u"file not found: {}".format(path)) testcases_list = [] TestcaseLoader.testcases_cache_mapping[path] = testcases_list return testcases_list
def _check_format(file_path, content): """ check testcase format if valid """ if not content: # testcase file content is empty err_msg = u"Testcase file content is empty: {}".format(file_path) logger.log_error(err_msg) raise exceptions.FileFormatError(err_msg) elif not isinstance(content, (list, dict)): # testcase file content does not match testcase format err_msg = u"Testcase file content format invalid: {}".format( file_path) logger.log_error(err_msg) raise exceptions.FileFormatError(err_msg)
def query_json(json_content, query, delimiter='.'): """ Do an xpath-like query with json_content. @param (dict/list/string) json_content json_content = { "ids": [1, 2, 3, 4], "person": { "name": { "first_name": "Leo", "last_name": "Lee", }, "age": 29, "cities": ["Guangzhou", "Shenzhen"] } } @param (str) query "person.name.first_name" => "Leo" "person.name.first_name.0" => "L" "person.cities.0" => "Guangzhou" @return queried result """ raise_flag = False response_body = u"response body: {}\n".format(json_content) try: for key in query.split(delimiter): if isinstance(json_content, (list, basestring)): json_content = json_content[int(key)] elif isinstance(json_content, dict): json_content = json_content[key] else: logger.log_error("invalid type value: {}({})".format( json_content, type(json_content))) raise_flag = True except (KeyError, ValueError, IndexError): raise_flag = True if raise_flag: err_msg = u"Failed to extract! => {}\n".format(query) err_msg += response_body logger.log_error(err_msg) raise exceptions.ExtractFailure(err_msg) return json_content
def do_validation(self, validator_dict): """ validate with functions """ # TODO: move comparator uniform to init_test_suites comparator = utils.get_uniform_comparator(validator_dict["comparator"]) validate_func = self.testcase_parser.get_bind_function(comparator) if not validate_func: raise exceptions.FunctionNotFound( "comparator not found: {}".format(comparator)) check_item = validator_dict["check"] check_value = validator_dict["check_value"] expect_value = validator_dict["expect"] if (check_value is None or expect_value is None) \ and comparator not in ["is", "eq", "equals", "=="]: raise exceptions.ParamsError( "Null value can only be compared with comparator: eq/equals/==" ) validate_msg = "validate: {} {} {}({})".format( check_item, comparator, expect_value, type(expect_value).__name__) try: validator_dict["check_result"] = "pass" validate_func(check_value, expect_value) validate_msg += "\t==> pass" logger.log_debug(validate_msg) except (AssertionError, TypeError): validate_msg += "\t==> fail" validate_msg += "\n{}({}) {} {}({})".format( check_value, type(check_value).__name__, comparator, expect_value, type(expect_value).__name__) logger.log_error(validate_msg) validator_dict["check_result"] = "fail" raise exceptions.ValidationFailure(validate_msg)
def run_test(self, testcase_dict): """ run single testcase. @param (dict) testcase_dict { "name": "testcase description", "skip": "skip this test unconditionally", "times": 3, "variables": [], # optional, override "request": { "url": "http://127.0.0.1:5000/api/users/1000", "method": "POST", "headers": { "Content-Type": "application/json", "authorization": "$authorization", "random": "$random" }, "body": '{"name": "user", "password": "******"}' }, "extract": [], # optional "validate": [], # optional "setup_hooks": [], # optional "teardown_hooks": [] # optional } @return True or raise exception during test """ # check skip self._handle_skip_feature(testcase_dict) # prepare parsed_request = self.init_config(testcase_dict, level="testcase") self.context.bind_testcase_variable("request", parsed_request) # setup hooks setup_hooks = testcase_dict.get("setup_hooks", []) setup_hooks.insert(0, "${setup_hook_prepare_kwargs($request)}") self.do_hook_actions(setup_hooks) try: url = parsed_request.pop('url') method = parsed_request.pop('method') group_name = parsed_request.pop("group", None) except KeyError: raise exceptions.ParamsError("URL or METHOD missed!") # TODO: move method validation to json schema valid_methods = [ "GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS" ] if method.upper() not in valid_methods: err_msg = u"Invalid HTTP method! => {}\n".format(method) err_msg += "Available HTTP methods: {}".format( "/".join(valid_methods)) logger.log_error(err_msg) raise exceptions.ParamsError(err_msg) logger.log_info("{method} {url}".format(method=method, url=url)) logger.log_debug( "request kwargs(raw): {kwargs}".format(kwargs=parsed_request)) # request resp = self.http_client_session.request(method, url, name=group_name, **parsed_request) resp_obj = response.ResponseObject(resp) # teardown hooks teardown_hooks = testcase_dict.get("teardown_hooks", []) if teardown_hooks: self.context.bind_testcase_variable("response", resp_obj) self.do_hook_actions(teardown_hooks) # extract extractors = testcase_dict.get("extract", []) or testcase_dict.get( "extractors", []) extracted_variables_mapping = resp_obj.extract_response(extractors) self.context.bind_extracted_variables(extracted_variables_mapping) # validate validators = testcase_dict.get("validate", []) or testcase_dict.get( "validators", []) try: self.context.validate(validators, resp_obj) except (exceptions.ParamsError, \ exceptions.ValidationFailure, exceptions.ExtractFailure): # log request err_req_msg = "request: \n" err_req_msg += "headers: {}\n".format( parsed_request.pop("headers", {})) for k, v in parsed_request.items(): err_req_msg += "{}: {}\n".format(k, repr(v)) logger.log_error(err_req_msg) # log response err_resp_msg = "response: \n" err_resp_msg += "status_code: {}\n".format(resp_obj.status_code) err_resp_msg += "headers: {}\n".format(resp_obj.headers) err_resp_msg += "body: {}\n".format(repr(resp_obj.text)) logger.log_error(err_resp_msg) raise