def __init__(self, testset, variables_mapping=None, http_client_session=None): super(ApiTestSuite, self).__init__() self.config_dict = testset.get("config", {}) variables = self.config_dict.get("variables", []) variables_mapping = variables_mapping or {} self.config_dict["variables"] = utils.override_variables_binds( variables, variables_mapping) parameters = self.config_dict.get("parameters", []) cartesian_product_parameters = testcase.gen_cartesian_product_parameters( parameters, self.config_dict["path"]) or [{}] for parameter_mapping in cartesian_product_parameters: if parameter_mapping: self.config_dict["variables"] = utils.override_variables_binds( self.config_dict["variables"], parameter_mapping) self.test_runner = runner.Runner(self.config_dict, http_client_session) testcases = testset.get("testcases", []) self._add_tests_to_suite(testcases)
def test_override_variables_binds(self): map_list = [ {"a": 1}, {"b": 2} ] override_mapping = {"a": 3, "c": 4} new_dict = utils.override_variables_binds(map_list, override_mapping) self.assertEqual(3, new_dict["a"]) self.assertEqual(4, new_dict["c"]) map_list = OrderedDict( { "a": 1, "b": 2 } ) override_mapping = {"a": 3, "c": 4} new_dict = utils.override_variables_binds(map_list, override_mapping) self.assertEqual(3, new_dict["a"]) self.assertEqual(4, new_dict["c"]) map_list = "invalid" override_mapping = {"a": 3, "c": 4} with self.assertRaises(exception.ParamsError): utils.override_variables_binds(map_list, override_mapping)
def __init__(self, testset, variables_mapping=None, http_client_session=None): super(TestSuite, self).__init__() self.test_runner_list = [] config_dict = testset.get("config", {}) self.output_variables_list = config_dict.get("output", []) self.testset_file_path = config_dict.get("path") config_dict_parameters = config_dict.get("parameters", []) config_dict_variables = config_dict.get("variables", []) variables_mapping = variables_mapping or {} config_dict_variables = utils.override_variables_binds( config_dict_variables, variables_mapping) #zhengchun 20180608 self.testcase_parser = testcase.TestcaseParser() config_parametered_variables_list = self._get_parametered_variables( config_dict_variables, config_dict_parameters, self.testcase_parser) testcases = testset.get("testcases", []) for config_variables in config_parametered_variables_list: # config level config_dict["variables"] = config_variables # update by zhengchun, can use myRunner, 20180608 test_runner = testset["runner"](config_dict, http_client_session) for testcase_dict in testcases: testcase_dict = copy.copy(testcase_dict) # testcase level testcase_parametered_variables_list = self._get_parametered_variables( testcase_dict.get("variables", []), testcase_dict.get("parameters", []), test_runner.context.testcase_parser) for testcase_variables in testcase_parametered_variables_list: testcase_dict["variables"] = testcase_variables # eval testcase name with bind variables variables = utils.override_variables_binds( config_variables, testcase_variables) self.testcase_parser.update_binded_variables(variables) try: testcase_name = self.testcase_parser.eval_content_with_bindings( testcase_dict["name"]) except (AssertionError, exception.ParamsError): logger.log_warning( "failed to eval testcase name: {}".format( testcase_dict["name"])) testcase_name = testcase_dict["name"] self.test_runner_list.append((test_runner, variables)) self._add_test_to_suite(testcase_name, test_runner, testcase_dict)
def __init__(self, testset, variables_mapping=None, http_client_session=None): super(TestSuite, self).__init__() self.test_runner_list = [] config_dict = testset.get("config", {}) self.output_variables_list = config_dict.get("output", []) self.testset_file_path = config_dict.get("path") config_dict_parameters = config_dict.get("parameters", []) config_dict_variables = config_dict.get("variables", []) variables_mapping = variables_mapping or {} config_dict_variables = utils.override_variables_binds(config_dict_variables, variables_mapping) config_parametered_variables_list = self._get_parametered_variables( config_dict_variables, config_dict_parameters ) self.testcase_parser = testcase.TestcaseParser() testcases = testset.get("testcases", []) for config_variables in config_parametered_variables_list: # config level config_dict["variables"] = config_variables test_runner = runner.Runner(config_dict, http_client_session) for testcase_dict in testcases: testcase_dict = copy.copy(testcase_dict) # testcase level testcase_parametered_variables_list = self._get_parametered_variables( testcase_dict.get("variables", []), testcase_dict.get("parameters", []) ) for testcase_variables in testcase_parametered_variables_list: testcase_dict["variables"] = testcase_variables # eval testcase name with bind variables variables = utils.override_variables_binds( config_variables, testcase_variables ) self.testcase_parser.update_binded_variables(variables) try: testcase_name = self.testcase_parser.eval_content_with_bindings(testcase_dict["name"]) except (AssertionError, exception.ParamsError): logger.log_warning("failed to eval testcase name: {}".format(testcase_dict["name"])) testcase_name = testcase_dict["name"] self.test_runner_list.append((test_runner, variables)) self._add_test_to_suite(testcase_name, test_runner, testcase_dict)
def __init__(self, testset, variables_mapping=None, http_client_session=None): super().__init__() self.test_runner_list = [] config_dict = testset.get("config", {}) self.output_variables_list = config_dict.get("output", []) self.testset_file_path = config_dict["path"] config_dict_parameters = config_dict.get("parameters", []) config_dict_variables = config_dict.get("variables", []) variables_mapping = variables_mapping or {} config_dict_variables = utils.override_variables_binds( config_dict_variables, variables_mapping) config_parametered_variables_list = self._get_parametered_variables( config_dict_variables, config_dict_parameters) self.testcase_parser = testcase.TestcaseParser() testcases = testset.get("testcases", []) for config_variables in config_parametered_variables_list: # config level config_dict["variables"] = config_variables # test_runner = runner.Runner(config_dict, http_client_session) test_runner = YDHrunner.YDHRunner(config_dict, http_client_session) for testcase_dict in testcases: testcase_dict = copy.copy(testcase_dict) # testcase level testcase_parametered_variables_list = self._get_parametered_variables( testcase_dict.get("variables", []), testcase_dict.get("parameters", [])) for testcase_variables in testcase_parametered_variables_list: testcase_dict["variables"] = testcase_variables # eval testcase name with bind variables variables = utils.override_variables_binds( config_variables, testcase_variables) self.testcase_parser.update_binded_variables(variables) testcase_name = self.testcase_parser.eval_content_with_bindings( testcase_dict["name"]) self.test_runner_list.append((test_runner, variables)) self._add_test_to_suite(testcase_name, test_runner, testcase_dict)
def _get_parametered_variables(self, variables, parameters): """ parameterize varaibles with parameters """ cartesian_product_parameters = testcase.parse_parameters( parameters, self.testset_file_path) or [{}] parametered_variables_list = [] for parameter_mapping in cartesian_product_parameters: parameter_mapping = parameter_mapping or {} variables = utils.override_variables_binds(variables, parameter_mapping) parametered_variables_list.append(variables) return parametered_variables_list
def _get_parametered_variables(self, variables, parameters): """ parameterize varaibles with parameters """ cartesian_product_parameters = testcase.parse_parameters( parameters, self.testset_file_path ) or [{}] parametered_variables_list = [] for parameter_mapping in cartesian_product_parameters: parameter_mapping = parameter_mapping or {} variables = utils.override_variables_binds( variables, parameter_mapping ) parametered_variables_list.append(variables) return parametered_variables_list
def _get_parametered_variables(self, variables, parameters, parser_handle=None): """ parameterize varaibles with parameters """ cartesian_product_parameters = testcase.parse_parameters( parameters, self.testset_file_path, parser_handle #modified by zhengchun, 解析动态加载函数 ) or [{}] parametered_variables_list = [] for parameter_mapping in cartesian_product_parameters: parameter_mapping = parameter_mapping or {} variables = utils.override_variables_binds(variables, parameter_mapping) parametered_variables_list.append(variables) return parametered_variables_list
def _run_testset(self, testset, variables_mapping=None): """ run single testset, including one or several testcases. @param (dict) testset { "name": "testset description", "config": { "name": "testset description", "requires": [], "function_binds": {}, "variables": [], "request": {} }, "testcases": [ { "name": "testcase description", "variables": [], # optional, override "request": {}, "extract": {}, # optional "validate": {} # optional }, testcase12 ] } (dict) variables_mapping: passed in variables mapping, it will override variables in config block @return (dict) test result of testset { "success": True, "output": {} # variables mapping } """ success = True config_dict = testset.get("config", {}) variables = config_dict.get("variables", []) variables_mapping = variables_mapping or {} config_dict["variables"] = utils.override_variables_binds( variables, variables_mapping) self.init_config(config_dict, level="testset") testcases = testset.get("testcases", []) for testcase_dict in testcases: try: self._run_test(testcase_dict) except exception.MyBaseError as ex: success = False if self.request_failure_hook: self.request_failure_hook.fire( request_type=testcase_dict.get("request", {}).get("method"), name=testcase_dict.get("request", {}).get("url"), response_time=0, exception=ex) else: logging.exception( "Exception occured in testcase: {}".format( testcase_dict.get("name"))) break output_variables_list = config_dict.get("output", []) return { "success": success, "output": self.generate_output(output_variables_list) }
def __init__(self, testset, variables_mapping=None, http_client_session=None): super(TestSuite, self).__init__() self.test_runner_list = [] self.config = testset.get("config", {}) self.output_variables_list = self.config.get("output", []) self.testset_file_path = self.config.get("path") config_dict_parameters = self.config.get("parameters", []) config_dict_variables = self.config.get("variables", []) variables_mapping = variables_mapping or {} config_dict_variables = utils.override_variables_binds( config_dict_variables, variables_mapping) config_parametered_variables_lists = self._get_parametered_variables( config_dict_variables, config_dict_parameters) config_parametered_variables_list = config_parametered_variables_lists[ 0] self.testcase_parser = testcase.TestcaseParser() testcases = testset.get("testcases", []) for config_variables in config_parametered_variables_list: # config level self.config["variables"] = config_variables test_runner = runner.Runner(self.config, http_client_session) for testcase_dict in testcases: testcase_dict = copy.copy(testcase_dict) # testcase level testcase_parametered_variables_lists = self._get_parametered_variables( testcase_dict.get("variables", []), testcase_dict.get("parameters", [])) testcase_parametered_variables_list = testcase_parametered_variables_lists[ 0] assert_list = testcase_parametered_variables_lists[1] testcase_index = 0 if 'validate' not in testcase_dict.keys(): testcase_dict['validate'] = [] basic_validate = testcase_dict['validate'] for testcase_variables in testcase_parametered_variables_list: testcase_dict["variables"] = testcase_variables asserts_str = assert_list[testcase_index] asserts = ast.literal_eval(asserts_str) # validates = [] checks = {} if isinstance(asserts, list): for validate in asserts: # validate = { # 'check': 'content.code', # 'comparator': 'equals', # 'expected': '100201' # } # validates.append(validate) checks[validate.get('check')] = 1 elif isinstance(asserts, dict): checks[asserts.get('check')] = 1 asserts = [asserts] else: asserts = [] validates = [] validates.extend(asserts) # 去重,以parameters中的断言优先 for validate in basic_validate: check = validate.get('check') if check not in checks.keys(): validates.append(validate) testcase_dict['validate'] = validates # eval testcase name with bind variables variables = utils.override_variables_binds( config_variables, testcase_variables) self.testcase_parser.update_binded_variables(variables) try: testcase_name = self.testcase_parser.eval_content_with_bindings( testcase_dict["name"]) except (AssertionError, exceptions.ParamsError): logger.log_warning( "failed to eval testcase name: {}".format( testcase_dict["name"])) testcase_name = testcase_dict["name"] self.test_runner_list.append((test_runner, variables)) self._add_test_to_suite(testcase_name, test_runner, testcase_dict) testcase_index = testcase_index + 1
def __init__(self, testset, variables_mapping=None, http_client_session=None): super(TestSuite, self).__init__() self.test_runner_list = [] config_dict = testset.get("config", {}) self.output_variables_list = config_dict.get("output", []) self.testset_file_path = config_dict.get("path") config_dict_parameters = config_dict.get("parameters", []) config_dict_variables = config_dict.get("variables", []) variables_mapping = variables_mapping or {} config_dict_variables = utils.override_variables_binds(config_dict_variables, variables_mapping) config_parametered_variables_list = self._get_parametered_variables( config_dict_variables, config_dict_parameters ) self.testcase_parser = testcase.TestcaseParser() testcases = testset.get("testcases", []) count = 0 self.all_case_list = [] self.variables_list = [] for config_variables in config_parametered_variables_list: # print("config_variables====>") # print(config_variables) # config level config_dict["variables"] = config_variables test_runner = runner.Runner(config_dict, http_client_session) # TODO 存储已添加的用例 self.testcase_dict_list = [] for testcase_dict in testcases: count = count + 1 testcase_dict = copy.copy(testcase_dict) # testcase level testcase_parametered_variables_list = self._get_parametered_variables( testcase_dict.get("variables", []), testcase_dict.get("parameters", []) ) for testcase_variables in testcase_parametered_variables_list: testcase_dict["variables"] = testcase_variables # eval testcase name with bind variables variables = utils.override_variables_binds( config_variables, testcase_variables ) self.testcase_parser.update_binded_variables(variables) try: testcase_name = self.testcase_parser.eval_content_with_bindings(testcase_dict["name"]) except (AssertionError, exception.ParamsError): logger.log_warning("failed to eval testcase name: {}".format(testcase_dict["name"])) testcase_name = testcase_dict["name"] # TODO 不重复添加用例 # print("testcase_dict") # print(testcase_dict) if config_variables not in self.variables_list: self.variables_list.append(config_variables) self.testcase_dict_list.append(testcase_dict) self.all_case_list.append(testcase_dict) self.test_runner_list.append((test_runner, variables)) self._add_test_to_suite(testcase_name, test_runner, testcase_dict) if testcase_dict not in self.testcase_dict_list and testcase_dict not in self.all_case_list: self.testcase_dict_list.append(testcase_dict) self.all_case_list.append(testcase_dict) self.test_runner_list.append((test_runner, variables)) self._add_test_to_suite(testcase_name, test_runner, testcase_dict) print('for ----count ===',count)
def _run_testset(self, testset, variables_mapping=None): """ run single testset, including one or several testcases. @param (dict) testset { "name": "testset description", "config": { "name": "testset description", "requires": [], "function_binds": {}, "variables": [], "request": {} }, "testcases": [ { "name": "testcase description", "variables": [], # optional, override "request": {}, "extract": {}, # optional "validate": {} # optional }, testcase12 ] } (dict) variables_mapping: passed in variables mapping, it will override variables in config block @return (dict) test result of testset { "success": True, "output": {} # variables mapping } """ success = True config_dict = testset.get("config", {}) variables = config_dict.get("variables", []) variables_mapping = variables_mapping or {} config_dict["variables"] = utils.override_variables_binds(variables, variables_mapping) self.init_config(config_dict, level="testset") testcases = testset.get("testcases", []) for testcase_dict in testcases: try: self._run_test(testcase_dict) except exception.MyBaseError as ex: success = False if self.request_failure_hook: self.request_failure_hook.fire( request_type=testcase_dict.get("request", {}).get("method"), name=testcase_dict.get("request", {}).get("url"), response_time=0, exception=ex ) else: logging.exception( "Exception occured in testcase: {}".format(testcase_dict.get("name"))) break output_variables_list = config_dict.get("output", []) return { "success": success, "output": self.generate_output(output_variables_list) }