def __init__(self,
testset,
variables_mapping=None,
http_client_session=None):
super(ApiTestSuite, self).__init__()
self.config_dict = testset.get("config", {})
variables = self.config_dict.get("variables", [])
variables_mapping = variables_mapping or {}
self.config_dict["variables"] = utils.override_variables_binds(
variables, variables_mapping)
parameters = self.config_dict.get("parameters", [])
cartesian_product_parameters = testcase.gen_cartesian_product_parameters(
parameters, self.config_dict["path"]) or [{}]
for parameter_mapping in cartesian_product_parameters:
if parameter_mapping:
self.config_dict["variables"] = utils.override_variables_binds(
self.config_dict["variables"], parameter_mapping)
self.test_runner = runner.Runner(self.config_dict,
http_client_session)
testcases = testset.get("testcases", [])
self._add_tests_to_suite(testcases)
def test_override_variables_binds(self):
map_list = [
{"a": 1},
{"b": 2}
]
override_mapping = {"a": 3, "c": 4}
new_dict = utils.override_variables_binds(map_list, override_mapping)
self.assertEqual(3, new_dict["a"])
self.assertEqual(4, new_dict["c"])
map_list = OrderedDict(
{
"a": 1,
"b": 2
}
)
override_mapping = {"a": 3, "c": 4}
new_dict = utils.override_variables_binds(map_list, override_mapping)
self.assertEqual(3, new_dict["a"])
self.assertEqual(4, new_dict["c"])
map_list = "invalid"
override_mapping = {"a": 3, "c": 4}
with self.assertRaises(exception.ParamsError):
utils.override_variables_binds(map_list, override_mapping)
def test_override_variables_binds(self):
map_list = [
{"a": 1},
{"b": 2}
]
override_mapping = {"a": 3, "c": 4}
new_dict = utils.override_variables_binds(map_list, override_mapping)
self.assertEqual(3, new_dict["a"])
self.assertEqual(4, new_dict["c"])
map_list = OrderedDict(
{
"a": 1,
"b": 2
}
)
override_mapping = {"a": 3, "c": 4}
new_dict = utils.override_variables_binds(map_list, override_mapping)
self.assertEqual(3, new_dict["a"])
self.assertEqual(4, new_dict["c"])
map_list = "invalid"
override_mapping = {"a": 3, "c": 4}
with self.assertRaises(exception.ParamsError):
utils.override_variables_binds(map_list, override_mapping)
def __init__(self,
testset,
variables_mapping=None,
http_client_session=None):
super(TestSuite, self).__init__()
self.test_runner_list = []
config_dict = testset.get("config", {})
self.output_variables_list = config_dict.get("output", [])
self.testset_file_path = config_dict.get("path")
config_dict_parameters = config_dict.get("parameters", [])
config_dict_variables = config_dict.get("variables", [])
variables_mapping = variables_mapping or {}
config_dict_variables = utils.override_variables_binds(
config_dict_variables, variables_mapping)
#zhengchun 20180608
self.testcase_parser = testcase.TestcaseParser()
config_parametered_variables_list = self._get_parametered_variables(
config_dict_variables, config_dict_parameters,
self.testcase_parser)
testcases = testset.get("testcases", [])
for config_variables in config_parametered_variables_list:
# config level
config_dict["variables"] = config_variables
# update by zhengchun, can use myRunner, 20180608
test_runner = testset["runner"](config_dict, http_client_session)
for testcase_dict in testcases:
testcase_dict = copy.copy(testcase_dict)
# testcase level
testcase_parametered_variables_list = self._get_parametered_variables(
testcase_dict.get("variables", []),
testcase_dict.get("parameters", []),
test_runner.context.testcase_parser)
for testcase_variables in testcase_parametered_variables_list:
testcase_dict["variables"] = testcase_variables
# eval testcase name with bind variables
variables = utils.override_variables_binds(
config_variables, testcase_variables)
self.testcase_parser.update_binded_variables(variables)
try:
testcase_name = self.testcase_parser.eval_content_with_bindings(
testcase_dict["name"])
except (AssertionError, exception.ParamsError):
logger.log_warning(
"failed to eval testcase name: {}".format(
testcase_dict["name"]))
testcase_name = testcase_dict["name"]
self.test_runner_list.append((test_runner, variables))
self._add_test_to_suite(testcase_name, test_runner,
testcase_dict)
def __init__(self, testset, variables_mapping=None, http_client_session=None):
super(TestSuite, self).__init__()
self.test_runner_list = []
config_dict = testset.get("config", {})
self.output_variables_list = config_dict.get("output", [])
self.testset_file_path = config_dict.get("path")
config_dict_parameters = config_dict.get("parameters", [])
config_dict_variables = config_dict.get("variables", [])
variables_mapping = variables_mapping or {}
config_dict_variables = utils.override_variables_binds(config_dict_variables, variables_mapping)
config_parametered_variables_list = self._get_parametered_variables(
config_dict_variables,
config_dict_parameters
)
self.testcase_parser = testcase.TestcaseParser()
testcases = testset.get("testcases", [])
for config_variables in config_parametered_variables_list:
# config level
config_dict["variables"] = config_variables
test_runner = runner.Runner(config_dict, http_client_session)
for testcase_dict in testcases:
testcase_dict = copy.copy(testcase_dict)
# testcase level
testcase_parametered_variables_list = self._get_parametered_variables(
testcase_dict.get("variables", []),
testcase_dict.get("parameters", [])
)
for testcase_variables in testcase_parametered_variables_list:
testcase_dict["variables"] = testcase_variables
# eval testcase name with bind variables
variables = utils.override_variables_binds(
config_variables,
testcase_variables
)
self.testcase_parser.update_binded_variables(variables)
try:
testcase_name = self.testcase_parser.eval_content_with_bindings(testcase_dict["name"])
except (AssertionError, exception.ParamsError):
logger.log_warning("failed to eval testcase name: {}".format(testcase_dict["name"]))
testcase_name = testcase_dict["name"]
self.test_runner_list.append((test_runner, variables))
self._add_test_to_suite(testcase_name, test_runner, testcase_dict)
def __init__(self,
testset,
variables_mapping=None,
http_client_session=None):
super().__init__()
self.test_runner_list = []
config_dict = testset.get("config", {})
self.output_variables_list = config_dict.get("output", [])
self.testset_file_path = config_dict["path"]
config_dict_parameters = config_dict.get("parameters", [])
config_dict_variables = config_dict.get("variables", [])
variables_mapping = variables_mapping or {}
config_dict_variables = utils.override_variables_binds(
config_dict_variables, variables_mapping)
config_parametered_variables_list = self._get_parametered_variables(
config_dict_variables, config_dict_parameters)
self.testcase_parser = testcase.TestcaseParser()
testcases = testset.get("testcases", [])
for config_variables in config_parametered_variables_list:
# config level
config_dict["variables"] = config_variables
# test_runner = runner.Runner(config_dict, http_client_session)
test_runner = YDHrunner.YDHRunner(config_dict, http_client_session)
for testcase_dict in testcases:
testcase_dict = copy.copy(testcase_dict)
# testcase level
testcase_parametered_variables_list = self._get_parametered_variables(
testcase_dict.get("variables", []),
testcase_dict.get("parameters", []))
for testcase_variables in testcase_parametered_variables_list:
testcase_dict["variables"] = testcase_variables
# eval testcase name with bind variables
variables = utils.override_variables_binds(
config_variables, testcase_variables)
self.testcase_parser.update_binded_variables(variables)
testcase_name = self.testcase_parser.eval_content_with_bindings(
testcase_dict["name"])
self.test_runner_list.append((test_runner, variables))
self._add_test_to_suite(testcase_name, test_runner,
testcase_dict)
def _get_parametered_variables(self, variables, parameters):
""" parameterize varaibles with parameters
"""
cartesian_product_parameters = testcase.parse_parameters(
parameters, self.testset_file_path) or [{}]
parametered_variables_list = []
for parameter_mapping in cartesian_product_parameters:
parameter_mapping = parameter_mapping or {}
variables = utils.override_variables_binds(variables,
parameter_mapping)
parametered_variables_list.append(variables)
return parametered_variables_list
def _get_parametered_variables(self, variables, parameters):
""" parameterize varaibles with parameters
"""
cartesian_product_parameters = testcase.parse_parameters(
parameters,
self.testset_file_path
) or [{}]
parametered_variables_list = []
for parameter_mapping in cartesian_product_parameters:
parameter_mapping = parameter_mapping or {}
variables = utils.override_variables_binds(
variables,
parameter_mapping
)
parametered_variables_list.append(variables)
return parametered_variables_list
def _get_parametered_variables(self,
variables,
parameters,
parser_handle=None):
""" parameterize varaibles with parameters
"""
cartesian_product_parameters = testcase.parse_parameters(
parameters,
self.testset_file_path,
parser_handle #modified by zhengchun, 解析动态加载函数
) or [{}]
parametered_variables_list = []
for parameter_mapping in cartesian_product_parameters:
parameter_mapping = parameter_mapping or {}
variables = utils.override_variables_binds(variables,
parameter_mapping)
parametered_variables_list.append(variables)
return parametered_variables_list
def _run_testset(self, testset, variables_mapping=None):
""" run single testset, including one or several testcases.
@param
(dict) testset
{
"name": "testset description",
"config": {
"name": "testset description",
"requires": [],
"function_binds": {},
"variables": [],
"request": {}
},
"testcases": [
{
"name": "testcase description",
"variables": [], # optional, override
"request": {},
"extract": {}, # optional
"validate": {} # optional
},
testcase12
]
}
(dict) variables_mapping:
passed in variables mapping, it will override variables in config block
@return (dict) test result of testset
{
"success": True,
"output": {} # variables mapping
}
"""
success = True
config_dict = testset.get("config", {})
variables = config_dict.get("variables", [])
variables_mapping = variables_mapping or {}
config_dict["variables"] = utils.override_variables_binds(
variables, variables_mapping)
self.init_config(config_dict, level="testset")
testcases = testset.get("testcases", [])
for testcase_dict in testcases:
try:
self._run_test(testcase_dict)
except exception.MyBaseError as ex:
success = False
if self.request_failure_hook:
self.request_failure_hook.fire(
request_type=testcase_dict.get("request",
{}).get("method"),
name=testcase_dict.get("request", {}).get("url"),
response_time=0,
exception=ex)
else:
logging.exception(
"Exception occured in testcase: {}".format(
testcase_dict.get("name")))
break
output_variables_list = config_dict.get("output", [])
return {
"success": success,
"output": self.generate_output(output_variables_list)
}
def __init__(self,
testset,
variables_mapping=None,
http_client_session=None):
super(TestSuite, self).__init__()
self.test_runner_list = []
self.config = testset.get("config", {})
self.output_variables_list = self.config.get("output", [])
self.testset_file_path = self.config.get("path")
config_dict_parameters = self.config.get("parameters", [])
config_dict_variables = self.config.get("variables", [])
variables_mapping = variables_mapping or {}
config_dict_variables = utils.override_variables_binds(
config_dict_variables, variables_mapping)
config_parametered_variables_lists = self._get_parametered_variables(
config_dict_variables, config_dict_parameters)
config_parametered_variables_list = config_parametered_variables_lists[
0]
self.testcase_parser = testcase.TestcaseParser()
testcases = testset.get("testcases", [])
for config_variables in config_parametered_variables_list:
# config level
self.config["variables"] = config_variables
test_runner = runner.Runner(self.config, http_client_session)
for testcase_dict in testcases:
testcase_dict = copy.copy(testcase_dict)
# testcase level
testcase_parametered_variables_lists = self._get_parametered_variables(
testcase_dict.get("variables", []),
testcase_dict.get("parameters", []))
testcase_parametered_variables_list = testcase_parametered_variables_lists[
0]
assert_list = testcase_parametered_variables_lists[1]
testcase_index = 0
if 'validate' not in testcase_dict.keys():
testcase_dict['validate'] = []
basic_validate = testcase_dict['validate']
for testcase_variables in testcase_parametered_variables_list:
testcase_dict["variables"] = testcase_variables
asserts_str = assert_list[testcase_index]
asserts = ast.literal_eval(asserts_str)
# validates = []
checks = {}
if isinstance(asserts, list):
for validate in asserts:
# validate = {
# 'check': 'content.code',
# 'comparator': 'equals',
# 'expected': '100201'
# }
# validates.append(validate)
checks[validate.get('check')] = 1
elif isinstance(asserts, dict):
checks[asserts.get('check')] = 1
asserts = [asserts]
else:
asserts = []
validates = []
validates.extend(asserts)
# 去重，以parameters中的断言优先
for validate in basic_validate:
check = validate.get('check')
if check not in checks.keys():
validates.append(validate)
testcase_dict['validate'] = validates
# eval testcase name with bind variables
variables = utils.override_variables_binds(
config_variables, testcase_variables)
self.testcase_parser.update_binded_variables(variables)
try:
testcase_name = self.testcase_parser.eval_content_with_bindings(
testcase_dict["name"])
except (AssertionError, exceptions.ParamsError):
logger.log_warning(
"failed to eval testcase name: {}".format(
testcase_dict["name"]))
testcase_name = testcase_dict["name"]
self.test_runner_list.append((test_runner, variables))
self._add_test_to_suite(testcase_name, test_runner,
testcase_dict)
testcase_index = testcase_index + 1
def __init__(self, testset, variables_mapping=None, http_client_session=None):
super(TestSuite, self).__init__()
self.test_runner_list = []
config_dict = testset.get("config", {})
self.output_variables_list = config_dict.get("output", [])
self.testset_file_path = config_dict.get("path")
config_dict_parameters = config_dict.get("parameters", [])
config_dict_variables = config_dict.get("variables", [])
variables_mapping = variables_mapping or {}
config_dict_variables = utils.override_variables_binds(config_dict_variables, variables_mapping)
config_parametered_variables_list = self._get_parametered_variables(
config_dict_variables,
config_dict_parameters
)
self.testcase_parser = testcase.TestcaseParser()
testcases = testset.get("testcases", [])
count = 0
self.all_case_list = []
self.variables_list = []
for config_variables in config_parametered_variables_list:
# print("config_variables====>")
# print(config_variables)
# config level
config_dict["variables"] = config_variables
test_runner = runner.Runner(config_dict, http_client_session)
# TODO 存储已添加的用例
self.testcase_dict_list = []
for testcase_dict in testcases:
count = count + 1
testcase_dict = copy.copy(testcase_dict)
# testcase level
testcase_parametered_variables_list = self._get_parametered_variables(
testcase_dict.get("variables", []),
testcase_dict.get("parameters", [])
)
for testcase_variables in testcase_parametered_variables_list:
testcase_dict["variables"] = testcase_variables
# eval testcase name with bind variables
variables = utils.override_variables_binds(
config_variables,
testcase_variables
)
self.testcase_parser.update_binded_variables(variables)
try:
testcase_name = self.testcase_parser.eval_content_with_bindings(testcase_dict["name"])
except (AssertionError, exception.ParamsError):
logger.log_warning("failed to eval testcase name: {}".format(testcase_dict["name"]))
testcase_name = testcase_dict["name"]
# TODO 不重复添加用例
# print("testcase_dict")
# print(testcase_dict)
if config_variables not in self.variables_list:
self.variables_list.append(config_variables)
self.testcase_dict_list.append(testcase_dict)
self.all_case_list.append(testcase_dict)
self.test_runner_list.append((test_runner, variables))
self._add_test_to_suite(testcase_name, test_runner, testcase_dict)
if testcase_dict not in self.testcase_dict_list and testcase_dict not in self.all_case_list:
self.testcase_dict_list.append(testcase_dict)
self.all_case_list.append(testcase_dict)
self.test_runner_list.append((test_runner, variables))
self._add_test_to_suite(testcase_name, test_runner, testcase_dict)
print('for ----count ===',count)
def _run_testset(self, testset, variables_mapping=None):
""" run single testset, including one or several testcases.
@param
(dict) testset
{
"name": "testset description",
"config": {
"name": "testset description",
"requires": [],
"function_binds": {},
"variables": [],
"request": {}
},
"testcases": [
{
"name": "testcase description",
"variables": [], # optional, override
"request": {},
"extract": {}, # optional
"validate": {} # optional
},
testcase12
]
}
(dict) variables_mapping:
passed in variables mapping, it will override variables in config block
@return (dict) test result of testset
{
"success": True,
"output": {} # variables mapping
}
"""
success = True
config_dict = testset.get("config", {})
variables = config_dict.get("variables", [])
variables_mapping = variables_mapping or {}
config_dict["variables"] = utils.override_variables_binds(variables, variables_mapping)
self.init_config(config_dict, level="testset")
testcases = testset.get("testcases", [])
for testcase_dict in testcases:
try:
self._run_test(testcase_dict)
except exception.MyBaseError as ex:
success = False
if self.request_failure_hook:
self.request_failure_hook.fire(
request_type=testcase_dict.get("request", {}).get("method"),
name=testcase_dict.get("request", {}).get("url"),
response_time=0,
exception=ex
)
else:
logging.exception(
"Exception occured in testcase: {}".format(testcase_dict.get("name")))
break
output_variables_list = config_dict.get("output", [])
return {
"success": success,
"output": self.generate_output(output_variables_list)
}