def __getattr__(self, key):
if key in ["json", "content", "body"]:
try:
value = self.resp_obj.json()
except ValueError:
value = self.resp_obj.content
elif key == "cookies":
value = self.resp_obj.cookies.get_dict()
else:
try:
value = getattr(self.resp_obj, key)
except AttributeError:
err_msg = "ResponseObject does not have attribute: {}".format(
key)
logger.error(err_msg)
raise exceptions.ParamsError(err_msg)
self.__dict__[key] = value
return value
def extract_field(self, field,context=""):
""" extract value from requests.Response.
"""
if not isinstance(field, basestring):
err_msg = u"Invalid extractor! => {}\n".format(field)
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
msg = "extract: {}".format(field)
if text_extractor_regexp_compile.match(field):
value = self._extract_field_with_regex(field,context=context)
else:
value = self._extract_field_with_delimiter(field,context=context)
msg += "\t=> {}".format(value)
logger.log_debug(msg)
return value
def run(self, path_or_tests, dot_env_path=None, mapping=None):
""" main interface.
Args:
path_or_tests:
str: testcase/testsuite file/foler path
dict: valid testcase/testsuite data
Returns:
dict: result summary
"""
logger.log_info("HttpRunner version: {}".format(__version__))
if loader.is_testcase_path(path_or_tests):
return self.run_path(path_or_tests, dot_env_path, mapping)
elif loader.is_testcases(path_or_tests):
return self.run_tests(path_or_tests)
else:
raise exceptions.ParamsError("Invalid testcase path or testcases: {}".format(path_or_tests))
def do_validation(self, validator_dict):
""" validate with functions
"""
# TODO: move comparator uniform to init_test_suites
comparator = utils.get_uniform_comparator(validator_dict["comparator"])
validate_func = self.testcase_parser.get_bind_function(comparator)
if not validate_func:
raise exceptions.FunctionNotFound("comparator not found: {}".format(comparator))
check_item = validator_dict["check"]
check_value = validator_dict["check_value"]
expect_value = validator_dict["expect"]
if (check_value is None or expect_value is None) \
and comparator not in ["is", "eq", "equals", "=="]:
raise exceptions.ParamsError("Null value can only be compared with comparator: eq/equals/==")
validate_msg = "validate: {} {} {}({})".format(
check_item,
comparator,
expect_value,
type(expect_value).__name__
)
try:
validator_dict["check_result"] = "pass"
validate_func(check_value, expect_value)
validate_msg += "\t==> pass"
logger.log_debug(validate_msg)
except (AssertionError, TypeError):
validate_msg += "\t==> fail"
validate_msg += "\n{}({}) {} {}({})".format(
check_value,
type(check_value).__name__,
comparator,
expect_value,
type(expect_value).__name__
)
logger.log_error(validate_msg)
validator_dict["check_result"] = "fail"
raise exceptions.ValidationFailure(validate_msg)
def override_mapping_list(variables, new_mapping):
""" override variables with new mapping.
Args:
variables (list): variables list
[
{"var_a": 1},
{"var_b": "world"}
]
new_mapping (dict): overrided variables mapping
{
"var_a": "hello"
}
Returns:
OrderedDict: overrided variables mapping.
Examples:
>>> variables = [
{"var_a": 1},
{"var_b": "world"}
]
>>> new_mapping = {
"var_a": "hello"
}
>>> override_mapping_list(variables, new_mapping)
OrderedDict(
{
"var_a": "hello",
"var_b": "world"
}
)
"""
if isinstance(variables, list):
variables_ordered_dict = convert_mappinglist_to_orderdict(variables)
elif isinstance(variables, (OrderedDict, dict)):
variables_ordered_dict = variables
else:
raise exceptions.ParamsError("variables error!")
return update_ordered_dict(variables_ordered_dict, new_mapping)
def convert_testcase_path(testcase_path: Text) -> Tuple[Text, Text]:
"""convert single YAML/JSON testcase path to python file"""
if os.path.isdir(testcase_path):
# folder does not need to convert
return testcase_path, ""
raw_file_name, file_suffix = os.path.splitext(os.path.basename(testcase_path))
file_suffix = file_suffix.lower()
if file_suffix not in [".json", ".yml", ".yaml"]:
raise exceptions.ParamsError("")
file_name = raw_file_name.replace(" ", "_").replace(".", "_").replace("-", "_")
testcase_dir = os.path.dirname(testcase_path)
testcase_python_path = os.path.join(testcase_dir, f"{file_name}_test.py")
# convert title case, e.g. request_with_variables => RequestWithVariables
name_in_title_case = file_name.title().replace("_", "")
return testcase_python_path, name_in_title_case
def _get_block_by_name(ref_call, ref_type, project_mapping):
""" get test content by reference name.
Args:
ref_call (str): call function.
e.g. api_v1_Account_Login_POST($UserName, $Password)
ref_type (enum): "def-api" or "def-testcase"
project_mapping (dict): project_mapping
Returns:
dict: api/testcase definition.
Raises:
exceptions.ParamsError: call args number is not equal to defined args number.
"""
function_meta = parser.parse_function(ref_call)
func_name = function_meta["func_name"]
call_args = function_meta["args"]
block = _get_test_definition(func_name, ref_type, project_mapping)
def_args = block.get("function_meta", {}).get("args", [])
if len(call_args) != len(def_args):
err_msg = "{}: call args number is not equal to defined args number!\n".format(
func_name)
err_msg += "defined args: {}\n".format(def_args)
err_msg += "reference args: {}".format(call_args)
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
args_mapping = {}
for index, item in enumerate(def_args):
if call_args[index] == item:
continue
args_mapping[item] = call_args[index]
if args_mapping:
block = parser.substitute_variables(block, args_mapping)
return block
def main_run(extra_args):
tests_path_list = []
for index, item in enumerate(extra_args):
if not os.path.exists(item):
# item is not file/folder path
continue
elif os.path.isfile(item):
# replace YAML/JSON file path with generated python file
extra_args[index] = convert_testcase_path(item)
tests_path_list.append(item)
if len(tests_path_list) == 0:
# has not specified any testcase path
raise exceptions.ParamsError("Missed testcase path")
main_make(tests_path_list)
if "-s" not in extra_args:
extra_args.insert(0, "-s")
pytest.main(extra_args)
def __ensure_absolute(path: Text) -> Text:
if path.startswith("./"):
# Linux/Darwin, hrun ./test.yml
path = path[len("./"):]
elif path.startswith(".\\"):
# Windows, hrun .\\test.yml
path = path[len(".\\"):]
path = ensure_path_sep(path)
project_meta = load_project_meta(path)
if os.path.isabs(path):
absolute_path = path
else:
absolute_path = os.path.join(project_meta.RootDir, path)
if not os.path.isfile(absolute_path):
raise exceptions.ParamsError(
f"Invalid testcase file path: {absolute_path}")
return absolute_path
def run(self, path_or_tests, dot_env_path=None, mapping=None):
""" main interface.
Args:
path_or_tests:
str: testcase/testsuite file/foler path
dict: valid testcase/testsuite data
dot_env_path (str): specified .env file path.
mapping (dict): if mapping is specified, it will override variables in config block.
Returns:
dict: result summary
"""
logger.log_info("HttpRunner version: {}".format(__version__))
if loader.is_test_path(path_or_tests):
return self.run_path(path_or_tests, dot_env_path, mapping)
elif loader.is_test_content(path_or_tests):
return self.run_tests(path_or_tests)
else:
raise exceptions.ParamsError("Invalid testcase path or testcases: {}".format(path_or_tests))
def extract_field(self, field):
""" extract value from requests.Response.
"""
if not isinstance(field, str):
err_msg = f"Invalid extractor! => {field}\n"
logger.error(err_msg)
raise exceptions.ParamsError(err_msg)
msg = f"extract: {field}"
if field.startswith("$"):
value = self._extract_field_with_jsonpath(field)
elif text_extractor_regexp_compile.match(field):
value = self._extract_field_with_regex(field)
else:
value = self._extract_field_with_delimiter(field)
msg += f"\t=> {value}"
logger.debug(msg)
return value
def load_test_dependencies():
""" load all api and suite definitions.
default api folder is "$CWD/tests/api/".
default suite folder is "$CWD/tests/suite/".
"""
# TODO: cache api and suite loading
# load api definitions
api_def_folder = os.path.join(os.getcwd(), "tests", "api")
for test_file in load_folder_files(api_def_folder):
load_api_file(test_file)
# load suite definitions
suite_def_folder = os.path.join(os.getcwd(), "tests", "suite")
for suite_file in load_folder_files(suite_def_folder):
suite = load_test_file(suite_file)
if "def" not in suite["config"]:
raise exceptions.ParamsError("def missed in suite file: {}!".format(suite_file))
call_func = suite["config"]["def"]
function_meta = parser.parse_function(call_func)
suite["function_meta"] = function_meta
overall_def_dict["suite"][function_meta["func_name"]] = suite
def __run_step_testcase(self, step: TStep) -> StepData:
"""run teststep: referenced testcase"""
step_data = StepData(name=step.name)
step_variables = step.variables
step_export = step.export
if hasattr(step.testcase, "config") and hasattr(
step.testcase, "teststeps"):
testcase_cls = step.testcase
case_result = (testcase_cls().with_session(
self.__session).with_case_id(self.__case_id).with_variables(
step_variables).with_export(step_export).run())
elif isinstance(step.testcase, Text):
if os.path.isabs(step.testcase):
ref_testcase_path = step.testcase
else:
ref_testcase_path = os.path.join(self.__project_meta.RootDir,
step.testcase)
case_result = (HttpRunner().with_session(
self.__session).with_case_id(
self.__case_id).with_variables(step_variables).with_export(
step_export).run_path(ref_testcase_path))
else:
raise exceptions.ParamsError(
f"Invalid teststep referenced testcase: {step.dict()}")
step_data.data = case_result.get_step_datas() # list of step data
step_data.export_vars = case_result.get_export_variables()
step_data.success = case_result.success
self.success = case_result.success
if step_data.export_vars:
logger.info(f"export variables: {step_data.export_vars}")
return step_data
def run(self, path_or_tests, dot_env_path=None, mapping=None):
""" main interface.
Args:
path_or_tests:
str: testcase/testsuite file/foler path
dict: valid testcase/testsuite data
dot_env_path (str): specified .env file path.
mapping (dict): if mapping is specified, it will override variables in config block.
Returns:
dict: result summary
"""
logger.info(f"HttpRunner version: {__version__}")
if loader.is_test_path(path_or_tests):
return self.run_path(path_or_tests, dot_env_path, mapping)
elif loader.is_test_content(path_or_tests):
project_working_directory = path_or_tests.get("project_mapping", {}).get("PWD", os.getcwd())
loader.init_pwd(project_working_directory)
return self.run_tests(path_or_tests)
else:
raise exceptions.ParamsError(f"Invalid testcase path or testcases: {path_or_tests}")
def extract_field(self, field):
""" extract value from requests.Response.
"""
if not isinstance(field, basestring):
err_msg = u"Invalid extractor! => {}\n".format(field)
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
msg = "extract: {}".format(field)
if field.startswith("$"):
value = self._extract_field_with_jsonpath(field)
elif text_extractor_regexp_compile.match(field):
value = self._extract_field_with_regex(field)
else:
value = self._extract_field_with_delimiter(field)
if is_py2 and isinstance(value, unicode):
value = value.encode("utf-8")
msg += "\t=> {}".format(value)
logger.log_debug(msg)
return value
def _extract_field_with_delimiter(self, field):
""" response content could be json or html text.
Args:
field (str): string joined by delimiter.
e.g.
"status_code"
"headers"
"cookies"
"content"
"headers.content-type"
"content.person.name.first_name"
"""
# string.split(sep=None, maxsplit=1) -> list of strings
# e.g. "content.person.name" => ["content", "person.name"]
try:
top_query, sub_query = field.split('.', 1)
except ValueError:
top_query = field
sub_query = None
# status_code
if top_query in ["status_code", "encoding", "ok", "reason", "url"]:
if sub_query:
# status_code.XX
err_msg = u"Failed to extract: {}\n".format(field)
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
return getattr(self, top_query)
# cookies
elif top_query == "cookies":
cookies = self.cookies
if not sub_query:
# extract cookies
return cookies
try:
return cookies[sub_query]
except KeyError:
err_msg = u"Failed to extract cookie! => {}\n".format(field)
err_msg += u"response cookies: {}\n".format(cookies)
logger.log_error(err_msg)
raise exceptions.ExtractFailure(err_msg)
# elapsed
elif top_query == "elapsed":
available_attributes = u"available attributes: days, seconds, microseconds, total_seconds"
if not sub_query:
err_msg = u"elapsed is datetime.timedelta instance, attribute should also be specified!\n"
err_msg += available_attributes
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
elif sub_query in ["days", "seconds", "microseconds"]:
return getattr(self.elapsed, sub_query)
elif sub_query == "total_seconds":
return self.elapsed.total_seconds()
else:
err_msg = "{} is not valid datetime.timedelta attribute.\n".format(
sub_query)
err_msg += available_attributes
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
# req_headers
elif top_query == "req_headers":
headers = self.resp_obj.request.headers
if not sub_query:
# extract headers
return headers
try:
return headers[sub_query]
except KeyError:
err_msg = u"Failed to extract req_header! => {}\n".format(
field)
err_msg += u"response headers: {}\n".format(headers)
logger.log_error(err_msg)
raise exceptions.ExtractFailure(err_msg)
# headers
elif top_query == "headers":
headers = self.headers
if not sub_query:
# extract headers
return headers
try:
return headers[sub_query]
except KeyError:
err_msg = u"Failed to extract header! => {}\n".format(field)
err_msg += u"response headers: {}\n".format(headers)
logger.log_error(err_msg)
raise exceptions.ExtractFailure(err_msg)
# response body
elif top_query in ["body", "content", "text", "json"]:
try:
body = self.json
except exceptions.JSONDecodeError:
body = self.text
if not sub_query:
# extract response body
return self.content
# return body
if isinstance(body, (dict, list)):
# content = {"xxx": 123}, content.xxx
return utils.query_json(body, sub_query)
elif sub_query.isdigit():
# content = "abcdefg", content.3 => d
return utils.query_json(body, sub_query)
else:
# content = "<html>abcdefg</html>", content.xxx
err_msg = u"Failed to extract attribute from response body! => {}\n".format(
field)
err_msg += u"response body: {}\n".format(body)
logger.log_error(err_msg)
raise exceptions.ExtractFailure(err_msg)
# new set response attributes in teardown_hooks
elif top_query in self.__dict__:
attributes = self.__dict__[top_query]
if not sub_query:
# extract response attributes
return attributes
if isinstance(attributes, (dict, list)):
# attributes = {"xxx": 123}, content.xxx
return utils.query_json(attributes, sub_query)
elif sub_query.isdigit():
# attributes = "abcdefg", attributes.3 => d
return utils.query_json(attributes, sub_query)
else:
# content = "attributes.new_attribute_not_exist"
err_msg = u"Failed to extract cumstom set attribute from teardown hooks! => {}\n".format(
field)
err_msg += u"response set attributes: {}\n".format(attributes)
logger.log_error(err_msg)
raise exceptions.TeardownHooksFailure(err_msg)
# others
else:
err_msg = u"Failed to extract attribute from response! => {}\n".format(
field)
err_msg += u"available response attributes: status_code, cookies, elapsed, headers, content, " \
u"text, json, encoding, ok, reason, url.\n\n"
err_msg += u"If you want to set attribute in teardown_hooks, take the following example as reference:\n"
err_msg += u"response.new_attribute = 'new_attribute_value'\n"
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
def parse_parameters(parameters: Dict, ) -> List[Dict]:
""" parse parameters and generate cartesian product.
Args:
parameters (Dict) parameters: parameter name and value mapping
parameter value may be in three types:
(1) data list, e.g. ["iOS/10.1", "iOS/10.2", "iOS/10.3"]
(2) call built-in parameterize function, "${parameterize(account.csv)}"
(3) call custom function in debugtalk.py, "${gen_app_version()}"
Returns:
list: cartesian product list
Examples:
>>> parameters = {
"user_agent": ["iOS/10.1", "iOS/10.2", "iOS/10.3"],
"username-password": "******",
"app_version": "${gen_app_version()}",
}
>>> parse_parameters(parameters)
"""
parsed_parameters_list: List[List[Dict]] = []
# load project_meta functions
project_meta = loader.load_project_meta(os.getcwd())
functions_mapping = project_meta.functions
for parameter_name, parameter_content in parameters.items():
parameter_name_list = parameter_name.split("-")
if isinstance(parameter_content, List):
# (1) data list
# e.g. {"app_version": ["2.8.5", "2.8.6"]}
# => [{"app_version": "2.8.5", "app_version": "2.8.6"}]
# e.g. {"username-password": [["user1", "111111"], ["test2", "222222"]}
# => [{"username": "******", "password": "******"}, {"username": "******", "password": "******"}]
parameter_content_list: List[Dict] = []
for parameter_item in parameter_content:
if not isinstance(parameter_item, (list, tuple)):
# "2.8.5" => ["2.8.5"]
parameter_item = [parameter_item]
# ["app_version"], ["2.8.5"] => {"app_version": "2.8.5"}
# ["username", "password"], ["user1", "111111"] => {"username": "******", "password": "******"}
parameter_content_dict = dict(zip(parameter_name_list, parameter_item))
parameter_content_list.append(parameter_content_dict)
elif isinstance(parameter_content, Text):
# (2) & (3)
parsed_parameter_content: List = parse_data(
parameter_content, {}, functions_mapping
)
if not isinstance(parsed_parameter_content, List):
raise exceptions.ParamsError(
f"parameters content should be in List type, got {parsed_parameter_content} for {parameter_content}"
)
parameter_content_list: List[Dict] = []
for parameter_item in parsed_parameter_content:
if isinstance(parameter_item, Dict):
# get subset by parameter name
# {"app_version": "${gen_app_version()}"}
# gen_app_version() => [{'app_version': '2.8.5'}, {'app_version': '2.8.6'}]
# {"username-password": "******"}
# get_account() => [
# {"username": "******", "password": "******"},
# {"username": "******", "password": "******"}
# ]
parameter_dict: Dict = {
key: parameter_item[key] for key in parameter_name_list
}
elif isinstance(parameter_item, (List, tuple)):
if len(parameter_name_list) == len(parameter_item):
# {"username-password": "******"}
# get_account() => [("user1", "111111"), ("user2", "222222")]
parameter_dict = dict(zip(parameter_name_list, parameter_item))
else:
raise exceptions.ParamsError(
f"parameter names length are not equal to value length.\n"
f"parameter names: {parameter_name_list}\n"
f"parameter values: {parameter_item}"
)
elif len(parameter_name_list) == 1:
# {"user_agent": "${get_user_agent()}"}
# get_user_agent() => ["iOS/10.1", "iOS/10.2"]
# parameter_dict will get: {"user_agent": "iOS/10.1", "user_agent": "iOS/10.2"}
parameter_dict = {parameter_name_list[0]: parameter_item}
else:
raise exceptions.ParamsError(
f"Invalid parameter names and values:\n"
f"parameter names: {parameter_name_list}\n"
f"parameter values: {parameter_item}"
)
parameter_content_list.append(parameter_dict)
else:
raise exceptions.ParamsError(
f"parameter content should be List or Text(variables or functions call), got {parameter_content}"
)
parsed_parameters_list.append(parameter_content_list)
return utils.gen_cartesian_product(*parsed_parameters_list)
def _add_tests(self, testcases):
""" initialize testcase with Runner() and add to test suite.
Args:
testcases (list): testcases list.
Returns:
unittest.TestSuite()
"""
def _add_test(test_runner, test_dict):
""" add test to testcase.
"""
def test(self):
try:
test_runner.run_test(test_dict)
except exceptions.MyBaseFailure as ex:
self.fail(str(ex))
finally:
self.meta_datas = test_runner.meta_datas
if "config" in test_dict:
# run nested testcase
test.__doc__ = test_dict["config"].get("name")
variables = test_dict["config"].get("variables", {})
else:
# run api test
test.__doc__ = test_dict.get("name")
variables = test_dict.get("variables", {})
if isinstance(test.__doc__, parser.LazyString):
try:
parsed_variables = parser.parse_variables_mapping(
variables)
test.__doc__ = parser.parse_lazy_data(
test.__doc__, parsed_variables)
except exceptions.VariableNotFound:
test.__doc__ = str(test.__doc__)
return test
test_suite = unittest.TestSuite()
for testcase in testcases:
config = testcase.get("config", {})
test_runner = runner.Runner(config)
TestSequense = type('TestSequense', (unittest.TestCase, ), {})
tests = testcase.get("teststeps", [])
for index, test_dict in enumerate(tests):
times = test_dict.get("times", 1)
try:
times = int(times)
except ValueError:
raise exceptions.ParamsError(
"times should be digit, given: {}".format(times))
for times_index in range(times):
# suppose one testcase should not have more than 9999 steps,
# and one step should not run more than 999 times.
test_method_name = 'test_{:04}_{:03}'.format(
index, times_index)
test_method = _add_test(test_runner, test_dict)
setattr(TestSequense, test_method_name, test_method)
loaded_testcase = self.test_loader.loadTestsFromTestCase(
TestSequense)
setattr(loaded_testcase, "config", config)
setattr(loaded_testcase, "teststeps", tests)
setattr(loaded_testcase, "runner", test_runner)
test_suite.addTest(loaded_testcase)
return test_suite
def parse_parameters(parameters,
variables_mapping=None,
functions_mapping=None):
""" parse parameters and generate cartesian product.
Args:
parameters (list) parameters: parameter name and value in list
parameter value may be in three types:
(1) data list, e.g. ["iOS/10.1", "iOS/10.2", "iOS/10.3"]
(2) call built-in parameterize function, "${parameterize(account.csv)}"
(3) call custom function in debugtalk.py, "${gen_app_version()}"
variables_mapping (dict): variables mapping loaded from testcase config
functions_mapping (dict): functions mapping loaded from debugtalk.py
Returns:
list: cartesian product list
Examples:
>>> parameters = [
{"user_agent": ["iOS/10.1", "iOS/10.2", "iOS/10.3"]},
{"username-password": "******"},
{"app_version": "${gen_app_version()}"}
]
>>> parse_parameters(parameters)
"""
variables_mapping = variables_mapping or {}
functions_mapping = functions_mapping or {}
parsed_parameters_list = []
parameters = utils.ensure_mapping_format(parameters)
for parameter_name, parameter_content in parameters.items():
parameter_name_list = parameter_name.split("-")
if isinstance(parameter_content, list):
# (1) data list
# e.g. {"app_version": ["2.8.5", "2.8.6"]}
# => [{"app_version": "2.8.5", "app_version": "2.8.6"}]
# e.g. {"username-password": [["user1", "111111"], ["test2", "222222"]}
# => [{"username": "******", "password": "******"}, {"username": "******", "password": "******"}]
parameter_content_list = []
for parameter_item in parameter_content:
if not isinstance(parameter_item, (list, tuple)):
# "2.8.5" => ["2.8.5"]
parameter_item = [parameter_item]
# ["app_version"], ["2.8.5"] => {"app_version": "2.8.5"}
# ["username", "password"], ["user1", "111111"] => {"username": "******", "password": "******"}
parameter_content_dict = dict(
zip(parameter_name_list, parameter_item))
parameter_content_list.append(parameter_content_dict)
else:
# (2) & (3)
parsed_variables_mapping = parse_variables_mapping(
variables_mapping)
parsed_parameter_content = eval_lazy_data(
parameter_content, parsed_variables_mapping, functions_mapping)
if not isinstance(parsed_parameter_content, list):
raise exceptions.ParamsError("parameters syntax error!")
parameter_content_list = []
for parameter_item in parsed_parameter_content:
if isinstance(parameter_item, dict):
# get subset by parameter name
# {"app_version": "${gen_app_version()}"}
# gen_app_version() => [{'app_version': '2.8.5'}, {'app_version': '2.8.6'}]
# {"username-password": "******"}
# get_account() => [
# {"username": "******", "password": "******"},
# {"username": "******", "password": "******"}
# ]
parameter_dict = {
key: parameter_item[key]
for key in parameter_name_list
}
elif isinstance(parameter_item, (list, tuple)):
# {"username-password": "******"}
# get_account() => [("user1", "111111"), ("user2", "222222")]
parameter_dict = dict(
zip(parameter_name_list, parameter_item))
elif len(parameter_name_list) == 1:
# {"user_agent": "${get_user_agent()}"}
# get_user_agent() => ["iOS/10.1", "iOS/10.2"]
parameter_dict = {parameter_name_list[0]: parameter_item}
parameter_content_list.append(parameter_dict)
parsed_parameters_list.append(parameter_content_list)
return utils.gen_cartesian_product(*parsed_parameters_list)
def _run_dubbo_test(self, test_dict):
# clear meta data first to ensure independence for each test
self.__clear_test_data()
# check skip
self._handle_skip_feature(test_dict)
# prepare
test_dict = utils.lower_test_dict_keys(test_dict)
test_variables = test_dict.get("variables", {})
self.session_context.init_test_variables(test_variables)
# parse test request
raw_request = test_dict.get('request', {})
parsed_test_request = self.session_context.eval_content(raw_request)
self.session_context.update_test_variables("request",
parsed_test_request)
# prepend url with base_url unless it's already an absolute URL
host = parsed_test_request.pop('host', '')
port = parsed_test_request.pop('port', '')
# setup hooks
setup_hooks = test_dict.get("setup_hooks", [])
if setup_hooks:
self.do_hook_actions(setup_hooks, HookTypeEnum.SETUP)
try:
service = parsed_test_request.pop('service')
method = parsed_test_request.pop('method')
params = parsed_test_request.pop("params", "")
parsed_test_request.setdefault("verify", self.verify)
group_name = parsed_test_request.pop("group", None)
except KeyError:
raise exceptions.ParamsError("SERVICE or METHOD missed!")
logger.log_info("{host}:{port} {service}.{method}".format(
host=host, port=port, service=service, method=method))
logger.log_debug(
"request kwargs(raw): {kwargs}".format(kwargs=parsed_test_request))
# request
resp = self.dubbo_client_session.request(
host,
port,
service,
method,
params,
name=(group_name or self.session_context.eval_content(
test_dict.get("name", ""))),
**parsed_test_request)
# TODO: responseObject
resp_obj = DubboResponseObject(resp)
def log_req_resp_details():
err_msg = "{} DETAILED REQUEST & RESPONSE {}\n".format(
"*" * 32, "*" * 32)
# log request
err_msg += "====== request details ======\n"
err_msg += "host: {}\n".format(host)
err_msg += "port: {}\n".format(port)
err_msg += "service: {}\n".format(service)
err_msg += "method: {}\n".format(method)
err_msg += "params: {}\n".format(params)
for k, v in parsed_test_request.items():
v = utils.omit_long_data(v)
err_msg += "{}: {}\n".format(k, repr(v))
err_msg += "\n"
# log response
err_msg += "====== response details ======\n"
err_msg += "body: {}\n".format(repr(resp_obj.json))
logger.log_error(err_msg)
# teardown hooks
teardown_hooks = test_dict.get("teardown_hooks", [])
if teardown_hooks:
self.session_context.update_test_variables("response", resp_obj)
self.do_hook_actions(teardown_hooks, HookTypeEnum.TEARDOWN)
self.dubbo_client_session.update_last_req_resp_record(resp_obj)
# extract
extractors = test_dict.get("extract", {})
try:
extracted_variables_mapping = resp_obj.extract_response(extractors)
self.session_context.update_session_variables(
extracted_variables_mapping)
except (exceptions.ParamsError, exceptions.ExtractFailure):
log_req_resp_details()
raise
# validate
validators = test_dict.get("validate") or test_dict.get(
"validators") or []
validate_script = test_dict.get("validate_script", [])
if validate_script:
validators.append({
"type": "python_script",
"script": validate_script
})
validator = Validator(self.session_context, resp_obj)
try:
validator.validate(validators)
except exceptions.ValidationFailure:
log_req_resp_details()
raise
return validator.validation_results
def _run_test(self, test_dict):
""" run single teststep.
Args:
test_dict (dict): teststep info
{
"name": "teststep description",
"skip": "skip this test unconditionally",
"times": 3,
"variables": [], # optional, override
"request": {
"url": "http://127.0.0.1:5000/api/users/1000",
"method": "POST",
"headers": {
"Content-Type": "application/json",
"authorization": "$authorization",
"random": "$random"
},
"json": {"name": "user", "password": "******"}
},
"extract": {}, # optional
"validate": [], # optional
"setup_hooks": [], # optional
"teardown_hooks": [] # optional
}
Raises:
exceptions.ParamsError
exceptions.ValidationFailure
exceptions.ExtractFailure
"""
# clear meta data first to ensure independence for each test
self.__clear_test_data()
# check skip
self._handle_skip_feature(test_dict)
# prepare
test_dict = utils.lower_test_dict_keys(test_dict)
test_variables = test_dict.get("variables", {})
self.session_context.init_test_variables(test_variables)
# teststep name
test_name = self.session_context.eval_content(test_dict.get(
"name", ""))
# parse test request
raw_request = test_dict.get('request', {})
parsed_test_request = self.session_context.eval_content(raw_request)
self.session_context.update_test_variables("request",
parsed_test_request)
# setup hooks
setup_hooks = test_dict.get("setup_hooks", [])
if setup_hooks:
self.do_hook_actions(setup_hooks, HookTypeEnum.SETUP)
# prepend url with base_url unless it's already an absolute URL
url = parsed_test_request.pop('url')
base_url = self.session_context.eval_content(
test_dict.get("base_url", ""))
parsed_url = utils.build_url(base_url, url)
request_headers = parsed_test_request.setdefault("headers", {})
if "HRUN-Request-ID" not in request_headers:
parsed_test_request["headers"]["HRUN-Request-ID"] = \
self.session_context.session_variables_mapping["HRUN-Request-ID"]
try:
method = parsed_test_request.pop('method')
parsed_test_request.setdefault("verify", self.verify)
group_name = parsed_test_request.pop("group", None)
except KeyError:
raise exceptions.ParamsError("URL or METHOD missed!")
logger.info(f"{method} {parsed_url}")
logger.debug(f"request kwargs(raw): {parsed_test_request}")
# request
resp = self.http_client_session.request(method,
parsed_url,
name=(group_name or test_name),
**parsed_test_request)
resp_obj = response.ResponseObject(resp)
def log_req_resp_details():
err_msg = "{} DETAILED REQUEST & RESPONSE {}\n".format(
"*" * 32, "*" * 32)
# log request
err_msg += "====== request details ======\n"
err_msg += f"url: {parsed_url}\n"
err_msg += f"method: {method}\n"
headers = parsed_test_request.pop("headers", {})
err_msg += f"headers: {headers}\n"
for k, v in parsed_test_request.items():
v = utils.omit_long_data(v)
err_msg += f"{k}: {repr(v)}\n"
err_msg += "\n"
# log response
err_msg += "====== response details ======\n"
err_msg += f"status_code: {resp_obj.status_code}\n"
err_msg += f"headers: {resp_obj.headers}\n"
err_msg += f"body: {repr(resp_obj.text)}\n"
logger.error(err_msg)
# teardown hooks
teardown_hooks = test_dict.get("teardown_hooks", [])
if teardown_hooks:
self.session_context.update_test_variables("response", resp_obj)
self.do_hook_actions(teardown_hooks, HookTypeEnum.TEARDOWN)
self.http_client_session.update_last_req_resp_record(resp_obj)
# extract
extractors = test_dict.get("extract", {})
try:
extracted_variables_mapping = resp_obj.extract_response(extractors)
self.session_context.update_session_variables(
extracted_variables_mapping)
except (exceptions.ParamsError, exceptions.ExtractFailure):
log_req_resp_details()
raise
# validate
validators = test_dict.get("validate") or test_dict.get(
"validators") or []
validate_script = test_dict.get("validate_script", [])
if validate_script:
validators.append({
"type": "python_script",
"script": validate_script
})
validator = Validator(self.session_context, resp_obj)
try:
validator.validate(validators)
except exceptions.ValidationFailure:
log_req_resp_details()
raise
finally:
self.validation_results = validator.validation_results
def _extract_field_with_delimiter(self, field):
""" response content could be json or html text.
@param (str) field should be string joined by delimiter.
e.g.
"status_code"
"headers"
"cookies"
"content"
"headers.content-type"
"content.person.name.first_name"
"""
# string.split(sep=None, maxsplit=-1) -> list of strings
# e.g. "content.person.name" => ["content", "person.name"]
try:
top_query, sub_query = field.split('.', 1)
except ValueError:
top_query = field
sub_query = None
# status_code
if top_query in ["status_code", "encoding", "ok", "reason", "url"]:
if sub_query:
# status_code.XX
err_msg = u"Failed to extract: {}\n".format(field)
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
return getattr(self, top_query)
# cookies
elif top_query == "cookies":
cookies = self.cookies.get_dict()
if not sub_query:
# extract cookies
return cookies
try:
return cookies[sub_query]
except KeyError:
err_msg = u"Failed to extract cookie! => {}\n".format(field)
err_msg += u"response cookies: {}\n".format(cookies)
logger.log_error(err_msg)
raise exceptions.ExtractFailure(err_msg)
# elapsed
elif top_query == "elapsed":
available_attributes = u"available attributes: days, seconds, microseconds, total_seconds"
if not sub_query:
err_msg = u"elapsed is datetime.timedelta instance, attribute should also be specified!\n"
err_msg += available_attributes
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
elif sub_query in ["days", "seconds", "microseconds"]:
return getattr(self.elapsed, sub_query)
elif sub_query == "total_seconds":
return self.elapsed.total_seconds()
else:
err_msg = "{} is not valid datetime.timedelta attribute.\n".format(
sub_query)
err_msg += available_attributes
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
# headers
elif top_query == "headers":
headers = self.headers
if not sub_query:
# extract headers
return headers
try:
return headers[sub_query]
except KeyError:
err_msg = u"Failed to extract header! => {}\n".format(field)
err_msg += u"response headers: {}\n".format(headers)
logger.log_error(err_msg)
raise exceptions.ExtractFailure(err_msg)
# response body
elif top_query in ["content", "text", "json"]:
try:
body = self.json
except exceptions.JSONDecodeError:
body = self.text
if not sub_query:
# extract response body
return body
if isinstance(body, (dict, list)):
# content = {"xxx": 123}, content.xxx
return utils.query_json(body, sub_query)
elif sub_query.isdigit():
# content = "abcdefg", content.3 => d
return utils.query_json(body, sub_query)
else:
# content = "<html>abcdefg</html>", content.xxx
err_msg = u"Failed to extract attribute from response body! => {}\n".format(
field)
err_msg += u"response body: {}\n".format(body)
logger.log_error(err_msg)
raise exceptions.ExtractFailure(err_msg)
# response jsonpath
elif top_query == 'jsonpath':
try:
body = self.json
except exceptions.JSONDecodeError:
body = self.text
if not sub_query:
# extract response body
return body
re = jsonpath.jsonpath(body, expr=sub_query)
if re:
# return first value of result(list)
return re[0]
else:
return 'null'
# others
else:
err_msg = u"Failed to extract attribute from response! => {}\n".format(
field)
err_msg += u"available response attributes: status_code, cookies, elapsed, headers, content, text, json, encoding, ok, reason, url."
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
def parse_parameters(parameters, testset_path=None):
""" parse parameters and generate cartesian product
@params
(list) parameters: parameter name and value in list
parameter value may be in three types:
(1) data list
(2) call built-in parameterize function
(3) call custom function in debugtalk.py
e.g.
[
{"user_agent": ["iOS/10.1", "iOS/10.2", "iOS/10.3"]},
{"username-password": "******"},
{"app_version": "${gen_app_version()}"}
]
(str) testset_path: testset file path, used for locating csv file and debugtalk.py
@return cartesian product in list
"""
testcase_parser = TestcaseParser(file_path=testset_path)
parsed_parameters_list = []
for parameter in parameters:
parameter_name, parameter_content = list(parameter.items())[0]
parameter_name_list = parameter_name.split("-")
if isinstance(parameter_content, list):
# (1) data list
# e.g. {"app_version": ["2.8.5", "2.8.6"]}
# => [{"app_version": "2.8.5", "app_version": "2.8.6"}]
# e.g. {"username-password": [["user1", "111111"], ["test2", "222222"]}
# => [{"username": "******", "password": "******"}, {"username": "******", "password": "******"}]
parameter_content_list = []
for parameter_item in parameter_content:
if not isinstance(parameter_item, (list, tuple)):
# "2.8.5" => ["2.8.5"]
parameter_item = [parameter_item]
# ["app_version"], ["2.8.5"] => {"app_version": "2.8.5"}
# ["username", "password"], ["user1", "111111"] => {"username": "******", "password": "******"}
parameter_content_dict = dict(
zip(parameter_name_list, parameter_item))
parameter_content_list.append(parameter_content_dict)
else:
# (2) & (3)
parsed_parameter_content = testcase_parser.eval_content_with_bindings(
parameter_content)
# e.g. [{'app_version': '2.8.5'}, {'app_version': '2.8.6'}]
# e.g. [{"username": "******", "password": "******"}, {"username": "******", "password": "******"}]
if not isinstance(parsed_parameter_content, list):
raise exceptions.ParamsError("parameters syntax error!")
parameter_content_list = [
# get subset by parameter name
{key: parameter_item[key]
for key in parameter_name_list}
for parameter_item in parsed_parameter_content
]
parsed_parameters_list.append(parameter_content_list)
return gen_cartesian_product(*parsed_parameters_list)
def _run_test(self, test_dict):
""" run single teststep.
Args:
test_dict (dict): teststep info
{
"name": "teststep description",
"skip": "skip this test unconditionally",
"times": 3,
"variables": [], # optional, override
"request": {
"url": "http://127.0.0.1:5000/api/users/1000",
"method": "POST",
"headers": {
"Content-Type": "application/json",
"authorization": "$authorization",
"random": "$random"
},
"json": {"name": "user", "password": "******"}
},
"extract": {}, # optional
"validate": [], # optional
"setup_hooks": [], # optional
"teardown_hooks": [], # optional
"wait":{} # optional
}
Raises:
exceptions.ParamsError
exceptions.ValidationFailure
exceptions.ExtractFailure
"""
# get wait parameter, if this step result needs to waif for a right one
wait_params = test_dict.get("wait", {})
validate_pass = None
failures = []
if wait_params:
wait_total_time = wait_params["Total_Time"] if wait_params.has_key(
'Total_Time') else 300
wait_interval = wait_params["Interval"] if wait_params.has_key(
'Interval') else 5
# validate for wait function
wait_validators = wait_params.get("validate", [])
for item in [wait_total_time, wait_interval]:
if not isinstance(item, int):
err_msg = u"Invalid wait function parameters! => {}\n".format(
item)
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
start_time = time.time()
current_time = time.time()
try:
while current_time - start_time < wait_total_time:
validate_pass, failures = self._run_test_once(
test_dict=test_dict, wait_validators=wait_validators)
# to break while when validate is OK
if validate_pass:
break
current_time = time.time()
time.sleep(wait_interval)
# to raise alarm when failures
except (exceptions.ParamsError, exceptions.ValidationFailure,
exceptions.ExtractFailure):
raise
else:
validate_pass, failures = self._run_test_once(test_dict=test_dict)
if not validate_pass:
failures_string = "\n".join([failure for failure in failures])
raise exceptions.ValidationFailure(failures_string)
def _add_tests(self, testcases):
""" initialize testcase with Runner() and add to test suite.
把测试用例加载到测试套件中
Args:
testcases (list): testcases list.
Returns:
unittest.TestSuite()
"""
def _add_test(test_runner, test_dict):
""" add test to testcase. 把步骤添加到用例，返回一个测试步骤
test_dict = parsed_testcases[0][teststeps][0]
test_dict = {
"name": "/mgmt/store/checkBusinessAddressIsExist",
"request": {
"headers": {"Authorization": "LazyString(${token_type} ${access_token})"},
"method": "GET",
"params": {
"provinceName": "LazyString(${provinceName})",
"cityName": "LazyString(${cityName})",
"areaName": LazyString(${areaName}),
"streetName": LazyString(${streetName}),
"detailAddress": LazyString(${detailAddress})},
"url": LazyString(${base_url}/mgmt/store/checkBusinessAddressIsExist),
"verify": True
},
"variables": {
"provinceName": "广东省",
"cityName": "广州市",
"areaName": "海珠区",
"streetName": "南州街道",
"detailAddress": "广州市海珠区南洲街道新滘中路88号唯品同创汇6区东三街17号自编23号",
"access_token": LazyString(${ENV(access_token)}),
"token_type": LazyString(${ENV(token_type)}),
"base_url": LazyString(${ENV(base_url)})
},
"validate": [LazyFunction(equals(status_code, 200))]
}
"""
def test(self):
try:
test_runner.run_test(test_dict)
pprint(test_runner)
except exceptions.MyBaseFailure as ex:
self.fail(str(ex))
finally:
self.meta_datas = test_runner.meta_datas
if "config" in test_dict:
# run nested testcase 嵌套testcase运行：testcase引用了testcase
test.__doc__ = test_dict["config"].get("name")
variables = test_dict["config"].get("variables", {})
else:
# run api test 运行api测试 testcase引用了api
test.__doc__ = test_dict.get("name")
variables = test_dict.get("variables", {})
if isinstance(test.__doc__, parser.LazyString): #懒惰的字符串：名字中有引用的变量
try:
parsed_variables = parser.parse_variables_mapping(
variables) # 所有的变量字典
test.__doc__ = parser.parse_lazy_data( # 找到引用变量对应的值
test.__doc__, parsed_variables)
except exceptions.VariableNotFound:
test.__doc__ = str(test.__doc__)
# 返回函数<function HttpRunner._add_tests.<locals>._add_test.<locals>.test at 0x000002D69C38A550>
return test
test_suite = unittest.TestSuite() # 用例集的子类
for testcase in testcases: #遍历用例中引用的每一个用例
config = testcase.get("config", {})
# <httprunner.runner.Runner object at 0x000002629251F520>
test_runner = runner.Runner(config)
TestSequense = type('TestSequense', (unittest.TestCase, ),
{}) #创建测试用例：unittest.TestCase子类
tests = testcase.get("teststeps", []) # 测试用例
for index, test_dict in enumerate(tests): # 遍历每一个测试步骤
times = test_dict.get("times", 1)
try:
times = int(times)
except ValueError:
raise exceptions.ParamsError(
"times should be digit, given: {}".format(times))
for times_index in range(times): #根据times设置，运行一个测试步骤N次
# suppose one testcase should not have more than 9999 steps,
# 假设一个测试用例不应该有超过9999个步骤
# and one step should not run more than 999 times.
# 一个步骤不应该运行超过999次
test_method_name = 'test_{:04}_{:03}'.format(
index, times_index) # 测试方法名
test_method = _add_test(test_runner, test_dict) # 测试方法
setattr(TestSequense, test_method_name,
test_method) #加载测试方法到测试用例中：unittest.TestCase子类
loaded_testcase = self.test_loader.loadTestsFromTestCase(
TestSequense) # 加载测试用例到用例集（小集：测试用例）
setattr(loaded_testcase, "config", config) # 加载属性到用例集（小集：测试用例）
setattr(loaded_testcase, "teststeps", tests) # 加载属性到用例集（小集：测试用例）
setattr(loaded_testcase, "runner",
test_runner) # 加载运行方法到用例集（小集：测试用例）
test_suite.addTest(loaded_testcase) # 加载用例集（小集：测试用例）到大用例集
return test_suite
def run_path(self, path: Text) -> "HttpRunner":
if not os.path.isfile(path):
raise exceptions.ParamsError(f"Invalid testcase path: {path}")
_, testcase_obj = load_testcase_file(path)
return self.run(testcase_obj)
def load_api_folder(api_folder_path):
""" load api definitions from api folder.
Args:
api_folder_path (str): api files folder.
api file should be in the following format:
[
{
"api": {
"def": "api_login",
"request": {},
"validate": []
}
},
{
"api": {
"def": "api_logout",
"request": {},
"validate": []
}
}
]
Returns:
dict: api definition mapping.
{
"api_login": {
"function_meta": {"func_name": "api_login", "args": [], "kwargs": {}}
"request": {}
},
"api_logout": {
"function_meta": {"func_name": "api_logout", "args": [], "kwargs": {}}
"request": {}
}
}
"""
api_definition_mapping = {}
api_items_mapping = load_folder_content(api_folder_path)
for api_file_path, api_items in api_items_mapping.items():
# TODO: add JSON schema validation
if isinstance(api_items, list):
for api_item in api_items:
key, api_dict = api_item.popitem()
api_id = api_dict.get("id") or api_dict.get("def") \
or api_dict.get("name")
if key != "api" or not api_id:
raise exceptions.ParamsError(
"Invalid API defined in {}".format(api_file_path))
if api_id in api_definition_mapping:
raise exceptions.ParamsError(
"Duplicated API ({}) defined in {}".format(
api_id, api_file_path))
else:
api_definition_mapping[api_id] = api_dict
elif isinstance(api_items, dict):
if api_file_path in api_definition_mapping:
raise exceptions.ParamsError(
"Duplicated API defined: {}".format(api_file_path))
else:
api_definition_mapping[api_file_path] = api_items
return api_definition_mapping
def _run_test_once(self, test_dict, wait_validators=None):
# clear meta data first to ensure independence for each test
self.__clear_test_data()
# check skip
self._handle_skip_feature(test_dict)
# prepare
test_dict = utils.lower_test_dict_keys(test_dict)
test_variables = test_dict.get("variables", {})
self.session_context.init_test_variables(test_variables)
# teststep name
test_name = test_dict.get("name", "")
# parse test request
raw_request = test_dict.get('request', {})
parsed_test_request = self.session_context.eval_content(raw_request)
self.session_context.update_test_variables("request",
parsed_test_request)
# setup hooks
setup_hooks = test_dict.get("setup_hooks", [])
if setup_hooks:
self.do_hook_actions(setup_hooks, "setup")
try:
url = parsed_test_request.pop('url')
method = parsed_test_request.pop('method')
parsed_test_request.setdefault("verify", self.verify)
group_name = parsed_test_request.pop("group", None)
except KeyError:
raise exceptions.ParamsError("URL or METHOD missed!")
# TODO: move method validation to json schema
valid_methods = [
"GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"
]
if method.upper() not in valid_methods:
err_msg = u"Invalid HTTP method! => {}\n".format(method)
err_msg += "Available HTTP methods: {}".format(
"/".join(valid_methods))
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
logger.log_info("{method} {url}".format(method=method, url=url))
logger.log_debug(
"request kwargs(raw): {kwargs}".format(kwargs=parsed_test_request))
# request
resp = self.http_client_session.request(method,
url,
name=(group_name or test_name),
**parsed_test_request)
resp_obj = response.ResponseObject(resp)
# teardown hooks
teardown_hooks = test_dict.get("teardown_hooks", [])
if teardown_hooks:
self.session_context.update_test_variables("response", resp_obj)
self.do_hook_actions(teardown_hooks, "teardown")
# extract
extractors = test_dict.get("extract", {})
extracted_variables_mapping = resp_obj.extract_response(extractors)
self.session_context.update_session_variables(
extracted_variables_mapping)
# validate
if wait_validators:
wait_validate_pass, wait_failures = self.session_context.validate(
wait_validators, resp_obj)
if not wait_validate_pass:
# err_msg = "{} DETAILED REQUEST & RESPONSE {}\n".format("*" * 32, "*" * 32)
#
# # log request
# err_msg += "====== request details ======\n"
# err_msg += "url: {}\n".format(url)
# err_msg += "method: {}\n".format(method)
# err_msg += "headers: {}\n".format(parsed_test_request.pop("headers", {}))
# for k, v in parsed_test_request.items():
# v = utils.omit_long_data(v)
# err_msg += "{}: {}\n".format(k, repr(v))
# err_msg += "\n"
# log response
# err_msg += "====== response details ======\n"
# err_msg += "status_code: {}\n".format(resp_obj.status_code)
# err_msg += "headers: {}\n".format(resp_obj.headers)
# err_msg += "body: {}\n".format(repr(resp_obj.text))
# logger.log_error("During the wait: \n" + err_msg)
logger.log_error(
"====== During the wait: the expect conditions can not Meet the requirements! ======"
)
failures_string = "\n".join(
[failure for failure in wait_failures])
self.validation_results = self.session_context.validation_results
raise exceptions.ValidationFailure(failures_string)
validators = test_dict.get("validate", [])
validate_pass, failures = self.session_context.validate(
validators, resp_obj)
self.validation_results = self.session_context.validation_results
# try:
# validate_pass, failures = self.session_context.validate(validators, resp_obj)
# except (exceptions.ParamsError, exceptions.ValidationFailure, exceptions.ExtractFailure):
# err_msg = "{} DETAILED REQUEST & RESPONSE {}\n".format("*" * 32, "*" * 32)
#
# # log request
# err_msg += "====== request details ======\n"
# err_msg += "url: {}\n".format(url)
# err_msg += "method: {}\n".format(method)
# err_msg += "headers: {}\n".format(parsed_test_request.pop("headers", {}))
# for k, v in parsed_test_request.items():
# v = utils.omit_long_data(v)
# err_msg += "{}: {}\n".format(k, repr(v))
#
# err_msg += "\n"
#
# # log response
# err_msg += "====== response details ======\n"
# err_msg += "status_code: {}\n".format(resp_obj.status_code)
# err_msg += "headers: {}\n".format(resp_obj.headers)
# err_msg += "body: {}\n".format(repr(resp_obj.text))
# logger.log_error(err_msg)
# raise
# finally:
# self.validation_results = self.session_context.validation_results
return validate_pass, failures
def run_test(self, testcase_dict):
""" run single testcase.
@param (dict) testcase_dict
{
"name": "testcase description",
"skip": "skip this test unconditionally",
"times": 3,
"variables": [], # optional, override
"request": {
"url": "http://127.0.0.1:5000/api/users/1000",
"method": "POST",
"headers": {
"Content-Type": "application/json",
"authorization": "$authorization",
"random": "$random"
},
"body": '{"name": "user", "password": "******"}'
},
"extract": [], # optional
"validate": [], # optional
"setup_hooks": [], # optional
"teardown_hooks": [] # optional
}
@return True or raise exception during test
"""
# check skip
self._handle_skip_feature(testcase_dict)
# prepare
parsed_request = self.init_config(testcase_dict, level="testcase")
self.context.bind_testcase_variable("request", parsed_request)
# setup hooks
setup_hooks = testcase_dict.get("setup_hooks", [])
setup_hooks.insert(0, "${setup_hook_prepare_kwargs($request)}")
self.do_hook_actions(setup_hooks)
try:
url = parsed_request.pop('url')
method = parsed_request.pop('method')
group_name = parsed_request.pop("group", None)
except KeyError:
raise exceptions.ParamsError("URL or METHOD missed!")
# TODO: move method validation to json schema
valid_methods = ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"]
if method.upper() not in valid_methods:
err_msg = u"Invalid HTTP method! => {}\n".format(method)
err_msg += "Available HTTP methods: {}".format("/".join(valid_methods))
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
logger.log_info("{method} {url}".format(method=method, url=url))
logger.log_debug("request kwargs(raw): {kwargs}".format(kwargs=parsed_request))
# request
resp = self.http_client_session.request(
method,
url,
name=group_name,
**parsed_request
)
resp_obj = response.ResponseObject(resp)
# teardown hooks
teardown_hooks = testcase_dict.get("teardown_hooks", [])
if teardown_hooks:
self.context.bind_testcase_variable("response", resp_obj)
self.do_hook_actions(teardown_hooks)
# extract
extractors = testcase_dict.get("extract", []) or testcase_dict.get("extractors", [])
extracted_variables_mapping = resp_obj.extract_response(extractors,context=self.context)
self.context.bind_extracted_variables(extracted_variables_mapping)
# validate
validators = testcase_dict.get("validate", []) or testcase_dict.get("validators", [])
try:
self.context.validate(validators, resp_obj)
except (exceptions.ParamsError, \
exceptions.ValidationFailure, exceptions.ExtractFailure):
# log request
err_req_msg = "request: \n"
err_req_msg += "headers: {}\n".format(parsed_request.pop("headers", {}))
for k, v in parsed_request.items():
err_req_msg += "{}: {}\n".format(k, repr(v))
logger.log_error(err_req_msg)
# log response
err_resp_msg = "response: \n"
err_resp_msg += "status_code: {}\n".format(resp_obj.status_code)
err_resp_msg += "headers: {}\n".format(resp_obj.headers)
err_resp_msg += "body: {}\n".format(repr(resp_obj.text))
logger.log_error(err_resp_msg)
raise
def _run_test(self, test_dict):
""" run single teststep.
Args:
test_dict (dict): teststep info
{
"name": "teststep description",
"skip": "skip this test unconditionally",
"times": 3,
"variables": [], # optional, override
"request": {
"url": "http://127.0.0.1:5000/api/users/1000",
"method": "POST",
"headers": {
"Content-Type": "application/json",
"authorization": "$authorization",
"random": "$random"
},
"json": {"name": "user", "password": "******"}
},
"extract": {}, # optional
"validate": [], # optional
"setup_hooks": [], # optional
"teardown_hooks": [] # optional
}
Raises:
exceptions.ParamsError
exceptions.ValidationFailure
exceptions.ExtractFailure
"""
# clear meta data first to ensure independence for each test
self.__clear_test_data()
# check skip
self._handle_skip_feature(test_dict)
# prepare
test_dict = utils.lower_test_dict_keys(test_dict)
test_variables = test_dict.get("variables", {})
self.session_context.init_test_variables(test_variables)
# teststep name
test_name = test_dict.get("name", "")
# parse test request
raw_request = test_dict.get('request', {})
parsed_test_request = self.session_context.eval_content(raw_request)
self.session_context.update_test_variables("request",
parsed_test_request)
# setup hooks
setup_hooks = test_dict.get("setup_hooks", [])
if setup_hooks:
self.do_hook_actions(setup_hooks, "setup")
try:
url = parsed_test_request.pop('url')
method = parsed_test_request.pop('method')
parsed_test_request.setdefault("verify", self.verify)
group_name = parsed_test_request.pop("group", None)
except KeyError:
raise exceptions.ParamsError("URL or METHOD missed!")
# TODO: move method validation to json schema
valid_methods = [
"GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"
]
if method.upper() not in valid_methods:
err_msg = u"Invalid HTTP method! => {}\n".format(method)
err_msg += "Available HTTP methods: {}".format(
"/".join(valid_methods))
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
logger.log_info("{method} {url}".format(method=method, url=url))
logger.log_debug(
"request kwargs(raw): {kwargs}".format(kwargs=parsed_test_request))
# request
resp = self.http_client_session.request(method,
url,
name=(group_name or test_name),
**parsed_test_request)
resp_obj = response.ResponseObject(resp)
# teardown hooks
teardown_hooks = test_dict.get("teardown_hooks", [])
if teardown_hooks:
self.session_context.update_test_variables("response", resp_obj)
self.do_hook_actions(teardown_hooks, "teardown")
# extract
extractors = test_dict.get("extract", {})
extracted_variables_mapping = resp_obj.extract_response(extractors)
self.session_context.update_session_variables(
extracted_variables_mapping)
# validate
validators = test_dict.get("validate", [])
try:
self.session_context.validate(validators, resp_obj)
except (exceptions.ParamsError, exceptions.ValidationFailure,
exceptions.ExtractFailure):
err_msg = "{} DETAILED REQUEST & RESPONSE {}\n".format(
"*" * 32, "*" * 32)
# log request
err_msg += "====== request details ======\n"
err_msg += "url: {}\n".format(url)
err_msg += "method: {}\n".format(method)
err_msg += "headers: {}\n".format(
parsed_test_request.pop("headers", {}))
for k, v in parsed_test_request.items():
v = utils.omit_long_data(v)
err_msg += "{}: {}\n".format(k, repr(v))
err_msg += "\n"
# log response
err_msg += "====== response details ======\n"
err_msg += "status_code: {}\n".format(resp_obj.status_code)
err_msg += "headers: {}\n".format(resp_obj.headers)
err_msg += "body: {}\n".format(repr(resp_obj.text))
logger.log_error(err_msg)
raise
finally:
self.validation_results = self.session_context.validation_results
