def __parse(self, function_meta):
""" init func as lazy functon instance
Args:
function_meta (dict): function meta including name, args and kwargs
"""
self._func = get_mapping_function(function_meta["func_name"],
self.functions_mapping)
self.func_name = self._func.__name__
self._args = prepare_lazy_data(function_meta.get("args", []),
self.functions_mapping,
self.check_variables_set)
self._kwargs = prepare_lazy_data(function_meta.get("kwargs", {}),
self.functions_mapping,
self.check_variables_set)
if self.func_name == "load_csv_file":
if len(self._args) != 1 or self._kwargs:
raise exceptions.ParamsError(
"P() should only pass in one argument!")
self._args = [self._args[0]]
elif self.func_name == "get_os_environ":
if len(self._args) != 1 or self._kwargs:
raise exceptions.ParamsError(
"ENV() should only pass in one argument!")
self._args = [self._args[0]]
def uniform_validator(validator):
""" unify validator
Args:
validator (dict): validator maybe in two formats:
format1: this is kept for compatiblity with the previous versions.
{"check": "status_code", "comparator": "eq", "expect": 201}
{"check": "$resp_body_success", "comparator": "eq", "expect": True}
format2: recommended new version, {comparator: [check_item, expected_value]}
{'eq': ['status_code', 201]}
{'eq': ['$resp_body_success', True]}
Returns
dict: validator info
{
"check": "status_code",
"expect": 201,
"comparator": "equals"
}
"""
if not isinstance(validator, dict):
raise exceptions.ParamsError("invalid validator: {}".format(validator))
if "check" in validator and "expect" in validator:
# format1
check_item = validator["check"]
expect_value = validator["expect"]
comparator = validator.get("comparator", "eq")
elif len(validator) == 1:
# format2
comparator = list(validator.keys())[0]
compare_values = validator[comparator]
if not isinstance(compare_values, list) or len(compare_values) != 2:
raise exceptions.ParamsError(
"invalid validator: {}".format(validator))
check_item, expect_value = compare_values
else:
raise exceptions.ParamsError("invalid validator: {}".format(validator))
# uniform comparator, e.g. lt => less_than, eq => equals
comparator = get_uniform_comparator(comparator)
return {
"check": check_item,
"expect": expect_value,
"comparator": comparator
}
def ensure_mapping_format(variables):
""" ensure variables are in mapping format.
Args:
variables (list/dict): original variables
Returns:
dict: ensured variables in dict format
Examples:
>>> variables = [
{"a": 1},
{"b": 2}
]
>>> print(ensure_mapping_format(variables))
{
"a": 1,
"b": 2
}
"""
if isinstance(variables, list):
variables_dict = {}
for map_dict in variables:
variables_dict.update(map_dict)
return variables_dict
elif isinstance(variables, dict):
return variables
else:
raise exceptions.ParamsError("variables format error!")
def __getattr__(self, key):
try:
if key == "json":
value = self.resp_obj.json()
elif key == "cookies":
value = self.resp_obj.cookies.get_dict()
else:
value = getattr(self.resp_obj, key)
self.__dict__[key] = value
return value
except AttributeError:
err_msg = "ResponseObject does not have attribute: {}".format(key)
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
def run(self, path_or_tests, dot_env_path=None, mapping=None):
""" main interface.
Args:
path_or_tests:
str: testcase/testsuite file/foler path
dict: valid testcase/testsuite data
"""
if validator.is_testcase_path(path_or_tests):
return self.run_path(path_or_tests, dot_env_path, mapping)
elif validator.is_testcases(path_or_tests):
return self.run_tests(path_or_tests)
else:
raise exceptions.ParamsError(
"Invalid testcase path or testcases: {}".format(path_or_tests))
def extract_field(self, field):
""" extract value from requests.Response.
"""
if not isinstance(field, basestring):
err_msg = u"Invalid extractor! => {}\n".format(field)
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
msg = "extract: {}".format(field)
if text_extractor_regexp_compile.match(field):
value = self._extract_field_with_regex(field)
else:
value = self._extract_field_with_delimiter(field)
if is_py2 and isinstance(value, unicode):
value = value.encode("utf-8")
msg += "\t=> {}".format(value)
logger.log_debug(msg)
return value
def parse_parameters(parameters,
variables_mapping=None,
functions_mapping=None):
""" parse parameters and generate cartesian product.
Args:
parameters (list) parameters: parameter name and value in list
parameter value may be in three types:
(1) data list, e.g. ["iOS/10.1", "iOS/10.2", "iOS/10.3"]
(2) call built-in parameterize function, "${parameterize(account.csv)}"
(3) call custom function in debugtalk.py, "${gen_app_version()}"
variables_mapping (dict): variables mapping loaded from testcase config
functions_mapping (dict): functions mapping loaded from debugtalk.py
Returns:
list: cartesian product list
Examples:
>>> parameters = [
{"user_agent": ["iOS/10.1", "iOS/10.2", "iOS/10.3"]},
{"username-password": "******"},
{"app_version": "${gen_app_version()}"}
]
>>> parse_parameters(parameters)
"""
variables_mapping = variables_mapping or {}
functions_mapping = functions_mapping or {}
parsed_parameters_list = []
parameters = utils.ensure_mapping_format(parameters)
cartesian = parameters.pop('cartesian', None)
for parameter_name, parameter_content in parameters.items():
parameter_name_list = parameter_name.split("-")
if isinstance(parameter_content, list):
# (1) data list
# e.g. {"app_version": ["2.8.5", "2.8.6"]}
# => [{"app_version": "2.8.5", "app_version": "2.8.6"}]
# e.g. {"username-password": [["user1", "111111"], ["test2", "222222"]}
# => [{"username": "******", "password": "******"}, {"username": "******", "password": "******"}]
parameter_content_list = []
for parameter_item in parameter_content:
if not isinstance(parameter_item, (list, tuple)):
# "2.8.5" => ["2.8.5"]
parameter_item = [parameter_item]
# ["app_version"], ["2.8.5"] => {"app_version": "2.8.5"}
# ["username", "password"], ["user1", "111111"] => {"username": "******", "password": "******"}
parameter_content_dict = dict(
zip(parameter_name_list, parameter_item))
parameter_content_list.append(parameter_content_dict)
else:
# (2) & (3)
parsed_variables_mapping = parse_variables_mapping(
variables_mapping)
parsed_parameter_content = eval_lazy_data(
parameter_content, parsed_variables_mapping, functions_mapping)
if not isinstance(parsed_parameter_content, list):
raise exceptions.ParamsError("parameters syntax error!")
parameter_content_list = []
for parameter_item in parsed_parameter_content:
if isinstance(parameter_item, dict):
# get subset by parameter name
# {"app_version": "${gen_app_version()}"}
# gen_app_version() => [{'app_version': '2.8.5'}, {'app_version': '2.8.6'}]
# {"username-password": "******"}
# get_account() => [
# {"username": "******", "password": "******"},
# {"username": "******", "password": "******"}
# ]
parameter_dict = {
key: parameter_item[key]
for key in parameter_name_list
}
elif isinstance(parameter_item, (list, tuple)):
# {"username-password": "******"}
# get_account() => [("user1", "111111"), ("user2", "222222")]
parameter_dict = dict(
zip(parameter_name_list, parameter_item))
elif len(parameter_name_list) == 1:
# {"user_agent": "${get_user_agent()}"}
# get_user_agent() => ["iOS/10.1", "iOS/10.2"]
parameter_dict = {parameter_name_list[0]: parameter_item}
parameter_content_list.append(parameter_dict)
parsed_parameters_list.append(parameter_content_list)
if cartesian:
return utils.gen_cartesian_product(*parsed_parameters_list)
else:
return utils.gen_dict_from_zip(*parsed_parameters_list)
def _run_test(self, test_dict):
""" run single teststep.
Args:
test_dict (dict): teststep info
{
"name": "teststep description",
"skip": "skip this test unconditionally",
"times": 3,
"variables": [], # optional, override
"request": {
"url": "http://127.0.0.1:5000/api/users/1000",
"method": "POST",
"headers": {
"Content-Type": "application/json",
"authorization": "$authorization",
"random": "$random"
},
"json": {"name": "user", "password": "******"}
},
"extract": {}, # optional
"validate": [], # optional
"setup_hooks": [], # optional
"teardown_hooks": [] # optional
}
Raises:
exceptions.ParamsError
exceptions.ValidationFailure
exceptions.ExtractFailure
"""
global tmp_extracted_variables
# clear meta data first to ensure independence for each test
self.__clear_test_data()
# check skip
self._handle_skip_feature(test_dict)
# prepare
test_dict = utils.lower_test_dict_keys(test_dict)
test_variables = test_dict.get("variables", {})
# override variables use former extracted_variables
test_variables.update(tmp_extracted_variables)
self.session_context.init_test_variables(test_variables)
# teststep name
test_name = self.session_context.eval_content(test_dict.get(
"name", ""))
# parse test request
raw_request = test_dict.get('request', {})
parsed_test_request = self.session_context.eval_content(raw_request)
self.session_context.update_test_variables("request",
parsed_test_request)
# prepend url with base_url unless it's already an absolute URL
url = parsed_test_request.pop('url')
base_url = self.session_context.eval_content(
test_dict.get("base_url", ""))
parsed_url = utils.build_url(base_url, url)
# setup hooks
setup_hooks = test_dict.get("setup_hooks", [])
if setup_hooks:
self.do_hook_actions(setup_hooks, "setup")
try:
method = parsed_test_request.pop('method')
parsed_test_request.setdefault("verify", self.verify)
group_name = parsed_test_request.pop("group", None)
except KeyError:
raise exceptions.ParamsError("URL or METHOD missed!")
# TODO: move method validation to json schema
valid_methods = [
"GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"
]
if method.upper() not in valid_methods:
err_msg = u"Invalid HTTP method! => {}\n".format(method)
err_msg += "Available HTTP methods: {}".format(
"/".join(valid_methods))
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
logger.log_info("{method} {url}".format(method=method, url=parsed_url))
logger.log_debug(
"request kwargs(raw): {kwargs}".format(kwargs=parsed_test_request))
# request
resp = self.http_client_session.request(method,
parsed_url,
name=(group_name or test_name),
**parsed_test_request)
resp_obj = response.ResponseObject(resp)
# extract
extractors = test_dict.get("extract", {})
if extractors:
extracted_variables_mapping = resp_obj.extract_response(extractors)
tmp_extracted_variables.update(extracted_variables_mapping)
self.session_context.update_session_variables(
extracted_variables_mapping)
self.session_context.update_session_variables(tmp_extracted_variables)
# validate
validators = test_dict.get("validate") or test_dict.get(
"validators") or []
try:
self.session_context.validate(validators, resp_obj)
except (exceptions.ParamsError, exceptions.ValidationFailure,
exceptions.ExtractFailure):
err_msg = "{} DETAILED REQUEST & RESPONSE {}\n".format(
"*" * 32, "*" * 32)
# log request
err_msg += "====== request details ======\n"
err_msg += "url: {}\n".format(parsed_url)
err_msg += "method: {}\n".format(method)
err_msg += "headers: {}\n".format(
parsed_test_request.pop("headers", {}))
for k, v in parsed_test_request.items():
v = utils.omit_long_data(v)
err_msg += "{}: {}\n".format(k, repr(v))
err_msg += "\n"
# log response
err_msg += "====== response details ======\n"
err_msg += "status_code: {}\n".format(resp_obj.status_code)
err_msg += "headers: {}\n".format(resp_obj.headers)
err_msg += "body: {}\n".format(repr(resp_obj.text))
logger.log_error(err_msg)
raise
finally:
self.validation_results = self.session_context.validation_results
# teardown hooks
teardown_hooks = test_dict.get("teardown_hooks", [])
if teardown_hooks:
self.session_context.update_test_variables(
"response", resp_obj)
self.do_hook_actions(teardown_hooks, "teardown")
def load_api_folder(api_folder_path):
""" load api definitions from api folder.
Args:
api_folder_path (str): api files folder.
api file should be in the following format:
[
{
"api": {
"def": "api_login",
"request": {},
"validate": []
}
},
{
"api": {
"def": "api_logout",
"request": {},
"validate": []
}
}
]
Returns:
dict: api definition mapping.
{
"api_login": {
"function_meta": {"func_name": "api_login", "args": [], "kwargs": {}}
"request": {}
},
"api_logout": {
"function_meta": {"func_name": "api_logout", "args": [], "kwargs": {}}
"request": {}
}
}
"""
api_definition_mapping = {}
api_items_mapping = load_folder_content(api_folder_path)
for api_file_path, api_items in api_items_mapping.items():
# TODO: add JSON schema validation
if isinstance(api_items, list):
for api_item in api_items:
key, api_dict = api_item.popitem()
api_id = api_dict.get("id") or api_dict.get(
"def") or api_dict.get("name")
if key != "api" or not api_id:
raise exceptions.ParamsError(
"Invalid API defined in {}".format(api_file_path))
if api_id in api_definition_mapping:
raise exceptions.ParamsError(
"Duplicated API ({}) defined in {}".format(
api_id, api_file_path))
else:
api_definition_mapping[api_id] = api_dict
elif isinstance(api_items, dict):
if api_file_path in api_definition_mapping:
raise exceptions.ParamsError(
"Duplicated API defined: {}".format(api_file_path))
else:
api_definition_mapping[api_file_path] = api_items
return api_definition_mapping
def _extract_field_with_delimiter(self, field):
""" response content could be json or html text.
Args:
field (str): string joined by delimiter.
e.g.
"status_code"
"headers"
"cookies"
"content"
"headers.content-type"
"content.person.name.first_name"
"""
# string.split(sep=None, maxsplit=-1) -> list of strings
# e.g. "content.person.name" => ["content", "person.name"]
try:
top_query, sub_query = field.split('.', 1)
except ValueError:
top_query = field
sub_query = None
# status_code
if top_query in ["status_code", "encoding", "ok", "reason", "url"]:
if sub_query:
# status_code.XX
err_msg = u"Failed to extract: {}\n".format(field)
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
return getattr(self, top_query)
# cookies
elif top_query == "cookies":
cookies = self.cookies
if not sub_query:
# extract cookies
return cookies
try:
return cookies[sub_query]
except KeyError:
err_msg = u"Failed to extract cookie! => {}\n".format(field)
err_msg += u"response cookies: {}\n".format(cookies)
logger.log_error(err_msg)
raise exceptions.ExtractFailure(err_msg)
# elapsed
elif top_query == "elapsed":
available_attributes = u"available attributes: days, seconds, microseconds, total_seconds"
if not sub_query:
err_msg = u"elapsed is datetime.timedelta instance, attribute should also be specified!\n"
err_msg += available_attributes
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
elif sub_query in ["days", "seconds", "microseconds"]:
return getattr(self.elapsed, sub_query)
elif sub_query == "total_seconds":
return self.elapsed.total_seconds()
else:
err_msg = "{} is not valid datetime.timedelta attribute.\n".format(sub_query)
err_msg += available_attributes
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
# headers
elif top_query == "headers":
headers = self.headers
if not sub_query:
# extract headers
return headers
try:
return headers[sub_query]
except KeyError:
err_msg = u"Failed to extract header! => {}\n".format(field)
err_msg += u"response headers: {}\n".format(headers)
logger.log_error(err_msg)
raise exceptions.ExtractFailure(err_msg)
# response body
elif top_query in ["content", "text", "json"]:
try:
body = self.json
except exceptions.JSONDecodeError:
body = self.text
if not sub_query:
# extract response body
return body
if isinstance(body, (dict, list)):
# content = {"xxx": 123}, content.xxx
return utils.query_json(body, sub_query)
elif sub_query.isdigit():
# content = "abcdefg", content.3 => d
return utils.query_json(body, sub_query)
else:
# content = "<html>abcdefg</html>", content.xxx
err_msg = u"Failed to extract attribute from response body! => {}\n".format(field)
err_msg += u"response body: {}\n".format(body)
logger.log_error(err_msg)
raise exceptions.ExtractFailure(err_msg)
# new set response attributes in teardown_hooks
elif top_query in self.__dict__:
attributes = self.__dict__[top_query]
if not sub_query:
# extract response attributes
return attributes
if isinstance(attributes, (dict, list)):
# attributes = {"xxx": 123}, content.xxx
return utils.query_json(attributes, sub_query)
elif sub_query.isdigit():
# attributes = "abcdefg", attributes.3 => d
return utils.query_json(attributes, sub_query)
else:
# content = "attributes.new_attribute_not_exist"
err_msg = u"Failed to extract cumstom set attribute from teardown hooks! => {}\n".format(field)
err_msg += u"response set attributes: {}\n".format(attributes)
logger.log_error(err_msg)
raise exceptions.TeardownHooksFailure(err_msg)
# others
else:
err_msg = u"Failed to extract attribute from response! => {}\n".format(field)
err_msg += u"available response attributes: status_code, cookies, elapsed, headers, content, text, json, encoding, ok, reason, url.\n\n"
err_msg += u"If you want to set attribute in teardown_hooks, take the following example as reference:\n"
err_msg += u"response.new_attribute = 'new_attribute_value'\n"
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)