def extract_response(self, extractors):
""" extract value from requests.Response and store in OrderedDict.
@param (list) extractors
[
{"resp_status_code": "status_code"},
{"resp_headers_content_type": "headers.content-type"},
{"resp_content": "content"},
{"resp_content_person_first_name": "content.person.name.first_name"}
]
@return (OrderDict) variable binds ordered dict
"""
if not extractors:
return {}
logger.log_info("start to extract from response object.")
extracted_variables_mapping = OrderedDict()
extract_binds_order_dict = utils.convert_to_order_dict(extractors)
for key, field in extract_binds_order_dict.items():
if not isinstance(field, basestring):
raise exception.ParamsError("invalid extractors in testcase!")
result = self.extract_field(field)
extracted_variables_mapping[key] = result
if not (isinstance(result, bool)
or bool(result)): # False 可以return
err_msg = u"extract data with delimiter can be None!\n"
err_msg += u"response: {}\n".format(self.parsed_dict())
err_msg += u"regex: {}\n".format(field)
logger.log_error(err_msg)
raise exception.ParamsError(err_msg)
return extracted_variables_mapping
def _get_bind_item(self, item_type, item_name):
if item_type == "function":
if item_name in self.functions:
return self.functions[item_name]
try:
# check if builtin functions
item_func = eval(item_name)
if callable(item_func):
# is builtin function
return item_func
except (NameError, TypeError):
# is not builtin function, continue to search
pass
elif item_type == "variable":
if item_name in self.variables:
return self.variables[item_name]
else:
raise exception.ParamsError(
"bind item should only be function or variable.")
try:
assert self.file_path is not None
return utils.search_conf_item(self.file_path, item_type, item_name)
except (AssertionError, exception.FunctionNotFound):
raise exception.ParamsError(
"{} is not defined in bind {}s!".format(item_name, item_type))
def _extract_field_with_delimiter(self, field):
""" response content could be json or html text.
@param (str) field should be string joined by delimiter.
e.g.
"status_code"
"content"
"headers.content-type"
"content.person.name.first_name"
"""
try:
# string.split(sep=None, maxsplit=-1) -> list of strings
# e.g. "content.person.name" => ["content", "person.name"]
try:
top_query, sub_query = field.split('.', 1)
except ValueError:
top_query = field
sub_query = None
if top_query in ["body", "content", "text"]:
top_query_content = self.parsed_body()
elif top_query == "cookies":
cookies = self.resp_obj.cookies
try:
return cookies[sub_query]
except KeyError:
err_msg = u"Failed to extract attribute from cookies!\n"
err_msg += u"cookies: {}\n".format(cookies)
err_msg += u"attribute: {}".format(sub_query)
logger.log_error(err_msg)
raise exception.ParamsError(err_msg)
else:
try:
top_query_content = getattr(self.resp_obj, top_query)
except AttributeError:
err_msg = u"Failed to extract attribute from response object: resp_obj.{}".format(
top_query)
logger.log_error(err_msg)
raise exception.ParamsError(err_msg)
if sub_query:
if not isinstance(top_query_content,
(dict, CaseInsensitiveDict, list)):
err_msg = u"Failed to extract data with delimiter!\n"
err_msg += u"response: {}\n".format(self.parsed_dict())
err_msg += u"regex: {}\n".format(field)
logger.log_error(err_msg)
raise exception.ParamsError(err_msg)
# e.g. key: resp_headers_content_type, sub_query = "content-type"
return utils.query_json(top_query_content, sub_query)
else:
# e.g. key: resp_status_code, resp_content
return top_query_content
except AttributeError:
err_msg = u"Failed to extract value from response!\n"
err_msg += u"response: {}\n".format(self.parsed_dict())
err_msg += u"extract field: {}\n".format(field)
logger.log_error(err_msg)
raise exception.ParamsError(err_msg)
def parse_validator(validator):
""" parse validator, validator maybe in two format
@param (dict) validator
format1: this is kept for compatiblity with the previous versions.
{"check": "status_code", "comparator": "eq", "expect": 201}
{"check": "$resp_body_success", "comparator": "eq", "expect": True}
format2: recommended new version
{'eq': ['status_code', 201]}
{'eq': ['$resp_body_success', True]}
@return (dict) validator info
{
"check": "status_code",
"expect": 201,
"comparator": "eq"
}
"""
if not isinstance(validator, dict):
raise exception.ParamsError("invalid validator: {}".format(validator))
if "check" in validator and len(validator) > 1:
# format1
check_item = validator.get("check")
if "expect" in validator:
expect_value = validator.get("expect")
elif "expected" in validator:
expect_value = validator.get("expected")
else:
raise exception.ParamsError(
"invalid validator: {}".format(validator))
comparator = validator.get("comparator", "eq")
elif len(validator) == 1:
# format2
comparator = list(validator.keys())[0]
compare_values = validator[comparator]
if not isinstance(compare_values, list) or len(compare_values) != 2:
raise exception.ParamsError(
"invalid validator: {}".format(validator))
check_item, expect_value = compare_values
else:
raise exception.ParamsError("invalid validator: {}".format(validator))
return {
"check": check_item,
"expect": expect_value,
"comparator": comparator
}
def validate(self, validators, variables_mapping):
""" Bind named validators to value within the context.
@param (list) validators
[
{"check": "status_code", "comparator": "eq", "expect": 201},
{"check": "resp_body_success", "comparator": "eq", "expect": True}
]
@param (dict) variables_mapping
{
"resp_body_success": True
}
@return (list) content differences
[
{
"check": "status_code",
"comparator": "eq", "expect": 201, "value": 200
}
]
"""
for validator_dict in validators:
check_item = validator_dict.get("check")
if not check_item:
raise exception.ParamsError(
"check item invalid: {}".format(check_item))
if "expect" in validator_dict:
expect_value = validator_dict.get("expect")
elif "expected" in validator_dict:
expect_value = validator_dict.get("expected")
else:
raise exception.ParamsError(
"expected value missed in testcase validator!")
comparator = validator_dict.get("comparator", "eq")
if check_item in variables_mapping:
validator_dict["actual_value"] = variables_mapping[check_item]
else:
try:
validator_dict["actual_value"] = self.extract_field(
check_item)
except exception.ParseResponseError:
raise exception.ParseResponseError(
"failed to extract check item in response!")
utils.match_expected(validator_dict["actual_value"], expect_value,
comparator, check_item)
return True
def _extract_field_with_jsonpath(self, field):
result = jsonpath.jsonpath(self.resp_body, field)
if result:
return result
else:
raise exception.ParamsError(
"jsonpath {} get nothing".format(field))
def _parse_steps(self, step_content):
""" parse steps from list or xls
@params
(list) steps: value in list
step value may be in three types:
(1) data list
(2) call custom function
e.g.
[
{"user_agent": ["iOS/10.1", "iOS/10.2", "iOS/10.3"]},
{"username-password": "******"},
{"app_version": "${gen_app_version()}"}
]
@return steps in list
"""
testcase_parser = self.context.testcase_parser
steps_list = []
if isinstance(step_content, list):
# (1) data list
return step_content
else:
steps_list = testcase_parser.eval_content_with_bindings(
step_content)
# e.g. [{'app_version': '2.8.5'}, {'app_version': '2.8.6'}]
# e.g. [{"username": "******", "password": "******"}, {"username": "******", "password": "******"}]
if not isinstance(steps_list, list):
raise exception.ParamsError("step syntax error!")
return steps_list
def _get_block_by_name(ref_call, ref_type):
""" get test content by reference name
@params:
ref_call: e.g. api_v1_Account_Login_POST($UserName, $Password)
ref_type: "api" or "suite"
"""
function_meta = parse_function(ref_call)
func_name = function_meta["func_name"]
call_args = function_meta["args"]
block = TestcaseLoader._get_test_definition(func_name, ref_type)
def_args = block.get("function_meta").get("args", [])
if len(call_args) != len(def_args):
raise exception.ParamsError("call args mismatch defined args!")
args_mapping = {}
for index, item in enumerate(def_args):
if call_args[index] == item:
continue
args_mapping[item] = call_args[index]
if args_mapping:
block = substitute_variables_with_mapping(block, args_mapping)
return block
def extract_response(self, extractors):
""" extract value from requests.Response and store in OrderedDict.
@param (list) extractors
[
{"resp_status_code": "status_code"},
{"resp_headers_content_type": "headers.content-type"},
{"resp_content": "content"},
{"resp_content_person_first_name": "content.person.name.first_name"}
]
@return (OrderDict) variable binds ordered dict
"""
if not extractors:
return {}
logger.log_info("start to extract from response object.")
extracted_variables_mapping = OrderedDict()
extract_binds_order_dict = utils.convert_to_order_dict(extractors)
for key, field in extract_binds_order_dict.items():
if not isinstance(field, basestring):
raise exception.ParamsError("invalid extractors in testcase!")
extracted_variables_mapping[key] = self.extract_field(field)
return extracted_variables_mapping
def do_validation(self, validator_dict):
""" validate with functions
"""
comparator = utils.get_uniform_comparator(validator_dict["comparator"])
validate_func = self.testcase_parser.get_bind_item(
"function", comparator)
if not validate_func:
raise exception.FunctionNotFound(
"comparator not found: {}".format(comparator))
check_item = validator_dict["check"]
check_value = validator_dict["check_value"]
expect_value = validator_dict["expect"]
if (check_value is None or expect_value is None) \
and comparator not in ["is", "eq", "equals", "=="]:
raise exception.ParamsError(
"Null value can only be compared with comparator: eq/equals/=="
)
try:
validate_func(validator_dict["check_value"],
validator_dict["expect"])
except (AssertionError, TypeError):
err_msg = "\n" + "\n".join([
"\tcheck item name: %s;" % check_item,
"\tcheck item value: %s (%s);" %
(check_value, type(check_value).__name__),
"\tcomparator: %s;" % comparator,
"\texpected value: %s (%s)." %
(expect_value, type(expect_value).__name__)
])
raise exception.ValidationError(err_msg)
def get_bind_item(self, item_type, item_name):
if item_type == "function":
if item_name in self.functions:
return self.functions[item_name]
elif item_type == "variable":
if item_name in self.variables:
return self.variables[item_name]
else:
raise exception.ParamsError("bind item should only be function or variable.")
try:
assert self.file_path is not None
return utils.search_conf_item(self.file_path, item_type, item_name)
except (AssertionError, exception.FunctionNotFound):
raise exception.ParamsError(
"{} is not defined in bind {}s!".format(item_name, item_type))
def load_test_dependencies():
""" load all api and suite definitions.
default api folder is "$CWD/tests/api/".
default suite folder is "$CWD/tests/suite/".
"""
test_def_overall_dict["loaded"] = True
test_def_overall_dict["api"] = {}
test_def_overall_dict["suite"] = {}
# load api definitions
api_def_folder = os.path.join(os.getcwd(), "tests", "api")
api_files = utils.load_folder_files(api_def_folder)
for test_file in api_files:
testset = load_test_file(test_file)
test_def_overall_dict["api"].update(testset["api"])
# load suite definitions
suite_def_folder = os.path.join(os.getcwd(), "tests", "suite")
suite_files = utils.load_folder_files(suite_def_folder)
for suite_file in suite_files:
suite = load_test_file(suite_file)
if "def" not in suite["config"]:
raise exception.ParamsError(
"def missed in suite file: {}!".format(suite_file))
call_func = suite["config"]["def"]
function_meta = parse_function(call_func)
suite["function_meta"] = function_meta
test_def_overall_dict["suite"][function_meta["func_name"]] = suite
def parse_parameters(parameters, testset_path=None):
""" parse parameters and generate cartesian product
@params
(list) parameters: parameter name and value in list
parameter value may be in three types:
(1) data list
(2) call built-in parameterize function
(3) call custom function in debugtalk.py
e.g.
[
{"user_agent": ["iOS/10.1", "iOS/10.2", "iOS/10.3"]},
{"username-password": "******"},
{"app_version": "${gen_app_version()}"}
]
(str) testset_path: testset file path, used for locating csv file and debugtalk.py
@return cartesian product in list
"""
testcase_parser = TestcaseParser(file_path=testset_path)
parsed_parameters_list = []
for parameter in parameters:
parameter_name, parameter_content = list(parameter.items())[0]
parameter_name_list = parameter_name.split("-")
if isinstance(parameter_content, list):
# (1) data list
# e.g. {"app_version": ["2.8.5", "2.8.6"]}
# => [{"app_version": "2.8.5", "app_version": "2.8.6"}]
# e.g. {"username-password": [["user1", "111111"], ["test2", "222222"]}
# => [{"username": "******", "password": "******"}, {"username": "******", "password": "******"}]
parameter_content_list = []
for parameter_item in parameter_content:
if not isinstance(parameter_item, (list, tuple)):
# "2.8.5" => ["2.8.5"]
parameter_item = [parameter_item]
# ["app_version"], ["2.8.5"] => {"app_version": "2.8.5"}
# ["username", "password"], ["user1", "111111"] => {"username": "******", "password": "******"}
parameter_content_dict = dict(zip(parameter_name_list, parameter_item))
parameter_content_list.append(parameter_content_dict)
else:
# (2) & (3)
parsed_parameter_content = testcase_parser.eval_content_with_bindings(parameter_content)
# e.g. [{'app_version': '2.8.5'}, {'app_version': '2.8.6'}]
# e.g. [{"username": "******", "password": "******"}, {"username": "******", "password": "******"}]
if not isinstance(parsed_parameter_content, list):
raise exception.ParamsError("parameters syntax error!")
parameter_content_list = [
# get subset by parameter name
{key: parameter_item[key] for key in parameter_name_list}
for parameter_item in parsed_parameter_content
]
parsed_parameters_list.append(parameter_content_list)
return gen_cartesian_product(*parsed_parameters_list)
def override_variables_binds(variables, new_mapping):
""" convert variables in testcase to ordered mapping, with new_mapping overrided
"""
if isinstance(variables, list):
variables_ordered_dict = convert_to_order_dict(variables)
elif isinstance(variables, (OrderedDict, dict)):
variables_ordered_dict = variables
else:
raise exception.ParamsError("variables error!")
return update_ordered_dict(variables_ordered_dict, new_mapping)
def __getattr__(self, key):
try:
if key == "json":
value = self.resp_obj.json()
else:
value = getattr(self.resp_obj, key)
self.__dict__[key] = value
return value
except AttributeError:
err_msg = "ResponseObject does not have attribute: {}".format(key)
logger.log_error(err_msg)
raise exception.ParamsError(err_msg)
def _extract_field_with_regex(self, field):
""" extract field from response content with regex.
requests.Response body could be json or html text.
@param (str) field should only be regex string that matched r".*\(.*\).*"
e.g.
self.resp_text: "LB123abcRB789"
field: "LB[\d]*(.*)RB[\d]*"
return: abc
"""
matched = re.search(field, self.resp_text)
if not matched:
err_msg = u"Failed to extract data with regex!\n"
err_msg += u"response body: {}\n".format(self.resp_text)
err_msg += u"regex: {}\n".format(field)
logger.log_error(err_msg)
raise exception.ParamsError(err_msg)
return matched.group(1)
def load_test_dependencies():
""" load all api and suite definitions.
default api folder is "$CWD/tests/api/".
default suite folder is "$CWD/tests/suite/".
"""
# TODO: cache api and suite loading
# load api definitions
api_def_folder = os.path.join(os.getcwd(), "tests", "api")
for test_file in FileUtils.load_folder_files(api_def_folder):
TestcaseLoader.load_api_file(test_file)
# load suite definitions
suite_def_folder = os.path.join(os.getcwd(), "tests", "suite")
for suite_file in FileUtils.load_folder_files(suite_def_folder):
suite = TestcaseLoader.load_test_file(suite_file)
if "def" not in suite["config"]:
raise exception.ParamsError("def missed in suite file: {}!".format(suite_file))
call_func = suite["config"]["def"]
function_meta = parse_function(call_func)
suite["function_meta"] = function_meta
TestcaseLoader.overall_def_dict["suite"][function_meta["func_name"]] = suite
def get_test_definition(name, ref_type):
""" get expected api or suite.
@params:
name: api or suite name
ref_type: "api" or "suite"
@return
expected api info if found, otherwise raise ApiNotFound exception
"""
if not test_def_overall_dict.get("loaded", False):
load_test_dependencies()
test_info = test_def_overall_dict.get(ref_type, {}).get(name)
if not test_info:
err_msg = "{} {} not found!".format(ref_type, name)
if ref_type == "api":
raise exception.ApiNotFound(err_msg)
elif ref_type == "suite":
raise exception.SuiteNotFound(err_msg)
else:
raise exception.ParamsError("ref_type can only be api or suite!")
return test_info
def _extract_field_with_delimiter(self, field):
""" response content could be json or html text.
@param (str) field should be string joined by delimiter.
e.g.
"status_code"
"headers"
"cookies"
"content"
"headers.content-type"
"content.person.name.first_name"
"""
try:
# string.split(sep=None, maxsplit=-1) -> list of strings
# e.g. "content.person.name" => ["content", "person.name"]
try:
top_query, sub_query = field.split('.', 1)
except ValueError:
top_query = field
sub_query = None
if top_query == "cookies":
cookies = self.cookies
try:
return cookies[sub_query]
except KeyError:
err_msg = u"Failed to extract attribute from cookies!\n"
err_msg += u"cookies: {}\n".format(cookies)
err_msg += u"attribute: {}".format(sub_query)
logger.log_error(err_msg)
raise exception.ParamsError(err_msg)
elif top_query == "elapsed":
if sub_query in ["days", "seconds", "microseconds"]:
return getattr(self.elapsed, sub_query)
elif sub_query == "total_seconds":
return self.elapsed.total_seconds()
else:
err_msg = "{}: {} is not valid timedelta attribute.\n".format(
field, sub_query)
err_msg += "elapsed only support attributes: days, seconds, microseconds, total_seconds.\n"
logger.log_error(err_msg)
raise exception.ParamsError(err_msg)
try:
top_query_content = getattr(self, top_query)
except AttributeError:
err_msg = u"Failed to extract attribute from response object: resp_obj.{}".format(
top_query)
logger.log_error(err_msg)
raise exception.ParamsError(err_msg)
if sub_query:
if not isinstance(top_query_content,
(dict, CaseInsensitiveDict, list)):
try:
# TODO: remove compatibility for content, text
if isinstance(top_query_content, bytes):
top_query_content = top_query_content.decode(
"utf-8")
if isinstance(top_query_content, PreparedRequest):
top_query_content = top_query_content.__dict__
else:
top_query_content = json.loads(top_query_content)
except json.decoder.JSONDecodeError:
err_msg = u"Failed to extract data with delimiter!\n"
err_msg += u"response content: {}\n".format(
self.content)
err_msg += u"regex: {}\n".format(field)
logger.log_error(err_msg)
raise exception.ParamsError(err_msg)
# e.g. key: resp_headers_content_type, sub_query = "content-type"
return utils.query_json(top_query_content, sub_query)
else:
# e.g. key: resp_status_code, resp_content
return top_query_content
except AttributeError:
err_msg = u"Failed to extract value from response!\n"
err_msg += u"response content: {}\n".format(self.content)
err_msg += u"extract field: {}\n".format(field)
logger.log_error(err_msg)
raise exception.ParamsError(err_msg)
def run_test(self, testcase_dict):
""" run single testcase.
@param (dict) testcase_dict
{
"name": "testcase description",
"skip": "skip this test unconditionally",
"times": 3,
"requires": [], # optional, override
"function_binds": {}, # optional, override
"variables": [], # optional, override
"request": {
"url": "http://127.0.0.1:5000/api/users/1000",
"method": "POST",
"headers": {
"Content-Type": "application/json",
"authorization": "$authorization",
"random": "$random"
},
"body": '{"name": "user", "password": "******"}'
},
"extract": [], # optional
"validate": [], # optional
"setup": [], # optional
"teardown": [] # optional
}
@return True or raise exception during test
"""
parsed_request = self.init_config(testcase_dict, level="testcase")
try:
url = parsed_request.pop('url')
method = parsed_request.pop('method')
group_name = parsed_request.pop("group", None)
except KeyError:
raise exception.ParamsError("URL or METHOD missed!")
extractors = testcase_dict.get("extract", [])
validators = testcase_dict.get("validate", [])
setup_actions = testcase_dict.get("setup", [])
teardown_actions = testcase_dict.get("teardown", [])
self._handle_skip_feature(testcase_dict)
def setup_teardown(actions):
for action in actions:
self.context.eval_content(action)
setup_teardown(setup_actions)
resp = self.http_client_session.request(
method,
url,
name=group_name,
**parsed_request
)
resp_obj = response.ResponseObject(resp)
extracted_variables_mapping = resp_obj.extract_response(extractors)
self.context.bind_extracted_variables(extracted_variables_mapping)
try:
self.context.validate(validators, resp_obj)
except (exception.ParamsError, exception.ResponseError, exception.ValidationError):
# log request
err_req_msg = "request: \n"
err_req_msg += "headers: {}\n".format(parsed_request.pop("headers", {}))
for k, v in parsed_request.items():
err_req_msg += "{}: {}\n".format(k, v)
logger.log_error(err_req_msg)
# log response
err_resp_msg = "response: \n"
err_resp_msg += "status_code: {}\n".format(resp.status_code)
err_resp_msg += "headers: {}\n".format(resp.headers)
err_resp_msg += "body: {}\n".format(resp.text)
logger.log_error(err_resp_msg)
raise
finally:
setup_teardown(teardown_actions)
def run_test(self, testcase_dict):
""" run single testcase.
@param (dict) testcase_dict
{
"name": "testcase description",
"skip": "skip this test unconditionally",
"times": 3,
"requires": [], # optional, override
"function_binds": {}, # optional, override
"variables": [], # optional, override
"request": {
"url": "http://127.0.0.1:5000/api/users/1000",
"method": "POST",
"headers": {
"Content-Type": "application/json",
"authorization": "$authorization",
"random": "$random"
},
"body": '{"name": "user", "password": "******"}'
},
"extract": [
{"error_code": "errorcode"},
{"ID": "Data.AreaList.Area.ID"},
], # optional
"validate": [], # optional
"setup_hooks": [], # optional
"teardown_hooks": [] # optional
}
@return True or raise exception during test
"""
# check skip
self._handle_skip_feature(testcase_dict)
# prepare
parsed_request = self.init_config(testcase_dict, level="testcase")
self.context.bind_testcase_variable("request", parsed_request)
# setup hooks
setup_hooks = testcase_dict.get("setup_hooks", [])
setup_hooks.insert(0, "${setup_hook_prepare_kwargs($request)}")
self.do_hook_actions(setup_hooks)
try:
url = parsed_request.pop('url')
method = parsed_request.pop('method')
group_name = parsed_request.pop("group", None)
except KeyError:
raise exception.ParamsError("URL or METHOD missed!")
logger.log_info("{method} {url}".format(method=method, url=url))
logger.log_debug(
"request kwargs(raw): {kwargs}".format(kwargs=parsed_request))
# request
resp = self.http_client_session.request(method,
url,
name=group_name,
**parsed_request)
resp_obj = response.ResponseObject(resp)
# teardown hooks
teardown_hooks = testcase_dict.get("teardown_hooks", [])
if teardown_hooks:
self.context.bind_testcase_variable("response", resp_obj)
self.do_hook_actions(teardown_hooks)
# extract
extractors = testcase_dict.get("extract", []) or testcase_dict.get(
"extractors", [])
extracted_variables_mapping = resp_obj.extract_response(extractors)
self.context.bind_extracted_variables(extracted_variables_mapping)
# validate
validators = testcase_dict.get("validate", []) or testcase_dict.get(
"validators", [])
try:
self.context.validate(validators, resp_obj)
except (exception.ParamsError, exception.ResponseError, \
exception.ValidationError, exception.ParseResponseError):
# log request
err_req_msg = "request: \n"
err_req_msg += "headers: {}\n".format(
parsed_request.pop("headers", {}))
for k, v in parsed_request.items():
err_req_msg += "{}: {}\n".format(k, v)
logger.log_error(err_req_msg)
# log response
err_resp_msg = "response: \n"
err_resp_msg += "status_code: {}\n".format(resp_obj.status_code)
err_resp_msg += "headers: {}\n".format(resp_obj.headers)
err_resp_msg += "content: {}\n".format(resp_obj.content)
logger.log_error(err_resp_msg)
raise
def _run_test(self, testcase_dict):
""" run single testcase.
@param (dict) testcase_dict
{
"name": "testcase description",
"times": 3,
"requires": [], # optional, override
"function_binds": {}, # optional, override
"variables": [], # optional, override
"request": {
"url": "http://127.0.0.1:5000/api/users/1000",
"method": "POST",
"headers": {
"Content-Type": "application/json",
"authorization": "$authorization",
"random": "$random"
},
"body": '{"name": "user", "password": "******"}'
},
"extract": [], # optional
"validate": [], # optional
"setup": [], # optional
"teardown": [] # optional
}
@return True or raise exception during test
"""
parsed_request = self.init_config(testcase_dict, level="testcase")
try:
url = parsed_request.pop('url')
method = parsed_request.pop('method')
group_name = parsed_request.pop("group", None)
except KeyError:
raise exception.ParamsError("URL or METHOD missed!")
run_times = int(testcase_dict.get("times", 1))
extractors = testcase_dict.get("extract") \
or testcase_dict.get("extractors") \
or testcase_dict.get("extract_binds", [])
validators = testcase_dict.get("validate") \
or testcase_dict.get("validators", [])
setup_actions = testcase_dict.get("setup", [])
teardown_actions = testcase_dict.get("teardown", [])
def setup_teardown(actions):
for action in actions:
self.context.exec_content_functions(action)
for _ in range(run_times):
setup_teardown(setup_actions)
resp = self.http_client_session.request(method,
url,
name=group_name,
**parsed_request)
resp_obj = response.ResponseObject(resp)
extracted_variables_mapping = resp_obj.extract_response(extractors)
self.context.bind_extracted_variables(extracted_variables_mapping)
try:
self.context.validate(validators, resp_obj)
except (exception.ParamsError, exception.ResponseError,
exception.ValidationError):
err_msg = u"Exception occured.\n"
err_msg += u"HTTP request url: {}\n".format(url)
err_msg += u"HTTP request kwargs: {}\n".format(parsed_request)
err_msg += u"HTTP response status_code: {}\n".format(
resp.status_code)
err_msg += u"HTTP response content: \n{}".format(resp.text)
logging.error(err_msg)
raise
finally:
setup_teardown(teardown_actions)
return True
def load_test_file(file_path):
""" load testcase file or suite file
@param file_path: absolute valid file path
file_path should be in format below:
[
{
"config": {
"name": "",
"def": "suite_order()",
"request": {}
}
},
{
"test": {
"name": "add product to cart",
"api": "api_add_cart()",
"validate": []
}
},
{
"test": {
"name": "checkout cart",
"request": {},
"validate": []
}
}
]
@return testset dict
{
"name": "desc1",
"config": {},
"testcases": [testcase11, testcase12]
}
"""
testset = {
"name": "",
"config": {
"path": file_path
},
"testcases": [] # TODO: rename to tests
}
for item in FileUtils.load_file(file_path):
if not isinstance(item, dict) or len(item) != 1:
raise exception.FileFormatError(
"Testcase format error: {}".format(file_path))
key, test_block = item.popitem()
if not isinstance(test_block, dict):
raise exception.FileFormatError(
"Testcase format error: {}".format(file_path))
if key == "config":
testset["config"].update(test_block)
testset["name"] = test_block.get("name", "")
#add by zhengchun, can define runner
if "runner" in test_block:
# module_name, cls_name=item["config"]["runner"].split(".")[0:2]
s = test_block["runner"]
inx = s.rfind(".")
if (inx <= 0):
raise exception.ParamsError("runner format error[%s]" %
(s))
module_name = s[0:inx]
cls_name = s[inx + 1:]
imported_module = utils.get_imported_module(module_name)
ip_module_cls = getattr(imported_module, cls_name)
testset["runner"] = ip_module_cls
else:
#defalut runner
imported_module = utils.get_imported_module("runner")
ip_module_cls = getattr(imported_module, "Runner")
testset["runner"] = ip_module_cls
elif key == "test":
if "api" in test_block:
ref_call = test_block["api"]
def_block = TestcaseLoader._get_block_by_name(
ref_call, "api")
TestcaseLoader._override_block(def_block, test_block)
testset["testcases"].append(test_block)
elif "suite" in test_block:
ref_call = test_block["suite"]
block = TestcaseLoader._get_block_by_name(
ref_call, "suite")
testset["testcases"].extend(block["testcases"])
else:
testset["testcases"].append(test_block)
else:
logger.log_warning(
"unexpected block key: {}. block key should only be 'config' or 'test'."
.format(key))
return testset
def match_expected(value, expected, comparator="eq", check_item=""):
""" check if value matches expected value.
@param value: actual value that get from response.
@param expected: expected result described in testcase
@param comparator: compare method
@param check_item: check item name
"""
try:
if value is None or expected is None:
assert comparator in ["is", "eq", "equals", "=="]
assert value is None
assert expected is None
if comparator in ["eq", "equals", "=="]:
assert value == expected
elif comparator in ["lt", "less_than"]:
assert value < expected
elif comparator in ["le", "less_than_or_equals"]:
assert value <= expected
elif comparator in ["gt", "greater_than"]:
assert value > expected
elif comparator in ["ge", "greater_than_or_equals"]:
assert value >= expected
elif comparator in ["ne", "not_equals"]:
assert value != expected
elif comparator in ["str_eq", "string_equals"]:
assert str(value) == str(expected)
elif comparator in ["len_eq", "length_equals", "count_eq"]:
assert isinstance(expected, int)
assert len(value) == expected
elif comparator in [
"len_gt", "count_gt", "length_greater_than",
"count_greater_than"
]:
assert isinstance(expected, int)
assert len(value) > expected
elif comparator in ["len_ge", "count_ge", "length_greater_than_or_equals", \
"count_greater_than_or_equals"]:
assert isinstance(expected, int)
assert len(value) >= expected
elif comparator in [
"len_lt", "count_lt", "length_less_than", "count_less_than"
]:
assert isinstance(expected, int)
assert len(value) < expected
elif comparator in ["len_le", "count_le", "length_less_than_or_equals", \
"count_less_than_or_equals"]:
assert isinstance(expected, int)
assert len(value) <= expected
elif comparator in ["contains"]:
assert isinstance(value, (list, tuple, dict, string_type))
assert expected in value
elif comparator in ["contained_by"]:
assert isinstance(expected, (list, tuple, dict, string_type))
assert value in expected
elif comparator in ["type"]:
assert isinstance(value, expected)
elif comparator in ["regex"]:
assert isinstance(expected, string_type)
assert isinstance(value, string_type)
assert re.match(expected, value)
elif comparator in ["startswith"]:
assert str(value).startswith(str(expected))
elif comparator in ["endswith"]:
assert str(value).endswith(str(expected))
else:
raise exception.ParamsError("comparator not supported!")
return True
except (AssertionError, TypeError):
err_msg = "\n".join([
"check item name: %s;" % check_item,
"check item value: %s (%s);" % (value, type(value).__name__),
"comparator: %s;" % comparator,
"expected value: %s (%s)." % (expected, type(expected).__name__)
])
raise exception.ValidationError(err_msg)
def parse_validator(self, validator, resp_obj):
""" parse validator, validator maybe in two format
@param (dict) validator
format1: this is kept for compatiblity with the previous versions.
{"check": "status_code", "comparator": "eq", "expect": 201}
{"check": "$resp_body_success", "comparator": "eq", "expect": True}
format2: recommended new version
{'eq': ['status_code', 201]}
{'eq': ['$resp_body_success', True]}
@param (object) resp_obj
@return (dict) validator info
{
"check_item": check_item,
"check_value": check_value,
"expect_value": expect_value,
"comparator": comparator
}
"""
if not isinstance(validator, dict):
raise exception.ParamsError(
"invalid validator: {}".format(validator))
if "check" in validator and len(validator) > 1:
# format1
check_item = validator.get("check")
if "expect" in validator:
expect_value = validator.get("expect")
elif "expected" in validator:
expect_value = validator.get("expected")
else:
raise exception.ParamsError(
"invalid validator: {}".format(validator))
comparator = validator.get("comparator", "eq")
elif len(validator) == 1:
# format2
comparator = list(validator.keys())[0]
compare_values = validator[comparator]
if not isinstance(compare_values,
list) or len(compare_values) != 2:
raise exception.ParamsError(
"invalid validator: {}".format(validator))
check_item, expect_value = compare_values
else:
raise exception.ParamsError(
"invalid validator: {}".format(validator))
# check_item should only be in 3 type:
# 1, variable reference, e.g. $token
# 2, string joined by delimiter. e.g. "status_code", "headers.content-type"
# 3, regex string, e.g. "LB[\d]*(.*)RB[\d]*"
if testcase.extract_variables(check_item):
# type 1
check_value = self.testcase_parser.eval_content_variables(
check_item)
else:
try:
# type 2 or type 3
check_value = resp_obj.extract_field(check_item)
except exception.ParseResponseError:
raise exception.ParseResponseError(
"failed to extract check item in response!")
expect_value = self.testcase_parser.eval_content_variables(
expect_value)
validator_dict = {
"check_item": check_item,
"check_value": check_value,
"expect_value": expect_value,
"comparator": comparator
}
return validator_dict
