def run(self, path_or_testsets, mapping=None):
""" start to run test with varaibles mapping
@param path_or_testsets: YAML/JSON testset file path or testset list
path: path could be in several type
- absolute/relative file path
- absolute/relative folder path
- list/set container with file(s) and/or folder(s)
testsets: testset or list of testset
- (dict) testset_dict
- (list) list of testset_dict
[
testset_dict_1,
testset_dict_2
]
@param (dict) mapping:
if mapping specified, it will override variables in config block
"""
try:
test_suite_list = init_test_suites(path_or_testsets, mapping)
except exceptions.TestcaseNotFound:
logger.log_error(
"Testcases not found in {}".format(path_or_testsets))
sys.exit(1)
self.summary = {"success": True, "stat": {}, "time": {}, "details": []}
def accumulate_stat(origin_stat, new_stat):
""" accumulate new_stat to origin_stat
"""
for key in new_stat:
if key not in origin_stat:
origin_stat[key] = new_stat[key]
elif key == "start_at":
# start datetime
origin_stat[key] = min(origin_stat[key], new_stat[key])
else:
origin_stat[key] += new_stat[key]
for test_suite in test_suite_list:
result = self.runner.run(test_suite)
# test_suite_summary = get_summary(result)
#
# self.summary["success"] &= test_suite_summary["success"]
# test_suite_summary["name"] = test_suite.config.get("name")
# test_suite_summary["base_url"] = test_suite.config.get("request", {}).get("base_url", "")
# test_suite_summary["output"] = test_suite.output
# print_output(test_suite_summary["output"])
#
# accumulate_stat(self.summary["stat"], test_suite_summary["stat"])
# accumulate_stat(self.summary["time"], test_suite_summary["time"])
#
# self.summary["details"].append(test_suite_summary)
return self
def _load_json_file(json_file):
""" load json file and check file content format
"""
with io.open(json_file, encoding='utf-8') as data_file:
try:
json_content = json.load(data_file)
except exceptions.JSONDecodeError:
err_msg = u"JSONDecodeError: JSON file format error: {}".format(
json_file)
logger.log_error(err_msg)
raise exceptions.FileFormatError(err_msg)
FileUtils._check_format(json_file, json_content)
return json_content
def load_testsets_by_path(path):
""" load testcases from file path
@param path: path could be in several type
- absolute/relative file path
- absolute/relative folder path
- list/set container with file(s) and/or folder(s)
@return testcase sets list, each testset is corresponding to a file
[
testset_dict_1,
testset_dict_2
]
"""
if isinstance(path, (list, set)):
testsets = []
for file_path in set(path):
testset = TestcaseLoader.load_testsets_by_path(file_path)
if not testset:
continue
testsets.extend(testset)
return testsets
if not os.path.isabs(path):
path = os.path.join(os.getcwd(), path)
if path in TestcaseLoader.testcases_cache_mapping:
return TestcaseLoader.testcases_cache_mapping[path]
if os.path.isdir(path):
files_list = FileUtils.load_folder_files(path)
testcases_list = TestcaseLoader.load_testsets_by_path(files_list)
elif os.path.isfile(path):
try:
testset = TestcaseLoader.load_test_file(path)
if testset["testcases"] or testset["api"]:
testcases_list = [testset]
else:
testcases_list = []
except exceptions.FileFormatError:
testcases_list = []
else:
logger.log_error(u"file not found: {}".format(path))
testcases_list = []
TestcaseLoader.testcases_cache_mapping[path] = testcases_list
return testcases_list
def _check_format(file_path, content):
""" check testcase format if valid
"""
if not content:
# testcase file content is empty
err_msg = u"Testcase file content is empty: {}".format(file_path)
logger.log_error(err_msg)
raise exceptions.FileFormatError(err_msg)
elif not isinstance(content, (list, dict)):
# testcase file content does not match testcase format
err_msg = u"Testcase file content format invalid: {}".format(
file_path)
logger.log_error(err_msg)
raise exceptions.FileFormatError(err_msg)
def query_json(json_content, query, delimiter='.'):
""" Do an xpath-like query with json_content.
@param (dict/list/string) json_content
json_content = {
"ids": [1, 2, 3, 4],
"person": {
"name": {
"first_name": "Leo",
"last_name": "Lee",
},
"age": 29,
"cities": ["Guangzhou", "Shenzhen"]
}
}
@param (str) query
"person.name.first_name" => "Leo"
"person.name.first_name.0" => "L"
"person.cities.0" => "Guangzhou"
@return queried result
"""
raise_flag = False
response_body = u"response body: {}\n".format(json_content)
try:
for key in query.split(delimiter):
if isinstance(json_content, (list, basestring)):
json_content = json_content[int(key)]
elif isinstance(json_content, dict):
json_content = json_content[key]
else:
logger.log_error("invalid type value: {}({})".format(
json_content, type(json_content)))
raise_flag = True
except (KeyError, ValueError, IndexError):
raise_flag = True
if raise_flag:
err_msg = u"Failed to extract! => {}\n".format(query)
err_msg += response_body
logger.log_error(err_msg)
raise exceptions.ExtractFailure(err_msg)
return json_content
def do_validation(self, validator_dict):
""" validate with functions
"""
# TODO: move comparator uniform to init_test_suites
comparator = utils.get_uniform_comparator(validator_dict["comparator"])
validate_func = self.testcase_parser.get_bind_function(comparator)
if not validate_func:
raise exceptions.FunctionNotFound(
"comparator not found: {}".format(comparator))
check_item = validator_dict["check"]
check_value = validator_dict["check_value"]
expect_value = validator_dict["expect"]
if (check_value is None or expect_value is None) \
and comparator not in ["is", "eq", "equals", "=="]:
raise exceptions.ParamsError(
"Null value can only be compared with comparator: eq/equals/=="
)
validate_msg = "validate: {} {} {}({})".format(
check_item, comparator, expect_value,
type(expect_value).__name__)
try:
validator_dict["check_result"] = "pass"
validate_func(check_value, expect_value)
validate_msg += "\t==> pass"
logger.log_debug(validate_msg)
except (AssertionError, TypeError):
validate_msg += "\t==> fail"
validate_msg += "\n{}({}) {} {}({})".format(
check_value,
type(check_value).__name__, comparator, expect_value,
type(expect_value).__name__)
logger.log_error(validate_msg)
validator_dict["check_result"] = "fail"
raise exceptions.ValidationFailure(validate_msg)
def run_test(self, testcase_dict):
""" run single testcase.
@param (dict) testcase_dict
{
"name": "testcase description",
"skip": "skip this test unconditionally",
"times": 3,
"variables": [], # optional, override
"request": {
"url": "http://127.0.0.1:5000/api/users/1000",
"method": "POST",
"headers": {
"Content-Type": "application/json",
"authorization": "$authorization",
"random": "$random"
},
"body": '{"name": "user", "password": "******"}'
},
"extract": [], # optional
"validate": [], # optional
"setup_hooks": [], # optional
"teardown_hooks": [] # optional
}
@return True or raise exception during test
"""
# check skip
self._handle_skip_feature(testcase_dict)
# prepare
parsed_request = self.init_config(testcase_dict, level="testcase")
self.context.bind_testcase_variable("request", parsed_request)
# setup hooks
setup_hooks = testcase_dict.get("setup_hooks", [])
setup_hooks.insert(0, "${setup_hook_prepare_kwargs($request)}")
self.do_hook_actions(setup_hooks)
try:
url = parsed_request.pop('url')
method = parsed_request.pop('method')
group_name = parsed_request.pop("group", None)
except KeyError:
raise exceptions.ParamsError("URL or METHOD missed!")
# TODO: move method validation to json schema
valid_methods = [
"GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"
]
if method.upper() not in valid_methods:
err_msg = u"Invalid HTTP method! => {}\n".format(method)
err_msg += "Available HTTP methods: {}".format(
"/".join(valid_methods))
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
logger.log_info("{method} {url}".format(method=method, url=url))
logger.log_debug(
"request kwargs(raw): {kwargs}".format(kwargs=parsed_request))
# request
resp = self.http_client_session.request(method,
url,
name=group_name,
**parsed_request)
resp_obj = response.ResponseObject(resp)
# teardown hooks
teardown_hooks = testcase_dict.get("teardown_hooks", [])
if teardown_hooks:
self.context.bind_testcase_variable("response", resp_obj)
self.do_hook_actions(teardown_hooks)
# extract
extractors = testcase_dict.get("extract", []) or testcase_dict.get(
"extractors", [])
extracted_variables_mapping = resp_obj.extract_response(extractors)
self.context.bind_extracted_variables(extracted_variables_mapping)
# validate
validators = testcase_dict.get("validate", []) or testcase_dict.get(
"validators", [])
try:
self.context.validate(validators, resp_obj)
except (exceptions.ParamsError, \
exceptions.ValidationFailure, exceptions.ExtractFailure):
# log request
err_req_msg = "request: \n"
err_req_msg += "headers: {}\n".format(
parsed_request.pop("headers", {}))
for k, v in parsed_request.items():
err_req_msg += "{}: {}\n".format(k, repr(v))
logger.log_error(err_req_msg)
# log response
err_resp_msg = "response: \n"
err_resp_msg += "status_code: {}\n".format(resp_obj.status_code)
err_resp_msg += "headers: {}\n".format(resp_obj.headers)
err_resp_msg += "body: {}\n".format(repr(resp_obj.text))
logger.log_error(err_resp_msg)
raise