def validate(self, validators, resp_obj):
""" make validations
"""
evaluated_validators = []
if not validators:
return evaluated_validators
logger.log_info("start to validate.")
validate_pass = True
failures = []
for validator in validators:
# evaluate validators with context variable mapping.
evaluated_validator = self.__eval_check_item(
parser.parse_validator(validator), resp_obj)
try:
self._do_validation(evaluated_validator)
except exceptions.ValidationFailure as ex:
validate_pass = False
failures.append(str(ex))
evaluated_validators.append(evaluated_validator)
if not validate_pass:
failures_string = "\n".join([failure for failure in failures])
raise exceptions.ValidationFailure(failures_string)
return evaluated_validators
def _do_validation(self, validator_dict):
""" validate with functions
Args:
validator_dict (dict): validator dict
{
"check": "status_code",
"check_value": 200,
"expect": 201,
"comparator": "eq"
}
"""
# TODO: move comparator uniform to init_test_suites
comparator = utils.get_uniform_comparator(validator_dict["comparator"])
validate_func = self.TESTCASE_SHARED_FUNCTIONS_MAPPING.get(comparator)
if not validate_func:
raise exceptions.FunctionNotFound(
"comparator not found: {}".format(comparator))
check_item = validator_dict["check"]
check_value = validator_dict["check_value"]
expect_value = validator_dict["expect"]
if (check_value is None or expect_value is None) \
and comparator not in ["is", "eq", "equals", "=="]:
raise exceptions.ParamsError(
"Null value can only be compared with comparator: eq/equals/=="
)
validate_msg = "validate: {} {} {}({})".format(
check_item, comparator, expect_value,
type(expect_value).__name__)
try:
validator_dict["check_result"] = "pass"
validate_func(check_value, expect_value)
validate_msg += "\t==> pass"
logger.log_debug(validate_msg)
except (AssertionError, TypeError):
validate_msg += "\t==> fail"
validate_msg += "\n{}({}) {} {}({})".format(
check_value,
type(check_value).__name__, comparator, expect_value,
type(expect_value).__name__)
logger.log_error(validate_msg)
validator_dict["check_result"] = "fail"
raise exceptions.ValidationFailure(validate_msg)
def validate(self, validators):
""" make validation with comparators
"""
self.validation_results = {}
if not validators:
return
logger.debug("start to validate.")
validate_pass = True
failures = []
for validator in validators:
if isinstance(validator,
dict) and validator.get("type") == "python_script":
script = self.session_context.eval_content(validator["script"])
result = self.validate_script(script)
if result["check_result"] == "fail":
validate_pass = False
failures.append(result["output"])
self.validation_results["validate_script"] = result
continue
if "validate_extractor" not in self.validation_results:
self.validation_results["validate_extractor"] = []
# validator should be LazyFunction object
if not isinstance(validator, parser.LazyFunction):
raise exceptions.ValidationFailure(
f"validator should be parsed first: {validators}")
# evaluate validator args with context variable mapping.
validator_args = validator.get_args()
check_item, expect_item = validator_args
check_value = self.__eval_validator_check(check_item)
expect_value = self.__eval_validator_expect(expect_item)
validator.update_args([check_value, expect_value])
comparator = validator.func_name
validator_dict = {
"comparator": comparator,
"check": check_item,
"check_value": check_value,
"expect": expect_item,
"expect_value": expect_value
}
validate_msg = f"\nvalidate: {check_item} {comparator} {expect_value}({type(expect_value).__name__})"
try:
validator.to_value(self.session_context.test_variables_mapping)
validator_dict["check_result"] = "pass"
validate_msg += "\t==> pass"
logger.debug(validate_msg)
except (AssertionError, TypeError):
validate_pass = False
validator_dict["check_result"] = "fail"
validate_msg += "\t==> fail"
validate_msg += "\n{}({}) {} {}({})".format(
check_value,
type(check_value).__name__, comparator, expect_value,
type(expect_value).__name__)
logger.error(validate_msg)
failures.append(validate_msg)
self.validation_results["validate_extractor"].append(
validator_dict)
# restore validator args, in case of running multiple times
validator.update_args(validator_args)
if not validate_pass:
failures_string = "\n".join([failure for failure in failures])
raise exceptions.ValidationFailure(failures_string)
def validate(self, validators, resp_obj):
""" make validation with comparators
"""
self.validation_results = []
if not validators:
return
logger.log_debug("start to validate.")
validate_pass = True
failures = []
for validator in validators:
# validator should be LazyFunction object
if not isinstance(validator, parser.LazyFunction):
raise exceptions.ValidationFailure(
"validator should be parsed first: {}".format(validators))
# evaluate validator args with context variable mapping.
validator_args = validator.get_args()
check_item, expect_item = validator_args
check_value = self.__eval_validator_check(check_item, resp_obj)
expect_value = self.__eval_validator_expect(expect_item)
validator.update_args([check_value, expect_value])
comparator = validator.func_name
validator_dict = {
"comparator": comparator,
"check": check_item,
"check_value": check_value,
"expect": expect_item,
"expect_value": expect_value
}
validate_msg = "\nvalidate: {} {} {}({})".format(
check_item, comparator, expect_value,
type(expect_value).__name__)
try:
validator.to_value(self.test_variables_mapping)
validator_dict["check_result"] = "pass"
validate_msg += "\t==> pass"
logger.log_debug(validate_msg)
except (AssertionError, TypeError):
validate_pass = False
validator_dict["check_result"] = "fail"
validate_msg += "\t==> fail"
validate_msg += "\n{}({}) {} {}({})".format(
check_value,
type(check_value).__name__, comparator, expect_value,
type(expect_value).__name__)
logger.log_error(validate_msg)
failures.append(validate_msg)
self.validation_results.append(validator_dict)
# restore validator args, in case of running multiple times
validator.update_args(validator_args)
if not validate_pass:
failures_string = "\n".join([failure for failure in failures])
raise exceptions.ValidationFailure(failures_string)
def _run_test(self, test_dict):
""" run single teststep.
Args:
test_dict (dict): teststep info
{
"name": "teststep description",
"skip": "skip this test unconditionally",
"times": 3,
"variables": [], # optional, override
"request": {
"url": "http://127.0.0.1:5000/api/users/1000",
"method": "POST",
"headers": {
"Content-Type": "application/json",
"authorization": "$authorization",
"random": "$random"
},
"json": {"name": "user", "password": "******"}
},
"extract": {}, # optional
"validate": [], # optional
"setup_hooks": [], # optional
"teardown_hooks": [], # optional
"wait":{} # optional
}
Raises:
exceptions.ParamsError
exceptions.ValidationFailure
exceptions.ExtractFailure
"""
# get wait parameter, if this step result needs to waif for a right one
wait_params = test_dict.get("wait", {})
validate_pass = None
failures = []
if wait_params:
wait_total_time = wait_params["Total_Time"] if wait_params.has_key(
'Total_Time') else 300
wait_interval = wait_params["Interval"] if wait_params.has_key(
'Interval') else 5
# validate for wait function
wait_validators = wait_params.get("validate", [])
for item in [wait_total_time, wait_interval]:
if not isinstance(item, int):
err_msg = u"Invalid wait function parameters! => {}\n".format(
item)
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
start_time = time.time()
current_time = time.time()
try:
while current_time - start_time < wait_total_time:
validate_pass, failures = self._run_test_once(
test_dict=test_dict, wait_validators=wait_validators)
# to break while when validate is OK
if validate_pass:
break
current_time = time.time()
time.sleep(wait_interval)
# to raise alarm when failures
except (exceptions.ParamsError, exceptions.ValidationFailure,
exceptions.ExtractFailure):
raise
else:
validate_pass, failures = self._run_test_once(test_dict=test_dict)
if not validate_pass:
failures_string = "\n".join([failure for failure in failures])
raise exceptions.ValidationFailure(failures_string)
def _run_test_once(self, test_dict, wait_validators=None):
# clear meta data first to ensure independence for each test
self.__clear_test_data()
# check skip
self._handle_skip_feature(test_dict)
# prepare
test_dict = utils.lower_test_dict_keys(test_dict)
test_variables = test_dict.get("variables", {})
self.session_context.init_test_variables(test_variables)
# teststep name
test_name = test_dict.get("name", "")
# parse test request
raw_request = test_dict.get('request', {})
parsed_test_request = self.session_context.eval_content(raw_request)
self.session_context.update_test_variables("request",
parsed_test_request)
# setup hooks
setup_hooks = test_dict.get("setup_hooks", [])
if setup_hooks:
self.do_hook_actions(setup_hooks, "setup")
try:
url = parsed_test_request.pop('url')
method = parsed_test_request.pop('method')
parsed_test_request.setdefault("verify", self.verify)
group_name = parsed_test_request.pop("group", None)
except KeyError:
raise exceptions.ParamsError("URL or METHOD missed!")
# TODO: move method validation to json schema
valid_methods = [
"GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"
]
if method.upper() not in valid_methods:
err_msg = u"Invalid HTTP method! => {}\n".format(method)
err_msg += "Available HTTP methods: {}".format(
"/".join(valid_methods))
logger.log_error(err_msg)
raise exceptions.ParamsError(err_msg)
logger.log_info("{method} {url}".format(method=method, url=url))
logger.log_debug(
"request kwargs(raw): {kwargs}".format(kwargs=parsed_test_request))
# request
resp = self.http_client_session.request(method,
url,
name=(group_name or test_name),
**parsed_test_request)
resp_obj = response.ResponseObject(resp)
# teardown hooks
teardown_hooks = test_dict.get("teardown_hooks", [])
if teardown_hooks:
self.session_context.update_test_variables("response", resp_obj)
self.do_hook_actions(teardown_hooks, "teardown")
# extract
extractors = test_dict.get("extract", {})
extracted_variables_mapping = resp_obj.extract_response(extractors)
self.session_context.update_session_variables(
extracted_variables_mapping)
# validate
if wait_validators:
wait_validate_pass, wait_failures = self.session_context.validate(
wait_validators, resp_obj)
if not wait_validate_pass:
# err_msg = "{} DETAILED REQUEST & RESPONSE {}\n".format("*" * 32, "*" * 32)
#
# # log request
# err_msg += "====== request details ======\n"
# err_msg += "url: {}\n".format(url)
# err_msg += "method: {}\n".format(method)
# err_msg += "headers: {}\n".format(parsed_test_request.pop("headers", {}))
# for k, v in parsed_test_request.items():
# v = utils.omit_long_data(v)
# err_msg += "{}: {}\n".format(k, repr(v))
# err_msg += "\n"
# log response
# err_msg += "====== response details ======\n"
# err_msg += "status_code: {}\n".format(resp_obj.status_code)
# err_msg += "headers: {}\n".format(resp_obj.headers)
# err_msg += "body: {}\n".format(repr(resp_obj.text))
# logger.log_error("During the wait: \n" + err_msg)
logger.log_error(
"====== During the wait: the expect conditions can not Meet the requirements! ======"
)
failures_string = "\n".join(
[failure for failure in wait_failures])
self.validation_results = self.session_context.validation_results
raise exceptions.ValidationFailure(failures_string)
validators = test_dict.get("validate", [])
validate_pass, failures = self.session_context.validate(
validators, resp_obj)
self.validation_results = self.session_context.validation_results
# try:
# validate_pass, failures = self.session_context.validate(validators, resp_obj)
# except (exceptions.ParamsError, exceptions.ValidationFailure, exceptions.ExtractFailure):
# err_msg = "{} DETAILED REQUEST & RESPONSE {}\n".format("*" * 32, "*" * 32)
#
# # log request
# err_msg += "====== request details ======\n"
# err_msg += "url: {}\n".format(url)
# err_msg += "method: {}\n".format(method)
# err_msg += "headers: {}\n".format(parsed_test_request.pop("headers", {}))
# for k, v in parsed_test_request.items():
# v = utils.omit_long_data(v)
# err_msg += "{}: {}\n".format(k, repr(v))
#
# err_msg += "\n"
#
# # log response
# err_msg += "====== response details ======\n"
# err_msg += "status_code: {}\n".format(resp_obj.status_code)
# err_msg += "headers: {}\n".format(resp_obj.headers)
# err_msg += "body: {}\n".format(repr(resp_obj.text))
# logger.log_error(err_msg)
# raise
# finally:
# self.validation_results = self.session_context.validation_results
return validate_pass, failures