def validate(self, validators, resp_obj): """ make validations """ evaluated_validators = [] if not validators: return evaluated_validators logger.log_info("start to validate.") validate_pass = True failures = [] for validator in validators: # evaluate validators with context variable mapping. evaluated_validator = self.__eval_check_item( parser.parse_validator(validator), resp_obj) try: self._do_validation(evaluated_validator) except exceptions.ValidationFailure as ex: validate_pass = False failures.append(str(ex)) evaluated_validators.append(evaluated_validator) if not validate_pass: failures_string = "\n".join([failure for failure in failures]) raise exceptions.ValidationFailure(failures_string) return evaluated_validators
def _do_validation(self, validator_dict): """ validate with functions Args: validator_dict (dict): validator dict { "check": "status_code", "check_value": 200, "expect": 201, "comparator": "eq" } """ # TODO: move comparator uniform to init_test_suites comparator = utils.get_uniform_comparator(validator_dict["comparator"]) validate_func = self.TESTCASE_SHARED_FUNCTIONS_MAPPING.get(comparator) if not validate_func: raise exceptions.FunctionNotFound( "comparator not found: {}".format(comparator)) check_item = validator_dict["check"] check_value = validator_dict["check_value"] expect_value = validator_dict["expect"] if (check_value is None or expect_value is None) \ and comparator not in ["is", "eq", "equals", "=="]: raise exceptions.ParamsError( "Null value can only be compared with comparator: eq/equals/==" ) validate_msg = "validate: {} {} {}({})".format( check_item, comparator, expect_value, type(expect_value).__name__) try: validator_dict["check_result"] = "pass" validate_func(check_value, expect_value) validate_msg += "\t==> pass" logger.log_debug(validate_msg) except (AssertionError, TypeError): validate_msg += "\t==> fail" validate_msg += "\n{}({}) {} {}({})".format( check_value, type(check_value).__name__, comparator, expect_value, type(expect_value).__name__) logger.log_error(validate_msg) validator_dict["check_result"] = "fail" raise exceptions.ValidationFailure(validate_msg)
def validate(self, validators): """ make validation with comparators """ self.validation_results = {} if not validators: return logger.debug("start to validate.") validate_pass = True failures = [] for validator in validators: if isinstance(validator, dict) and validator.get("type") == "python_script": script = self.session_context.eval_content(validator["script"]) result = self.validate_script(script) if result["check_result"] == "fail": validate_pass = False failures.append(result["output"]) self.validation_results["validate_script"] = result continue if "validate_extractor" not in self.validation_results: self.validation_results["validate_extractor"] = [] # validator should be LazyFunction object if not isinstance(validator, parser.LazyFunction): raise exceptions.ValidationFailure( f"validator should be parsed first: {validators}") # evaluate validator args with context variable mapping. validator_args = validator.get_args() check_item, expect_item = validator_args check_value = self.__eval_validator_check(check_item) expect_value = self.__eval_validator_expect(expect_item) validator.update_args([check_value, expect_value]) comparator = validator.func_name validator_dict = { "comparator": comparator, "check": check_item, "check_value": check_value, "expect": expect_item, "expect_value": expect_value } validate_msg = f"\nvalidate: {check_item} {comparator} {expect_value}({type(expect_value).__name__})" try: validator.to_value(self.session_context.test_variables_mapping) validator_dict["check_result"] = "pass" validate_msg += "\t==> pass" logger.debug(validate_msg) except (AssertionError, TypeError): validate_pass = False validator_dict["check_result"] = "fail" validate_msg += "\t==> fail" validate_msg += "\n{}({}) {} {}({})".format( check_value, type(check_value).__name__, comparator, expect_value, type(expect_value).__name__) logger.error(validate_msg) failures.append(validate_msg) self.validation_results["validate_extractor"].append( validator_dict) # restore validator args, in case of running multiple times validator.update_args(validator_args) if not validate_pass: failures_string = "\n".join([failure for failure in failures]) raise exceptions.ValidationFailure(failures_string)
def validate(self, validators, resp_obj): """ make validation with comparators """ self.validation_results = [] if not validators: return logger.log_debug("start to validate.") validate_pass = True failures = [] for validator in validators: # validator should be LazyFunction object if not isinstance(validator, parser.LazyFunction): raise exceptions.ValidationFailure( "validator should be parsed first: {}".format(validators)) # evaluate validator args with context variable mapping. validator_args = validator.get_args() check_item, expect_item = validator_args check_value = self.__eval_validator_check(check_item, resp_obj) expect_value = self.__eval_validator_expect(expect_item) validator.update_args([check_value, expect_value]) comparator = validator.func_name validator_dict = { "comparator": comparator, "check": check_item, "check_value": check_value, "expect": expect_item, "expect_value": expect_value } validate_msg = "\nvalidate: {} {} {}({})".format( check_item, comparator, expect_value, type(expect_value).__name__) try: validator.to_value(self.test_variables_mapping) validator_dict["check_result"] = "pass" validate_msg += "\t==> pass" logger.log_debug(validate_msg) except (AssertionError, TypeError): validate_pass = False validator_dict["check_result"] = "fail" validate_msg += "\t==> fail" validate_msg += "\n{}({}) {} {}({})".format( check_value, type(check_value).__name__, comparator, expect_value, type(expect_value).__name__) logger.log_error(validate_msg) failures.append(validate_msg) self.validation_results.append(validator_dict) # restore validator args, in case of running multiple times validator.update_args(validator_args) if not validate_pass: failures_string = "\n".join([failure for failure in failures]) raise exceptions.ValidationFailure(failures_string)
def _run_test(self, test_dict): """ run single teststep. Args: test_dict (dict): teststep info { "name": "teststep description", "skip": "skip this test unconditionally", "times": 3, "variables": [], # optional, override "request": { "url": "http://127.0.0.1:5000/api/users/1000", "method": "POST", "headers": { "Content-Type": "application/json", "authorization": "$authorization", "random": "$random" }, "json": {"name": "user", "password": "******"} }, "extract": {}, # optional "validate": [], # optional "setup_hooks": [], # optional "teardown_hooks": [], # optional "wait":{} # optional } Raises: exceptions.ParamsError exceptions.ValidationFailure exceptions.ExtractFailure """ # get wait parameter, if this step result needs to waif for a right one wait_params = test_dict.get("wait", {}) validate_pass = None failures = [] if wait_params: wait_total_time = wait_params["Total_Time"] if wait_params.has_key( 'Total_Time') else 300 wait_interval = wait_params["Interval"] if wait_params.has_key( 'Interval') else 5 # validate for wait function wait_validators = wait_params.get("validate", []) for item in [wait_total_time, wait_interval]: if not isinstance(item, int): err_msg = u"Invalid wait function parameters! => {}\n".format( item) logger.log_error(err_msg) raise exceptions.ParamsError(err_msg) start_time = time.time() current_time = time.time() try: while current_time - start_time < wait_total_time: validate_pass, failures = self._run_test_once( test_dict=test_dict, wait_validators=wait_validators) # to break while when validate is OK if validate_pass: break current_time = time.time() time.sleep(wait_interval) # to raise alarm when failures except (exceptions.ParamsError, exceptions.ValidationFailure, exceptions.ExtractFailure): raise else: validate_pass, failures = self._run_test_once(test_dict=test_dict) if not validate_pass: failures_string = "\n".join([failure for failure in failures]) raise exceptions.ValidationFailure(failures_string)
def _run_test_once(self, test_dict, wait_validators=None): # clear meta data first to ensure independence for each test self.__clear_test_data() # check skip self._handle_skip_feature(test_dict) # prepare test_dict = utils.lower_test_dict_keys(test_dict) test_variables = test_dict.get("variables", {}) self.session_context.init_test_variables(test_variables) # teststep name test_name = test_dict.get("name", "") # parse test request raw_request = test_dict.get('request', {}) parsed_test_request = self.session_context.eval_content(raw_request) self.session_context.update_test_variables("request", parsed_test_request) # setup hooks setup_hooks = test_dict.get("setup_hooks", []) if setup_hooks: self.do_hook_actions(setup_hooks, "setup") try: url = parsed_test_request.pop('url') method = parsed_test_request.pop('method') parsed_test_request.setdefault("verify", self.verify) group_name = parsed_test_request.pop("group", None) except KeyError: raise exceptions.ParamsError("URL or METHOD missed!") # TODO: move method validation to json schema valid_methods = [ "GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS" ] if method.upper() not in valid_methods: err_msg = u"Invalid HTTP method! => {}\n".format(method) err_msg += "Available HTTP methods: {}".format( "/".join(valid_methods)) logger.log_error(err_msg) raise exceptions.ParamsError(err_msg) logger.log_info("{method} {url}".format(method=method, url=url)) logger.log_debug( "request kwargs(raw): {kwargs}".format(kwargs=parsed_test_request)) # request resp = self.http_client_session.request(method, url, name=(group_name or test_name), **parsed_test_request) resp_obj = response.ResponseObject(resp) # teardown hooks teardown_hooks = test_dict.get("teardown_hooks", []) if teardown_hooks: self.session_context.update_test_variables("response", resp_obj) self.do_hook_actions(teardown_hooks, "teardown") # extract extractors = test_dict.get("extract", {}) extracted_variables_mapping = resp_obj.extract_response(extractors) self.session_context.update_session_variables( extracted_variables_mapping) # validate if wait_validators: wait_validate_pass, wait_failures = self.session_context.validate( wait_validators, resp_obj) if not wait_validate_pass: # err_msg = "{} DETAILED REQUEST & RESPONSE {}\n".format("*" * 32, "*" * 32) # # # log request # err_msg += "====== request details ======\n" # err_msg += "url: {}\n".format(url) # err_msg += "method: {}\n".format(method) # err_msg += "headers: {}\n".format(parsed_test_request.pop("headers", {})) # for k, v in parsed_test_request.items(): # v = utils.omit_long_data(v) # err_msg += "{}: {}\n".format(k, repr(v)) # err_msg += "\n" # log response # err_msg += "====== response details ======\n" # err_msg += "status_code: {}\n".format(resp_obj.status_code) # err_msg += "headers: {}\n".format(resp_obj.headers) # err_msg += "body: {}\n".format(repr(resp_obj.text)) # logger.log_error("During the wait: \n" + err_msg) logger.log_error( "====== During the wait: the expect conditions can not Meet the requirements! ======" ) failures_string = "\n".join( [failure for failure in wait_failures]) self.validation_results = self.session_context.validation_results raise exceptions.ValidationFailure(failures_string) validators = test_dict.get("validate", []) validate_pass, failures = self.session_context.validate( validators, resp_obj) self.validation_results = self.session_context.validation_results # try: # validate_pass, failures = self.session_context.validate(validators, resp_obj) # except (exceptions.ParamsError, exceptions.ValidationFailure, exceptions.ExtractFailure): # err_msg = "{} DETAILED REQUEST & RESPONSE {}\n".format("*" * 32, "*" * 32) # # # log request # err_msg += "====== request details ======\n" # err_msg += "url: {}\n".format(url) # err_msg += "method: {}\n".format(method) # err_msg += "headers: {}\n".format(parsed_test_request.pop("headers", {})) # for k, v in parsed_test_request.items(): # v = utils.omit_long_data(v) # err_msg += "{}: {}\n".format(k, repr(v)) # # err_msg += "\n" # # # log response # err_msg += "====== response details ======\n" # err_msg += "status_code: {}\n".format(resp_obj.status_code) # err_msg += "headers: {}\n".format(resp_obj.headers) # err_msg += "body: {}\n".format(repr(resp_obj.text)) # logger.log_error(err_msg) # raise # finally: # self.validation_results = self.session_context.validation_results return validate_pass, failures