def runtest(self): # Do a deep copy because this sometimes still retains things from previous tests(?) self.global_cfg = copy.deepcopy(load_global_cfg(self.config)) self.global_cfg.setdefault("variables", {}) load_plugins(self.global_cfg) self.global_cfg["tavern_internal"] = {"pytest_hook_caller": self.config.hook} # INTERNAL # NOTE - now that we can 'mark' tests, we could use pytest.mark.xfail # instead. This doesn't differentiate between an error in verification # and an error when running the test though. xfail = self.spec.get("_xfail", False) try: fixture_values = self._load_fixture_values() self.global_cfg["variables"].update(fixture_values) call_hook( self.global_cfg, "pytest_tavern_beta_before_every_test_run", test_dict=self.spec, variables=self.global_cfg["variables"], ) verify_tests(self.spec) for stage in self.spec["stages"]: if not stage.get("name"): if not stage.get("id"): # Should never actually reach here, should be caught at schema check time raise exceptions.BadSchemaError( "One of name or ID must be specified" ) stage["name"] = stage["id"] run_test(self.path, self.spec, self.global_cfg) except exceptions.BadSchemaError: if xfail == "verify": logger.info("xfailing test while verifying schema") else: raise except exceptions.TavernException: if xfail == "run": logger.info("xfailing test when running") else: raise else: if xfail: logger.error("Expected test to fail") raise exceptions.TestFailError( "Expected test to fail at {} stage".format(xfail) )
def runtest(self): self.global_cfg = load_global_cfg(self.config) self.global_cfg.setdefault("variables", {}) load_plugins(self.global_cfg) self.global_cfg["tavern_internal"] = { "pytest_hook_caller": self.config.hook } # INTERNAL # NOTE - now that we can 'mark' tests, we could use pytest.mark.xfail # instead. This doesn't differentiate between an error in verification # and an error when running the test though. xfail = self.spec.get("_xfail", False) try: fixture_values = self._load_fixture_values() self.global_cfg["variables"].update(fixture_values) call_hook( self.global_cfg, "pytest_tavern_beta_before_every_test_run", test_dict=self.spec, variables=self.global_cfg["variables"], ) verify_tests(self.spec) run_test(self.path, self.spec, self.global_cfg) except exceptions.BadSchemaError: if xfail == "verify": logger.info("xfailing test while verifying schema") else: raise except exceptions.TavernException: if xfail == "run": logger.info("xfailing test when running") else: raise else: if xfail: logger.error("Expected test to fail") raise exceptions.TestFailError( "Expected test to fail at {} stage".format(xfail))
def verify(self, response): """Verify response against expected values and returns any values that we wanted to save for use in future requests There are various ways to 'validate' a block - a specific function, just matching values, validating a schema, etc... Args: response (requests.Response): response object Returns: dict: Any saved values Raises: TestFailError: Something went wrong with validating the response """ self._verbose_log_response(response) call_hook( self.test_block_config, "pytest_tavern_beta_after_every_response", expected=self.expected, response=response, ) self.response = response self.status_code = response.status_code # Get things to use from the response try: body = response.json() except ValueError: body = None redirect_query_params = self._get_redirect_query_params(response) # Run validation on response self._check_status_code(response.status_code, body) self._validate_block("json", body) self._validate_block("headers", response.headers) self._validate_block("redirect_query_params", redirect_query_params) self._maybe_run_validate_functions(response) # Get any keys to save saved = {} saved.update(self.maybe_get_save_values_from_save_block("json", body)) saved.update( self.maybe_get_save_values_from_save_block("headers", response.headers)) saved.update( self.maybe_get_save_values_from_save_block("redirect_query_params", redirect_query_params)) saved.update( self.maybe_get_save_values_from_ext(response, self.expected)) # Check cookies for cookie in self.expected.get("cookies", []): if cookie not in response.cookies: self._adderr("No cookie named '%s' in response", cookie) if self.errors: raise exceptions.TestFailError( "Test '{:s}' failed:\n{:s}".format(self.name, self._str_errors()), failures=self.errors, ) return saved
def _await_response(self): """Actually wait for response""" # pylint: disable=too-many-statements topic = self.expected["topic"] timeout = self.expected.get("timeout", 1) expected_payload, expect_json_payload = self._get_payload_vals() # Any warnings to do with the request # eg, if a message was received but it didn't match, message had payload, etc. warnings = [] def addwarning(w, *args, **kwargs): logger.warning(w, *args, **kwargs) warnings.append(w % args) time_spent = 0 msg = None while time_spent < timeout: t0 = time.time() msg = self._client.message_received(timeout - time_spent) if not msg: # timed out break call_hook( self.test_block_config, "pytest_tavern_beta_after_every_response", expected=self.expected, response=msg, ) self.received_messages.append(msg) msg.payload = msg.payload.decode("utf8") if expect_json_payload: try: msg.payload = json.loads(msg.payload) except LoadException: addwarning( "Expected a json payload but got '%s'", msg.payload, exc_info=True, ) msg = None continue if expected_payload is None: if msg.payload is None or msg.payload == "": logger.info( "Got message with no payload (as expected) on '%s'", topic) break else: addwarning( "Message had payload '%s' but we expected no payload", msg.payload, ) elif expected_payload is ANYTHING: logger.info("Got message on %s matching !anything token", topic) break elif msg.payload != expected_payload: if expect_json_payload: try: check_keys_match_recursive(expected_payload, msg.payload, []) except exceptions.KeyMismatchError: # Just want to log the mismatch pass else: logger.info( "Got expected message in '%s' with payload '%s'", msg.topic, msg.payload, ) break addwarning( "Got unexpected payload on topic '%s': '%s' (expected '%s')", msg.topic, msg.payload, expected_payload, ) elif msg.topic != topic: addwarning( "Got unexpected message in '%s' with payload '%s'", msg.topic, msg.payload, ) else: logger.info( "Got expected message in '%s' with payload '%s'", msg.topic, msg.payload, ) break msg = None time_spent += time.time() - t0 if not msg: self._adderr( "Expected '%s' on topic '%s' but no such message received", expected_payload, topic, ) if self.errors: if warnings: self._adderr("\n".join(warnings)) raise exceptions.TestFailError( "Test '{:s}' failed:\n{:s}".format(self.name, self._str_errors()), failures=self.errors, ) saved = {} return saved
def verify(self, response): """Verify response against expected values and returns any values that we wanted to save for use in future requests There are various ways to 'validate' a block - a specific function, just matching values, validating a schema, etc... Args: response (requests.Response): response object Returns: dict: Any saved values Raises: TestFailError: Something went wrong with validating the response """ self._verbose_log_response(response) call_hook( self.test_block_config, "pytest_tavern_beta_after_every_response", expected=self.expected, response=response, ) self.response = response self.status_code = response.status_code try: body = response.json() except ValueError: body = None self._check_status_code(response.status_code, body) if self.validate_function: try: self.validate_function(response) except Exception as e: # pylint: disable=broad-except self._adderr( "Error calling validate function '%s':\n%s", self.validate_function.func, indent_err_text(traceback.format_exc()), e=e, ) # Get any keys to save saved = {} redirect_query_params = self._get_redirect_query_params(response) saved.update(self._save_value("body", body)) saved.update(self._save_value("headers", response.headers)) saved.update(self._save_value("redirect_query_params", redirect_query_params)) for cookie in self.expected.get("cookies", []): if cookie not in response.cookies: self._adderr("No cookie named '%s' in response", cookie) try: wrapped = get_wrapped_response_function(self.expected["save"]["$ext"]) except KeyError: logger.debug("No save function for this stage") else: try: to_save = wrapped(response) except Exception as e: # pylint: disable=broad-except self._adderr( "Error calling save function '%s':\n%s", wrapped.func, indent_err_text(traceback.format_exc()), e=e, ) else: if isinstance(to_save, dict): saved.update(to_save) elif to_save is not None: self._adderr("Unexpected return value '%s' from $ext save function") self._validate_block("body", body) self._validate_block("headers", response.headers) self._validate_block("redirect_query_params", redirect_query_params) if self.errors: raise TestFailError( "Test '{:s}' failed:\n{:s}".format(self.name, self._str_errors()), failures=self.errors, ) return saved