def runtest(self):
self.global_cfg = self._parse_arguments()
self.global_cfg.setdefault("variables", {})
load_plugins(self.global_cfg)
# INTERNAL
# NOTE - now that we can 'mark' tests, we could use pytest.mark.xfail
# instead. This doesn't differentiate between an error in verification
# and an error when running the test though.
xfail = self.spec.get("_xfail", False)
try:
verify_tests(self.spec)
fixture_values = self._load_fixture_values()
self.global_cfg["variables"].update(fixture_values)
run_test(self.path, self.spec, self.global_cfg)
except exceptions.BadSchemaError:
if xfail == "verify":
logger.info("xfailing test while verifying schema")
else:
raise
except exceptions.TavernException:
if xfail == "run":
logger.info("xfailing test when running")
else:
raise
else:
if xfail:
logger.error("Expected test to fail")
raise exceptions.TestFailError(
"Expected test to fail at {} stage".format(xfail))
def runtest(self):
# Do a deep copy because this sometimes still retains things from previous tests(?)
self.global_cfg = copy.deepcopy(load_global_cfg(self.config))
self.global_cfg.setdefault("variables", {})
load_plugins(self.global_cfg)
self.global_cfg["tavern_internal"] = {"pytest_hook_caller": self.config.hook}
# INTERNAL
# NOTE - now that we can 'mark' tests, we could use pytest.mark.xfail
# instead. This doesn't differentiate between an error in verification
# and an error when running the test though.
xfail = self.spec.get("_xfail", False)
try:
fixture_values = self._load_fixture_values()
self.global_cfg["variables"].update(fixture_values)
call_hook(
self.global_cfg,
"pytest_tavern_beta_before_every_test_run",
test_dict=self.spec,
variables=self.global_cfg["variables"],
)
verify_tests(self.spec)
for stage in self.spec["stages"]:
if not stage.get("name"):
if not stage.get("id"):
# Should never actually reach here, should be caught at schema check time
raise exceptions.BadSchemaError(
"One of name or ID must be specified"
)
stage["name"] = stage["id"]
run_test(self.path, self.spec, self.global_cfg)
except exceptions.BadSchemaError:
if xfail == "verify":
logger.info("xfailing test while verifying schema")
else:
raise
except exceptions.TavernException:
if xfail == "run":
logger.info("xfailing test when running")
else:
raise
else:
if xfail:
logger.error("Expected test to fail")
raise exceptions.TestFailError(
"Expected test to fail at {} stage".format(xfail)
)
def runtest(self):
self.global_cfg = load_global_cfg(self.config)
self.global_cfg.setdefault("variables", {})
load_plugins(self.global_cfg)
self.global_cfg["tavern_internal"] = {
"pytest_hook_caller": self.config.hook
}
# INTERNAL
# NOTE - now that we can 'mark' tests, we could use pytest.mark.xfail
# instead. This doesn't differentiate between an error in verification
# and an error when running the test though.
xfail = self.spec.get("_xfail", False)
try:
fixture_values = self._load_fixture_values()
self.global_cfg["variables"].update(fixture_values)
call_hook(
self.global_cfg,
"pytest_tavern_beta_before_every_test_run",
test_dict=self.spec,
variables=self.global_cfg["variables"],
)
verify_tests(self.spec)
run_test(self.path, self.spec, self.global_cfg)
except exceptions.BadSchemaError:
if xfail == "verify":
logger.info("xfailing test while verifying schema")
else:
raise
except exceptions.TavernException:
if xfail == "run":
logger.info("xfailing test when running")
else:
raise
else:
if xfail:
logger.error("Expected test to fail")
raise exceptions.TestFailError(
"Expected test to fail at {} stage".format(xfail))
def verify(self, response):
"""Verify response against expected values and returns any values that
we wanted to save for use in future requests
There are various ways to 'validate' a block - a specific function, just
matching values, validating a schema, etc...
Args:
response (requests.Response): response object
Returns:
dict: Any saved values
Raises:
TestFailError: Something went wrong with validating the response
"""
self._verbose_log_response(response)
call_hook(
self.test_block_config,
"pytest_tavern_beta_after_every_response",
expected=self.expected,
response=response,
)
self.response = response
self.status_code = response.status_code
# Get things to use from the response
try:
body = response.json()
except ValueError:
body = None
redirect_query_params = self._get_redirect_query_params(response)
# Run validation on response
self._check_status_code(response.status_code, body)
self._validate_block("json", body)
self._validate_block("headers", response.headers)
self._validate_block("redirect_query_params", redirect_query_params)
self._maybe_run_validate_functions(response)
# Get any keys to save
saved = {}
saved.update(self.maybe_get_save_values_from_save_block("json", body))
saved.update(
self.maybe_get_save_values_from_save_block("headers",
response.headers))
saved.update(
self.maybe_get_save_values_from_save_block("redirect_query_params",
redirect_query_params))
saved.update(
self.maybe_get_save_values_from_ext(response, self.expected))
# Check cookies
for cookie in self.expected.get("cookies", []):
if cookie not in response.cookies:
self._adderr("No cookie named '%s' in response", cookie)
if self.errors:
raise exceptions.TestFailError(
"Test '{:s}' failed:\n{:s}".format(self.name,
self._str_errors()),
failures=self.errors,
)
return saved
def _await_response(self):
"""Actually wait for response"""
topic = self.expected["topic"]
timeout = self.expected.get("timeout", 1)
payload, json_payload = self._get_payload_vals()
time_spent = 0
msg = None
while time_spent < timeout:
t0 = time.time()
msg = self._client.message_received(timeout - time_spent)
if not msg:
# timed out
break
self.received_messages.append(msg)
msg.payload = msg.payload.decode("utf8")
if json_payload:
try:
msg.payload = json.loads(msg.payload)
except LoadException:
logger.warning(
"Expected a json payload but got '%s'",
msg.payload,
exc_info=True,
)
msg = None
continue
if not payload:
if not msg.payload:
logger.info(
"Got message with no payload (as expected) on '%s'",
topic)
break
else:
logger.warning(
"Message had payload '%s' but we expected no payload")
elif payload is ANYTHING:
logger.info("Got message on %s matching !anything token",
topic)
break
elif msg.payload != payload:
if json_payload:
try:
check_keys_match_recursive(payload, msg.payload, [])
except exceptions.KeyMismatchError:
# Just want to log the mismatch
pass
else:
logger.info(
"Got expected message in '%s' with payload '%s'",
msg.topic,
msg.payload,
)
break
logger.warning(
"Got unexpected payload on topic '%s': '%s' (expected '%s')",
msg.topic,
msg.payload,
payload,
)
elif msg.topic != topic:
logger.warning(
"Got unexpected message in '%s' with payload '%s'",
msg.topic,
msg.payload,
)
else:
logger.info(
"Got expected message in '%s' with payload '%s'",
msg.topic,
msg.payload,
)
break
msg = None
time_spent += time.time() - t0
if not msg:
self._adderr(
"Expected '%s' on topic '%s' but no such message received",
payload,
topic,
)
if self.errors:
raise exceptions.TestFailError(
"Test '{:s}' failed:\n{:s}".format(self.name,
self._str_errors()),
failures=self.errors,
)
return {}
def verify(self, response):
"""Ensure mqtt message has arrived
Args:
response: not used
"""
self.response = response
topic = self.expected["topic"]
timeout = self.expected.get("timeout", 1)
# TODO move this check to initialisation/schema checking
if "json" in self.expected:
if "payload" in self.expected:
raise exceptions.BadSchemaError(
"Can only specify one of 'payload' or 'json' in MQTT response"
)
payload = self.expected["json"]
json_payload = True
else:
payload = self.expected["payload"]
json_payload = False
time_spent = 0
while time_spent < timeout:
t0 = time.time()
msg = self._client.message_received(timeout - time_spent)
if not msg:
# timed out
break
self.received_messages.append(msg)
msg.payload = msg.payload.decode("utf8")
if json_payload:
try:
msg.payload = json.loads(msg.payload)
except LoadException:
logger.warning("Expected a json payload but got '%s'",
msg.payload)
msg = None
continue
if msg.payload != payload:
logger.warning(
"Got unexpected payload on topic '%s': '%s' (expected '%s')",
msg.topic, msg.payload, payload)
elif msg.topic != topic:
logger.warning(
"Got unexpected message in '%s' with payload '%s'",
msg.topic, msg.payload)
else:
logger.info("Got expected message in '%s' with payload '%s'",
msg.topic, msg.payload)
break
msg = None
time_spent += time.time() - t0
if not msg:
self._adderr(
"Expected '%s' on topic '%s' but no such message received",
payload, topic)
if self.errors:
raise exceptions.TestFailError("Test '{:s}' failed:\n{:s}".format(
self.name, self._str_errors()))
return {}
def _await_response(self):
"""Actually wait for response"""
# pylint: disable=too-many-statements
topic = self.expected["topic"]
timeout = self.expected.get("timeout", 1)
expected_payload, expect_json_payload = self._get_payload_vals()
# Any warnings to do with the request
# eg, if a message was received but it didn't match, message had payload, etc.
warnings = []
def addwarning(w, *args, **kwargs):
logger.warning(w, *args, **kwargs)
warnings.append(w % args)
time_spent = 0
msg = None
while time_spent < timeout:
t0 = time.time()
msg = self._client.message_received(timeout - time_spent)
if not msg:
# timed out
break
call_hook(
self.test_block_config,
"pytest_tavern_beta_after_every_response",
expected=self.expected,
response=msg,
)
self.received_messages.append(msg)
msg.payload = msg.payload.decode("utf8")
if expect_json_payload:
try:
msg.payload = json.loads(msg.payload)
except LoadException:
addwarning(
"Expected a json payload but got '%s'",
msg.payload,
exc_info=True,
)
msg = None
continue
if expected_payload is None:
if msg.payload is None or msg.payload == "":
logger.info(
"Got message with no payload (as expected) on '%s'",
topic)
break
else:
addwarning(
"Message had payload '%s' but we expected no payload",
msg.payload,
)
elif expected_payload is ANYTHING:
logger.info("Got message on %s matching !anything token",
topic)
break
elif msg.payload != expected_payload:
if expect_json_payload:
try:
check_keys_match_recursive(expected_payload,
msg.payload, [])
except exceptions.KeyMismatchError:
# Just want to log the mismatch
pass
else:
logger.info(
"Got expected message in '%s' with payload '%s'",
msg.topic,
msg.payload,
)
break
addwarning(
"Got unexpected payload on topic '%s': '%s' (expected '%s')",
msg.topic,
msg.payload,
expected_payload,
)
elif msg.topic != topic:
addwarning(
"Got unexpected message in '%s' with payload '%s'",
msg.topic,
msg.payload,
)
else:
logger.info(
"Got expected message in '%s' with payload '%s'",
msg.topic,
msg.payload,
)
break
msg = None
time_spent += time.time() - t0
if not msg:
self._adderr(
"Expected '%s' on topic '%s' but no such message received",
expected_payload,
topic,
)
if self.errors:
if warnings:
self._adderr("\n".join(warnings))
raise exceptions.TestFailError(
"Test '{:s}' failed:\n{:s}".format(self.name,
self._str_errors()),
failures=self.errors,
)
saved = {}
return saved
