def test_run_all_tests(plan):
"""
Test running all tests.
"""
host, port = plan.interactive.http_handler_info
assert host == "0.0.0.0"
report_url = "http://localhost:{}/api/v1/interactive/report".format(port)
rsp = requests.get(report_url)
assert rsp.status_code == 200
report_json = rsp.json()
last_hash = report_json["hash"]
# Trigger all tests to run by updating the report status to RUNNING
# and PUTting back the data.
report_json["runtime_status"] = report.RuntimeStatus.RUNNING
rsp = requests.put(report_url, json=report_json)
assert rsp.status_code == 200
updated_json = rsp.json()
test_api.compare_json(updated_json, report_json)
assert updated_json["hash"] != last_hash
timing.wait(
functools.partial(
_check_test_status, report_url, "failed", updated_json["hash"]
),
interval=0.2,
timeout=300,
raise_on_timeout=True,
)
# After running all tests, check that we can retrieve the attached file.
_test_attachments(port)
def test_run_suite(plan):
"""Test running a single test suite."""
host, port = plan.interactive.http_handler_info
assert host == "0.0.0.0"
suite_url = (
"http://localhost:{}/api/v1/interactive/report/tests/ExampleMTest/"
"suites/ExampleSuite".format(port)
)
rsp = requests.get(suite_url)
assert rsp.status_code == 200
suite_json = rsp.json()
# Trigger all tests to run by updating the report status to RUNNING
# and PUTting back the data.
suite_json["runtime_status"] = report.RuntimeStatus.RUNNING
rsp = requests.put(suite_url, json=suite_json)
assert rsp.status_code == 200
updated_json = rsp.json()
test_api.compare_json(updated_json, suite_json)
assert updated_json["hash"] != suite_json["hash"]
timing.wait(
functools.partial(
_check_test_status, suite_url, "failed", updated_json["hash"]
),
interval=0.2,
timeout=300,
raise_on_timeout=True,
)
def test_run_param_testcase(plan):
"""Test running a single parametrized testcase."""
host, port = plan.interactive.http_handler_info
assert host == "0.0.0.0"
for param_name, expected_result in EXPECTED_PARAM_TESTCASE_RESULTS:
testcase_url = (
"http://localhost:{port}/api/v1/interactive/report/tests/"
"ExampleMTest/suites/ExampleSuite/testcases/test_parametrized/"
"parametrizations/{param}".format(port=port, param=param_name))
rsp = requests.get(testcase_url)
assert rsp.status_code == 200
testcase_json = rsp.json()
# Trigger all tests to run by updating the report status to RUNNING
# and PUTting back the data.
testcase_json["runtime_status"] = report.RuntimeStatus.RUNNING
rsp = requests.put(testcase_url, json=testcase_json)
assert rsp.status_code == 200
updated_json = rsp.json()
test_api.compare_json(updated_json, testcase_json)
assert updated_json["hash"] != testcase_json["hash"]
timing.wait(
functools.partial(
_check_test_status,
testcase_url,
expected_result,
updated_json["hash"],
),
interval=0.2,
timeout=60,
raise_on_timeout=True,
)
def test_environment_control(plan):
"""Test starting and stopping the environment."""
host, port = plan.interactive.http_handler_info
assert host == "0.0.0.0"
mtest_url = (
"http://localhost:{}/api/v1/interactive/report/tests/"
"ExampleMTest".format(port)
)
rsp = requests.get(mtest_url)
assert rsp.status_code == 200
mtest_json = rsp.json()
# Trigger the environment to start by setting the env_status to STARTING
# and PUTting back the data.
mtest_json["env_status"] = entity.ResourceStatus.STARTING
rsp = requests.put(mtest_url, json=mtest_json)
assert rsp.status_code == 200
updated_json = rsp.json()
test_api.compare_json(updated_json, mtest_json)
assert updated_json["hash"] != mtest_json["hash"]
# Wait for the environment to become STARTED.
timing.wait(
functools.partial(
_check_env_status,
mtest_url,
entity.ResourceStatus.STARTED,
updated_json["hash"],
),
interval=0.2,
timeout=60,
raise_on_timeout=True,
)
# Now trigger the environment to stop by setting the env_status to STOPPING
# and PUTting back the data.
mtest_json = updated_json
mtest_json["env_status"] = entity.ResourceStatus.STOPPING
rsp = requests.put(mtest_url, json=mtest_json)
assert rsp.status_code == 200
updated_json = rsp.json()
test_api.compare_json(updated_json, mtest_json)
assert updated_json["hash"] != mtest_json["hash"]
# Wait for the environment to become STOPPED.
timing.wait(
functools.partial(
_check_env_status,
mtest_url,
entity.ResourceStatus.STOPPED,
updated_json["hash"],
),
interval=0.2,
timeout=30,
raise_on_timeout=True,
)
def test_initial_get(plan):
"""
Test GETting the report state through each of the API resources at the
start of day, i.e. before any tests have been run.
"""
host, port = plan.interactive.http_handler_info
assert host == "0.0.0.0"
for resource_path, expected_json in EXPECTED_INITIAL_GET:
rsp = requests.get(
"http://localhost:{port}/api/v1/interactive{resource}".format(
port=port, resource=resource_path))
assert rsp.status_code == 200
test_api.compare_json(rsp.json(), expected_json)
def test_run_testcase(plan):
"""Test running a single testcase."""
host, port = plan.interactive.http_handler_info
assert host == "0.0.0.0"
for (
testcase_name,
expected_status,
expected_runtime_status,
) in EXPECTED_TESTCASE_RESULTS:
testcase_url = (
"http://localhost:{port}/api/v1/interactive/report/tests/"
"ExampleMTest/suites/ExampleSuite/testcases/{testcase}".format(
port=port, testcase=testcase_name
)
)
rsp = requests.get(testcase_url)
assert rsp.status_code == 200
testcase_json = rsp.json()
# Trigger testcase to run by updating the report status to RUNNING
# and PUTting back the data.
testcase_json["runtime_status"] = RuntimeStatus.RUNNING
rsp = requests.put(testcase_url, json=testcase_json)
assert rsp.status_code == 200
updated_json = rsp.json()
assert updated_json["hash"] != testcase_json["hash"]
assert updated_json["runtime_status"] == RuntimeStatus.WAITING
test_api.compare_json(
updated_json, testcase_json, ignored_keys=["runtime_status"]
)
timing.wait(
functools.partial(
_check_test_status,
testcase_url,
expected_status,
expected_runtime_status,
updated_json["hash"],
),
interval=0.2,
timeout=60,
raise_on_timeout=True,
)
def test_cannot_run_mtest(plan2):
"""Test running a single MultiTest and then reset the test report."""
host, port = plan2.interactive.http_handler_info
assert host == "0.0.0.0"
mtest_url = (
"http://localhost:{}/api/v1/interactive/report/tests/"
"BrokenMTest".format(port)
)
rsp = requests.get(mtest_url)
assert rsp.status_code == 200
mtest_json = rsp.json()
# Trigger multitest to run by updating the report status to RUNNING
# and PUTting back the data.
mtest_json["runtime_status"] = RuntimeStatus.RUNNING
rsp = requests.put(mtest_url, json=mtest_json)
assert rsp.status_code == 200
updated_json = rsp.json()
assert updated_json["hash"] != mtest_json["hash"]
assert updated_json["runtime_status"] == RuntimeStatus.WAITING
test_api.compare_json(
updated_json, mtest_json, ignored_keys=["runtime_status"]
)
timing.wait(
functools.partial(
_check_test_status,
mtest_url,
Status.ERROR,
RuntimeStatus.NOT_RUN,
updated_json["hash"],
),
interval=0.2,
timeout=60,
raise_on_timeout=True,
)
# Check the error message
rsp = requests.get(mtest_url)
assert rsp.status_code == 200
mtest_json = rsp.json()
assert len(mtest_json["logs"]) == 1
assert "Failed to start with no reason" in mtest_json["logs"][0]["message"]
def test_cannot_start_environment(plan2):
"""Test starting the environment but fails."""
host, port = plan2.interactive.http_handler_info
assert host == "0.0.0.0"
mtest_url = (
"http://localhost:{}/api/v1/interactive/report/tests/"
"BrokenMTest".format(port)
)
rsp = requests.get(mtest_url)
assert rsp.status_code == 200
mtest_json = rsp.json()
# Trigger the environment to start by setting the env_status to STARTING
# and PUTting back the data.
mtest_json["env_status"] = entity.ResourceStatus.STARTING
rsp = requests.put(mtest_url, json=mtest_json)
assert rsp.status_code == 200
updated_json = rsp.json()
test_api.compare_json(updated_json, mtest_json)
assert updated_json["hash"] != mtest_json["hash"]
# Wait for the environment to become STOPPED.
timing.wait(
functools.partial(
_check_env_status,
mtest_url,
entity.ResourceStatus.STOPPED,
updated_json["hash"],
),
interval=0.2,
timeout=60,
raise_on_timeout=True,
)
# Check the error message
rsp = requests.get(mtest_url)
assert rsp.status_code == 200
mtest_json = rsp.json()
assert len(mtest_json["logs"]) == 1
assert "Failed to start with no reason" in mtest_json["logs"][0]["message"]
def test_run_and_reset_mtest(plan):
"""Test running a single MultiTest and then reset the test report."""
host, port = plan.interactive.http_handler_info
assert host == "0.0.0.0"
mtest_url = ("http://localhost:{}/api/v1/interactive/report/tests/"
"ExampleMTest".format(port))
rsp = requests.get(mtest_url)
assert rsp.status_code == 200
mtest_json = rsp.json()
# Trigger multitest to run by updating the report status to RUNNING
# and PUTting back the data.
mtest_json["runtime_status"] = RuntimeStatus.RUNNING
rsp = requests.put(mtest_url, json=mtest_json)
assert rsp.status_code == 200
updated_json = rsp.json()
assert updated_json["hash"] != mtest_json["hash"]
assert updated_json["runtime_status"] == RuntimeStatus.WAITING
test_api.compare_json(updated_json,
mtest_json,
ignored_keys=["runtime_status"])
timing.wait(
functools.partial(
_check_test_status,
mtest_url,
Status.FAILED,
RuntimeStatus.FINISHED,
updated_json["hash"],
),
interval=0.2,
timeout=60,
raise_on_timeout=True,
)
# Get the updated report
rsp = requests.get(mtest_url)
assert rsp.status_code == 200
mtest_json = rsp.json()
# Trigger multitest to run by updating the report status to RESETTING
# and PUTting back the data.
mtest_json["runtime_status"] = RuntimeStatus.RESETTING
rsp = requests.put(mtest_url, json=mtest_json)
assert rsp.status_code == 200
updated_json = rsp.json()
assert updated_json["hash"] != mtest_json["hash"]
assert updated_json["runtime_status"] == RuntimeStatus.WAITING
test_api.compare_json(updated_json,
mtest_json,
ignored_keys=["runtime_status", "env_status"])
timing.wait(
functools.partial(
_check_test_status,
mtest_url,
Status.UNKNOWN,
RuntimeStatus.READY,
updated_json["hash"],
),
interval=0.2,
timeout=60,
raise_on_timeout=True,
)
rsp = requests.get(mtest_url)
assert rsp.status_code == 200
mtest_json = rsp.json()
assert mtest_json["runtime_status"] == RuntimeStatus.READY
assert mtest_json["env_status"] == entity.ResourceStatus.STOPPED