Example #1
0
def test_run_all_tests(plan):
    """
    Test running all tests.
    """
    host, port = plan.interactive.http_handler_info
    assert host == "0.0.0.0"

    report_url = "http://localhost:{}/api/v1/interactive/report".format(port)
    rsp = requests.get(report_url)
    assert rsp.status_code == 200
    report_json = rsp.json()
    last_hash = report_json["hash"]

    # Trigger all tests to run by updating the report status to RUNNING
    # and PUTting back the data.
    report_json["runtime_status"] = report.RuntimeStatus.RUNNING
    rsp = requests.put(report_url, json=report_json)
    assert rsp.status_code == 200

    updated_json = rsp.json()
    test_api.compare_json(updated_json, report_json)
    assert updated_json["hash"] != last_hash

    timing.wait(
        functools.partial(
            _check_test_status, report_url, "failed", updated_json["hash"]
        ),
        interval=0.2,
        timeout=300,
        raise_on_timeout=True,
    )

    # After running all tests, check that we can retrieve the attached file.
    _test_attachments(port)
Example #2
0
def test_run_suite(plan):
    """Test running a single test suite."""
    host, port = plan.interactive.http_handler_info
    assert host == "0.0.0.0"

    suite_url = (
        "http://localhost:{}/api/v1/interactive/report/tests/ExampleMTest/"
        "suites/ExampleSuite".format(port)
    )
    rsp = requests.get(suite_url)
    assert rsp.status_code == 200
    suite_json = rsp.json()

    # Trigger all tests to run by updating the report status to RUNNING
    # and PUTting back the data.
    suite_json["runtime_status"] = report.RuntimeStatus.RUNNING
    rsp = requests.put(suite_url, json=suite_json)
    assert rsp.status_code == 200
    updated_json = rsp.json()
    test_api.compare_json(updated_json, suite_json)
    assert updated_json["hash"] != suite_json["hash"]

    timing.wait(
        functools.partial(
            _check_test_status, suite_url, "failed", updated_json["hash"]
        ),
        interval=0.2,
        timeout=300,
        raise_on_timeout=True,
    )
Example #3
0
def test_run_param_testcase(plan):
    """Test running a single parametrized testcase."""
    host, port = plan.interactive.http_handler_info
    assert host == "0.0.0.0"

    for param_name, expected_result in EXPECTED_PARAM_TESTCASE_RESULTS:
        testcase_url = (
            "http://localhost:{port}/api/v1/interactive/report/tests/"
            "ExampleMTest/suites/ExampleSuite/testcases/test_parametrized/"
            "parametrizations/{param}".format(port=port, param=param_name))

        rsp = requests.get(testcase_url)
        assert rsp.status_code == 200
        testcase_json = rsp.json()

        # Trigger all tests to run by updating the report status to RUNNING
        # and PUTting back the data.
        testcase_json["runtime_status"] = report.RuntimeStatus.RUNNING
        rsp = requests.put(testcase_url, json=testcase_json)
        assert rsp.status_code == 200
        updated_json = rsp.json()
        test_api.compare_json(updated_json, testcase_json)
        assert updated_json["hash"] != testcase_json["hash"]

        timing.wait(
            functools.partial(
                _check_test_status,
                testcase_url,
                expected_result,
                updated_json["hash"],
            ),
            interval=0.2,
            timeout=60,
            raise_on_timeout=True,
        )
Example #4
0
def test_environment_control(plan):
    """Test starting and stopping the environment."""
    host, port = plan.interactive.http_handler_info
    assert host == "0.0.0.0"

    mtest_url = (
        "http://localhost:{}/api/v1/interactive/report/tests/"
        "ExampleMTest".format(port)
    )
    rsp = requests.get(mtest_url)
    assert rsp.status_code == 200
    mtest_json = rsp.json()

    # Trigger the environment to start by setting the env_status to STARTING
    # and PUTting back the data.
    mtest_json["env_status"] = entity.ResourceStatus.STARTING
    rsp = requests.put(mtest_url, json=mtest_json)
    assert rsp.status_code == 200
    updated_json = rsp.json()
    test_api.compare_json(updated_json, mtest_json)
    assert updated_json["hash"] != mtest_json["hash"]

    # Wait for the environment to become STARTED.
    timing.wait(
        functools.partial(
            _check_env_status,
            mtest_url,
            entity.ResourceStatus.STARTED,
            updated_json["hash"],
        ),
        interval=0.2,
        timeout=60,
        raise_on_timeout=True,
    )

    # Now trigger the environment to stop by setting the env_status to STOPPING
    # and PUTting back the data.
    mtest_json = updated_json
    mtest_json["env_status"] = entity.ResourceStatus.STOPPING
    rsp = requests.put(mtest_url, json=mtest_json)
    assert rsp.status_code == 200
    updated_json = rsp.json()
    test_api.compare_json(updated_json, mtest_json)
    assert updated_json["hash"] != mtest_json["hash"]

    # Wait for the environment to become STOPPED.
    timing.wait(
        functools.partial(
            _check_env_status,
            mtest_url,
            entity.ResourceStatus.STOPPED,
            updated_json["hash"],
        ),
        interval=0.2,
        timeout=30,
        raise_on_timeout=True,
    )
Example #5
0
def test_initial_get(plan):
    """
    Test GETting the report state through each of the API resources at the
    start of day, i.e. before any tests have been run.
    """
    host, port = plan.interactive.http_handler_info
    assert host == "0.0.0.0"

    for resource_path, expected_json in EXPECTED_INITIAL_GET:
        rsp = requests.get(
            "http://localhost:{port}/api/v1/interactive{resource}".format(
                port=port, resource=resource_path))
        assert rsp.status_code == 200
        test_api.compare_json(rsp.json(), expected_json)
Example #6
0
def test_run_testcase(plan):
    """Test running a single testcase."""
    host, port = plan.interactive.http_handler_info
    assert host == "0.0.0.0"

    for (
        testcase_name,
        expected_status,
        expected_runtime_status,
    ) in EXPECTED_TESTCASE_RESULTS:
        testcase_url = (
            "http://localhost:{port}/api/v1/interactive/report/tests/"
            "ExampleMTest/suites/ExampleSuite/testcases/{testcase}".format(
                port=port, testcase=testcase_name
            )
        )

        rsp = requests.get(testcase_url)
        assert rsp.status_code == 200
        testcase_json = rsp.json()

        # Trigger testcase to run by updating the report status to RUNNING
        # and PUTting back the data.
        testcase_json["runtime_status"] = RuntimeStatus.RUNNING
        rsp = requests.put(testcase_url, json=testcase_json)
        assert rsp.status_code == 200
        updated_json = rsp.json()
        assert updated_json["hash"] != testcase_json["hash"]
        assert updated_json["runtime_status"] == RuntimeStatus.WAITING
        test_api.compare_json(
            updated_json, testcase_json, ignored_keys=["runtime_status"]
        )

        timing.wait(
            functools.partial(
                _check_test_status,
                testcase_url,
                expected_status,
                expected_runtime_status,
                updated_json["hash"],
            ),
            interval=0.2,
            timeout=60,
            raise_on_timeout=True,
        )
Example #7
0
def test_cannot_run_mtest(plan2):
    """Test running a single MultiTest and then reset the test report."""
    host, port = plan2.interactive.http_handler_info
    assert host == "0.0.0.0"

    mtest_url = (
        "http://localhost:{}/api/v1/interactive/report/tests/"
        "BrokenMTest".format(port)
    )
    rsp = requests.get(mtest_url)
    assert rsp.status_code == 200
    mtest_json = rsp.json()

    # Trigger multitest to run by updating the report status to RUNNING
    # and PUTting back the data.
    mtest_json["runtime_status"] = RuntimeStatus.RUNNING
    rsp = requests.put(mtest_url, json=mtest_json)
    assert rsp.status_code == 200
    updated_json = rsp.json()
    assert updated_json["hash"] != mtest_json["hash"]
    assert updated_json["runtime_status"] == RuntimeStatus.WAITING
    test_api.compare_json(
        updated_json, mtest_json, ignored_keys=["runtime_status"]
    )

    timing.wait(
        functools.partial(
            _check_test_status,
            mtest_url,
            Status.ERROR,
            RuntimeStatus.NOT_RUN,
            updated_json["hash"],
        ),
        interval=0.2,
        timeout=60,
        raise_on_timeout=True,
    )

    # Check the error message
    rsp = requests.get(mtest_url)
    assert rsp.status_code == 200
    mtest_json = rsp.json()
    assert len(mtest_json["logs"]) == 1
    assert "Failed to start with no reason" in mtest_json["logs"][0]["message"]
Example #8
0
def test_cannot_start_environment(plan2):
    """Test starting the environment but fails."""
    host, port = plan2.interactive.http_handler_info
    assert host == "0.0.0.0"

    mtest_url = (
        "http://localhost:{}/api/v1/interactive/report/tests/"
        "BrokenMTest".format(port)
    )
    rsp = requests.get(mtest_url)
    assert rsp.status_code == 200
    mtest_json = rsp.json()

    # Trigger the environment to start by setting the env_status to STARTING
    # and PUTting back the data.
    mtest_json["env_status"] = entity.ResourceStatus.STARTING
    rsp = requests.put(mtest_url, json=mtest_json)
    assert rsp.status_code == 200
    updated_json = rsp.json()
    test_api.compare_json(updated_json, mtest_json)
    assert updated_json["hash"] != mtest_json["hash"]

    # Wait for the environment to become STOPPED.
    timing.wait(
        functools.partial(
            _check_env_status,
            mtest_url,
            entity.ResourceStatus.STOPPED,
            updated_json["hash"],
        ),
        interval=0.2,
        timeout=60,
        raise_on_timeout=True,
    )

    # Check the error message
    rsp = requests.get(mtest_url)
    assert rsp.status_code == 200
    mtest_json = rsp.json()
    assert len(mtest_json["logs"]) == 1
    assert "Failed to start with no reason" in mtest_json["logs"][0]["message"]
Example #9
0
def test_run_and_reset_mtest(plan):
    """Test running a single MultiTest and then reset the test report."""
    host, port = plan.interactive.http_handler_info
    assert host == "0.0.0.0"

    mtest_url = ("http://localhost:{}/api/v1/interactive/report/tests/"
                 "ExampleMTest".format(port))
    rsp = requests.get(mtest_url)
    assert rsp.status_code == 200
    mtest_json = rsp.json()

    # Trigger multitest to run by updating the report status to RUNNING
    # and PUTting back the data.
    mtest_json["runtime_status"] = RuntimeStatus.RUNNING
    rsp = requests.put(mtest_url, json=mtest_json)
    assert rsp.status_code == 200
    updated_json = rsp.json()
    assert updated_json["hash"] != mtest_json["hash"]
    assert updated_json["runtime_status"] == RuntimeStatus.WAITING
    test_api.compare_json(updated_json,
                          mtest_json,
                          ignored_keys=["runtime_status"])

    timing.wait(
        functools.partial(
            _check_test_status,
            mtest_url,
            Status.FAILED,
            RuntimeStatus.FINISHED,
            updated_json["hash"],
        ),
        interval=0.2,
        timeout=60,
        raise_on_timeout=True,
    )

    # Get the updated report
    rsp = requests.get(mtest_url)
    assert rsp.status_code == 200
    mtest_json = rsp.json()

    # Trigger multitest to run by updating the report status to RESETTING
    # and PUTting back the data.
    mtest_json["runtime_status"] = RuntimeStatus.RESETTING
    rsp = requests.put(mtest_url, json=mtest_json)
    assert rsp.status_code == 200
    updated_json = rsp.json()
    assert updated_json["hash"] != mtest_json["hash"]
    assert updated_json["runtime_status"] == RuntimeStatus.WAITING
    test_api.compare_json(updated_json,
                          mtest_json,
                          ignored_keys=["runtime_status", "env_status"])

    timing.wait(
        functools.partial(
            _check_test_status,
            mtest_url,
            Status.UNKNOWN,
            RuntimeStatus.READY,
            updated_json["hash"],
        ),
        interval=0.2,
        timeout=60,
        raise_on_timeout=True,
    )

    rsp = requests.get(mtest_url)
    assert rsp.status_code == 200
    mtest_json = rsp.json()
    assert mtest_json["runtime_status"] == RuntimeStatus.READY
    assert mtest_json["env_status"] == entity.ResourceStatus.STOPPED