Exemplo n.º 1
0
def check_sets(old, ostat, new, nstat, set_):
    """ Check that the statuses are added to the correct set """
    old['tests']['sometest']['result'] = ostat
    new['tests']['sometest']['result'] = nstat

    with utils.with_tempfile(json.dumps(old)) as ofile:
        with utils.with_tempfile(json.dumps(new)) as nfile:
            summ = summary.Summary([ofile, nfile])

            print(summ.tests)
            nt.assert_equal(1, len(summ.tests[set_]),
                            msg="{0} was not appended".format(set_))
Exemplo n.º 2
0
def check_sets(old, ostat, new, nstat, set_):
    """ Check that the statuses are added to the correct set """
    old['tests']['sometest']['result'] = ostat
    new['tests']['sometest']['result'] = nstat

    with utils.with_tempfile(json.dumps(old)) as ofile:
        with utils.with_tempfile(json.dumps(new)) as nfile:
            summ = summary.Summary([ofile, nfile])

            print(summ.tests)
            nt.assert_equal(1,
                            len(summ.tests[set_]),
                            msg="{0} was not appended".format(set_))
Exemplo n.º 3
0
def glslparser_exetensions_seperators():
    """ GlslParserTest() can only have [A-Za-z_] as characters

    This test generates a number of tests that should catch the majority of
    errors relating to seperating extensions in the config block of a
    glslparser test

    """
    problems = [
        ('comma seperator', '// require_extensions: ARB_ham, ARB_turkey\n'),
        ('semi-colon seperator', '// require_extensions: ARB_ham; ARB_turkey\n'),
        ('trailing semi-colon', '// require_extensions: ARB_ham ARB_turkey\n;'),
        ('Non-alpha character', '// require_extensions: ARB_$$$\n'),
    ]

    content = ('// [config]\n'
               '// expect_result: pass\n'
               '// glsl_version: 1.00\n'
               '{}'
               '// [end config]\n')

    for name, value in problems:
        test = content.format(value)
        with utils.with_tempfile(test) as tfile:
            check_bad_character.description = (
                'require_extensions: {0} should raise an error'.format(name))
            yield check_bad_character, tfile
def glslparser_exetensions_seperators():
    """ GlslParserTest() can only have [A-Za-z_] as characters

    This test generates a number of tests that should catch the majority of
    errors relating to seperating extensions in the config block of a
    glslparser test

    """
    problems = [
        ('comma seperator', '// require_extensions: ARB_ham, ARB_turkey\n'),
        ('semi-colon seperator',
         '// require_extensions: ARB_ham; ARB_turkey\n'),
        ('trailing semi-colon',
         '// require_extensions: ARB_ham ARB_turkey\n;'),
        ('Non-alpha character', '// require_extensions: ARB_$$$\n'),
    ]

    content = ('// [config]\n'
               '// expect_result: pass\n'
               '// glsl_version: 1.00\n'
               '{}'
               '// [end config]\n')

    for name, value in problems:
        test = content.format(value)
        with utils.with_tempfile(test) as tfile:
            check_bad_character.description = (
                'require_extensions: {0} should raise an error'.format(name))
            yield check_bad_character, tfile
Exemplo n.º 5
0
def test_subtest_handling():
    data = copy.deepcopy(utils.JSON_DATA)
    data['tests']['with_subtests'] = {}
    data['tests']['with_subtests']['result'] = 'pass'

    data['tests']['with_subtests']['subtest'] = {}
    data['tests']['with_subtests']['subtest']['subtest1'] = 'fail'
    data['tests']['with_subtests']['subtest']['subtest2'] = 'warn'
    data['tests']['with_subtests']['subtest']['subtest3'] = 'crash'
    data['tests']['is_skip'] = {}
    data['tests']['is_skip']['result'] = 'skip'

    with utils.with_tempfile(json.dumps(data)) as sumfile:
        summ = summary.Summary([sumfile])

        check_subtests_are_tests.description = \
            "Subtests should be treated as full tests "
        yield check_subtests_are_tests, summ

        check_tests_w_subtests_are_groups.description = \
            "Tests with subtests should be a group"
        yield check_tests_w_subtests_are_groups, summ

        test_removed_from_all.description = \
            "Tests with subtests should not be in the tests['all'] name"
        yield test_removed_from_all, summ

        subtest_not_skip_notrun.description = \
            "Skip's should not become NotRun"
        yield subtest_not_skip_notrun, summ
Exemplo n.º 6
0
def test_subtest_handling():
    data = copy.deepcopy(utils.JSON_DATA)
    data['tests']['with_subtests'] = {}
    data['tests']['with_subtests']['result'] = 'pass'

    data['tests']['with_subtests']['subtest'] = {}
    data['tests']['with_subtests']['subtest']['subtest1'] = 'fail'
    data['tests']['with_subtests']['subtest']['subtest2'] = 'warn'
    data['tests']['with_subtests']['subtest']['subtest3'] = 'crash'
    data['tests']['is_skip'] = {}
    data['tests']['is_skip']['result'] = 'skip'

    with utils.with_tempfile(json.dumps(data)) as sumfile:
        summ = summary.Summary([sumfile])

        check_subtests_are_tests.description = \
            "Subtests should be treated as full tests "
        yield check_subtests_are_tests, summ

        check_tests_w_subtests_are_groups.description = \
            "Tests with subtests should be a group"
        yield check_tests_w_subtests_are_groups, summ

        test_removed_from_all.description = \
            "Tests with subtests should not be in the tests['all'] name"
        yield test_removed_from_all, summ

        subtest_not_skip_notrun.description = \
            "Skip's should not become NotRun"
        yield subtest_not_skip_notrun, summ
Exemplo n.º 7
0
def test_no_config_end():
    """ GLSLParserTest requires [end config] """
    with utils.with_tempfile('// [config]\n') as tfile:
        with nt.assert_raises(glsl.GLSLParserException) as exc:
            glsl.GLSLParserTest(tfile)
            nt.assert_equal(
                exc.exception, 'No [end config] section found!',
                msg="config section not closed, no exception raised")
Exemplo n.º 8
0
def test_info_split():
    """ Version 1: info can split into any number of elements """
    data = copy.copy(DATA)
    data['tests']['sometest']['info'] = \
        'Returncode: 1\n\nErrors:stderr\n\nOutput: stdout\n\nmore\n\nstuff'

    with utils.with_tempfile(json.dumps(data)) as f:
        results._update_zero_to_one(results.load_results(f))
Exemplo n.º 9
0
def check_no_duplicates(content, dup):
    """ Ensure that duplicate entries raise an error """
    with nt.assert_raises(glsl.GLSLParserException) as e:
        with utils.with_tempfile(content) as tfile:
            glsl.GLSLParserTest(tfile)

            nt.eq_(
                e.exception.message,
                'Duplicate entry for key {0} in file {1}'.format(dup, tfile))
def check_no_duplicates(content, dup):
    """ Ensure that duplicate entries raise an error """
    with utils.with_tempfile(content) as tfile:
        with nt.assert_raises(SystemExit) as e:
            glsl.GLSLParserTest(tfile)

            nt.eq_(
                e.exception.message,
                'Duplicate entry for key {0} in file {1}'.format(dup, tfile))
def test_find_config_start():
    """ GLSLParserTest finds [config] """
    content = ('// [config]\n' '// glsl_version: 1.00\n' '//\n')
    with utils.with_tempfile(content) as tfile:
        with nt.assert_raises(SystemExit) as exc:
            glsl.GLSLParserTest(tfile)
            nt.assert_not_equal(exc.exception,
                                'No [config] section found!',
                                msg="Config section not parsed")
def test_no_config_end():
    """ GLSLParserTest requires [end config] """
    with utils.with_tempfile('// [config]\n') as tfile:
        with nt.assert_raises(SystemExit) as exc:
            glsl.GLSLParserTest(tfile)
            nt.assert_equal(
                exc.exception,
                'No [end config] section found!',
                msg="config section not closed, no exception raised")
def test_no_glsl_version():
    """ glsl_version section is required """
    content = ('//\n' '// expect_result: pass\n' '// [end config]\n')
    with utils.with_tempfile(content) as tfile:
        with nt.assert_raises(SystemExit) as exc:
            glsl.GLSLParserTest(tfile)
            nt.assert_equal(
                exc.exception,
                'Missing required section glsl_version from config',
                msg="config section not closed, no exception raised")
Exemplo n.º 14
0
def test_no_expect_result():
    """ expect_result section is required """
    content = ('// [config]\n' '// glsl_version: 1.00\n' '//\n')
    with utils.with_tempfile(content) as tfile:
        with nt.assert_raises(glsl.GLSLParserException) as exc:
            glsl.GLSLParserTest(tfile)
            nt.assert_equal(
                exc.exception,
                'Missing required section expect_result from config',
                msg="config section not closed, no exception raised")
Exemplo n.º 15
0
def test_no_config_start():
    """ GLSLParserTest requires [config] """
    content = ('// expect_result: pass\n'
               '// glsl_version: 1.00\n'
               '// [end config]\n')
    with utils.with_tempfile(content) as tfile:
        with nt.assert_raises(glsl.GLSLParserException) as exc:
            glsl.GLSLParserTest(tfile)
            nt.assert_equal(
                exc.exception, 'No [config] section found!',
                msg="No config section found, no exception raised")
Exemplo n.º 16
0
def test_find_config_start():
    """ GLSLParserTest finds [config] """
    content = ('// [config]\n'
               '// glsl_version: 1.00\n'
               '//\n')
    with utils.with_tempfile(content) as tfile:
        with nt.assert_raises(glsl.GLSLParserException) as exc:
            glsl.GLSLParserTest(tfile)
            nt.assert_not_equal(
                exc.exception, 'No [config] section found!',
                msg="Config section not parsed")
Exemplo n.º 17
0
def test_parse_gl_test_no_decimal():
    """ The GL Parser raises an exception if GL version lacks decimal """
    data = ('[require]\n' 'GL = 2\n')
    with utils.with_tempfile(data) as temp:
        with nt.assert_raises(testm.ShaderTestParserException) as exc:
            testm.ShaderTest(temp)
            nt.assert_equal(exc.exception,
                            "No GL version set",
                            msg="A GL version was passed without a decimal, "
                            "which should have raised an exception, but "
                            "did not")
def test_no_config_start():
    """ GLSLParserTest requires [config] """
    content = ('// expect_result: pass\n'
               '// glsl_version: 1.00\n'
               '// [end config]\n')
    with utils.with_tempfile(content) as tfile:
        with nt.assert_raises(SystemExit) as exc:
            glsl.GLSLParserTest(tfile)
            nt.assert_equal(exc.exception,
                            'No [config] section found!',
                            msg="No config section found, no exception raised")
Exemplo n.º 19
0
def test_parse_gl_test_no_decimal():
    """ The GL Parser raises an exception if GL version lacks decimal """
    data = ('[require]\n'
            'GL = 2\n')
    with utils.with_tempfile(data) as temp:
        with nt.assert_raises(shader_test.ShaderTestParserException) as exc:
            shader_test.ShaderTest(temp)
            nt.assert_equal(exc.exception, "No GL version set",
                            msg="A GL version was passed without a decimal, "
                                "which should have raised an exception, but "
                                "did not")
Exemplo n.º 20
0
def test_parse_gles3_test():
    """ Tests the parser for GLES3 tests """
    data = ('[require]\n' 'GL ES >= 3.0\n' 'GLSL ES >= 3.00\n')
    with utils.with_tempfile(data) as temp:
        test = testm.ShaderTest(temp)

    nt.assert_equal(os.path.basename(test.command[0]),
                    "shader_runner_gles3",
                    msg="This test should have run with shader_runner_gles3, "
                    "but instead ran with " +
                    os.path.basename(test.command[0]))
Exemplo n.º 21
0
def test_no_glsl_version():
    """ glsl_version section is required """
    content = ('//\n'
               '// expect_result: pass\n'
               '// [end config]\n')
    with utils.with_tempfile(content) as tfile:
        with nt.assert_raises(glsl.GLSLParserException) as exc:
            glsl.GLSLParserTest(tfile)
            nt.assert_equal(
                exc.exception,
                'Missing required section glsl_version from config',
                msg="config section not closed, no exception raised")
Exemplo n.º 22
0
def test_parse_gles3_test():
    """ Tests the parser for GLES3 tests """
    data = ('[require]\n'
            'GL ES >= 3.0\n'
            'GLSL ES >= 3.00\n')
    with utils.with_tempfile(data) as temp:
        test = shader_test.ShaderTest(temp)

    nt.assert_equal(
        os.path.basename(test.command[0]), "shader_runner_gles3",
        msg="This test should have run with shader_runner_gles3, "
            "but instead ran with " + os.path.basename(test.command[0]))
Exemplo n.º 23
0
def test_no_expect_result():
    """ expect_result section is required """
    content = ('// [config]\n'
               '// glsl_version: 1.00\n'
               '//\n')
    with utils.with_tempfile(content) as tfile:
        with nt.assert_raises(SystemExit) as exc:
            glsl.GLSLParserTest(tfile)
            nt.assert_equal(
                exc.exception,
                'Missing required section expect_result from config',
                msg="config section not closed, no exception raised")
Exemplo n.º 24
0
def test_parse_listfile_return():
    """ Test that parse_listfile returns a container

    Given a file with a newline seperated list of results, parse_listfile
    should return a list of files with no whitespace

    """
    contents = "/tmp/foo\n/tmp/bar\n"

    with utils.with_tempfile(contents) as tfile:
        results = core.parse_listfile(tfile)

    assert isinstance(results, collections.Container)
def test_parse_listfile_return():
    """ Test that parse_listfile returns a container

    Given a file with a newline seperated list of results, parse_listfile
    should return a list of files with no whitespace

    """
    contents = "/tmp/foo\n/tmp/bar\n"

    with utils.with_tempfile(contents) as tfile:
        results = core.parse_listfile(tfile)

    assert isinstance(results, collections.Container)
def test_parse_listfile_tilde():
    """ Test that parse_listfile properly expands tildes

    According to the python docs for python 2.7
    (http://docs.python.org/2/library/os.path.html#module-os.path), both
    os.path.expanduser and os.path.expandvars work on both *nix systems (Linux,
    *BSD, OSX) and Windows.

    """
    contents = "~/foo\n"

    with utils.with_tempfile(contents) as tfile:
        results = core.parse_listfile(tfile)

    assert results[0] == os.path.expandvars("$HOME/foo")
Exemplo n.º 27
0
def test_bad_section_name():
    """ A section name not in the _CONFIG_KEYS name raises an error """
    content = ('// [config]\n'
               '// expect_result: pass\n'
               '// glsl_version: 1.00\n'
               '// new_awesome_key: foo\n'
               '// [end config]\n')

    with utils.with_tempfile(content) as tfile:
        with nt.assert_raises(SystemExit) as e:
            glsl.GLSLParserTest(tfile)

            nt.eq_(e.exception.message,
                   'Key new_awesome_key in file {0 is not a valid key for a '
                   'glslparser test config block'.format(tfile))
Exemplo n.º 28
0
def test_parse_listfile_tilde():
    """ Test that parse_listfile properly expands tildes

    According to the python docs for python 2.7
    (http://docs.python.org/2/library/os.path.html#module-os.path), both
    os.path.expanduser and os.path.expandvars work on both *nix systems (Linux,
    *BSD, OSX) and Windows.

    """
    contents = "~/foo\n"

    with utils.with_tempfile(contents) as tfile:
        results = core.parse_listfile(tfile)

    assert results[0] == os.path.expandvars("$HOME/foo")
def test_bad_section_name():
    """ A section name not in the _CONFIG_KEYS name raises an error """
    content = ('// [config]\n'
               '// expect_result: pass\n'
               '// glsl_version: 1.00\n'
               '// new_awesome_key: foo\n'
               '// [end config]\n')

    with utils.with_tempfile(content) as tfile:
        with nt.assert_raises(SystemExit) as e:
            glsl.GLSLParserTest(tfile)

            nt.eq_(
                e.exception.message,
                'Key new_awesome_key in file {0 is not a valid key for a '
                'glslparser test config block'.format(tfile))
def _load_with_update(data):
    """If the file is not results.json, it will be renamed.

    This ensures that the right file is removed.

    """
    try:
        with utils.with_tempfile(json.dumps(data)) as t:
            result = backends.json.load_results(t)
    except OSError as e:
        # There is the potential that the file will be renamed. In that event
        # remove the renamed files
        if e.errno == 2:
            os.unlink(os.path.join(tempfile.tempdir, 'results.json'))
            os.unlink(os.path.join(tempfile.tempdir, 'results.json.old'))
        else:
            raise

    return result
Exemplo n.º 31
0
def _load_with_update(data):
    """If the file is not results.json, it will be renamed.

    This ensures that the right file is removed.

    """
    try:
        with utils.with_tempfile(json.dumps(data)) as t:
            result = backends.json.load_results(t)
    except OSError as e:
        # There is the potential that the file will be renamed. In that event
        # remove the renamed files
        if e.errno == 2:
            os.unlink(os.path.join(tempfile.tempdir, 'results.json'))
            os.unlink(os.path.join(tempfile.tempdir, 'results.json.old'))
        else:
            raise

    return result
Exemplo n.º 32
0
def test_good_extensions():
    """ Generates tests with good extensions which shouldn't raise errors """
    content = ('// [config]\n'
               '// expect_result: pass\n'
               '// glsl_version: 1.00\n'
               '// require_extensions: {}\n'
               '// [end config]\n')
    options = [
        'GL_EXT_texture_array',
        'GL_EXT_texture_array ARB_example',
        '!GL_ARB_ham_sandwhich',
    ]

    for x in options:
        test = content.format(x)
        check_good_extension.description = \
            'require_extension {} is valid'.format(x)

        with utils.with_tempfile(test) as tfile:
            yield check_good_extension, tfile, x
def test_good_extensions():
    """ Generates tests with good extensions which shouldn't raise errors """
    content = ('// [config]\n'
               '// expect_result: pass\n'
               '// glsl_version: 1.00\n'
               '// require_extensions: {}\n'
               '// [end config]\n')
    options = [
        'GL_EXT_texture_array',
        'GL_EXT_texture_array ARB_example',
        '!GL_ARB_ham_sandwhich',
    ]

    for x in options:
        test = content.format(x)
        check_good_extension.description = \
            'require_extension {} is valid'.format(x)

        with utils.with_tempfile(test) as tfile:
            yield check_good_extension, tfile, x
Exemplo n.º 34
0
def test_parse_listfile_whitespace():
    """ Test that parse_listfile remove whitespace """
    contents = "/tmp/foo\n/tmp/foo  \n/tmp/foo\t\n"

    with utils.with_tempfile(contents) as tfile:
        results = core.parse_listfile(tfile)

    yld = check_whitespace

    # Test for newlines
    yld.description = "Test that trailing newlines are removed by " "parse_listfile"
    yield yld, results[0], "/tmp/foo", "Trailing newline not removed!"

    # test for normal spaces
    yld.description = "Test that trailing spaces are removed by parse_listfile"
    yield yld, results[1], "/tmp/foo", "Trailing spaces not removed!"

    # test for tabs
    yld.description = "Test that trailing tabs are removed by parse_listfile"
    yield yld, results[2], "/tmp/foo", "Trailing tab not removed!"
Exemplo n.º 35
0
def test_parse_listfile_whitespace():
    """ Test that parse_listfile remove whitespace """
    contents = "/tmp/foo\n/tmp/foo  \n/tmp/foo\t\n"

    with utils.with_tempfile(contents) as tfile:
        results = core.parse_listfile(tfile)

    yld = check_whitespace

    # Test for newlines
    yld.description = ("Test that trailing newlines are removed by "
                       "parse_listfile")
    yield yld, results[0], "/tmp/foo", "Trailing newline not removed!"

    # test for normal spaces
    yld.description = "Test that trailing spaces are removed by parse_listfile"
    yield yld, results[1], "/tmp/foo", "Trailing spaces not removed!"

    # test for tabs
    yld.description = "Test that trailing tabs are removed by parse_listfile"
    yield yld, results[2], "/tmp/foo", "Trailing tab not removed!"
Exemplo n.º 36
0
    def setup_class(cls):
        data = {
            "results_version": 1,
            "name": "test",
            "options": {
                "profile": ['quick'],
                "dmesg": False,
                "verbose": False,
                "platform": "gbm",
                "sync": False,
                "valgrind": False,
                "filter": [],
                "concurrent": "all",
                "test_count": 0,
                "exclude_tests": [],
                "exclude_filter": [],
                "env": {
                    "lspci": "stuff",
                    "uname": "more stuff",
                    "glxinfo": "and stuff",
                    "wglinfo": "stuff"
                }
            },
            "tests": {
                "test/is/a/test": {
                    "returncode": 0,
                    "err": None,
                    "environment": None,
                    "command": "foo",
                    "result": "skip",
                    "time": 0.123,
                    "out": None,
                }
            }
        }

        with utils.with_tempfile(json.dumps(data)) as t:
            with open(t, 'r') as f:
                cls.result = results._update_one_to_two(
                    results.TestrunResult.load(f))
Exemplo n.º 37
0
    def setup_class(cls):
        data = {
            "results_version": 1,
            "name": "test",
            "options": {
                "profile": ['quick'],
                "dmesg": False,
                "verbose": False,
                "platform": "gbm",
                "sync": False,
                "valgrind": False,
                "filter": [],
                "concurrent": "all",
                "test_count": 0,
                "exclude_tests": [],
                "exclude_filter": [],
                "env": {
                    "lspci": "stuff",
                    "uname": "more stuff",
                    "glxinfo": "and stuff",
                    "wglinfo": "stuff"
                }
            },
            "tests": {
                "test/is/a/test": {
                    "returncode": 0,
                    "err": None,
                    "environment": None,
                    "command": "foo",
                    "result": "skip",
                    "time": 0.123,
                    "out": None,
                }
            }
        }

        with utils.with_tempfile(json.dumps(data)) as t:
            with open(t, 'r') as f:
                cls.result = results._update_one_to_two(
                    results.TestrunResult.load(f))
def make_result(data):
    """Write data to a file and return a result.TestrunResult object."""
    with utils.with_tempfile(json.dumps(data)) as t:
        with open(t, 'r') as f:
            # pylint: disable=protected-access
            return backends.json._update_four_to_five(backends.json._load(f))
def _check_config(content):
    """ This is the test that actually checks the glsl config section """
    with utils.with_tempfile(content) as tfile:
        return glsl.GLSLParserTest(tfile), tfile
Exemplo n.º 40
0
def test_resume_non_folder():
    """ TestrunResult.resume doesn't accept a file """
    with utils.with_tempfile('') as f:
        with nt.assert_raises(AssertionError):
            results.TestrunResult.resume(f)
Exemplo n.º 41
0
def _check_config(content):
    """ This is the test that actually checks the glsl config section """
    with utils.with_tempfile(content) as tfile:
        return glsl.GLSLParserTest(tfile), tfile
Exemplo n.º 42
0
def make_result(data):
    """Write data to a file and return a result.TestrunResult object."""
    with utils.with_tempfile(json.dumps(data)) as t:
        with open(t, 'r') as f:
            # pylint: disable=protected-access
            return backends.json._update_three_to_four(backends.json._load(f))
Exemplo n.º 43
0
    },
    'group3/groupA/test': {
        'info': 'Returncode: 1\n\nErrors:stderr\n\nOutput:stdout\n',
        'subtest': {
            'subtest 1': 'pass',
            'subtest 2': 'pass',
            'subtest 3': 'pass',
        },
        'returncode': 0,
        'command': 'this is a command',
        'result': 'pass',
        'time': 0.1
    },
})

with utils.with_tempfile(json.dumps(DATA)) as f:
    RESULT = results._update_zero_to_one(results.load_results(f))


def test_dmesg():
    """ version 1: dmesg is converted from a list to a string """
    assert RESULT.tests['sometest']['dmesg'] == 'this\nis\ndmesg'


def test_subtests_remove_duplicates():
    """ Version 1: Removes duplicate entries """
    assert 'group1/groupA/test/subtest 1' not in RESULT.tests
    assert 'group1/groupA/test/subtest 2' not in RESULT.tests


def test_subtests_add_test():
Exemplo n.º 44
0
def test_json_resume_non_folder():
    """ TestrunResult.resume doesn't accept a file """
    with utils.with_tempfile('') as f:
        with nt.assert_raises(AssertionError):
            backends.json._resume(f)