def check_sets(old, ostat, new, nstat, set_):
""" Check that the statuses are added to the correct set """
old['tests']['sometest']['result'] = ostat
new['tests']['sometest']['result'] = nstat
with utils.with_tempfile(json.dumps(old)) as ofile:
with utils.with_tempfile(json.dumps(new)) as nfile:
summ = summary.Summary([ofile, nfile])
print(summ.tests)
nt.assert_equal(1, len(summ.tests[set_]),
msg="{0} was not appended".format(set_))
def check_sets(old, ostat, new, nstat, set_):
""" Check that the statuses are added to the correct set """
old['tests']['sometest']['result'] = ostat
new['tests']['sometest']['result'] = nstat
with utils.with_tempfile(json.dumps(old)) as ofile:
with utils.with_tempfile(json.dumps(new)) as nfile:
summ = summary.Summary([ofile, nfile])
print(summ.tests)
nt.assert_equal(1,
len(summ.tests[set_]),
msg="{0} was not appended".format(set_))
def glslparser_exetensions_seperators():
""" GlslParserTest() can only have [A-Za-z_] as characters
This test generates a number of tests that should catch the majority of
errors relating to seperating extensions in the config block of a
glslparser test
"""
problems = [
('comma seperator', '// require_extensions: ARB_ham, ARB_turkey\n'),
('semi-colon seperator', '// require_extensions: ARB_ham; ARB_turkey\n'),
('trailing semi-colon', '// require_extensions: ARB_ham ARB_turkey\n;'),
('Non-alpha character', '// require_extensions: ARB_$$$\n'),
]
content = ('// [config]\n'
'// expect_result: pass\n'
'// glsl_version: 1.00\n'
'{}'
'// [end config]\n')
for name, value in problems:
test = content.format(value)
with utils.with_tempfile(test) as tfile:
check_bad_character.description = (
'require_extensions: {0} should raise an error'.format(name))
yield check_bad_character, tfile
def glslparser_exetensions_seperators():
""" GlslParserTest() can only have [A-Za-z_] as characters
This test generates a number of tests that should catch the majority of
errors relating to seperating extensions in the config block of a
glslparser test
"""
problems = [
('comma seperator', '// require_extensions: ARB_ham, ARB_turkey\n'),
('semi-colon seperator',
'// require_extensions: ARB_ham; ARB_turkey\n'),
('trailing semi-colon',
'// require_extensions: ARB_ham ARB_turkey\n;'),
('Non-alpha character', '// require_extensions: ARB_$$$\n'),
]
content = ('// [config]\n'
'// expect_result: pass\n'
'// glsl_version: 1.00\n'
'{}'
'// [end config]\n')
for name, value in problems:
test = content.format(value)
with utils.with_tempfile(test) as tfile:
check_bad_character.description = (
'require_extensions: {0} should raise an error'.format(name))
yield check_bad_character, tfile
def test_subtest_handling():
data = copy.deepcopy(utils.JSON_DATA)
data['tests']['with_subtests'] = {}
data['tests']['with_subtests']['result'] = 'pass'
data['tests']['with_subtests']['subtest'] = {}
data['tests']['with_subtests']['subtest']['subtest1'] = 'fail'
data['tests']['with_subtests']['subtest']['subtest2'] = 'warn'
data['tests']['with_subtests']['subtest']['subtest3'] = 'crash'
data['tests']['is_skip'] = {}
data['tests']['is_skip']['result'] = 'skip'
with utils.with_tempfile(json.dumps(data)) as sumfile:
summ = summary.Summary([sumfile])
check_subtests_are_tests.description = \
"Subtests should be treated as full tests "
yield check_subtests_are_tests, summ
check_tests_w_subtests_are_groups.description = \
"Tests with subtests should be a group"
yield check_tests_w_subtests_are_groups, summ
test_removed_from_all.description = \
"Tests with subtests should not be in the tests['all'] name"
yield test_removed_from_all, summ
subtest_not_skip_notrun.description = \
"Skip's should not become NotRun"
yield subtest_not_skip_notrun, summ
def test_subtest_handling():
data = copy.deepcopy(utils.JSON_DATA)
data['tests']['with_subtests'] = {}
data['tests']['with_subtests']['result'] = 'pass'
data['tests']['with_subtests']['subtest'] = {}
data['tests']['with_subtests']['subtest']['subtest1'] = 'fail'
data['tests']['with_subtests']['subtest']['subtest2'] = 'warn'
data['tests']['with_subtests']['subtest']['subtest3'] = 'crash'
data['tests']['is_skip'] = {}
data['tests']['is_skip']['result'] = 'skip'
with utils.with_tempfile(json.dumps(data)) as sumfile:
summ = summary.Summary([sumfile])
check_subtests_are_tests.description = \
"Subtests should be treated as full tests "
yield check_subtests_are_tests, summ
check_tests_w_subtests_are_groups.description = \
"Tests with subtests should be a group"
yield check_tests_w_subtests_are_groups, summ
test_removed_from_all.description = \
"Tests with subtests should not be in the tests['all'] name"
yield test_removed_from_all, summ
subtest_not_skip_notrun.description = \
"Skip's should not become NotRun"
yield subtest_not_skip_notrun, summ
def test_no_config_end():
""" GLSLParserTest requires [end config] """
with utils.with_tempfile('// [config]\n') as tfile:
with nt.assert_raises(glsl.GLSLParserException) as exc:
glsl.GLSLParserTest(tfile)
nt.assert_equal(
exc.exception, 'No [end config] section found!',
msg="config section not closed, no exception raised")
def test_info_split():
""" Version 1: info can split into any number of elements """
data = copy.copy(DATA)
data['tests']['sometest']['info'] = \
'Returncode: 1\n\nErrors:stderr\n\nOutput: stdout\n\nmore\n\nstuff'
with utils.with_tempfile(json.dumps(data)) as f:
results._update_zero_to_one(results.load_results(f))
def check_no_duplicates(content, dup):
""" Ensure that duplicate entries raise an error """
with nt.assert_raises(glsl.GLSLParserException) as e:
with utils.with_tempfile(content) as tfile:
glsl.GLSLParserTest(tfile)
nt.eq_(
e.exception.message,
'Duplicate entry for key {0} in file {1}'.format(dup, tfile))
def check_no_duplicates(content, dup):
""" Ensure that duplicate entries raise an error """
with utils.with_tempfile(content) as tfile:
with nt.assert_raises(SystemExit) as e:
glsl.GLSLParserTest(tfile)
nt.eq_(
e.exception.message,
'Duplicate entry for key {0} in file {1}'.format(dup, tfile))
def test_find_config_start():
""" GLSLParserTest finds [config] """
content = ('// [config]\n' '// glsl_version: 1.00\n' '//\n')
with utils.with_tempfile(content) as tfile:
with nt.assert_raises(SystemExit) as exc:
glsl.GLSLParserTest(tfile)
nt.assert_not_equal(exc.exception,
'No [config] section found!',
msg="Config section not parsed")
def test_no_config_end():
""" GLSLParserTest requires [end config] """
with utils.with_tempfile('// [config]\n') as tfile:
with nt.assert_raises(SystemExit) as exc:
glsl.GLSLParserTest(tfile)
nt.assert_equal(
exc.exception,
'No [end config] section found!',
msg="config section not closed, no exception raised")
def test_no_glsl_version():
""" glsl_version section is required """
content = ('//\n' '// expect_result: pass\n' '// [end config]\n')
with utils.with_tempfile(content) as tfile:
with nt.assert_raises(SystemExit) as exc:
glsl.GLSLParserTest(tfile)
nt.assert_equal(
exc.exception,
'Missing required section glsl_version from config',
msg="config section not closed, no exception raised")
def test_no_expect_result():
""" expect_result section is required """
content = ('// [config]\n' '// glsl_version: 1.00\n' '//\n')
with utils.with_tempfile(content) as tfile:
with nt.assert_raises(glsl.GLSLParserException) as exc:
glsl.GLSLParserTest(tfile)
nt.assert_equal(
exc.exception,
'Missing required section expect_result from config',
msg="config section not closed, no exception raised")
def test_no_config_start():
""" GLSLParserTest requires [config] """
content = ('// expect_result: pass\n'
'// glsl_version: 1.00\n'
'// [end config]\n')
with utils.with_tempfile(content) as tfile:
with nt.assert_raises(glsl.GLSLParserException) as exc:
glsl.GLSLParserTest(tfile)
nt.assert_equal(
exc.exception, 'No [config] section found!',
msg="No config section found, no exception raised")
def test_find_config_start():
""" GLSLParserTest finds [config] """
content = ('// [config]\n'
'// glsl_version: 1.00\n'
'//\n')
with utils.with_tempfile(content) as tfile:
with nt.assert_raises(glsl.GLSLParserException) as exc:
glsl.GLSLParserTest(tfile)
nt.assert_not_equal(
exc.exception, 'No [config] section found!',
msg="Config section not parsed")
def test_parse_gl_test_no_decimal():
""" The GL Parser raises an exception if GL version lacks decimal """
data = ('[require]\n' 'GL = 2\n')
with utils.with_tempfile(data) as temp:
with nt.assert_raises(testm.ShaderTestParserException) as exc:
testm.ShaderTest(temp)
nt.assert_equal(exc.exception,
"No GL version set",
msg="A GL version was passed without a decimal, "
"which should have raised an exception, but "
"did not")
def test_no_config_start():
""" GLSLParserTest requires [config] """
content = ('// expect_result: pass\n'
'// glsl_version: 1.00\n'
'// [end config]\n')
with utils.with_tempfile(content) as tfile:
with nt.assert_raises(SystemExit) as exc:
glsl.GLSLParserTest(tfile)
nt.assert_equal(exc.exception,
'No [config] section found!',
msg="No config section found, no exception raised")
def test_parse_gl_test_no_decimal():
""" The GL Parser raises an exception if GL version lacks decimal """
data = ('[require]\n'
'GL = 2\n')
with utils.with_tempfile(data) as temp:
with nt.assert_raises(shader_test.ShaderTestParserException) as exc:
shader_test.ShaderTest(temp)
nt.assert_equal(exc.exception, "No GL version set",
msg="A GL version was passed without a decimal, "
"which should have raised an exception, but "
"did not")
def test_parse_gles3_test():
""" Tests the parser for GLES3 tests """
data = ('[require]\n' 'GL ES >= 3.0\n' 'GLSL ES >= 3.00\n')
with utils.with_tempfile(data) as temp:
test = testm.ShaderTest(temp)
nt.assert_equal(os.path.basename(test.command[0]),
"shader_runner_gles3",
msg="This test should have run with shader_runner_gles3, "
"but instead ran with " +
os.path.basename(test.command[0]))
def test_no_glsl_version():
""" glsl_version section is required """
content = ('//\n'
'// expect_result: pass\n'
'// [end config]\n')
with utils.with_tempfile(content) as tfile:
with nt.assert_raises(glsl.GLSLParserException) as exc:
glsl.GLSLParserTest(tfile)
nt.assert_equal(
exc.exception,
'Missing required section glsl_version from config',
msg="config section not closed, no exception raised")
def test_parse_gles3_test():
""" Tests the parser for GLES3 tests """
data = ('[require]\n'
'GL ES >= 3.0\n'
'GLSL ES >= 3.00\n')
with utils.with_tempfile(data) as temp:
test = shader_test.ShaderTest(temp)
nt.assert_equal(
os.path.basename(test.command[0]), "shader_runner_gles3",
msg="This test should have run with shader_runner_gles3, "
"but instead ran with " + os.path.basename(test.command[0]))
def test_no_expect_result():
""" expect_result section is required """
content = ('// [config]\n'
'// glsl_version: 1.00\n'
'//\n')
with utils.with_tempfile(content) as tfile:
with nt.assert_raises(SystemExit) as exc:
glsl.GLSLParserTest(tfile)
nt.assert_equal(
exc.exception,
'Missing required section expect_result from config',
msg="config section not closed, no exception raised")
def test_parse_listfile_return():
""" Test that parse_listfile returns a container
Given a file with a newline seperated list of results, parse_listfile
should return a list of files with no whitespace
"""
contents = "/tmp/foo\n/tmp/bar\n"
with utils.with_tempfile(contents) as tfile:
results = core.parse_listfile(tfile)
assert isinstance(results, collections.Container)
def test_parse_listfile_return():
""" Test that parse_listfile returns a container
Given a file with a newline seperated list of results, parse_listfile
should return a list of files with no whitespace
"""
contents = "/tmp/foo\n/tmp/bar\n"
with utils.with_tempfile(contents) as tfile:
results = core.parse_listfile(tfile)
assert isinstance(results, collections.Container)
def test_parse_listfile_tilde():
""" Test that parse_listfile properly expands tildes
According to the python docs for python 2.7
(http://docs.python.org/2/library/os.path.html#module-os.path), both
os.path.expanduser and os.path.expandvars work on both *nix systems (Linux,
*BSD, OSX) and Windows.
"""
contents = "~/foo\n"
with utils.with_tempfile(contents) as tfile:
results = core.parse_listfile(tfile)
assert results[0] == os.path.expandvars("$HOME/foo")
def test_bad_section_name():
""" A section name not in the _CONFIG_KEYS name raises an error """
content = ('// [config]\n'
'// expect_result: pass\n'
'// glsl_version: 1.00\n'
'// new_awesome_key: foo\n'
'// [end config]\n')
with utils.with_tempfile(content) as tfile:
with nt.assert_raises(SystemExit) as e:
glsl.GLSLParserTest(tfile)
nt.eq_(e.exception.message,
'Key new_awesome_key in file {0 is not a valid key for a '
'glslparser test config block'.format(tfile))
def test_parse_listfile_tilde():
""" Test that parse_listfile properly expands tildes
According to the python docs for python 2.7
(http://docs.python.org/2/library/os.path.html#module-os.path), both
os.path.expanduser and os.path.expandvars work on both *nix systems (Linux,
*BSD, OSX) and Windows.
"""
contents = "~/foo\n"
with utils.with_tempfile(contents) as tfile:
results = core.parse_listfile(tfile)
assert results[0] == os.path.expandvars("$HOME/foo")
def test_bad_section_name():
""" A section name not in the _CONFIG_KEYS name raises an error """
content = ('// [config]\n'
'// expect_result: pass\n'
'// glsl_version: 1.00\n'
'// new_awesome_key: foo\n'
'// [end config]\n')
with utils.with_tempfile(content) as tfile:
with nt.assert_raises(SystemExit) as e:
glsl.GLSLParserTest(tfile)
nt.eq_(
e.exception.message,
'Key new_awesome_key in file {0 is not a valid key for a '
'glslparser test config block'.format(tfile))
def _load_with_update(data):
"""If the file is not results.json, it will be renamed.
This ensures that the right file is removed.
"""
try:
with utils.with_tempfile(json.dumps(data)) as t:
result = backends.json.load_results(t)
except OSError as e:
# There is the potential that the file will be renamed. In that event
# remove the renamed files
if e.errno == 2:
os.unlink(os.path.join(tempfile.tempdir, 'results.json'))
os.unlink(os.path.join(tempfile.tempdir, 'results.json.old'))
else:
raise
return result
def _load_with_update(data):
"""If the file is not results.json, it will be renamed.
This ensures that the right file is removed.
"""
try:
with utils.with_tempfile(json.dumps(data)) as t:
result = backends.json.load_results(t)
except OSError as e:
# There is the potential that the file will be renamed. In that event
# remove the renamed files
if e.errno == 2:
os.unlink(os.path.join(tempfile.tempdir, 'results.json'))
os.unlink(os.path.join(tempfile.tempdir, 'results.json.old'))
else:
raise
return result
def test_good_extensions():
""" Generates tests with good extensions which shouldn't raise errors """
content = ('// [config]\n'
'// expect_result: pass\n'
'// glsl_version: 1.00\n'
'// require_extensions: {}\n'
'// [end config]\n')
options = [
'GL_EXT_texture_array',
'GL_EXT_texture_array ARB_example',
'!GL_ARB_ham_sandwhich',
]
for x in options:
test = content.format(x)
check_good_extension.description = \
'require_extension {} is valid'.format(x)
with utils.with_tempfile(test) as tfile:
yield check_good_extension, tfile, x
def test_good_extensions():
""" Generates tests with good extensions which shouldn't raise errors """
content = ('// [config]\n'
'// expect_result: pass\n'
'// glsl_version: 1.00\n'
'// require_extensions: {}\n'
'// [end config]\n')
options = [
'GL_EXT_texture_array',
'GL_EXT_texture_array ARB_example',
'!GL_ARB_ham_sandwhich',
]
for x in options:
test = content.format(x)
check_good_extension.description = \
'require_extension {} is valid'.format(x)
with utils.with_tempfile(test) as tfile:
yield check_good_extension, tfile, x
def test_parse_listfile_whitespace():
""" Test that parse_listfile remove whitespace """
contents = "/tmp/foo\n/tmp/foo \n/tmp/foo\t\n"
with utils.with_tempfile(contents) as tfile:
results = core.parse_listfile(tfile)
yld = check_whitespace
# Test for newlines
yld.description = "Test that trailing newlines are removed by " "parse_listfile"
yield yld, results[0], "/tmp/foo", "Trailing newline not removed!"
# test for normal spaces
yld.description = "Test that trailing spaces are removed by parse_listfile"
yield yld, results[1], "/tmp/foo", "Trailing spaces not removed!"
# test for tabs
yld.description = "Test that trailing tabs are removed by parse_listfile"
yield yld, results[2], "/tmp/foo", "Trailing tab not removed!"
def test_parse_listfile_whitespace():
""" Test that parse_listfile remove whitespace """
contents = "/tmp/foo\n/tmp/foo \n/tmp/foo\t\n"
with utils.with_tempfile(contents) as tfile:
results = core.parse_listfile(tfile)
yld = check_whitespace
# Test for newlines
yld.description = ("Test that trailing newlines are removed by "
"parse_listfile")
yield yld, results[0], "/tmp/foo", "Trailing newline not removed!"
# test for normal spaces
yld.description = "Test that trailing spaces are removed by parse_listfile"
yield yld, results[1], "/tmp/foo", "Trailing spaces not removed!"
# test for tabs
yld.description = "Test that trailing tabs are removed by parse_listfile"
yield yld, results[2], "/tmp/foo", "Trailing tab not removed!"
def setup_class(cls):
data = {
"results_version": 1,
"name": "test",
"options": {
"profile": ['quick'],
"dmesg": False,
"verbose": False,
"platform": "gbm",
"sync": False,
"valgrind": False,
"filter": [],
"concurrent": "all",
"test_count": 0,
"exclude_tests": [],
"exclude_filter": [],
"env": {
"lspci": "stuff",
"uname": "more stuff",
"glxinfo": "and stuff",
"wglinfo": "stuff"
}
},
"tests": {
"test/is/a/test": {
"returncode": 0,
"err": None,
"environment": None,
"command": "foo",
"result": "skip",
"time": 0.123,
"out": None,
}
}
}
with utils.with_tempfile(json.dumps(data)) as t:
with open(t, 'r') as f:
cls.result = results._update_one_to_two(
results.TestrunResult.load(f))
def setup_class(cls):
data = {
"results_version": 1,
"name": "test",
"options": {
"profile": ['quick'],
"dmesg": False,
"verbose": False,
"platform": "gbm",
"sync": False,
"valgrind": False,
"filter": [],
"concurrent": "all",
"test_count": 0,
"exclude_tests": [],
"exclude_filter": [],
"env": {
"lspci": "stuff",
"uname": "more stuff",
"glxinfo": "and stuff",
"wglinfo": "stuff"
}
},
"tests": {
"test/is/a/test": {
"returncode": 0,
"err": None,
"environment": None,
"command": "foo",
"result": "skip",
"time": 0.123,
"out": None,
}
}
}
with utils.with_tempfile(json.dumps(data)) as t:
with open(t, 'r') as f:
cls.result = results._update_one_to_two(
results.TestrunResult.load(f))
def make_result(data):
"""Write data to a file and return a result.TestrunResult object."""
with utils.with_tempfile(json.dumps(data)) as t:
with open(t, 'r') as f:
# pylint: disable=protected-access
return backends.json._update_four_to_five(backends.json._load(f))
def _check_config(content):
""" This is the test that actually checks the glsl config section """
with utils.with_tempfile(content) as tfile:
return glsl.GLSLParserTest(tfile), tfile
def test_resume_non_folder():
""" TestrunResult.resume doesn't accept a file """
with utils.with_tempfile('') as f:
with nt.assert_raises(AssertionError):
results.TestrunResult.resume(f)
def _check_config(content):
""" This is the test that actually checks the glsl config section """
with utils.with_tempfile(content) as tfile:
return glsl.GLSLParserTest(tfile), tfile
def make_result(data):
"""Write data to a file and return a result.TestrunResult object."""
with utils.with_tempfile(json.dumps(data)) as t:
with open(t, 'r') as f:
# pylint: disable=protected-access
return backends.json._update_three_to_four(backends.json._load(f))
},
'group3/groupA/test': {
'info': 'Returncode: 1\n\nErrors:stderr\n\nOutput:stdout\n',
'subtest': {
'subtest 1': 'pass',
'subtest 2': 'pass',
'subtest 3': 'pass',
},
'returncode': 0,
'command': 'this is a command',
'result': 'pass',
'time': 0.1
},
})
with utils.with_tempfile(json.dumps(DATA)) as f:
RESULT = results._update_zero_to_one(results.load_results(f))
def test_dmesg():
""" version 1: dmesg is converted from a list to a string """
assert RESULT.tests['sometest']['dmesg'] == 'this\nis\ndmesg'
def test_subtests_remove_duplicates():
""" Version 1: Removes duplicate entries """
assert 'group1/groupA/test/subtest 1' not in RESULT.tests
assert 'group1/groupA/test/subtest 2' not in RESULT.tests
def test_subtests_add_test():
def test_json_resume_non_folder():
""" TestrunResult.resume doesn't accept a file """
with utils.with_tempfile('') as f:
with nt.assert_raises(AssertionError):
backends.json._resume(f)