def check_sets(old, ostat, new, nstat, set_): """ Check that the statuses are added to the correct set """ old['tests']['sometest']['result'] = ostat new['tests']['sometest']['result'] = nstat with utils.with_tempfile(json.dumps(old)) as ofile: with utils.with_tempfile(json.dumps(new)) as nfile: summ = summary.Summary([ofile, nfile]) print(summ.tests) nt.assert_equal(1, len(summ.tests[set_]), msg="{0} was not appended".format(set_))
def glslparser_exetensions_seperators(): """ GlslParserTest() can only have [A-Za-z_] as characters This test generates a number of tests that should catch the majority of errors relating to seperating extensions in the config block of a glslparser test """ problems = [ ('comma seperator', '// require_extensions: ARB_ham, ARB_turkey\n'), ('semi-colon seperator', '// require_extensions: ARB_ham; ARB_turkey\n'), ('trailing semi-colon', '// require_extensions: ARB_ham ARB_turkey\n;'), ('Non-alpha character', '// require_extensions: ARB_$$$\n'), ] content = ('// [config]\n' '// expect_result: pass\n' '// glsl_version: 1.00\n' '{}' '// [end config]\n') for name, value in problems: test = content.format(value) with utils.with_tempfile(test) as tfile: check_bad_character.description = ( 'require_extensions: {0} should raise an error'.format(name)) yield check_bad_character, tfile
def test_subtest_handling(): data = copy.deepcopy(utils.JSON_DATA) data['tests']['with_subtests'] = {} data['tests']['with_subtests']['result'] = 'pass' data['tests']['with_subtests']['subtest'] = {} data['tests']['with_subtests']['subtest']['subtest1'] = 'fail' data['tests']['with_subtests']['subtest']['subtest2'] = 'warn' data['tests']['with_subtests']['subtest']['subtest3'] = 'crash' data['tests']['is_skip'] = {} data['tests']['is_skip']['result'] = 'skip' with utils.with_tempfile(json.dumps(data)) as sumfile: summ = summary.Summary([sumfile]) check_subtests_are_tests.description = \ "Subtests should be treated as full tests " yield check_subtests_are_tests, summ check_tests_w_subtests_are_groups.description = \ "Tests with subtests should be a group" yield check_tests_w_subtests_are_groups, summ test_removed_from_all.description = \ "Tests with subtests should not be in the tests['all'] name" yield test_removed_from_all, summ subtest_not_skip_notrun.description = \ "Skip's should not become NotRun" yield subtest_not_skip_notrun, summ
def test_no_config_end(): """ GLSLParserTest requires [end config] """ with utils.with_tempfile('// [config]\n') as tfile: with nt.assert_raises(glsl.GLSLParserException) as exc: glsl.GLSLParserTest(tfile) nt.assert_equal( exc.exception, 'No [end config] section found!', msg="config section not closed, no exception raised")
def test_info_split(): """ Version 1: info can split into any number of elements """ data = copy.copy(DATA) data['tests']['sometest']['info'] = \ 'Returncode: 1\n\nErrors:stderr\n\nOutput: stdout\n\nmore\n\nstuff' with utils.with_tempfile(json.dumps(data)) as f: results._update_zero_to_one(results.load_results(f))
def check_no_duplicates(content, dup): """ Ensure that duplicate entries raise an error """ with nt.assert_raises(glsl.GLSLParserException) as e: with utils.with_tempfile(content) as tfile: glsl.GLSLParserTest(tfile) nt.eq_( e.exception.message, 'Duplicate entry for key {0} in file {1}'.format(dup, tfile))
def check_no_duplicates(content, dup): """ Ensure that duplicate entries raise an error """ with utils.with_tempfile(content) as tfile: with nt.assert_raises(SystemExit) as e: glsl.GLSLParserTest(tfile) nt.eq_( e.exception.message, 'Duplicate entry for key {0} in file {1}'.format(dup, tfile))
def test_find_config_start(): """ GLSLParserTest finds [config] """ content = ('// [config]\n' '// glsl_version: 1.00\n' '//\n') with utils.with_tempfile(content) as tfile: with nt.assert_raises(SystemExit) as exc: glsl.GLSLParserTest(tfile) nt.assert_not_equal(exc.exception, 'No [config] section found!', msg="Config section not parsed")
def test_no_config_end(): """ GLSLParserTest requires [end config] """ with utils.with_tempfile('// [config]\n') as tfile: with nt.assert_raises(SystemExit) as exc: glsl.GLSLParserTest(tfile) nt.assert_equal( exc.exception, 'No [end config] section found!', msg="config section not closed, no exception raised")
def test_no_glsl_version(): """ glsl_version section is required """ content = ('//\n' '// expect_result: pass\n' '// [end config]\n') with utils.with_tempfile(content) as tfile: with nt.assert_raises(SystemExit) as exc: glsl.GLSLParserTest(tfile) nt.assert_equal( exc.exception, 'Missing required section glsl_version from config', msg="config section not closed, no exception raised")
def test_no_expect_result(): """ expect_result section is required """ content = ('// [config]\n' '// glsl_version: 1.00\n' '//\n') with utils.with_tempfile(content) as tfile: with nt.assert_raises(glsl.GLSLParserException) as exc: glsl.GLSLParserTest(tfile) nt.assert_equal( exc.exception, 'Missing required section expect_result from config', msg="config section not closed, no exception raised")
def test_no_config_start(): """ GLSLParserTest requires [config] """ content = ('// expect_result: pass\n' '// glsl_version: 1.00\n' '// [end config]\n') with utils.with_tempfile(content) as tfile: with nt.assert_raises(glsl.GLSLParserException) as exc: glsl.GLSLParserTest(tfile) nt.assert_equal( exc.exception, 'No [config] section found!', msg="No config section found, no exception raised")
def test_find_config_start(): """ GLSLParserTest finds [config] """ content = ('// [config]\n' '// glsl_version: 1.00\n' '//\n') with utils.with_tempfile(content) as tfile: with nt.assert_raises(glsl.GLSLParserException) as exc: glsl.GLSLParserTest(tfile) nt.assert_not_equal( exc.exception, 'No [config] section found!', msg="Config section not parsed")
def test_parse_gl_test_no_decimal(): """ The GL Parser raises an exception if GL version lacks decimal """ data = ('[require]\n' 'GL = 2\n') with utils.with_tempfile(data) as temp: with nt.assert_raises(testm.ShaderTestParserException) as exc: testm.ShaderTest(temp) nt.assert_equal(exc.exception, "No GL version set", msg="A GL version was passed without a decimal, " "which should have raised an exception, but " "did not")
def test_no_config_start(): """ GLSLParserTest requires [config] """ content = ('// expect_result: pass\n' '// glsl_version: 1.00\n' '// [end config]\n') with utils.with_tempfile(content) as tfile: with nt.assert_raises(SystemExit) as exc: glsl.GLSLParserTest(tfile) nt.assert_equal(exc.exception, 'No [config] section found!', msg="No config section found, no exception raised")
def test_parse_gl_test_no_decimal(): """ The GL Parser raises an exception if GL version lacks decimal """ data = ('[require]\n' 'GL = 2\n') with utils.with_tempfile(data) as temp: with nt.assert_raises(shader_test.ShaderTestParserException) as exc: shader_test.ShaderTest(temp) nt.assert_equal(exc.exception, "No GL version set", msg="A GL version was passed without a decimal, " "which should have raised an exception, but " "did not")
def test_parse_gles3_test(): """ Tests the parser for GLES3 tests """ data = ('[require]\n' 'GL ES >= 3.0\n' 'GLSL ES >= 3.00\n') with utils.with_tempfile(data) as temp: test = testm.ShaderTest(temp) nt.assert_equal(os.path.basename(test.command[0]), "shader_runner_gles3", msg="This test should have run with shader_runner_gles3, " "but instead ran with " + os.path.basename(test.command[0]))
def test_no_glsl_version(): """ glsl_version section is required """ content = ('//\n' '// expect_result: pass\n' '// [end config]\n') with utils.with_tempfile(content) as tfile: with nt.assert_raises(glsl.GLSLParserException) as exc: glsl.GLSLParserTest(tfile) nt.assert_equal( exc.exception, 'Missing required section glsl_version from config', msg="config section not closed, no exception raised")
def test_parse_gles3_test(): """ Tests the parser for GLES3 tests """ data = ('[require]\n' 'GL ES >= 3.0\n' 'GLSL ES >= 3.00\n') with utils.with_tempfile(data) as temp: test = shader_test.ShaderTest(temp) nt.assert_equal( os.path.basename(test.command[0]), "shader_runner_gles3", msg="This test should have run with shader_runner_gles3, " "but instead ran with " + os.path.basename(test.command[0]))
def test_no_expect_result(): """ expect_result section is required """ content = ('// [config]\n' '// glsl_version: 1.00\n' '//\n') with utils.with_tempfile(content) as tfile: with nt.assert_raises(SystemExit) as exc: glsl.GLSLParserTest(tfile) nt.assert_equal( exc.exception, 'Missing required section expect_result from config', msg="config section not closed, no exception raised")
def test_parse_listfile_return(): """ Test that parse_listfile returns a container Given a file with a newline seperated list of results, parse_listfile should return a list of files with no whitespace """ contents = "/tmp/foo\n/tmp/bar\n" with utils.with_tempfile(contents) as tfile: results = core.parse_listfile(tfile) assert isinstance(results, collections.Container)
def test_parse_listfile_tilde(): """ Test that parse_listfile properly expands tildes According to the python docs for python 2.7 (http://docs.python.org/2/library/os.path.html#module-os.path), both os.path.expanduser and os.path.expandvars work on both *nix systems (Linux, *BSD, OSX) and Windows. """ contents = "~/foo\n" with utils.with_tempfile(contents) as tfile: results = core.parse_listfile(tfile) assert results[0] == os.path.expandvars("$HOME/foo")
def test_bad_section_name(): """ A section name not in the _CONFIG_KEYS name raises an error """ content = ('// [config]\n' '// expect_result: pass\n' '// glsl_version: 1.00\n' '// new_awesome_key: foo\n' '// [end config]\n') with utils.with_tempfile(content) as tfile: with nt.assert_raises(SystemExit) as e: glsl.GLSLParserTest(tfile) nt.eq_(e.exception.message, 'Key new_awesome_key in file {0 is not a valid key for a ' 'glslparser test config block'.format(tfile))
def test_bad_section_name(): """ A section name not in the _CONFIG_KEYS name raises an error """ content = ('// [config]\n' '// expect_result: pass\n' '// glsl_version: 1.00\n' '// new_awesome_key: foo\n' '// [end config]\n') with utils.with_tempfile(content) as tfile: with nt.assert_raises(SystemExit) as e: glsl.GLSLParserTest(tfile) nt.eq_( e.exception.message, 'Key new_awesome_key in file {0 is not a valid key for a ' 'glslparser test config block'.format(tfile))
def _load_with_update(data): """If the file is not results.json, it will be renamed. This ensures that the right file is removed. """ try: with utils.with_tempfile(json.dumps(data)) as t: result = backends.json.load_results(t) except OSError as e: # There is the potential that the file will be renamed. In that event # remove the renamed files if e.errno == 2: os.unlink(os.path.join(tempfile.tempdir, 'results.json')) os.unlink(os.path.join(tempfile.tempdir, 'results.json.old')) else: raise return result
def test_good_extensions(): """ Generates tests with good extensions which shouldn't raise errors """ content = ('// [config]\n' '// expect_result: pass\n' '// glsl_version: 1.00\n' '// require_extensions: {}\n' '// [end config]\n') options = [ 'GL_EXT_texture_array', 'GL_EXT_texture_array ARB_example', '!GL_ARB_ham_sandwhich', ] for x in options: test = content.format(x) check_good_extension.description = \ 'require_extension {} is valid'.format(x) with utils.with_tempfile(test) as tfile: yield check_good_extension, tfile, x
def test_parse_listfile_whitespace(): """ Test that parse_listfile remove whitespace """ contents = "/tmp/foo\n/tmp/foo \n/tmp/foo\t\n" with utils.with_tempfile(contents) as tfile: results = core.parse_listfile(tfile) yld = check_whitespace # Test for newlines yld.description = "Test that trailing newlines are removed by " "parse_listfile" yield yld, results[0], "/tmp/foo", "Trailing newline not removed!" # test for normal spaces yld.description = "Test that trailing spaces are removed by parse_listfile" yield yld, results[1], "/tmp/foo", "Trailing spaces not removed!" # test for tabs yld.description = "Test that trailing tabs are removed by parse_listfile" yield yld, results[2], "/tmp/foo", "Trailing tab not removed!"
def test_parse_listfile_whitespace(): """ Test that parse_listfile remove whitespace """ contents = "/tmp/foo\n/tmp/foo \n/tmp/foo\t\n" with utils.with_tempfile(contents) as tfile: results = core.parse_listfile(tfile) yld = check_whitespace # Test for newlines yld.description = ("Test that trailing newlines are removed by " "parse_listfile") yield yld, results[0], "/tmp/foo", "Trailing newline not removed!" # test for normal spaces yld.description = "Test that trailing spaces are removed by parse_listfile" yield yld, results[1], "/tmp/foo", "Trailing spaces not removed!" # test for tabs yld.description = "Test that trailing tabs are removed by parse_listfile" yield yld, results[2], "/tmp/foo", "Trailing tab not removed!"
def setup_class(cls): data = { "results_version": 1, "name": "test", "options": { "profile": ['quick'], "dmesg": False, "verbose": False, "platform": "gbm", "sync": False, "valgrind": False, "filter": [], "concurrent": "all", "test_count": 0, "exclude_tests": [], "exclude_filter": [], "env": { "lspci": "stuff", "uname": "more stuff", "glxinfo": "and stuff", "wglinfo": "stuff" } }, "tests": { "test/is/a/test": { "returncode": 0, "err": None, "environment": None, "command": "foo", "result": "skip", "time": 0.123, "out": None, } } } with utils.with_tempfile(json.dumps(data)) as t: with open(t, 'r') as f: cls.result = results._update_one_to_two( results.TestrunResult.load(f))
def make_result(data): """Write data to a file and return a result.TestrunResult object.""" with utils.with_tempfile(json.dumps(data)) as t: with open(t, 'r') as f: # pylint: disable=protected-access return backends.json._update_four_to_five(backends.json._load(f))
def _check_config(content): """ This is the test that actually checks the glsl config section """ with utils.with_tempfile(content) as tfile: return glsl.GLSLParserTest(tfile), tfile
def test_resume_non_folder(): """ TestrunResult.resume doesn't accept a file """ with utils.with_tempfile('') as f: with nt.assert_raises(AssertionError): results.TestrunResult.resume(f)
def make_result(data): """Write data to a file and return a result.TestrunResult object.""" with utils.with_tempfile(json.dumps(data)) as t: with open(t, 'r') as f: # pylint: disable=protected-access return backends.json._update_three_to_four(backends.json._load(f))
}, 'group3/groupA/test': { 'info': 'Returncode: 1\n\nErrors:stderr\n\nOutput:stdout\n', 'subtest': { 'subtest 1': 'pass', 'subtest 2': 'pass', 'subtest 3': 'pass', }, 'returncode': 0, 'command': 'this is a command', 'result': 'pass', 'time': 0.1 }, }) with utils.with_tempfile(json.dumps(DATA)) as f: RESULT = results._update_zero_to_one(results.load_results(f)) def test_dmesg(): """ version 1: dmesg is converted from a list to a string """ assert RESULT.tests['sometest']['dmesg'] == 'this\nis\ndmesg' def test_subtests_remove_duplicates(): """ Version 1: Removes duplicate entries """ assert 'group1/groupA/test/subtest 1' not in RESULT.tests assert 'group1/groupA/test/subtest 2' not in RESULT.tests def test_subtests_add_test():
def test_json_resume_non_folder(): """ TestrunResult.resume doesn't accept a file """ with utils.with_tempfile('') as f: with nt.assert_raises(AssertionError): backends.json._resume(f)