def test_update_results_old():
"""backends.json.update_results(): updates results
Because of the design of the our updates (namely that they silently
incrementally update from x to y) it's impossible to konw exactly what
we'll get at th end without having tests that have to be reworked each time
updates are run. Since there already is (at least for v0 -> v1) a fairly
comprehensive set of tests, this test only tests that update_results() has
been set equal to the CURRENT_JSON_VERSION, (which is one of the effects of
runing update_results() with the assumption that there is sufficient other
testing of the update process.
"""
data = utils.JSON_DATA.copy()
data["results_version"] = 0
with utils.tempdir() as d:
with open(os.path.join(d, "main"), "w") as f:
json.dump(data, f)
with open(os.path.join(d, "main"), "r") as f:
base = backends.json._load(f)
res = backends.json._update_results(base, f.name)
nt.assert_equal(res.results_version, backends.json.CURRENT_JSON_VERSION)
def test_load_results_folder_as_main():
""" Test that load_results takes a folder with a file named main in it """
with utils.tempdir() as tdir:
with open(os.path.join(tdir, 'main'), 'w') as tfile:
tfile.write(json.dumps(utils.JSON_DATA))
backends.json.load_results(tdir)
def test_json_initialize_metadata():
"""backends.json.JSONBackend.initialize(): produces a metadata.json file"""
with utils.tempdir() as f:
test = backends.json.JSONBackend(f)
test.initialize(BACKEND_INITIAL_META)
nt.ok_(os.path.exists(os.path.join(f, "metadata.json")))
def test_load_results_folder():
"""backends.json.load_results: takes a folder with a file named results.json"""
with utils.tempdir() as tdir:
with open(os.path.join(tdir, "results.json"), "w") as tfile:
tfile.write(json.dumps(utils.JSON_DATA))
backends.json.load_results(tdir, "none")
def test_update_results_old():
""" update_results() updates results
Because of the design of the our updates (namely that they silently
incrementally update from x to y) it's impossible to konw exactly what
we'll get at th end without having tests that have to be reworked each time
updates are run. Since there already is (at least for v0 -> v1) a fairly
comprehensive set of tests, this test only tests that update_results() has
been set equal to the CURRENT_JSON_VERSION, (which is one of the effects of
runing update_results() with the assumption that there is sufficient other
testing of the update process.
"""
data = utils.JSON_DATA.copy()
data['results_version'] = 0
with utils.tempdir() as d:
with open(os.path.join(d, 'main'), 'w') as f:
json.dump(data, f)
with open(os.path.join(d, 'main'), 'r') as f:
base = backends.json._load(f)
res = backends.json._update_results(base, f.name)
nt.assert_equal(res.results_version, backends.json.CURRENT_JSON_VERSION)
def test_load_results_folder_as_main():
""" Test that load_results takes a folder with a file named main in it """
with utils.tempdir() as tdir:
with open(os.path.join(tdir, 'main'), 'w') as tfile:
tfile.write(json.dumps(utils.JSON_DATA))
results.load_results(tdir)
def test_load_results_folder():
""" Test that load_results takes a folder with a file named results.json """
with utils.tempdir() as tdir:
with open(os.path.join(tdir, 'results.json'), 'w') as tfile:
tfile.write(json.dumps(utils.JSON_DATA))
backends.json.load_results(tdir)
def test_json_initialize_metadata():
""" JSONBackend.initialize() produces a metadata.json file """
with utils.tempdir() as f:
test = backends.json.JSONBackend(f)
test.initialize(BACKEND_INITIAL_META)
nt.ok_(os.path.exists(os.path.join(f, 'metadata.json')))
def test_load_results_folder():
""" Test that load_results takes a folder with a file named results.json """
with utils.tempdir() as tdir:
with open(os.path.join(tdir, 'results.json'), 'w') as tfile:
tfile.write(json.dumps(utils.JSON_DATA))
results.load_results(tdir)
def test_resume_load_incomplete():
"""backends.json._resume: loads incomplete results.
Because resume, aggregate, and summary all use the function called _resume
we can't remove incomplete tests here. It's probably worth doing a refactor
to split some code out and allow this to be done in the resume path.
"""
with utils.tempdir() as f:
backend = backends.json.JSONBackend(f)
backend.initialize(BACKEND_INITIAL_META)
with backend.write_test("group1/test1") as t:
t({"result": "fail"})
with backend.write_test("group1/test2") as t:
t({"result": "pass"})
with backend.write_test("group2/test3") as t:
t({"result": "crash"})
with backend.write_test("group2/test4") as t:
t({"result": "incomplete"})
test = backends.json._resume(f)
nt.assert_set_equal(
set(test.tests.keys()), set(["group1/test1", "group1/test2", "group2/test3", "group2/test4"])
)
def test_load_results_folder():
"""backends.json.load_results: takes a folder with a file named results.json"""
with utils.tempdir() as tdir:
with open(os.path.join(tdir, 'results.json'), 'w') as tfile:
tfile.write(json.dumps(utils.JSON_DATA,
default=backends.json.piglit_encoder))
backends.json.load_results(tdir, 'none')
def test_write_compressed_one_suffix_mixed():
"""backends.abstract.write_compressed: does not generate two different compression suffixes
"""
with utils.tempdir() as d:
with abstract.write_compressed(os.path.join(d, 'results.txt.bz2')) as f:
f.write('foo')
nt.eq_(os.listdir(d)[0], 'results.txt.gz')
def test_write_compressed_one_suffix_gz():
"""backends.abstract.write_compressed: gz Does not duplicate compression suffixes
"""
with utils.tempdir() as d:
with abstract.write_compressed(os.path.join(d, 'results.txt.gz')) as f:
f.write('foo')
nt.eq_(os.listdir(d)[0], 'results.txt.gz')
def test_load_results(self):
"""backends.json.update_results (6 -> 7): load_results properly updates."""
with utils.tempdir() as d:
tempfile = os.path.join(d, 'results.json')
with open(tempfile, 'w') as f:
json.dump(self.DATA, f, default=backends.json.piglit_encoder)
with open(tempfile, 'r') as f:
result = backends.json.load_results(tempfile, 'none')
nt.eq_(result.results_version, 7)
def test_load_results():
"""Version 4: load_results properly updates."""
with utils.tempdir() as d:
tempfile = os.path.join(d, 'results.json')
with open(tempfile, 'w') as f:
json.dump(DATA, f)
with open(tempfile, 'r') as f:
result = backends.json.load_results(tempfile)
nt.assert_equal(result.results_version, 5)
def test_xdg_config_home(self):
""" get_config() finds $XDG_CONFIG_HOME/piglit.conf """
with utils.tempdir() as tdir:
os.environ["XDG_CONFIG_HOME"] = tdir
with open(os.path.join(tdir, "piglit.conf"), "w") as f:
f.write(TestGetConfig.CONF_FILE)
core.get_config()
nt.ok_(core.PIGLIT_CONFIG.has_section("nose-test"), msg="$XDG_CONFIG_HOME not found")
def test_option_conf(self):
""" Run parser: platform option replaces conf """
with utils.tempdir() as tdir:
os.environ['XDG_CONFIG_HOME'] = tdir
with open(os.path.join(tdir, 'piglit.conf'), 'w') as f:
f.write(self._CONF)
args = run._run_parser(['-p', 'x11_egl', 'quick.py', 'foo'])
nt.assert_equal(args.platform, 'x11_egl')
def test_load_results():
"""Version 4: load_results properly updates."""
with utils.tempdir() as d:
tempfile = os.path.join(d, 'results.json')
with open(tempfile, 'w') as f:
json.dump(DATA, f)
with open(tempfile, 'r') as f:
result = backends.json.load_results(tempfile)
nt.assert_equal(result.results_version, 5)
def test_option_conf(self):
""" Run parser: backend option replaces conf """
with utils.tempdir() as tdir:
os.environ['XDG_CONFIG_HOME'] = tdir
with open(os.path.join(tdir, 'piglit.conf'), 'w') as f:
f.write(self._CONF)
args = run._run_parser(['-b', 'json', 'quick.py', 'foo'])
nt.assert_equal(args.backend, 'json')
def test_config_home_fallback(self):
""" get_config() finds $HOME/.config/piglit.conf """
with utils.tempdir() as tdir:
os.environ["HOME"] = tdir
os.mkdir(os.path.join(tdir, ".config"))
with open(os.path.join(tdir, ".config/piglit.conf"), "w") as f:
f.write(TestGetConfig.CONF_FILE)
core.get_config()
nt.ok_(core.PIGLIT_CONFIG.has_section("nose-test"), msg="$HOME/.config/piglit.conf not found")
def test_resume_load():
""" TestrunResult.resume loads with good results """
with utils.tempdir() as f:
backend = backends.json.JSONBackend(f)
backend.initialize(BACKEND_INITIAL_META)
backend.write_test("group1/test1", {'result': 'fail'})
backend.write_test("group1/test2", {'result': 'pass'})
backend.write_test("group2/test3", {'result': 'fail'})
backends.json._resume(f)
def test_initialize_jsonbackend():
"""backends.json.JSONBackend: Class initializes
This needs to be handled separately from the others because it requires
arguments
"""
with utils.tempdir() as tdir:
func = backends.json.JSONBackend(tdir)
nt.ok_(isinstance(func, backends.json.JSONBackend))
def test_load_file_name():
"""backend.junit._load: uses the filename for name if filename != 'results'
"""
with utils.tempdir() as tdir:
filename = os.path.join(tdir, 'foobar.xml')
with open(filename, 'w') as f:
f.write(_XML)
test = backends.junit.REGISTRY.load(filename)
nt.assert_equal(test.name, 'foobar')
def test_initialize_jsonbackend():
""" Test that JSONBackend initializes
This needs to be handled separately from the others because it requires
arguments
"""
with utils.tempdir() as tdir:
func = backends.json.JSONBackend(tdir)
assert isinstance(func, backends.json.JSONBackend)
def test_resume_load():
""" TestrunResult.resume loads with good results """
with utils.tempdir() as f:
backend = backends.json.JSONBackend(f)
backend.initialize(BACKEND_INITIAL_META)
backend.write_test("group1/test1", {'result': 'fail'})
backend.write_test("group1/test2", {'result': 'pass'})
backend.write_test("group2/test3", {'result': 'fail'})
backends.json._resume(f)
def test_xdg_config_home():
"""core.get_config() finds $XDG_CONFIG_HOME/piglit.conf"""
with utils.tempdir() as tdir:
os.environ['XDG_CONFIG_HOME'] = tdir
with open(os.path.join(tdir, 'piglit.conf'), 'w') as f:
f.write(_CONF_FILE)
core.get_config()
nt.ok_(core.PIGLIT_CONFIG.has_section('nose-test'),
msg='$XDG_CONFIG_HOME not found')
def test_load_folder_name():
"""backends.junit._load: uses the folder name if the result is 'results'"""
with utils.tempdir() as tdir:
os.mkdir(os.path.join(tdir, 'a cool test'))
filename = os.path.join(tdir, 'a cool test', 'results.xml')
with open(filename, 'w') as f:
f.write(_XML)
test = backends.junit.REGISTRY.load(filename, 'none')
nt.assert_equal(test.name, 'a cool test')
def test_load_file_name():
"""backends.junit._load: uses the filename for name if filename != 'results'
"""
with utils.tempdir() as tdir:
filename = os.path.join(tdir, 'foobar.xml')
with open(filename, 'w') as f:
f.write(_XML)
test = backends.junit.REGISTRY.load(filename, 'none')
nt.assert_equal(test.name, 'foobar')
def test_initialize_jsonbackend():
""" Test that JSONBackend initializes
This needs to be handled separately from the others because it requires
arguments
"""
with utils.tempdir() as tdir:
func = results.JSONBackend(tdir)
assert isinstance(func, results.JSONBackend)
def test_xdg_config_home(self):
""" get_config() finds $XDG_CONFIG_HOME/piglit.conf """
with utils.tempdir() as tdir:
os.environ['XDG_CONFIG_HOME'] = tdir
with open(os.path.join(tdir, 'piglit.conf'), 'w') as f:
f.write(TestGetConfig.CONF_FILE)
core.get_config()
nt.ok_(core.PIGLIT_CONFIG.has_section('nose-test'),
msg='$XDG_CONFIG_HOME not found')
def test_initialize_jsonbackend():
""" Test that JSONBackend initializes
This needs to be handled separately from the others because it requires
arguments
"""
with utils.tempdir() as tdir:
func = results.JSONBackend(tdir, BACKEND_INITIAL_META)
assert isinstance(func, results.JSONBackend)
def test_load_json():
"""backends.load(): Loads .json files."""
with utils.tempdir() as tdir:
filename = os.path.join(tdir, 'results.json')
with open(filename, 'w') as f:
json.dump(utils.JSON_DATA, f, default=backends.json.piglit_encoder)
result = backends.load(filename)
nt.assert_is_instance(result, results.TestrunResult)
nt.assert_in('sometest', result.tests)
def test_load_json():
"""backends.load(): Loads .json files."""
with utils.tempdir() as tdir:
filename = os.path.join(tdir, "results.json")
with open(filename, "w") as f:
json.dump(utils.JSON_DATA, f)
result = backends.load(filename)
nt.assert_is_instance(result, results.TestrunResult)
nt.assert_in("sometest", result.tests)
def test_config_home_fallback():
"""core.get_config() finds $HOME/.config/piglit.conf"""
with utils.tempdir() as tdir:
os.environ['HOME'] = tdir
os.mkdir(os.path.join(tdir, '.config'))
with open(os.path.join(tdir, '.config/piglit.conf'), 'w') as f:
f.write(_CONF_FILE)
core.get_config()
nt.ok_(core.PIGLIT_CONFIG.has_section('nose-test'),
msg='$HOME/.config/piglit.conf not found')
def test_load_folder_name():
"""backend.junit._load: uses the foldername if the result is 'results'
"""
with utils.tempdir() as tdir:
os.mkdir(os.path.join(tdir, 'a cool test'))
filename = os.path.join(tdir, 'a cool test', 'results.xml')
with open(filename, 'w') as f:
f.write(_XML)
test = backends.junit.REGISTRY.load(filename)
nt.assert_equal(test.name, 'a cool test')
def test_config_home_fallback(self):
""" get_config() finds $HOME/.config/piglit.conf """
with utils.tempdir() as tdir:
os.environ['HOME'] = tdir
os.mkdir(os.path.join(tdir, '.config'))
with open(os.path.join(tdir, '.config/piglit.conf'), 'w') as f:
f.write(TestGetConfig.CONF_FILE)
core.get_config()
nt.ok_(core.PIGLIT_CONFIG.has_section('nose-test'),
msg='$HOME/.config/piglit.conf not found')
def test_load_json():
"""backends.load(): Loads .json files."""
with utils.tempdir() as tdir:
filename = os.path.join(tdir, 'results.json')
with open(filename, 'w') as f:
json.dump(utils.JSON_DATA, f)
result = backends.load(filename)
nt.assert_is_instance(result, results.TestrunResult)
nt.assert_in('sometest', result.tests)
def test_conf_default(self):
""" Run parser platform: conf is used as a default when applicable """
self._unset_config()
self._move_piglit_conf()
with utils.tempdir() as tdir:
os.environ['XDG_CONFIG_HOME'] = tdir
with open(os.path.join(tdir, 'piglit.conf'), 'w') as f:
f.write(self._CONF)
args = run._run_parser(['quick.py', 'foo'])
nt.assert_equal(args.backend, 'junit')
def test_local():
"""core.get_config() finds ./piglit.conf"""
with utils.tempdir() as tdir:
os.chdir(tdir)
with open(os.path.join(tdir, 'piglit.conf'), 'w') as f:
f.write(_CONF_FILE)
core.get_config()
nt.ok_(core.PIGLIT_CONFIG.has_section('nose-test'),
msg='./piglit.conf not found')
def test_load_default_name():
"""backends.junit._load: uses 'junit result' for name as fallback"""
with utils.tempdir() as tdir:
os.chdir(tdir)
filename = 'results.xml'
with open(filename, 'w') as f:
f.write(_XML)
test = backends.junit.REGISTRY.load(filename, 'none')
nt.assert_equal(test.name, 'junit result')
def test_load_default_name():
"""backend.junit._load: uses 'junit result' for name as fallback"""
with utils.tempdir() as tdir:
os.chdir(tdir)
filename = 'results.xml'
with open(filename, 'w') as f:
f.write(_XML)
test = backends.junit.REGISTRY.load(filename)
nt.assert_equal(test.name, 'junit result')
def test_conf_default(self):
""" Run parser platform: conf is used as a default when applicable """
self._unset_config()
self._move_piglit_conf()
with utils.tempdir() as tdir:
os.environ['XDG_CONFIG_HOME'] = tdir
with open(os.path.join(tdir, 'piglit.conf'), 'w') as f:
f.write(self._CONF)
args = run._run_parser(['quick.py', 'foo'])
nt.assert_equal(args.platform, 'gbm')
def test_local():
""" get_config() finds ./piglit.conf """
with utils.tempdir() as tdir:
os.chdir(tdir)
with open(os.path.join(tdir, 'piglit.conf'), 'w') as f:
f.write(_CONF_FILE)
core.get_config()
nt.ok_(core.PIGLIT_CONFIG.has_section('nose-test'),
msg='./piglit.conf not found')
def test_local(self):
""" get_config() finds ./piglit.conf """
with utils.tempdir() as tdir:
self.defer(os.chdir, os.getcwd())
os.chdir(tdir)
with open(os.path.join(tdir, "piglit.conf"), "w") as f:
f.write(TestGetConfig.CONF_FILE)
core.get_config()
nt.ok_(core.PIGLIT_CONFIG.has_section("nose-test"), msg="./piglit.conf not found")
def _test_decompressor(mode):
"""helper to simplify testing decompressors."""
func = compression.COMPRESSORS[mode]
dec = compression.DECOMPRESSORS[mode]
with utils.tempdir() as t:
path = os.path.join(t, 'file')
with func(path) as f:
f.write('foo')
with dec(path) as f:
nt.eq_(f.read(), 'foo')
def test_resume_load():
""" TestrunResult.resume loads with good results """
with utils.tempdir() as f:
backend = backends.JSONBackend(f)
backend.initialize(BACKEND_INITIAL_META)
backend.write_test("group1/test1", {'result': 'fail'})
backend.write_test("group1/test2", {'result': 'pass'})
backend.write_test("group2/test3", {'result': 'fail'})
try:
results.TestrunResult.resume(f)
except Exception as e:
raise AssertionError(e)
def test_env_conf(self):
""" Run parser: env overwrides a conf value """
self._unset_config()
self._move_piglit_conf()
self.__set_env()
with utils.tempdir() as tdir:
os.environ['XDG_CONFIG_HOME'] = tdir
with open(os.path.join(tdir, 'piglit.conf'), 'w') as f:
f.write(self._CONF)
args = run._run_parser(['quick.py', 'foo'])
nt.assert_equal(args.platform, 'glx')
def test_env_conf(self):
""" Run parser: env overwrides a conf value """
self._unset_config()
self._move_piglit_conf()
self.__set_env()
with utils.tempdir() as tdir:
os.environ['XDG_CONFIG_HOME'] = tdir
with open(os.path.join(tdir, 'piglit.conf'), 'w') as f:
f.write(self._CONF)
args = run._run_parser(['quick.py', 'foo'])
nt.assert_equal(args.platform, 'glx')
def test_update_results_current():
""" update_results() returns early when the results_version is current """
data = utils.JSON_DATA.copy()
data['results_version'] = backends.json.CURRENT_JSON_VERSION
with utils.tempdir() as d:
with open(os.path.join(d, 'main'), 'w') as f:
json.dump(data, f)
with open(os.path.join(d, 'main'), 'r') as f:
base = backends.json._load(f)
res = backends.json._update_results(base, f.name)
nt.assert_dict_equal(res.__dict__, base.__dict__)
def test_testrunresult_write():
""" TestrunResult.write() works
This tests for a bug where TestrunResult.write() wrote a file containing
{}, essentially if it dumps a file that is equal to what was provided then
it's probably working
"""
with utils.resultfile() as f:
result = results.load_results(f.name)
with utils.tempdir() as tdir:
result.write(os.path.join(tdir, 'results.json'))
new = results.load_results(os.path.join(tdir, 'results.json'))
nt.assert_dict_equal(result.__dict__, new.__dict__)
def test_resume_load_valid():
""" TestrunResult.resume loads valid results """
with utils.tempdir() as f:
backend = backends.json.JSONBackend(f)
backend.initialize(BACKEND_INITIAL_META)
backend.write_test("group1/test1", {'result': 'fail'})
backend.write_test("group1/test2", {'result': 'pass'})
backend.write_test("group2/test3", {'result': 'fail'})
test = backends.json._resume(f)
nt.assert_set_equal(
set(test.tests.keys()),
set(['group1/test1', 'group1/test2', 'group2/test3']),
)
def test(self):
""" get_config() finds $XDG_CONFIG_HOME/piglit.conf """
self.defer(lambda: core.PIGLIT_CONFIG == ConfigParser.SafeConfigParser)
self.add_teardown('XDG_CONFIG_HOME')
if os.path.exists('piglit.conf'):
shutil.move('piglit.conf', 'piglit.conf.restore')
self.defer(shutil.move, 'piglit.conf.restore', 'piglit.conf')
with utils.tempdir() as tdir:
os.environ['XDG_CONFIG_HOME'] = tdir
with open(os.path.join(tdir, 'piglit.conf'), 'w') as f:
f.write(CONF_FILE)
core.get_config()
nt.ok_(core.PIGLIT_CONFIG.has_section('nose-test'),
msg='$XDG_CONFIG_HOME not found')
def test_resume_load_invalid():
""" TestrunResult.resume ignores invalid results """
with utils.tempdir() as f:
backend = backends.json.JSONBackend(f)
backend.initialize(BACKEND_INITIAL_META)
backend.write_test("group1/test1", {'result': 'fail'})
backend.write_test("group1/test2", {'result': 'pass'})
backend.write_test("group2/test3", {'result': 'fail'})
with open(os.path.join(f, 'tests', 'x.json'), 'w') as w:
w.write('foo')
test = backends.json._resume(f)
nt.assert_set_equal(
set(test.tests.keys()),
set(['group1/test1', 'group1/test2', 'group2/test3']),
)
def test_bad_value_in_conf(self):
""" run parser: an error is raised when the platform in conf is bad """
self._unset_config()
self._move_piglit_conf()
# This has sideffects, it shouldn't effect anything in this module, but
# it may cause later problems. But without this we get ugly error spew
# from this test.
sys.stderr = open(os.devnull, 'w')
with utils.tempdir() as tdir:
with open(os.path.join(tdir, 'piglit.conf'), 'w') as f:
f.write('[core]\nplatform=foobar')
with nt.assert_raises(SystemExit):
run._run_parser([
'-f',
os.path.join(tdir, 'piglit.conf'), 'quick.py', 'foo'
])
def test_junit_replace():
"""JUnitBackend.write_test: '{separator}' is replaced with '.'"""
with utils.tempdir() as tdir:
test = backends.junit.JUnitBackend(tdir)
test.initialize(BACKEND_INITIAL_META)
test.write_test(
grouptools.join('a', 'test', 'group', 'test1'),
results.TestResult({
'time': 1.2345,
'result': 'pass',
'out': 'this is stdout',
'err': 'this is stderr',
'command': 'foo',
}))
test.finalize()
test_value = etree.parse(os.path.join(tdir, 'results.xml')).getroot()
nt.assert_equal(
test_value.find('.//testcase').attrib['classname'],
'piglit.a.test.group')
def test_junit_skips_bad_tests():
""" backends.JUnitBackend skips illformed tests """
with utils.tempdir() as tdir:
test = backends.JUnitBackend(tdir)
test.initialize(BACKEND_INITIAL_META)
test.write_test(
'a/test/group/test1',
results.TestResult({
'time': 1.2345,
'result': 'pass',
'out': 'this is stdout',
'err': 'this is stderr',
})
)
with open(os.path.join(tdir, 'tests', '1.xml'), 'w') as f:
f.write('bad data')
try:
test.finalize()
except etree.XMLSyntaxError as e:
raise AssertionError(e)
