def test_resume_load_incomplete():
"""backends.json._resume: loads incomplete results.
Because resume, aggregate, and summary all use the function called _resume
we can't remove incomplete tests here. It's probably worth doing a refactor
to split some code out and allow this to be done in the resume path.
"""
with utils.tempdir() as f:
backend = backends.json.JSONBackend(f)
backend.initialize(BACKEND_INITIAL_META)
with backend.write_test("group1/test1") as t:
t(results.TestResult('fail'))
with backend.write_test("group1/test2") as t:
t(results.TestResult('pass'))
with backend.write_test("group2/test3") as t:
t(results.TestResult('crash'))
with backend.write_test("group2/test4") as t:
t(results.TestResult('incomplete'))
test = backends.json._resume(f)
nt.assert_set_equal(
set(test.tests.keys()),
set([
'group1/test1', 'group1/test2', 'group2/test3', 'group2/test4'
]),
)
def setup_class(cls):
"""class fixture."""
res1 = results.TestrunResult()
res1.tests['foo'] = results.TestResult('pass')
res1.tests['bar'] = results.TestResult('fail')
res1.tests['oink'] = results.TestResult('crash')
res1.tests['bonk'] = results.TestResult('warn')
res1.tests['bor'] = results.TestResult('crash')
res1.tests['bor'].subtests['1'] = 'pass'
res1.tests['bor'].subtests['2'] = 'skip'
res1.tests['bor'].subtests['3'] = 'fail'
res1.tests['bor'].subtests['4'] = 'pass'
res2 = results.TestrunResult()
res2.tests['foo'] = results.TestResult('fail')
res2.tests['bar'] = results.TestResult('pass')
res2.tests['oink'] = results.TestResult('crash')
res2.tests['tonk'] = results.TestResult('incomplete')
res2.tests['bor'] = results.TestResult('crash')
res2.tests['bor'].subtests['1'] = 'fail'
res2.tests['bor'].subtests['2'] = 'skip'
res2.tests['bor'].subtests['3'] = 'pass'
res2.tests['bor'].subtests['5'] = 'pass'
cls.test = summary.Results([res1, res2])
def setup_class(cls):
res = results.TestrunResult()
res.tests['foo'] = results.TestResult('fail')
res.tests['bar'] = results.TestResult('pass')
res.tests['oink'] = results.TestResult('skip')
res.tests['tonk'] = results.TestResult('incomplete')
cls.test = summary.Results([res])
def result(self):
"""Returns a result object with no subtests."""
res1 = results.TestrunResult()
res1.tests['foo'] = results.TestResult('pass')
res2 = results.TestrunResult()
res2.tests['foo'] = results.TestResult('fail')
res2.tests['bar'] = results.TestResult('fail')
return summary.Results([res1, res2])
def setup_class(cls):
tr = results.TestResult('crash')
tr.subtests['foo'] = status.PASS
run = results.TestrunResult()
run.tests['sub'] = tr
run.tests['test'] = results.TestResult('pass')
run.calculate_group_totals()
cls.inst = run
def test_Results_get_results():
"""summary.Results.get_results: returns list of statuses"""
res1 = results.TestrunResult()
res1.tests['foo'] = results.TestResult('pass')
res2 = results.TestrunResult()
res2.tests['foo'] = results.TestResult('fail')
res = summary.Results([res1, res2])
nt.eq_(res.get_result('foo'), [status.PASS, status.FAIL])
def test_Results_get_results_missing():
"""summary.Results.get_results: handles KeyErrors"""
res1 = results.TestrunResult()
res1.tests['foo'] = results.TestResult('pass')
res2 = results.TestrunResult()
res2.tests['bar'] = results.TestResult('fail')
res = summary.Results([res1, res2])
nt.eq_(res.get_result('foo'), [status.PASS, status.NOTRUN])
def subtest(self):
"""results a Result object with subtests."""
res1 = results.TestrunResult()
res1.tests['foo'] = results.TestResult('notrun')
res1.tests['foo'].subtests['1'] = 'pass'
res1.tests['bar'] = results.TestResult('notrun')
res1.tests['bar'].subtests['1'] = 'pass'
res2 = results.TestrunResult()
res2.tests['foo'] = results.TestResult('notrun')
res2.tests['foo'].subtests['1'] = 'fail'
return summary.Results([res1, res2])
def test_Results_get_results_missing_subtest():
"""summary.Results.get_results (subtest): handles KeyErrors"""
res1 = results.TestrunResult()
res1.tests['foo'] = results.TestResult('pass')
res1.tests['foo'].subtests['1'] = 'pass'
res2 = results.TestrunResult()
res2.tests['bar'] = results.TestResult('fail')
res = summary.Results([res1, res2])
nt.eq_(res.get_result(grouptools.join('foo', '1')),
[status.PASS, status.NOTRUN])
def setup_class(cls):
"""class fixture."""
res = results.TestrunResult()
res.tests['foo'] = results.TestResult('pass')
res.tests['bar'] = results.TestResult('fail')
res.tests['oink'] = results.TestResult('crash')
res.tests['bonk'] = results.TestResult('warn')
res.tests['bor'] = results.TestResult('crash')
res.tests['bor'].subtests['1'] = 'pass'
res.tests['bor'].subtests['2'] = 'skip'
res.tests['bor'].subtests['3'] = 'fail'
res.tests['bor'].subtests['4'] = 'pass'
cls.results = res
def test_print_result():
"""summary.console_._print_result: prints expected values"""
res1 = results.TestrunResult()
res1.tests['foo'] = results.TestResult('pass')
res2 = results.TestrunResult()
res2.tests['foo'] = results.TestResult('fail')
reses = common.Results([res1, res2])
expected = 'foo: pass fail\n'
actual = get_stdout(lambda: console_._print_result(reses, reses.names.all))
nt.eq_(expected, actual)
def test_load_valid_folder(self, tmpdir):
"""backends.json._resume: loads valid results."""
backend = backends.json.JSONBackend(str(tmpdir))
backend.initialize(shared.INITIAL_METADATA)
with backend.write_test("group1/test1") as t:
t(results.TestResult('fail'))
with backend.write_test("group1/test2") as t:
t(results.TestResult('pass'))
with backend.write_test("group2/test3") as t:
t(results.TestResult('fail'))
test = backends.json._resume(str(tmpdir))
assert set(test.tests.keys()) == \
{'group1/test1', 'group1/test2', 'group2/test3'}
def test_Results_get_results_subtest():
"""summary.Results.get_results (subtest): returns list of statuses"""
res1 = results.TestrunResult()
res1.tests['foo'] = results.TestResult('notrun')
res1.tests['foo'].subtests['1'] = 'pass'
res2 = results.TestrunResult()
res2.tests['foo'] = results.TestResult('notrun')
res2.tests['foo'].subtests['1'] = 'fail'
res = summary.Results([res1, res2])
nt.eq_(res.get_result(grouptools.join('foo', '1')),
[status.PASS, status.FAIL])
def test_basic(self, capsys):
"""summary.console_._print_result: prints expected values."""
res1 = results.TestrunResult()
res1.tests['foo'] = results.TestResult('pass')
res2 = results.TestrunResult()
res2.tests['foo'] = results.TestResult('fail')
reses = common.Results([res1, res2])
expected = 'foo: pass fail\n'
console_._print_result(reses, reses.names.all)
actual, _ = capsys.readouterr()
assert expected == actual
def test_TestResult_result_getter_subtests():
"""results.TestResult.result: Getter returns worst subtest when subtests are present"""
test = results.TestResult('pass')
test.subtests['a'] = 'fail'
test.subtests['b'] = 'crash'
test.subtests['c'] = 'incomplete'
nt.eq_(test.result, 'incomplete')
def setup_class(cls):
test = results.TestResult()
test.returncode = 100
test.err = 'this is an err'
test.out = 'this is some text'
test.time.start = 0.3
test.time.end = 0.5
test.environment = 'some env stuff'
test.subtests.update({
'a': 'pass',
'b': 'fail'})
test.result = 'crash'
test.exception = 'an exception'
test.dmesg = 'this is dmesg'
test.pid = 1934
test.traceback = 'a traceback'
cls.test = test
cls.json = test.to_json()
# the TimeAttribute needs to be dict-ified as well. There isn't really
# a good way to do this that doesn't introduce a lot of complexity,
# such as:
# json.loads(json.dumps(test, default=piglit_encoder),
# object_hook=piglit_decoder)
cls.json['time'] = cls.json['time'].to_json()
def test_file(self, tmpdir):
tmpdir.mkdir('foo')
p = tmpdir.join('foo')
result = results.TestResult()
result.time.end = 1.2345
result.out = 'this is stdout'
result.err = 'this is stderr'
result.command = 'foo'
result.pid = 1034
result.subtests['foo'] = 'pass'
result.subtests['bar'] = 'fail'
test = backends.junit.JUnitBackend(six.text_type(p),
junit_subtests=True)
test.initialize(shared.INITIAL_METADATA)
with test.write_test(grouptools.join('a', 'group', 'test1')) as t:
t(result)
result.result = 'fail'
with test.write_test(grouptools.join('a', 'test', 'test1')) as t:
t(result)
test.finalize()
return six.text_type(p.join('results.xml'))
def test_junit_replace_suffix(self, tmpdir):
"""backends.junit.JUnitBackend.write_test: grouptools.SEPARATOR is
replaced with '.'.
"""
result = results.TestResult()
result.time.end = 1.2345
result.out = 'this is stdout'
result.err = 'this is stderr'
result.command = 'foo'
result.subtests['foo'] = 'pass'
result.subtests['bar'] = 'fail'
test = backends.junit.JUnitBackend(six.text_type(tmpdir),
junit_subtests=True,
junit_suffix='.foo')
test.initialize(shared.INITIAL_METADATA)
with test.write_test(grouptools.join('a', 'group', 'test1')) as t:
t(result)
test.finalize()
test_value = etree.parse(six.text_type(tmpdir.join('results.xml')))
test_value = test_value.getroot()
suite = test_value.find('.//testsuite/testsuite')
assert suite.attrib['name'] == 'piglit.a.group.test1'
assert suite.find('.//testcase[@name="{}"]'.format('foo.foo')) is not None
def setup_class(cls):
pass_ = results.TestResult('pass')
fail = results.TestResult('fail')
#warn = results.TestResult('warn')
crash = results.TestResult('crash')
skip = results.TestResult('skip')
tr = results.TestrunResult()
tr.tests = {
'oink': pass_,
grouptools.join('foo', 'bar'): fail,
grouptools.join('foo', 'foo', 'bar'): crash,
grouptools.join('foo', 'foo', 'oink'): skip,
}
tr.calculate_group_totals()
cls.test = tr.totals
class TestV6toV7(object):
DATA = {
"results_version": 6,
"name": "test",
"options": {
"profile": ['quick'],
"dmesg": False,
"verbose": False,
"platform": "gbm",
"sync": False,
"valgrind": False,
"filter": [],
"concurrent": "all",
"test_count": 0,
"exclude_tests": [],
"exclude_filter": [],
"env": {
"lspci": "stuff",
"uname": "more stuff",
"glxinfo": "and stuff",
"wglinfo": "stuff"
}
},
"tests": {
'a@test': results.TestResult('pass'),
'a@nother@test': results.TestResult('fail'),
'a@nother@thing': results.TestResult('crash'),
}
}
@classmethod
def setup_class(cls):
"""Class setup. Create a TestrunResult with v4 data."""
with utils.tempfile(
json.dumps(cls.DATA,
default=backends.json.piglit_encoder)) as t:
with open(t, 'r') as f:
cls.result = backends.json._update_six_to_seven(
backends.json._load(f))
def test_is_TestrunResult(self):
"""backends.json.update_results (6 -> 7): makes TestrunResult"""
nt.ok_(isinstance(self.result, results.TestrunResult))
def test_totals(self):
"""backends.json.update_results (6 -> 7): Totals are populated"""
nt.ok_(self.result.totals != {})
def test_update_result_regex_no_match():
"""dmesg.BaseDmesg.update_result: if no regex matches dont change status"""
dmesg_ = TestDmesg()
dmesg_.regex = re.compile(r'nomatchforthisreally')
result = results.TestResult('pass')
dmesg_.update_result(result)
nt.eq_(result.result, 'pass')
def test_load_invalid_folder(self, tmpdir):
"""backends.json._resume: ignores invalid results"""
f = six.text_type(tmpdir)
backend = backends.json.JSONBackend(f)
backend.initialize(shared.INITIAL_METADATA)
with backend.write_test("group1/test1") as t:
t(results.TestResult('fail'))
with backend.write_test("group1/test2") as t:
t(results.TestResult('pass'))
with backend.write_test("group2/test3") as t:
t(results.TestResult('fail'))
with open(os.path.join(f, 'tests', 'x.json'), 'w') as w:
w.write('foo')
test = backends.json._resume(f)
assert set(test.tests.keys()) == \
{'group1/test1', 'group1/test2', 'group2/test3'}
def test_update_result_regex_match():
"""dmesg.BaseDmesg.update_result: if regex matches change status"""
dmesg_ = TestDmesg()
dmesg_.regex = re.compile(r'.*')
result = results.TestResult('pass')
dmesg_.update_result(result)
nt.assert_not_equal(result.result, 'pass')
class TestFinalize(object):
"""Tests for the finalize method."""
name = grouptools.join('a', 'test', 'group', 'test1')
result = results.TestResult('pass')
@pytest.fixture(scope='class')
def result_dir(self, tmpdir_factory):
directory = tmpdir_factory.mktemp('main')
test = backends.json.JSONBackend(str(directory))
test.initialize(shared.INITIAL_METADATA)
with test.write_test(self.name) as t:
t(self.result)
test.finalize({
'time_elapsed':
results.TimeAttribute(start=0.0, end=1.0).to_json()
})
return directory
def test_metadata_removed(self, result_dir):
assert not result_dir.join('metadata.json').check()
def test_tests_directory_removed(self, result_dir):
assert not result_dir.join('tests').check()
def test_results_file_created(self, result_dir):
# Normally this would also have a compression extension, but this
# module has a setup fixture that forces the compression to None.
assert result_dir.join('results.json').check()
def test_results_are_json(self, result_dir):
# This only checks that the output is valid JSON, not that the
# schema is correct
with result_dir.join('results.json').open('r') as f:
json.load(f)
def test_results_are_valid(self, result_dir):
"""Test that the values produced are valid."""
with result_dir.join('results.json').open('r') as f:
json_ = json.load(f)
with open(SCHEMA, 'r') as f:
schema = json.load(f)
jsonschema.validate(json_, schema)
def test_ignores_invalid(self, tmpdir):
test = backends.json.JSONBackend(str(tmpdir))
test.initialize(shared.INITIAL_METADATA)
with test.write_test(self.name) as t:
t(self.result)
tmpdir.join('tests/2.json.tmp').write('{}')
test.finalize({
'time_elapsed':
results.TimeAttribute(start=0.0, end=1.0).to_json()
})
def test_resume_load_valid():
"""backends.json._resume: loads valid results"""
with utils.tempdir() as f:
backend = backends.json.JSONBackend(f)
backend.initialize(BACKEND_INITIAL_META)
with backend.write_test("group1/test1") as t:
t(results.TestResult('fail'))
with backend.write_test("group1/test2") as t:
t(results.TestResult('pass'))
with backend.write_test("group2/test3") as t:
t(results.TestResult('fail'))
test = backends.json._resume(f)
nt.assert_set_equal(
set(test.tests.keys()),
set(['group1/test1', 'group1/test2', 'group2/test3']),
)
def test_write(self, tmpdir):
"""The write method should create a file."""
p = str(tmpdir)
test = backends.json.JSONBackend(p)
test.initialize(shared.INITIAL_METADATA)
with test.write_test('bar') as t:
t(results.TestResult())
assert tmpdir.join('tests/0.json').check()
def setup_class(cls):
cls.test_name = grouptools.join('a', 'test', 'group', 'test1')
cls.result = results.TestResult('pass')
super(TestJSONTestFinalize, cls).setup_class()
test = backends.json.JSONBackend(cls.tdir)
test.initialize(BACKEND_INITIAL_META)
with test.write_test(cls.test_name) as t:
t(cls.result)
test.finalize()
def test_dummydmesg_update_result():
"""dmesg.DummyDmesg.update_result: returns result unmodified"""
dmesg_ = dmesg.DummyDmesg()
result = mock.MagicMock(spec=results.TestResult())
result.dmesg = mock.sentinel.dmesg
result.result = mock.sentinel.result
dmesg_.update_result(result)
nt.eq_(result.dmesg, mock.sentinel.dmesg)
nt.eq_(result.result, mock.sentinel.result)
def setup_class(cls):
tr = results.TestResult('crash')
tr.subtests['foo'] = status.PASS
tr.subtests['bar'] = status.CRASH
tr.subtests['oink'] = status.FAIL
run = results.TestrunResult()
run.tests[grouptools.join('sub', 'test')] = tr
run.calculate_group_totals()
cls.test = run.totals
def test_load_invalid_ext(self, tmpdir):
"""backends.json._resume: ignores invalid results extensions.
This gets triggered by an incomplete atomic write
"""
f = str(tmpdir)
backend = backends.json.JSONBackend(f)
backend.initialize(shared.INITIAL_METADATA)
with backend.write_test("group1/test1") as t:
t(results.TestResult('fail'))
with backend.write_test("group1/test2") as t:
t(results.TestResult('pass'))
with backend.write_test("group2/test3") as t:
t(results.TestResult('fail'))
with open(os.path.join(f, 'tests', '3.json.tmp'), 'w') as w:
w.write('foo')
test = backends.json._resume(f)
assert set(test.tests.keys()) == \
{'group1/test1', 'group1/test2', 'group2/test3'}
