def __init__(self):
self.data = {
grouptools.join('group1', 'test1'): 'thingy',
grouptools.join('group1', 'group3', 'test2'): 'thing',
grouptools.join('group3', 'test5'): 'other',
grouptools.join('group4', 'Test9'): 'is_caps',
}
def __init__(self):
self.data = {
grouptools.join("group1", "test1"): "thingy",
grouptools.join("group1", "group3", "test2"): "thing",
grouptools.join("group3", "test5"): "other",
grouptools.join("group4", "Test9"): "is_caps",
}
def test_file(self, tmpdir):
tmpdir.mkdir('foo')
p = tmpdir.join('foo')
result = results.TestResult()
result.time.end = 1.2345
result.out = 'this is stdout'
result.err = 'this is stderr'
result.command = 'foo'
result.pid = 1034
result.subtests['foo'] = 'pass'
result.subtests['bar'] = 'fail'
test = backends.junit.JUnitBackend(six.text_type(p),
junit_subtests=True)
test.initialize(shared.INITIAL_METADATA)
with test.write_test(grouptools.join('a', 'group', 'test1')) as t:
t(result)
result.result = 'fail'
with test.write_test(grouptools.join('a', 'test', 'test1')) as t:
t(result)
test.finalize()
return six.text_type(p.join('results.xml'))
def __init__(self):
self.data = profile.TestDict()
self.data[grouptools.join('group1', 'test1')] = utils.Test(['thingy'])
self.data[grouptools.join('group1', 'group3', 'test2')] = utils.Test(['thing'])
self.data[grouptools.join('group3', 'test5')] = utils.Test(['other'])
self.data[grouptools.join('group4', 'Test9')] = utils.Test(['is_caps'])
self.opts = None
self.__patcher = mock.patch('framework.profile.options.OPTIONS',
new_callable=options._Options)
def __init__(self):
self.data = {
grouptools.join('group1', 'test1'): 'thingy',
grouptools.join('group1', 'group3', 'test2'): 'thing',
grouptools.join('group3', 'test5'): 'other',
grouptools.join('group4', 'Test9'): 'is_caps',
}
self.opts = None
self.__patcher = mock.patch('framework.profile.options.OPTIONS',
new_callable=options._Options)
def generate_tests():
"""Generate tests for the groups tools module.
This cannot test all corners of the more complicated members.
"""
data = [
('testname', grouptools.testname, grouptools.join('g1', 'g2', 't1'),
't1'),
('groupname', grouptools.groupname, grouptools.join('g1', 'g2', 't1'),
grouptools.join('g1', 'g2')),
('splitname', grouptools.splitname, grouptools.join('g1', 'g2', 't1'),
(grouptools.join('g1', 'g2'), 't1')),
('commonprefix', grouptools.commonprefix,
[grouptools.join('g1', 'g2', '1'), grouptools.join('g1', 'g2', '2')],
grouptools.join('g1', 'g2', '')),
('split', grouptools.split, grouptools.join('g1', 'g2', 't1'),
['g1', 'g2', 't1']),
]
test = lambda f, i, e: nt.assert_equal(f(i), e)
for name, func, args, expect in data:
test.description = 'grouptools.{}: works'.format(name)
yield test, func, args, expect
def test_matches_env_exclude(self):
"""profile.TestProfile.prepare_test_list: 'not path in env.exclude_tests'"""
self.opts.exclude_tests.add(grouptools.join('group3', 'test5'))
baseline = copy.deepcopy(self.data)
del baseline[grouptools.join('group3', 'test5')]
profile_ = profile.TestProfile()
profile_.test_list = self.data
profile_._prepare_test_list()
nt.assert_dict_equal(dict(profile_.test_list), dict(baseline))
def test_matches_env_exclude(self):
"""TestProfile.prepare_test_list: 'not path in env.exclude_tests"""
env = core.Options()
env.exclude_tests.add(grouptools.join('group3', 'test5'))
profile_ = profile.TestProfile()
profile_.test_list = self.data
profile_._prepare_test_list(env)
baseline = copy.deepcopy(self.data)
del baseline[grouptools.join('group3', 'test5')]
nt.assert_dict_equal(profile_.test_list, baseline)
def test_testprofile_update_test_list():
"""profile.TestProfile.update(): updates TestProfile.test_list"""
profile1 = profile.TestProfile()
group1 = grouptools.join("group1", "test1")
group2 = grouptools.join("group1", "test2")
profile1.test_list[group1] = utils.Test(["test1"])
profile2 = profile.TestProfile()
profile2.test_list[group1] = utils.Test(["test3"])
profile2.test_list[group2] = utils.Test(["test2"])
profile1.update(profile2)
nt.assert_dict_equal(profile1.test_list, profile2.test_list)
def add_shader_test(shader):
"""Given an adder, creates a group and adds a shader test."""
for d in (
TESTS_DIR,
GENERATED_TESTS_DIR,
):
s = os.path.abspath(os.path.join(d, shader))
if os.path.exists(s):
basedir = d
abs_path = s
break
dirpath, filename = os.path.split(os.path.join(basedir, shader))
dirname = os.path.relpath(dirpath, basepath)
filepath = os.path.join(dirname, filename)
if os.path.commonprefix([abs_path,
GENERATED_TESTS_DIR]) == GENERATED_TESTS_DIR:
installpath = os.path.relpath(filepath, gen_basepath)
else:
installpath = None
groupname = grouptools.from_path(os.path.relpath(dirpath, basedir))
testname = os.path.splitext(os.path.basename(shader))[0]
group = grouptools.join(groupname, testname)
profile.test_list[group] = ShaderTest.new(filepath, installpath)
def test_junit_replace_suffix(self, tmpdir):
"""backends.junit.JUnitBackend.write_test: grouptools.SEPARATOR is
replaced with '.'.
"""
result = results.TestResult()
result.time.end = 1.2345
result.out = 'this is stdout'
result.err = 'this is stderr'
result.command = 'foo'
result.subtests['foo'] = 'pass'
result.subtests['bar'] = 'fail'
test = backends.junit.JUnitBackend(str(tmpdir),
junit_subtests=True,
junit_suffix='.foo')
test.initialize(shared.INITIAL_METADATA)
with test.write_test(grouptools.join('a', 'group', 'test1')) as t:
t(result)
test.finalize()
test_value = etree.parse(str(tmpdir.join('results.xml')))
test_value = test_value.getroot()
suite = test_value.find('.//testsuite/testsuite')
assert suite.attrib['name'] == 'piglit.a.group.test1'
assert suite.find(
'.//testcase[@name="{}"]'.format('foo.foo')) is not None
class TestReturned(object):
"""Test that the returned object is as expected."""
testname = grouptools.join('foo', 'bar', 'a-test')
@pytest.fixture
def result(self, tmpdir):
p = tmpdir.join('test.xml')
p.write(_XML)
return backends.junit._load(str(p))
def test_testrunresult(self, result):
"""backends.junit._load: returns a TestrunResult instance."""
assert isinstance(result, results.TestrunResult)
def test_replace_sep(self, result):
"""backends.junit._load: replaces '.' with grouptools.SEPARATOR."""
assert self.testname in result.tests
def test_testresult_instance(self, result):
"""backends.junit._load: replaces result with TestResult instance.
"""
assert isinstance(result.tests[self.testname], results.TestResult)
def test_status_instance(self, result):
"""backends.junit._load: a status is found and loaded."""
assert isinstance(result.tests[self.testname].result,
status.Status)
def test_time(self, result):
time = result.tests[self.testname].time
assert isinstance(time, results.TimeAttribute)
assert time.start == 1.0
assert time.end == 4.5
def test_command(self, result):
"""backends.junit._load: command is loaded correctly."""
assert result.tests[self.testname].command == 'this/is/a/command'
def test_out(self, result):
"""backends.junit._load: stdout is loaded correctly."""
assert result.tests[self.testname].out == 'This is stdout'
def test_err(self, result):
"""backends.junit._load: stderr is loaded correctly."""
expected = textwrap.dedent("""\
this is stderr
pid: [1934]
time start: 1.0
time end: 4.5""")
assert result.tests[self.testname].err.strip() == expected
def test_totals(self, result):
"""backends.junit._load: Totals are calculated."""
assert bool(result)
def test_pid(self, result):
"""backends.junit._load: pid is loaded correctly."""
assert result.tests[self.testname].pid == [1934]
def test_testprofile_group_manager_is_added():
"""profile.TestProfile.group_manager: Tests are added to the profile"""
prof = profile.TestProfile()
with prof.group_manager(utils.Test, "foo") as g:
g(["a", "b"], "a")
nt.assert_in(grouptools.join("foo", "a"), prof.test_list)
def _load(results_file):
"""Load a junit results instance and return a TestrunResult.
It's worth noting that junit is not as descriptive as piglit's own json
format, so some data structures will be empty compared to json.
This tries to not make too many assumptions about the strucuter of the
JUnit document.
"""
run_result = results.TestrunResult()
splitpath = os.path.splitext(results_file)[0].split(os.path.sep)
if splitpath[-1] != 'results':
run_result.name = splitpath[-1]
elif len(splitpath) > 1:
run_result.name = splitpath[-2]
else:
run_result.name = 'junit result'
tree = etree.parse(results_file).getroot().find('.//testsuite[@name="piglit"]')
for test in tree.iterfind('testcase'):
result = results.TestResult()
# Take the class name minus the 'piglit.' element, replace junit's '.'
# separator with piglit's separator, and join the group and test names
name = test.attrib['classname'].split('.', 1)[1]
name = name.replace('.', grouptools.SEPARATOR)
name = grouptools.join(name, test.attrib['name'])
# Remove the trailing _ if they were added (such as to api and search)
if name.endswith('_'):
name = name[:-1]
result.result = test.attrib['status']
# This is the fallback path, we'll try to overwrite this with the value
# in stderr
result.time = results.TimeAttribute(end=float(test.attrib['time']))
result.err = test.find('system-err').text
# The command is prepended to system-out, so we need to separate those
# into two separate elements
out = test.find('system-out').text.split('\n')
result.command = out[0]
result.out = '\n'.join(out[1:])
# Try to get the values in stderr for time
if 'time start' in result.err:
for line in result.err.split('\n'):
if line.startswith('time start:'):
result.time.start = float(line[len('time start: '):])
elif line.startswith('time end:'):
result.time.end = float(line[len('time end: '):])
break
run_result.tests[name] = result
run_result.calculate_group_totals()
return run_result
def test_testprofile_group_manager_no_name_args_gt_one():
"""profile.TestProfile.group_manager: no name and len(args) > 1 is valid"""
prof = profile.TestProfile()
with prof.group_manager(utils.Test, "foo") as g:
g(["a", "b"])
nt.assert_in(grouptools.join("foo", "a b"), prof.test_list)
def adder(args, name=None, **kwargs):
"""Helper function that actually adds the tests.
Arguments:
args -- arguments to be passed to the test_class constructor.
This must be appropriate for the underlying class
Keyword Arguments:
name -- If this is a a truthy value that value will be used as
the key for the test. If name is falsy then args will be
' '.join'd and used as name. Default: None
kwargs -- Any additional args will be passed directly to the test
constructor as keyword args.
"""
# If there is no name, then either
# a) join the arguments list together to make the name
# b) use the argument string as the name
# The former is used by the Piglit{G,C}LTest classes, the latter by
# GleanTest
if not name:
if isinstance(args, list):
name = ' '.join(args)
else:
assert isinstance(args, six.string_types)
name = args
assert isinstance(name, six.string_types)
lgroup = grouptools.join(group, name)
self[lgroup] = test_class(
args,
**dict(
itertools.chain(six.iteritems(default_args),
six.iteritems(kwargs))))
def adder(args, name=None, override_class=None, **kwargs):
"""Helper function that actually adds the tests.
Arguments:
args -- arguments to be passed to the test_class constructor.
This must be appropriate for the underlying class
Keyword Arguments:
name -- If this is a a truthy value that value will be used as
the key for the test. If name is falsy then args will be
' '.join'd and used as name. Default: None
kwargs -- Any additional args will be passed directly to the test
constructor as keyword args.
"""
# If there is no name, join the arguments list together to make
# the name
if not name:
assert isinstance(args, list) # //
name = ' '.join(args)
assert isinstance(name, str)
lgroup = grouptools.join(group, name)
class_ = override_class or test_class
self[lgroup] = class_(
args,
**dict(itertools.chain(default_args.items(), kwargs.items())))
def _populate_profile_xts(profile):
fpath = os.path.join(X_TEST_SUITE, 'xts5')
for dirpath, _, filenames in os.walk(fpath):
for fname in filenames:
# only look at the .m test files
testname, ext = os.path.splitext(fname)
if ext != '.m':
continue
# incrementing number generator
counts = itertools.count(start=1)
# Walk the file looking for >>ASSERTION, each of these corresponds
# to a generated subtest, there can be multiple subtests per .m
# file
with open(os.path.join(dirpath, fname), 'r') as rfile:
for line in rfile:
if line.startswith('>>ASSERTION'):
num = next(counts)
group = grouptools.join(
grouptools.from_path(
os.path.relpath(dirpath, X_TEST_SUITE)),
testname, str(num))
profile.test_list[group] = XTSTest(
os.path.join(dirpath, testname), testname,
os.path.relpath(dirpath, X_TEST_SUITE), num)
def setup_class(cls):
"""setup state for all tests."""
pass_ = results.TestResult('pass')
fail = results.TestResult('fail')
crash = results.TestResult('crash')
skip = results.TestResult('skip')
tr = results.TestrunResult()
tr.tests = {
'oink': pass_,
grouptools.join('foo', 'bar'): fail,
grouptools.join('foo', 'foo', 'bar'): crash,
grouptools.join('foo', 'foo', 'oink'): skip,
}
tr.calculate_group_totals()
cls.test = tr.totals
def test_two_parents(self):
"""Handles multiple parents correctly."""
expected = results.Totals()
expected['crash'] += 1
expected['skip'] += 1
assert dict(self.test[grouptools.join('foo', 'foo')]) == \
dict(expected)
def setup_class(cls):
super(TestJUnitMultiTest, cls).setup_class()
cls.test_file = os.path.join(cls.tdir, 'results.xml')
test = backends.junit.JUnitBackend(cls.tdir)
test.initialize(BACKEND_INITIAL_META)
test.write_test(
grouptools.join('a', 'test', 'group', 'test1'),
results.TestResult({
'time': 1.2345,
'result': 'pass',
'out': 'this is stdout',
'err': 'this is stderr',
'command': 'foo',
})
)
test.write_test(
'a/different/test/group/test2',
results.TestResult({
'time': 1.2345,
'result': 'fail',
'out': 'this is stdout',
'err': 'this is stderr',
'command': 'foo',
})
)
test.finalize()
def test_testprofile_group_manager_is_added():
"""TestProfile.group_manager: Tests are added to the profile"""
prof = profile.TestProfile()
with prof.group_manager(utils.Test, 'foo') as g:
g(['a', 'b'], 'a')
nt.assert_in(grouptools.join('foo', 'a'), prof.test_list)
def test_testprofile_group_manager_no_name_args_gt_one():
"""TestProfile.group_manager: no name and len(args) > 1 is valid"""
prof = profile.TestProfile()
with prof.group_manager(utils.Test, 'foo') as g:
g(['a', 'b'])
nt.assert_in(grouptools.join('foo', 'a b'), prof.test_list)
def test_junit_replace_suffix(self, tmpdir):
"""backends.junit.JUnitBackend.write_test: grouptools.SEPARATOR is
replaced with '.'.
"""
result = results.TestResult()
result.time.end = 1.2345
result.out = 'this is stdout'
result.err = 'this is stderr'
result.command = 'foo'
result.subtests['foo'] = 'pass'
result.subtests['bar'] = 'fail'
test = backends.junit.JUnitBackend(six.text_type(tmpdir),
junit_subtests=True,
junit_suffix='.foo')
test.initialize(shared.INITIAL_METADATA)
with test.write_test(grouptools.join('a', 'group', 'test1')) as t:
t(result)
test.finalize()
test_value = etree.parse(six.text_type(tmpdir.join('results.xml')))
test_value = test_value.getroot()
suite = test_value.find('.//testsuite/testsuite')
assert suite.attrib['name'] == 'piglit.a.group.test1'
assert suite.find('.//testcase[@name="{}"]'.format('foo.foo')) is not None
def test_two_parents(self):
"""Handles multiple parents correctly."""
expected = results.Totals()
expected['crash'] += 1
expected['skip'] += 1
assert dict(self.test[grouptools.join('foo', 'foo')]) == \
dict(expected)
def test_node_values(self):
"""results.TestrunResult.totals: Tests with subtests values are correct"""
expect = results.Totals()
expect['pass'] += 1
expect['crash'] += 1
expect['fail'] += 1
nt.assert_dict_equal(self.test[grouptools.join('sub', 'test')], expect)
def test_all(self):
"""summary.Names.all: Handles subtests as groups"""
baseline = {'foo', 'bar', 'oink', 'bonk', 'oink', 'tonk'}
for x in xrange(1, 6):
baseline.add(grouptools.join('bor', str(x)))
nt.eq_(self.test.names.all, baseline)
def test_sublcass_names(self):
"""summary.Names.all: subtest in all"""
expected = grouptools.join('bor', '1')
source = self.test.names.all
nt.ok_(expected in source,
msg='{} not found in {}'.format(expected, source))
def test_testprofile_update_test_list():
"""profile.TestProfile.update(): updates TestProfile.test_list"""
profile1 = profile.TestProfile()
group1 = grouptools.join('group1', 'test1')
group2 = grouptools.join('group1', 'test2')
profile1.test_list[group1] = utils.Test(['test1'])
profile2 = profile.TestProfile()
profile2.test_list[group1] = utils.Test(['test3'])
profile2.test_list[group2] = utils.Test(['test2'])
profile1.update(profile2)
nt.assert_dict_equal(profile1.test_list, profile2.test_list)
def test_from_path_posix():
"""grouptools.from_path: converts / to {seperator} in posix paths."""
# Since we already have tests for grouptools.join we can trust it to do the
# right thing here. This also means that the test doesn't need to be
# updated if the separator is changed.
nt.assert_equal(grouptools.from_path('foo/bar'),
grouptools.join('foo', 'bar'))
def adder(args, name=None, **kwargs):
"""Helper function that actually adds the tests.
Arguments:
args -- arguments to be passed to the test_class constructor.
This must be appropriate for the underlying class
Keyword Arguments:
name -- If this is a a truthy value that value will be used as
the key for the test. If name is falsy then args will be
' '.join'd and used as name. Default: None
kwargs -- Any additional args will be passed directly to the test
constructor as keyword args.
"""
# If there is no name, then either
# a) join the arguments list together to make the name
# b) use the argument string as the name
# The former is used by the Piglit{G,C}LTest classes, the latter by
# GleanTest
if not name:
if isinstance(args, list):
name = ' '.join(args)
else:
assert isinstance(args, six.string_types)
name = args
assert isinstance(name, six.string_types)
lgroup = grouptools.join(group, name)
self[lgroup] = test_class(
args,
**dict(itertools.chain(six.iteritems(default_args),
six.iteritems(kwargs))))
def _populate_profile_xts(profile):
fpath = os.path.join(X_TEST_SUITE, 'xts5')
for dirpath, _, filenames in os.walk(fpath):
for fname in filenames:
# only look at the .m test files
testname, ext = os.path.splitext(fname)
if ext != '.m':
continue
# incrementing number generator
counts = itertools.count(start=1)
# Walk the file looking for >>ASSERTION, each of these corresponds
# to a generated subtest, there can be multiple subtests per .m
# file
with open(os.path.join(dirpath, fname), 'r') as rfile:
for line in rfile:
if line.startswith('>>ASSERTION'):
num = next(counts)
group = grouptools.join(
grouptools.from_path(
os.path.relpath(dirpath, X_TEST_SUITE)),
testname, str(num))
profile.test_list[group] = XTSTest(
os.path.join(dirpath, testname),
testname,
num)
def test_all(self):
"""summary.Names.all: Handles subtests as groups"""
baseline = {'foo', 'bar', 'oink', 'bonk', 'oink', 'tonk'}
for x in range(1, 6):
baseline.add(grouptools.join('bor', str(x)))
assert self.test.names.all == baseline
def test_testprofile_group_manager_no_name_args_gt_one():
"""profile.TestProfile.group_manager: no name and len(args) > 1 is valid"""
prof = profile.TestProfile()
with prof.group_manager(utils.Test, 'foo') as g:
g(['a', 'b'])
nt.assert_in(grouptools.join('foo', 'a b'), prof.test_list)
def test_testprofile_update_test_list():
""" TestProfile.update() updates TestProfile.test_list """
profile1 = profile.TestProfile()
group1 = grouptools.join('group1', 'test1')
group2 = grouptools.join('group1', 'test2')
profile1.test_list[group1] = utils.Test(['test1'])
profile2 = profile.TestProfile()
profile2.test_list[group1] = utils.Test(['test3'])
profile2.test_list[group2] = utils.Test(['test2'])
profile1.update(profile2)
nt.assert_dict_equal(profile1.test_list, profile2.test_list)
def test_testprofile_group_manager_is_added():
"""profile.TestProfile.group_manager: Tests are added to the profile"""
prof = profile.TestProfile()
with prof.group_manager(utils.Test, 'foo') as g:
g(['a', 'b'], 'a')
nt.assert_in(grouptools.join('foo', 'a'), prof.test_list)
def setup_class(cls):
pass_ = results.TestResult('pass')
fail = results.TestResult('fail')
#warn = results.TestResult('warn')
crash = results.TestResult('crash')
skip = results.TestResult('skip')
tr = results.TestrunResult()
tr.tests = {
'oink': pass_,
grouptools.join('foo', 'bar'): fail,
grouptools.join('foo', 'foo', 'bar'): crash,
grouptools.join('foo', 'foo', 'oink'): skip,
}
tr.calculate_group_totals()
cls.test = tr.totals
def test_names_disabled(self):
"""summary.Names.disabled: subtests added properly"""
expected = grouptools.join('bor', '4')
source = self.test.names.disabled[1]
nt.ok_(expected in source,
msg='{} not found in "{}"'.format(expected, source))
def add_program_test_dir(group, dirpath):
for filename in os.listdir(dirpath):
testname, ext = os.path.splitext(filename)
if ext not in ['.cl', '.program_test']:
continue
profile.test_list[grouptools.join(group, testname)] = PiglitCLTest(
['cl-program-tester', os.path.join(dirpath, filename)])
def test_testprofile_groupmanager_kwargs_overwrite():
"""TestProfile.group_manager: default_args are overwritten by kwargs."""
prof = profile.TestProfile()
with prof.group_manager(utils.Test, 'foo', run_concurrent=True) as g:
g(['a'], run_concurrent=False)
test = prof.test_list[grouptools.join('foo', 'a')]
nt.assert_equal(test.run_concurrent, False)
def test_testprofile_groupmanager_name_str():
"""TestProfile.group_manager: if args is a string it is not joined."""
prof = profile.TestProfile()
# Yes, this is really about supporting gleantest anyway.
with prof.group_manager(GleanTest, 'foo') as g:
g('abc')
nt.ok_(grouptools.join('foo', 'abc') in prof.test_list)
def test_node_values(self):
"""Tests with subtests values are correct."""
expect = results.Totals()
expect['pass'] += 1
expect['crash'] += 1
expect['fail'] += 1
assert dict(self.test[grouptools.join('sub', 'test')]) == \
dict(expect)
def test_testprofile_groupmanager_default_args():
"""TestProfile.group_manager: group_manater kwargs are passed to the Test."""
prof = profile.TestProfile()
with prof.group_manager(utils.Test, 'foo', run_concurrent=True) as g:
g(['a'])
test = prof.test_list[grouptools.join('foo', 'a')]
nt.assert_equal(test.run_concurrent, True)
def test_testprofile_groupmanager_name_str():
"""profile.TestProfile.group_manager: if args is a string it is not joined"""
prof = profile.TestProfile()
# Yes, this is really about supporting gleantest anyway.
with prof.group_manager(GleanTest, 'foo') as g:
g('abc')
nt.ok_(grouptools.join('foo', 'abc') in prof.test_list)
def setup_class(cls):
super(TestJUnitLoad, cls).setup_class()
cls.xml_file = os.path.join(cls.tdir, 'results.xml')
with open(cls.xml_file, 'w') as f:
f.write(_XML)
cls.testname = grouptools.join('foo', 'bar', 'a-test')
def test_testprofile_groupmanager_default_args():
"""profile.TestProfile.group_manager: group_manater kwargs are passed to the Test"""
prof = profile.TestProfile()
with prof.group_manager(utils.Test, 'foo', run_concurrent=True) as g:
g(['a'])
test = prof.test_list[grouptools.join('foo', 'a')]
nt.assert_equal(test.run_concurrent, True)
def test_testprofile_groupmanager_kwargs_overwrite():
"""profile.TestProfile.group_manager: default_args are overwritten by kwargs"""
prof = profile.TestProfile()
with prof.group_manager(utils.Test, 'foo', run_concurrent=True) as g:
g(['a'], run_concurrent=False)
test = prof.test_list[grouptools.join('foo', 'a')]
nt.assert_equal(test.run_concurrent, False)
def add_program_test_dir(group, dirpath):
for filename in os.listdir(dirpath):
testname, ext = os.path.splitext(filename)
if ext not in ['.cl', '.program_test']:
continue
profile.test_list[grouptools.join(group, testname)] = PiglitCLTest(
['cl-program-tester', os.path.join(dirpath, filename)])
def add_program_test_dir(group, dirpath, buildbase, installbase):
for filename in os.listdir(os.path.join(buildbase, dirpath)):
testname, ext = os.path.splitext(filename)
if ext not in ['.cl', '.program_test']:
continue
profile.test_list[grouptools.join(group, testname)] = CLProgramTester(
os.path.join(installbase, dirpath, os.path.basename(filename)))
class TestFinalize(object):
"""Tests for the finalize method."""
name = grouptools.join('a', 'test', 'group', 'test1')
result = results.TestResult('pass')
@pytest.fixture(scope='class')
def result_dir(self, tmpdir_factory):
directory = tmpdir_factory.mktemp('main')
test = backends.json.JSONBackend(str(directory))
test.initialize(shared.INITIAL_METADATA)
with test.write_test(self.name) as t:
t(self.result)
test.finalize({
'time_elapsed':
results.TimeAttribute(start=0.0, end=1.0).to_json()
})
return directory
def test_metadata_removed(self, result_dir):
assert not result_dir.join('metadata.json').check()
def test_tests_directory_removed(self, result_dir):
assert not result_dir.join('tests').check()
def test_results_file_created(self, result_dir):
# Normally this would also have a compression extension, but this
# module has a setup fixture that forces the compression to None.
assert result_dir.join('results.json').check()
def test_results_are_json(self, result_dir):
# This only checks that the output is valid JSON, not that the
# schema is correct
with result_dir.join('results.json').open('r') as f:
json.load(f)
def test_results_are_valid(self, result_dir):
"""Test that the values produced are valid."""
with result_dir.join('results.json').open('r') as f:
json_ = json.load(f)
with open(SCHEMA, 'r') as f:
schema = json.load(f)
jsonschema.validate(json_, schema)
def test_ignores_invalid(self, tmpdir):
test = backends.json.JSONBackend(str(tmpdir))
test.initialize(shared.INITIAL_METADATA)
with test.write_test(self.name) as t:
t(self.result)
tmpdir.join('tests/2.json.tmp').write('{}')
test.finalize({
'time_elapsed':
results.TimeAttribute(start=0.0, end=1.0).to_json()
})
def test_matches_include_caps(self):
"""TestProfile.prepare_test_list: matches capitalized tests."""
env = core.Options(exclude_filter=['test9'])
profile_ = profile.TestProfile()
profile_.test_list = self.data
profile_._prepare_test_list(env)
nt.assert_not_in(grouptools.join('group4', 'Test9'), profile_.test_list)
def test_adder_kwargs_passed(self, inst):
"""Extra kwargs passed to the adder function are passed to the
Test.
"""
with inst.group_manager(utils.Test, 'foo') as g:
g(['a'], run_concurrent=True)
test = inst[grouptools.join('foo', 'a')]
assert test.run_concurrent is True
def test_context_manager_keyword_args_passed(self, inst):
"""kwargs passed to the context_manager are passed to the Test."""
with inst.group_manager(
utils.Test, 'foo', run_concurrent=True) as g: # pylint: disable=bad-continuation
# This is a pylint bug, there's an upstream report
g(['a'])
test = inst[grouptools.join('foo', 'a')]
assert test.run_concurrent is True
def test_adder_kwargs_overwrite_context_manager_kwargs(self, inst):
"""default_args are overwritten by kwargs."""
with inst.group_manager(
utils.Test, 'foo', run_concurrent=True) as g: # pylint: disable=bad-continuation
# This is a pylint bug, there's an upstream report
g(['a'], run_concurrent=False)
test = inst[grouptools.join('foo', 'a')]
assert test.run_concurrent is False
def test_values(self, index, capsys):
"""Create both an expected value and an actual value.
The expected value makes use of the template, this helps to minimize
the number of changes that need to be made if the output is altered.
"""
names = [
grouptools.join('foo', 'bar', 'oink', 'foobar', 'boink'), 'foo',
'bar'
]
template = '{: >20.20} {: >12.12}'
expected = console_._SUMMARY_TEMPLATE.format(
names=' '.join(['this is a really rea', 'another name']),
divider=' '.join(['--------------------', '------------']),
pass_=template.format('1', '2'),
fail=template.format('2', '0'),
crash=template.format('0', '0'),
skip=template.format('0', '1'),
timeout=template.format('0', '0'),
warn=template.format('0', '0'),
incomplete=template.format('0', '0'),
dmesg_warn=template.format('0', '0'),
dmesg_fail=template.format('0', '0'),
changes=template.format('0', '2'),
fixes=template.format('0', '1'),
regressions=template.format('0', '0'),
total=template.format('3', '3'),
time=template.format('00:01:39', '02:14:05')).split('\n')
res1 = results.TestrunResult()
res1.name = 'this is a really really really really long name'
res1.tests[names[0]] = results.TestResult('pass')
res1.tests[names[1]] = results.TestResult('fail')
res1.tests[names[2]] = results.TestResult('notrun')
res1.tests[names[2]].subtests['1'] = 'fail'
res1.time_elapsed = results.TimeAttribute(1509747121.4873962,
1509747220.544042)
res1.calculate_group_totals()
res2 = results.TestrunResult()
res2.name = 'another name'
res2.tests[names[0]] = results.TestResult('pass')
res2.tests[names[1]] = results.TestResult('pass')
res2.tests[names[2]] = results.TestResult('notrun')
res2.tests[names[2]].subtests['1'] = 'skip'
res2.time_elapsed = results.TimeAttribute(1464820707.4581327,
1464828753.201948)
res2.calculate_group_totals()
reses = common.Results([res1, res2])
console_._print_summary(reses)
actual = capsys.readouterr()[0].splitlines()
assert actual[index] == expected[index]
def test_testprofile_allow_reassignment_with_groupmanager():
"""TestProfile: allow_reassignment wrapper works with groupmanager."""
testname = grouptools.join('a', 'b')
prof = profile.TestProfile()
prof.test_list[testname] = utils.Test(['foo'])
with prof.allow_reassignment:
with prof.group_manager(utils.Test, 'a') as g:
g(['bar'], 'b')
nt.ok_(prof.test_list[testname].command == ['bar'])
def setup_class(cls):
cls.test_name = grouptools.join('a', 'test', 'group', 'test1')
cls.result = results.TestResult('pass')
super(TestJSONTestFinalize, cls).setup_class()
test = backends.json.JSONBackend(cls.tdir)
test.initialize(BACKEND_INITIAL_META)
with test.write_test(cls.test_name) as t:
t(cls.result)
test.finalize()
def test_matches_include_caps(self):
"""profile.TestProfile.prepare_test_list: matches capitalized tests"""
self.opts.exclude_filter = ['test9']
profile_ = profile.TestProfile()
profile_.test_list = self.data
profile_._prepare_test_list()
nt.assert_not_in(grouptools.join('group4', 'Test9'),
profile_.test_list)
def test_name_as_str(self):
"""if args is a string it is not joined.
This is a "feature" of glean, and no longer needs to be protected
whenever glean dies.
"""
with self.profile.group_manager(GleanTest, 'foo') as g:
g('abc')
assert grouptools.join('foo', 'abc') in self.profile.test_list
def setup_class(cls):
tr = results.TestResult('crash')
tr.subtests['foo'] = status.PASS
tr.subtests['bar'] = status.CRASH
tr.subtests['oink'] = status.FAIL
run = results.TestrunResult()
run.tests[grouptools.join('sub', 'test')] = tr
run.calculate_group_totals()
cls.test = run.totals