def test_values(self, index, capsys):
"""Create both an expected value and an actual value.
The expected value makes use of the template, this helps to minimize
the number of changes that need to be made if the output is altered.
"""
names = [
grouptools.join('foo', 'bar', 'oink', 'foobar', 'boink'), 'foo',
'bar'
]
template = '{: >20.20} {: >12.12}'
expected = console_._SUMMARY_TEMPLATE.format(
names=' '.join(['this is a really rea', 'another name']),
divider=' '.join(['--------------------', '------------']),
pass_=template.format('1', '2'),
fail=template.format('2', '0'),
crash=template.format('0', '0'),
skip=template.format('0', '1'),
timeout=template.format('0', '0'),
warn=template.format('0', '0'),
incomplete=template.format('0', '0'),
dmesg_warn=template.format('0', '0'),
dmesg_fail=template.format('0', '0'),
changes=template.format('0', '2'),
fixes=template.format('0', '1'),
regressions=template.format('0', '0'),
total=template.format('3', '3'),
time=template.format('00:01:39', '02:14:05')).split('\n')
res1 = results.TestrunResult()
res1.name = 'this is a really really really really long name'
res1.tests[names[0]] = results.TestResult('pass')
res1.tests[names[1]] = results.TestResult('fail')
res1.tests[names[2]] = results.TestResult('notrun')
res1.tests[names[2]].subtests['1'] = 'fail'
res1.time_elapsed = results.TimeAttribute(1509747121.4873962,
1509747220.544042)
res1.calculate_group_totals()
res2 = results.TestrunResult()
res2.name = 'another name'
res2.tests[names[0]] = results.TestResult('pass')
res2.tests[names[1]] = results.TestResult('pass')
res2.tests[names[2]] = results.TestResult('notrun')
res2.tests[names[2]].subtests['1'] = 'skip'
res2.time_elapsed = results.TimeAttribute(1464820707.4581327,
1464828753.201948)
res2.calculate_group_totals()
reses = common.Results([res1, res2])
console_._print_summary(reses)
actual = capsys.readouterr()[0].splitlines()
assert actual[index] == expected[index]
def test_TimeAttribute_to_json():
"""results.TimeAttribute.to_json(): returns expected dictionary"""
baseline = {'start': 0.1, 'end': 1.0}
test = results.TimeAttribute(**baseline)
baseline['__type__'] = 'TimeAttribute'
nt.assert_dict_equal(baseline, test.to_json())
def test_to_json(self):
"""returns expected dictionary."""
baseline = {'start': 0.1, 'end': 1.0}
test = results.TimeAttribute(**baseline)
baseline['__type__'] = 'TimeAttribute'
assert baseline == test.to_json()
def _update_seven_to_eight(result):
"""Update json results from version 7 to 8.
This update replaces the time attribute float with a TimeAttribute object,
which stores a start time and an end time, and provides methods for getting
total and delta.
This value is used for both TestResult.time and TestrunResult.time_elapsed.
"""
for test in compat.viewvalues(result.tests):
test.time = results.TimeAttribute(end=test.time)
result.time_elapsed = results.TimeAttribute(end=result.time_elapsed)
result.results_version = 8
return result
def test_ignores_invalid(self, tmpdir):
test = backends.json.JSONBackend(str(tmpdir))
test.initialize(shared.INITIAL_METADATA)
with test.write_test(self.name) as t:
t(self.result)
tmpdir.join('tests/2.json.tmp').write('{}')
test.finalize(
{'time_elapsed':
results.TimeAttribute(start=0.0, end=1.0).to_json()})
def setup(self, tmpdir):
test = backends.json.JSONBackend(six.text_type(tmpdir))
test.initialize(shared.INITIAL_METADATA)
with test.write_test(self.name) as t:
t(self.result)
test.finalize({
'time_elapsed':
results.TimeAttribute(start=0.0, end=1.0).to_json()
})
def result_dir(self, tmpdir_factory):
directory = tmpdir_factory.mktemp('main')
test = backends.json.JSONBackend(str(directory))
test.initialize(shared.INITIAL_METADATA)
with test.write_test(self.name) as t:
t(self.result)
test.finalize(
{'time_elapsed':
results.TimeAttribute(start=0.0, end=1.0).to_json()})
return directory
def setup_class(cls):
super(TestJsonOutput, cls).setup_class()
args = Namespace()
# pylint: disable=attribute-defined-outside-init
args.test_profile = ['fake.py']
args.platform = 'gbm'
args.log_level = 'verbose'
backend = JSONBackend(cls.tdir, file_fsync=True)
backend.initialize(_create_metadata(args, 'test'))
with backend.write_test('result') as t:
t(results.TestResult('pass'))
backend.finalize({'time_elapsed': results.TimeAttribute(end=1.2)})
with open(os.path.join(cls.tdir, 'results.json'), 'r') as f:
cls.json = json.load(f)
def test_TimeAttribute_delta():
"""results.TimeAttribute.delta: returns the delta of the values"""
test = results.TimeAttribute(1.0, 5.0)
nt.eq_(test.delta, '0:00:04')
def test_TimeAttribute_total():
"""results.TimeAttribute.total: returns the difference between end and start"""
test = results.TimeAttribute(1.0, 5.0)
nt.eq_(test.total, 4.0)
def _load(results_file):
"""Load a junit results instance and return a TestrunResult.
It's worth noting that junit is not as descriptive as piglit's own json
format, so some data structures will be empty compared to json.
This tries to not make too many assumptions about the strucuter of the
JUnit document.
"""
run_result = results.TestrunResult()
splitpath = os.path.splitext(results_file)[0].split(os.path.sep)
if splitpath[-1] != 'results':
run_result.name = splitpath[-1]
elif len(splitpath) > 1:
run_result.name = splitpath[-2]
else:
run_result.name = 'junit result'
tree = etree.parse(results_file).getroot().find(
'.//testsuite[@name="piglit"]')
for test in tree.iterfind('testcase'):
result = results.TestResult()
# Take the class name minus the 'piglit.' element, replace junit's '.'
# separator with piglit's separator, and join the group and test names
name = test.attrib['classname'].split('.', 1)[1]
name = name.replace('.', grouptools.SEPARATOR)
name = grouptools.join(name, test.attrib['name'])
# Remove the trailing _ if they were added (such as to api and search)
if name.endswith('_'):
name = name[:-1]
result.result = test.attrib['status']
# This is the fallback path, we'll try to overwrite this with the value
# in stderr
result.time = results.TimeAttribute(end=float(test.attrib['time']))
result.err = test.find('system-err').text
# The command is prepended to system-out, so we need to separate those
# into two separate elements
out = test.find('system-out').text.split('\n')
result.command = out[0]
result.out = '\n'.join(out[1:])
# Try to get the values in stderr for time and pid
for line in result.err.split('\n'):
if line.startswith('time start:'):
result.time.start = float(line[len('time start: '):])
continue
elif line.startswith('time end:'):
result.time.end = float(line[len('time end: '):])
continue
elif line.startswith('pid:'):
result.pid = json.loads(line[len('pid: '):])
continue
run_result.tests[name] = result
run_result.calculate_group_totals()
return run_result
def _load(results_file):
"""Load a junit results instance and return a TestrunResult.
It's worth noting that junit is not as descriptive as piglit's own json
format, so some data structures will be empty compared to json.
This tries to not make too many assumptions about the structure of the
JUnit document.
"""
run_result = results.TestrunResult()
splitpath = os.path.splitext(results_file)[0].split(os.path.sep)
if splitpath[-1] != 'results':
run_result.name = splitpath[-1]
elif len(splitpath) > 1:
run_result.name = splitpath[-2]
else:
run_result.name = 'junit result'
tree = etree.parse(results_file).getroot().find('.//testsuite')
for test in tree.iterfind('testcase'):
result = results.TestResult()
# Take the class name minus the 'piglit.' element, replace junit's '.'
# separator with piglit's separator, and join the group and test names
name = test.attrib['name']
if 'classname' in test.attrib:
name = grouptools.join(test.attrib['classname'], name)
name = name.replace('.', grouptools.SEPARATOR)
is_piglit = False
if name.startswith("piglit"):
is_piglit = True
name = name.split(grouptools.SEPARATOR, 1)[1]
# Remove the trailing _ if they were added (such as to api and search)
if name.endswith('_'):
name = name[:-1]
result.result = test.attrib['status']
# This is the fallback path, we'll try to overwrite this with the value
# in stderr
result.time = results.TimeAttribute()
if 'time' in test.attrib:
result.time = results.TimeAttribute(end=float(test.attrib['time']))
syserr = test.find('system-err')
if syserr is not None:
reversed_err = syserr.text.split('\n')
reversed_err.reverse()
else:
reversed_err = []
# The command is prepended to system-out, so we need to separate those
# into two separate elements
out_tag = test.find('system-out')
if out_tag is not None:
if is_piglit:
out = out_tag.text.split('\n')
result.command = out[0]
result.out = '\n'.join(out[1:])
else:
result.out = out_tag.text
err_list = []
skip = 0
# Try to get the values in stderr for time and pid
for line in reversed_err:
if skip > 0:
if line == '':
skip -= 1
continue
else:
skip = 0
if line.startswith(_START_TIME_STR):
result.time.start = float(line[len(_START_TIME_STR):])
continue
elif line.startswith(_END_TIME_STR):
result.time.end = float(line[len(_END_TIME_STR):])
continue
elif line.startswith(_PID_STR):
result.pid = json.loads(line[len(_PID_STR):])
skip = 2
continue
err_list.append(line)
err_list.reverse()
result.err = "\n".join(err_list)
run_result.tests[name] = result
run_result.calculate_group_totals()
return run_result
def test_delta(self):
"""results.TimeAttribute.delta: returns the delta of the values"""
test = results.TimeAttribute(1.0, 5.0)
assert test.delta == '0:00:04'
def test_total(self):
"""returns the difference between end and start."""
test = results.TimeAttribute(1.0, 5.0)
assert test.total == 4.0
