def test_max_statuses():
""" Verify that max() works between skip and non-skip statuses """
def _max_nochange_stat(nochange, stat):
""" max(nochange, stat) should = stat """
nt.assert_equal(
stat,
max(nochange, stat),
msg="max({nochange}, {stat}) = {stat}".format(**locals()))
def _max_stat_nochange(nochange, stat):
""" max(stat, nochange) should = stat """
nt.assert_equal(
stat,
max(stat, nochange),
msg="max({stat}, {nochange}) = {stat}".format(**locals()))
for nochange, stat in itertools.product(NO_OPS, STATUSES):
nochange = status.status_lookup(nochange)
stat = status.status_lookup(stat)
_max_nochange_stat.description = \
"status.Status: max({nochange}, {stat}) = {stat}".format(**locals())
yield _max_nochange_stat, nochange, stat
_max_stat_nochange.description = \
"status.Status: max({stat}, {nochange}) = {stat}".format(**locals())
yield _max_stat_nochange, nochange, stat
def test_not_change():
""" Skip and NotRun should not count as changes """
for nochange, stat in itertools.permutations(NO_OPS, 2):
check_not_change.description = \
"{0} -> {1} should not be a change".format(nochange, stat)
yield (check_not_change, status.status_lookup(nochange),
status.status_lookup(stat))
def test_max_statuses():
""" Verify that max() works between skip and non-skip statuses """
for nochange, stat in itertools.product(NO_OPS, STATUSES):
nochange = status.status_lookup(nochange)
stat = status.status_lookup(stat)
nt.assert_equal(
stat, max(nochange, stat),
msg="max({nochange}, {stat}) = {stat}".format(**locals()))
nt.assert_equal(
stat, max(stat, nochange),
msg="max({stat}, {nochange}) = {stat}".format(**locals()))
def test_max_statuses():
""" Verify that max() works between skip and non-skip statuses """
for nochange, stat in itertools.product(NO_OPS, STATUSES):
nochange = status.status_lookup(nochange)
stat = status.status_lookup(stat)
_max_nochange_stat.description = \
"max({nochange}, {stat}) = {stat}".format(**locals())
yield _max_nochange_stat, nochange, stat
_max_stat_nochange.description = \
"max({stat}, {nochange}) = {stat}".format(**locals())
yield _max_stat_nochange, nochange, stat
def test_max_statuses():
""" Verify that max() works between skip and non-skip statuses """
for nochange, stat in itertools.product(NO_OPS, STATUSES):
nochange = status.status_lookup(nochange)
stat = status.status_lookup(stat)
nt.assert_equal(
stat,
max(nochange, stat),
msg="max({nochange}, {stat}) = {stat}".format(**locals()))
nt.assert_equal(
stat,
max(stat, nochange),
msg="max({stat}, {nochange}) = {stat}".format(**locals()))
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-o", "--overwrite",
action="store_true",
help="Overwrite existing directories")
parser.add_argument("-l", "--list",
action="store",
help="Load a newline seperated list of results. These "
"results will be prepended to any Results "
"specified on the command line")
parser.add_argument("-e", "--exclude-details",
default=[],
action="append",
choices=['skip', 'pass', 'warn', 'crash' 'fail',
'all'],
help="Optionally exclude the generation of HTML pages "
"for individual test pages with the status(es) "
"given as arguments. This speeds up HTML "
"generation, but reduces the info in the HTML "
"pages. May be used multiple times")
parser.add_argument("summaryDir",
metavar="<Summary Directory>",
help="Directory to put HTML files in")
parser.add_argument("resultsFiles",
metavar="<Results Files>",
nargs="*",
help="Results files to include in HTML")
args = parser.parse_args()
# If args.list and args.resultsFiles are empty, then raise an error
if not args.list and not args.resultsFiles:
raise parser.error("Missing required option -l or <resultsFiles>")
# Convert the exclude_details list to status objects, without this using
# the -e option will except
if args.exclude_details:
# If exclude-results has all, then change it to be all
if 'all' in args.exclude_details:
args.exclude_details = [status.Skip(), status.Pass(), status.Warn(),
status.Crash(), status.Fail()]
else:
args.exclude_details = [status.status_lookup(i) for i in
args.exclude_details]
# if overwrite is requested delete the output directory
if path.exists(args.summaryDir) and args.overwrite:
shutil.rmtree(args.summaryDir)
# If the requested directory doesn't exist, create it or throw an error
checkDir(args.summaryDir, not args.overwrite)
# Merge args.list and args.resultsFiles
if args.list:
args.resultsFiles.extend(parse_listfile(args.list))
# Create the HTML output
output = summary.Summary(args.resultsFiles)
output.generate_html(args.summaryDir, args.exclude_details)
def html(input_):
# Make a copy of the status text list and add all. This is used as the
# argument list for -e/--exclude
statuses = set(str(s) for s in status.ALL)
statuses.add("all")
parser = argparse.ArgumentParser()
parser.add_argument("-o", "--overwrite", action="store_true", help="Overwrite existing directories")
parser.add_argument(
"-l",
"--list",
action="store",
help="Load a newline seperated list of results. These "
"results will be prepended to any Results "
"specified on the command line",
)
parser.add_argument(
"-e",
"--exclude-details",
default=[],
action="append",
choices=statuses,
help="Optionally exclude the generation of HTML pages "
"for individual test pages with the status(es) "
"given as arguments. This speeds up HTML "
"generation, but reduces the info in the HTML "
"pages. May be used multiple times",
)
parser.add_argument("summaryDir", metavar="<Summary Directory>", help="Directory to put HTML files in")
parser.add_argument("resultsFiles", metavar="<Results Files>", nargs="*", help="Results files to include in HTML")
args = parser.parse_args(input_)
# If args.list and args.resultsFiles are empty, then raise an error
if not args.list and not args.resultsFiles:
raise parser.error("Missing required option -l or <resultsFiles>")
# Convert the exclude_details list to status objects, without this using
# the -e option will except
if args.exclude_details:
# If exclude-results has all, then change it to be all
if "all" in args.exclude_details:
args.exclude_details = status.ALL
else:
args.exclude_details = frozenset(status.status_lookup(i) for i in args.exclude_details)
# if overwrite is requested delete the output directory
if path.exists(args.summaryDir) and args.overwrite:
shutil.rmtree(args.summaryDir)
# If the requested directory doesn't exist, create it or throw an error
core.checkDir(args.summaryDir, not args.overwrite)
# Merge args.list and args.resultsFiles
if args.list:
args.resultsFiles.extend(core.parse_listfile(args.list))
# Create the HTML output
output = summary.Summary(args.resultsFiles)
output.generate_html(args.summaryDir, args.exclude_details)
def __init__(self, *args):
super(TestResult, self).__init__(*args)
# Replace the result with a status object
try:
self['result'] = status.status_lookup(self['result'])
except KeyError:
# If there isn't a result (like when used by piglit-run), go on
# normally
pass
def __init__(self, *args):
super(TestResult, self).__init__(*args)
# Replace the result with a status object
try:
self['result'] = status.status_lookup(self['result'])
except KeyError:
# If there isn't a result (like when used by piglit-run), go on
# normally
pass
def test_not_change():
""" Skip and NotRun should not count as changes """
def check_not_change(new, old):
""" Check that a status doesn't count as a change
This checks that new < old and old < new do not return true. This is meant
for checking skip and notrun, which we don't want to show up as regressions
and fixes, but to go in their own special catagories.
"""
nt.assert_false(new < old,
msg="{new} -> {old}, is a change "
"but shouldn't be".format(**locals()))
nt.assert_false(new > old,
msg="{new} <- {old}, is a change "
"but shouldn't be".format(**locals()))
for nochange, stat in itertools.permutations(NO_OPS, 2):
check_not_change.description = \
"status.Status: {0} -> {1} is not a change".format(nochange, stat)
yield (check_not_change, status.status_lookup(nochange),
status.status_lookup(stat))
def test_not_change():
""" Skip and NotRun should not count as changes """
def check_not_change(new, old):
""" Check that a status doesn't count as a change
This checks that new < old and old < new do not return true. This is meant
for checking skip and notrun, which we don't want to show up as regressions
and fixes, but to go in their own special catagories.
"""
nt.assert_false(new < old,
msg="{new} -> {old}, is a change "
"but shouldn't be".format(**locals()))
nt.assert_false(new > old,
msg="{new} <- {old}, is a change "
"but shouldn't be".format(**locals()))
for nochange, stat in itertools.permutations(NO_OPS, 2):
check_not_change.description = \
"status.Status: {0} -> {1} is not a change".format(nochange, stat)
yield (check_not_change, status.status_lookup(nochange),
status.status_lookup(stat))
def test_max_statuses():
""" Verify that max() works between skip and non-skip statuses """
def _max_nochange_stat(nochange, stat):
""" max(nochange, stat) should = stat """
nt.assert_equal(
stat, max(nochange, stat),
msg="max({nochange}, {stat}) = {stat}".format(**locals()))
def _max_stat_nochange(nochange, stat):
""" max(stat, nochange) should = stat """
nt.assert_equal(
stat, max(stat, nochange),
msg="max({stat}, {nochange}) = {stat}".format(**locals()))
for nochange, stat in itertools.product(NO_OPS, STATUSES):
nochange = status.status_lookup(nochange)
stat = status.status_lookup(stat)
_max_nochange_stat.description = \
"status.Status: max({nochange}, {stat}) = {stat}".format(**locals())
yield _max_nochange_stat, nochange, stat
_max_stat_nochange.description = \
"status.Status: max({stat}, {nochange}) = {stat}".format(**locals())
yield _max_stat_nochange, nochange, stat
def load(cls, res):
"""Load an already generated result.
This is used as an alternate constructor which converts an existing
dictionary into a TestResult object. It converts a key 'result' into a
status.Status object
"""
result = cls(res)
# Replace the result with a status object. 'result' is a required key
# for results, so don't do any checking. This should fail if it doesn't
# exist.
result['result'] = status.status_lookup(result['result'])
return result
def load(cls, res):
"""Load an already generated result.
This is used as an alternate constructor which converts an existing
dictionary into a TestResult object. It converts a key 'result' into a
status.Status object
"""
result = cls(res)
# Replace the result with a status object. 'result' is a required key
# for results, so don't do any checking. This should fail if it doesn't
# exist.
result['result'] = status.status_lookup(result['result'])
return result
def _load(results_file):
"""Load a junit results instance and return a TestrunResult.
It's worth noting that junit is not as descriptive as piglit's own json
format, so some data structures will be empty compared to json.
This tries to not make too many assumptions about the strucuter of the
JUnit document.
"""
run_result = results.TestrunResult()
splitpath = os.path.splitext(results_file)[0].split(os.path.sep)
if splitpath[-1] != 'results':
run_result.name = splitpath[-1]
elif len(splitpath) > 1:
run_result.name = splitpath[-2]
else:
run_result.name = 'junit result'
tree = etree.parse(results_file).getroot().find('.//testsuite[@name="piglit"]')
for test in tree.iterfind('testcase'):
result = results.TestResult()
# Take the class name minus the 'piglit.' element, replace junit's '.'
# separator with piglit's separator, and join the group and test names
name = test.attrib['classname'].split('.', 1)[1]
name = name.replace('.', grouptools.SEPARATOR)
name = grouptools.join(name, test.attrib['name'])
# Remove the trailing _ if they were added (such as to api and search)
if name.endswith('_'):
name = name[:-1]
result['result'] = status.status_lookup(test.attrib['status'])
result['time'] = float(test.attrib['time'])
result['err'] = test.find('system-err').text
# The command is prepended to system-out, so we need to separate those
# into two separate elements
out = test.find('system-out').text.split('\n')
result['command'] = out[0]
result['out'] = '\n'.join(out[1:])
run_result.tests[name] = result
return run_result
def _load(results_file):
"""Load a junit results instance and return a TestrunResult.
It's worth noting that junit is not as descriptive as piglit's own json
format, so some data structures will be empty compared to json.
This tries to not make too many assumptions about the strucuter of the
JUnit document.
"""
run_result = results.TestrunResult()
splitpath = os.path.splitext(results_file)[0].split(os.path.sep)
if splitpath[-1] != 'results':
run_result.name = splitpath[-1]
elif len(splitpath) > 1:
run_result.name = splitpath[-2]
else:
run_result.name = 'junit result'
tree = etree.parse(results_file).getroot().find('.//testsuite[@name="piglit"]')
for test in tree.iterfind('testcase'):
result = results.TestResult()
# Take the class name minus the 'piglit.' element, replace junit's '.'
# separator with piglit's separator, and join the group and test names
name = test.attrib['classname'].split('.', 1)[1]
name = name.replace('.', grouptools.SEPARATOR)
name = grouptools.join(name, test.attrib['name'])
# Remove the trailing _ if they were added (such as to api and search)
if name.endswith('_'):
name = name[:-1]
result['result'] = status.status_lookup(test.attrib['status'])
result['time'] = float(test.attrib['time'])
result['err'] = test.find('system-err').text
# The command is prepended to system-out, so we need to separate those
# into two separate elements
out = test.find('system-out').text.split('\n')
result['command'] = out[0]
result['out'] = '\n'.join(out[1:])
run_result.tests[name] = result
return run_result
def test_update_result_match_regex():
""" Generates tests for update_result """
def create_test_result(res):
result = framework.results.TestResult()
result.result = res
result.subtests["test"] = res
return result
dmesg = TestDmesg()
for res in [status.status_lookup(x) for x in ["pass", "fail", "crash", "warn", "skip", "notrun"]]:
# check that the status is updated when Dmesg.regex is set and matches
# the dmesg output
dmesg.regex = re.compile("piglit.*test")
dmesg._new_messages = ["piglit.awesome.test", "and", "stuff"]
new_result = dmesg.update_result(create_test_result(res))
check_update_result.description = "dmesg.Dmesg.update_result: with matching regex '{0}'".format(res)
yield check_update_result, new_result.result, res
def test_update_result_replace():
""" Generates tests for update_result """
def create_test_result(res):
result = framework.results.TestResult()
result.result = res
result.subtests["test"] = res
return result
dmesg = TestDmesg()
for res in [status.status_lookup(x) for x in ["pass", "fail", "crash", "warn", "skip", "notrun"]]:
dmesg.regex = None
dmesg._new_messages = ["add", "some", "stuff"]
new_result = dmesg.update_result(create_test_result(res))
check_update_result.description = "dmesg.Dmesg.update_result: '{0}' replaced correctly".format(res)
yield check_update_result, new_result.result, res
check_update_result.description = "dmesg.Dmesg.update_result: subtest '{0}' replaced correctly".format(res)
yield check_update_result, new_result.subtests["test"], res
def test_update_result_match_regex():
""" Generates tests for update_result """
def create_test_result(res):
result = framework.results.TestResult()
result.result = res
result.subtests['test'] = res
return result
dmesg = TestDmesg()
for res in [status.status_lookup(x) for x in
['pass', 'fail', 'crash', 'warn', 'skip', 'notrun']]:
# check that the status is updated when Dmesg.regex is set and matches
# the dmesg output
dmesg.regex = re.compile("piglit.*test")
dmesg._new_messages = ['piglit.awesome.test', 'and', 'stuff']
new_result = dmesg.update_result(create_test_result(res))
check_update_result.description = \
"dmesg.Dmesg.update_result: with matching regex '{0}'".format(res)
yield check_update_result, new_result.result, res
def test_update_result_replace():
""" Generates tests for update_result """
def create_test_result(res):
result = framework.results.TestResult()
result.result = res
result.subtests['test'] = res
return result
dmesg = TestDmesg()
for res in [status.status_lookup(x) for x in
['pass', 'fail', 'crash', 'warn', 'skip', 'notrun']]:
dmesg.regex = None
dmesg._new_messages = ['add', 'some', 'stuff']
new_result = dmesg.update_result(create_test_result(res))
check_update_result.description = \
"dmesg.Dmesg.update_result: '{0}' replaced correctly".format(res)
yield check_update_result, new_result.result, res
check_update_result.description = \
"dmesg.Dmesg.update_result: subtest '{0}' replaced correctly".format(res)
yield check_update_result, new_result.subtests['test'], res
def is_equivalent(x, y):
# Test if status is equivalent. Note that this does not mean 'same', two
# statuses could be equivalent in terms of fixes and regressions, but that
# doesn't require that they are the same status
assert status.status_lookup(x) == status.status_lookup(y)
def test_changes(new, old):
assert status.status_lookup(new) != status.status_lookup(old)
def __init__(self, resultfiles):
"""
Create an initial object with all of the result information rolled up
in an easy to process form.
The constructor of the summary class has an attribute for each HTML
summary page, which are fed into the index.mako file to produce HTML
files. resultfiles is a list of paths to JSON results generated by
piglit-run.
"""
# Create a Result object for each piglit result and append it to the
# results list
self.results = [framework.results.load_results(i) for i in resultfiles]
self.status = {}
self.fractions = {}
self.totals = {}
self.tests = {
'all': set(),
'changes': set(),
'problems': set(),
'skipped': set(),
'regressions': set(),
'fixes': set(),
'enabled': set(),
'disabled': set()
}
def fgh(test, result):
""" Helper for updating the fractions and status lists """
fraction[test] = tuple(
[sum(i) for i in zip(fraction[test], result.fraction)])
# If the new status is worse update it, or if the new status is
# SKIP (which is equivalent to notrun) and the current is NOTRUN
# update it
if (status[test] < result
or (result == so.SKIP and status[test] == so.NOTRUN)):
status[test] = result
for results in self.results:
# Create a set of all of the tset names across all of the runs
self.tests['all'] = set(self.tests['all'] | set(results.tests))
# Create two dictionaries that have a default factory: they return
# a default value instead of a key error.
# This default key must be callable
self.fractions[results.name] = collections.defaultdict(lambda:
(0, 0))
self.status[results.name] = collections.defaultdict(
lambda: so.NOTRUN)
# short names
fraction = self.fractions[results.name]
status = self.status[results.name]
# store the results to be appeneded to results. Adding them in the
# loop will cause a RuntimeError
temp_results = {}
for key, value in results.tests.iteritems():
# if the first character of key is a / then our while loop will
# become an infinite loop. Beyond that / should never be the
# leading character, if it is then there is a bug in one of the
# test profiles.
assert key[0] != '/'
# Treat a test with subtests as if it is a group, assign the
# subtests' statuses and fractions down to the test, and then
# proceed like normal.
if 'subtest' in value:
for (subt, subv) in value['subtest'].iteritems():
subt = path.join(key, subt)
subv = so.status_lookup(subv)
# Add the subtest to the fractions and status lists
fraction[subt] = subv.fraction
status[subt] = subv
temp_results.update({subt: {'result': subv}})
self.tests['all'].add(subt)
while subt != '':
fgh(subt, subv)
subt = path.dirname(subt)
fgh('all', subv)
# remove the test from the 'all' list, this will cause to
# be treated as a group
self.tests['all'].discard(key)
else:
# Walk the test name as if it was a path, at each level
# update the tests passed over the total number of tests
# (fractions), and update the status of the current level
# if the status of the previous level was worse, but is not
# skip
while key != '':
fgh(key, value['result'])
key = path.dirname(key)
# when we hit the root update the 'all' group and stop
fgh('all', value['result'])
# Update the the results.tests dictionary with the subtests so that
# they are entered into the appropriate pages other than all.
# Updating it in the loop will raise a RuntimeError
for key, value in temp_results.iteritems():
results.tests[key] = value
# Create the lists of statuses like problems, regressions, fixes,
# changes and skips
for test in self.tests['all']:
status = []
for each in self.results:
try:
status.append(each.tests[test]['result'])
except KeyError:
status.append(so.NOTRUN)
# Problems include: warn, dmesg-warn, fail, dmesg-fail, and crash.
# Skip does not go on this page, it has the 'skipped' page
if max(status) > so.PASS:
self.tests['problems'].add(test)
# Find all tests with a status of skip
if so.SKIP in status:
self.tests['skipped'].add(test)
# find fixes, regressions, and changes
for i in xrange(len(status) - 1):
first = status[i]
last = status[i + 1]
if first in [so.SKIP, so.NOTRUN
] and last not in [so.SKIP, so.NOTRUN]:
self.tests['enabled'].add(test)
self.tests['changes'].add(test)
elif last in [so.SKIP, so.NOTRUN
] and first not in [so.SKIP, so.NOTRUN]:
self.tests['disabled'].add(test)
self.tests['changes'].add(test)
elif first < last:
self.tests['regressions'].add(test)
self.tests['changes'].add(test)
elif first > last:
self.tests['fixes'].add(test)
self.tests['changes'].add(test)
def is_regression(x, y):
# Test for regressions
assert status.status_lookup(x) < status.status_lookup(y)
def check_lookup(stat):
""" Lookup a status """
stt = status.status_lookup(stat)
assert stt
def test_bad_lookup():
""" A bad status raises a StatusException """
status.status_lookup('foobar')
def result(self, new):
try:
self.__result = status.status_lookup(new)
except exceptions.PiglitInternalError as e:
raise exceptions.PiglitFatalError(str(e))
def is_not_equivalent(new, old):
""" Test that new != old """
nt.ok_(status.status_lookup(new) != status.status_lookup(old))
def test(status_):
status.status_lookup(status_)
def is_regression(new, old):
""" Test that old -> new is a regression """
nt.ok_(status.status_lookup(new) < status.status_lookup(old))
def is_fix(new, old):
""" Test that new -> old is a fix """
nt.ok_(status.status_lookup(new) > status.status_lookup(old))
def is_not_equivalent(new, old):
""" Test that new != old """
assert status.status_lookup(new) != status.status_lookup(old)
def is_fix(new, old):
""" Test that new -> old is a fix """
assert status.status_lookup(new) > status.status_lookup(old)
def is_regression(new, old):
""" Test that old -> new is a regression """
assert status.status_lookup(new) < status.status_lookup(old)
def is_not_equivalent(x, y):
# Test that status is not equivalent.
assert status.status_lookup(x) != status.status_lookup(y)
def is_regression(new, old):
""" Test that old -> new is a regression """
assert status.status_lookup(new) < status.status_lookup(old)
def html(input_):
# Make a copy of the status text list and add all. This is used as the
# argument list for -e/--exclude
statuses = set(str(s) for s in status.ALL)
statuses.add('all')
"""Combine files in a tests/ directory into a single results file."""
unparsed = parsers.parse_config(input_)[1]
# Adding the parent is necissary to get the help options
parser = argparse.ArgumentParser(parents=[parsers.CONFIG])
parser.add_argument("-o", "--overwrite",
action="store_true",
help="Overwrite existing directories")
parser.add_argument("-l", "--list",
action="store",
help="Load a newline separated list of results. These "
"results will be prepended to any Results "
"specified on the command line")
parser.add_argument("-e", "--exclude-details",
default=[],
action="append",
choices=statuses,
help="Optionally exclude the generation of HTML pages "
"for individual test pages with the status(es) "
"given as arguments. This speeds up HTML "
"generation, but reduces the info in the HTML "
"pages. May be used multiple times")
parser.add_argument("summaryDir",
metavar="<Summary Directory>",
help="Directory to put HTML files in")
parser.add_argument("resultsFiles",
metavar="<Results Files>",
nargs="*",
help="Results files to include in HTML")
args = parser.parse_args(unparsed)
# If args.list and args.resultsFiles are empty, then raise an error
if not args.list and not args.resultsFiles:
raise parser.error("Missing required option -l or <resultsFiles>")
# Convert the exclude_details list to status objects, without this using
# the -e option will except
if args.exclude_details:
# If exclude-results has all, then change it to be all
if 'all' in args.exclude_details:
args.exclude_details = status.ALL
else:
args.exclude_details = frozenset(
status.status_lookup(i) for i in args.exclude_details)
# if overwrite is requested delete the output directory
if path.exists(args.summaryDir) and args.overwrite:
shutil.rmtree(args.summaryDir)
# If the requested directory doesn't exist, create it or throw an error
try:
core.check_dir(args.summaryDir, not args.overwrite)
except exceptions.PiglitException:
raise exceptions.PiglitFatalError(
'{} already exists.\n'
'use -o/--overwrite if you want to overwrite it.'.format(
args.summaryDir))
# Merge args.list and args.resultsFiles
if args.list:
args.resultsFiles.extend(core.parse_listfile(args.list))
# Create the HTML output
summary.html(args.resultsFiles, args.summaryDir, args.exclude_details)
def is_not_equivalent(new, old):
""" Test that new != old """
assert status.status_lookup(new) != status.status_lookup(old)
def is_fix(x, y):
# Test for fix
assert status.status_lookup(x) > status.status_lookup(y)
def result(self, new):
try:
self.__result = status.status_lookup(new)
except exceptions.PiglitInternalError as e:
raise exceptions.PiglitFatalError(str(e))
def __setitem__(self, name, value):
self.__container[name.lower()] = status.status_lookup(value)
def test_regression(new, old):
assert status.status_lookup(new) < status.status_lookup(old)
def test_bad_lookup():
"""status.status_lookup: An unexepcted value raises a StatusException"""
status.status_lookup('foobar')
def is_not_equivalent(x, y):
# Test that status is not equivalent.
assert status.status_lookup(x) != status.status_lookup(y)
def check_lookup(stat):
""" Lookup a status """
stt = status.status_lookup(stat)
assert stt
def is_not_equivalent(new, old):
""" Test that new != old """
nt.ok_(status.status_lookup(new) != status.status_lookup(old))
def is_fix(new, old):
""" Test that new -> old is a fix """
assert status.status_lookup(new) > status.status_lookup(old)
def test(status_):
status.status_lookup(status_)
def html(input_):
# Make a copy of the status text list and add all. This is used as the
# argument list for -e/--exclude
statuses = set(str(s) for s in status.ALL)
statuses.add('all')
parser = argparse.ArgumentParser()
parser.add_argument("-o", "--overwrite",
action="store_true",
help="Overwrite existing directories")
parser.add_argument("-l", "--list",
action="store",
help="Load a newline seperated list of results. These "
"results will be prepended to any Results "
"specified on the command line")
parser.add_argument("-e", "--exclude-details",
default=[],
action="append",
choices=statuses,
help="Optionally exclude the generation of HTML pages "
"for individual test pages with the status(es) "
"given as arguments. This speeds up HTML "
"generation, but reduces the info in the HTML "
"pages. May be used multiple times")
parser.add_argument("summaryDir",
metavar="<Summary Directory>",
help="Directory to put HTML files in")
parser.add_argument("resultsFiles",
metavar="<Results Files>",
nargs="*",
help="Results files to include in HTML")
args = parser.parse_args(input_)
# If args.list and args.resultsFiles are empty, then raise an error
if not args.list and not args.resultsFiles:
raise parser.error("Missing required option -l or <resultsFiles>")
# Convert the exclude_details list to status objects, without this using
# the -e option will except
if args.exclude_details:
# If exclude-results has all, then change it to be all
if 'all' in args.exclude_details:
args.exclude_details = status.ALL
else:
args.exclude_details = frozenset(
status.status_lookup(i) for i in args.exclude_details)
# if overwrite is requested delete the output directory
if path.exists(args.summaryDir) and args.overwrite:
shutil.rmtree(args.summaryDir)
# If the requested directory doesn't exist, create it or throw an error
core.checkDir(args.summaryDir, not args.overwrite)
# Merge args.list and args.resultsFiles
if args.list:
args.resultsFiles.extend(core.parse_listfile(args.list))
# Create the HTML output
output = summary.Summary(args.resultsFiles)
output.generate_html(args.summaryDir, args.exclude_details)
def test_bad_lookup():
"""status.status_lookup: An unexepcted value raises a StatusException"""
status.status_lookup('foobar')
def __setitem__(self, name, value):
self.__container[name.lower()] = status.status_lookup(value)
def is_equivalent(x, y):
# Test if status is equivalent. Note that this does not mean 'same', two
# statuses could be equivalent in terms of fixes and regressions, but that
# doesn't require that they are the same status
assert status.status_lookup(x) == status.status_lookup(y)
def test_fixes(new, old):
assert status.status_lookup(new) > status.status_lookup(old)
def is_regression(x, y):
# Test for regressions
assert status.status_lookup(x) < status.status_lookup(y)
def test_no_change(new, old):
new = status.status_lookup(new)
old = status.status_lookup(old)
assert not new < old
assert not new > old
def is_fix(x, y):
# Test for fix
assert status.status_lookup(x) > status.status_lookup(y)
def test_lookup(stat):
status.status_lookup(stat)
def is_fix(new, old):
""" Test that new -> old is a fix """
nt.ok_(status.status_lookup(new) > status.status_lookup(old))