def main():
parser = argparse.ArgumentParser()
parser.add_argument("results_path",
type=path.realpath,
metavar="<Results Path>",
help="Path to results folder")
args = parser.parse_args()
results = core.load_results(args.results_path)
env = core.Environment(concurrent=results.options['concurrent'],
exclude_filter=results.options['exclude_filter'],
include_filter=results.options['filter'],
execute=results.options['execute'],
valgrind=results.options['valgrind'],
dmesg=results.options['dmesg'],
verbose=results.options['verbose'])
# Change working directory to the piglit directory
os.chdir(path.dirname(path.realpath(sys.argv[0])))
# attempt to restore a saved platform, if there is no saved platform just
# go on
try:
os.environ['PIGLIT_PLATFORM'] = results.options['platform']
except KeyError:
pass
results_path = path.join(args.results_path, "main")
json_writer = core.JSONWriter(open(results_path, 'w+'))
json_writer.open_dict()
json_writer.write_dict_key("options")
json_writer.open_dict()
for key, value in results.options.iteritems():
json_writer.write_dict_item(key, value)
json_writer.close_dict()
json_writer.write_dict_item('name', results.name)
for (key, value) in env.collectData().items():
json_writer.write_dict_item(key, value)
json_writer.write_dict_key('tests')
json_writer.open_dict()
for key, value in results.tests.iteritems():
json_writer.write_dict_item(key, value)
env.exclude_tests.add(key)
profile = core.merge_test_profiles(results.options['profile'])
if env.dmesg:
profile.dmesg = env.dmesg
# This is resumed, don't bother with time since it wont be accurate anyway
profile.run(env, json_writer)
json_writer.close_dict()
json_writer.close_dict()
json_writer.file.close()
print("\n"
"Thank you for running Piglit!\n"
"Results have ben wrriten to {0}".format(results_path))
def main():
parser = argparse.ArgumentParser()
parser.add_argument("results_path",
type=path.realpath,
metavar="<Results Path>",
help="Path to results folder")
args = parser.parse_args()
results = core.load_results(args.results_path)
env = core.Environment(concurrent=results.options['concurrent'],
exclude_filter=results.options['exclude_filter'],
include_filter=results.options['filter'],
execute=results.options['execute'],
valgrind=results.options['valgrind'],
dmesg=results.options['dmesg'],
verbose=results.options['verbose'])
# Change working directory to the piglit directory
os.chdir(path.dirname(path.realpath(sys.argv[0])))
# attempt to restore a saved platform, if there is no saved platform just
# go on
try:
os.environ['PIGLIT_PLATFORM'] = results.options['platform']
except KeyError:
pass
results_path = path.join(args.results_path, "main")
json_writer = core.JSONWriter(open(results_path, 'w+'))
json_writer.open_dict()
json_writer.write_dict_key("options")
json_writer.open_dict()
for key, value in results.options.iteritems():
json_writer.write_dict_item(key, value)
json_writer.close_dict()
json_writer.write_dict_item('name', results.name)
for (key, value) in env.collectData().items():
json_writer.write_dict_item(key, value)
json_writer.write_dict_key('tests')
json_writer.open_dict()
for key, value in results.tests.iteritems():
json_writer.write_dict_item(key, value)
env.exclude_tests.add(key)
profile = framework.profile.merge_test_profiles(results.options['profile'])
profile.results_dir = args.results_path
if env.dmesg:
profile.dmesg = env.dmesg
# This is resumed, don't bother with time since it wont be accurate anyway
profile.run(env, json_writer)
json_writer.close_dict()
json_writer.close_dict()
json_writer.file.close()
print("Thank you for running Piglit!\n"
"Results have ben wrriten to {0}".format(results_path))
def main():
parser = argparse.ArgumentParser()
parser.add_argument("results",
metavar="<First Results File>",
nargs="*",
help="Space seperated list of results files")
args = parser.parse_args()
combined = core.load_results(args.results.pop(0))
for resultsDir in args.results:
results = core.load_results(resultsDir)
for testname, result in results.tests.items():
combined.tests[testname] = result
combined.write(sys.stdout)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("results",
metavar="<First Results File>",
nargs="*",
help="Space seperated list of results files")
args = parser.parse_args()
combined = core.load_results(args.results.pop(0))
for resultsDir in args.results:
results = core.load_results(resultsDir)
for testname, result in results.tests.items():
combined.tests[testname] = result
combined.write(sys.stdout)
def test_load_results_folder():
""" Test that load_results takes a folder with a file named main in it """
with utils.tempdir() as tdir:
with open(os.path.join(tdir, 'main'), 'w') as tfile:
tfile.write(json.dumps(utils.JSON_DATA))
results = core.load_results(tdir)
assert results
def write(self, arg):
testrun = core.load_results(arg)
self.report.start()
self.report.startSuite('piglit')
try:
for (path, result) in testrun.tests.items():
self.write_test(testrun, path, result)
finally:
self.enter_path([])
self.report.stopSuite()
self.report.stop()
def write(self, arg):
testrun = core.load_results(arg)
self.report.start()
self.report.startSuite('piglit')
try:
for (path, result) in testrun.tests.items():
self.write_test(testrun, path, result)
finally:
self.enter_path([])
self.report.stopSuite()
self.report.stop()
def main():
parser = argparse.ArgumentParser()
parser.add_argument("results_path",
type=path.realpath,
metavar="<Results Path>",
help="Path to results folder")
args = parser.parse_args()
results = core.load_results(args.results_path)
env = core.Environment(concurrent=results.options['concurrent'],
exclude_filter=results.options['exclude_filter'],
include_filter=results.options['filter'],
execute=results.options['execute'],
valgrind=results.options['valgrind'],
dmesg=results.options['dmesg'])
# Change working directory to the piglit directory
os.chdir(path.dirname(path.realpath(sys.argv[0])))
results_path = path.join(args.results_path, "main")
json_writer = core.JSONWriter(open(results_path, 'w+'))
json_writer.open_dict()
json_writer.write_dict_key("options")
json_writer.open_dict()
for key, value in results.options.iteritems():
json_writer.write_dict_item(key, value)
json_writer.close_dict()
json_writer.write_dict_item('name', results.name)
for (key, value) in env.collectData().items():
json_writer.write_dict_item(key, value)
json_writer.write_dict_key('tests')
json_writer.open_dict()
for key, value in results.tests.iteritems():
json_writer.write_dict_item(key, value)
env.exclude_tests.add(key)
json_writer.close_dict()
profile = core.loadTestProfile(results.options['profile'])
# This is resumed, don't bother with time since it wont be accurate anyway
profile.run(env, json_writer)
json_writer.close_dict()
json_writer.close_dict()
json_writer.file.close()
print("\n"
"Thank you for running Piglit!\n"
"Results have ben wrriten to {0}".format(results_path))
def __init__(self, resultfiles):
"""
Create an initial object with all of the result information rolled up
in an easy to process form.
The constructor of the summary class has an attribute for each HTML
summary page, which are fed into the index.mako file to produce HTML
files. resultfiles is a list of paths to JSON results generated by
piglit-run.
"""
# Create a Result object for each piglit result and append it to the
# results list
self.results = [core.load_results(i) for i in resultfiles]
self.status = {}
self.fractions = {}
self.totals = {}
self.tests = {'all': set(), 'changes': set(), 'problems': set(),
'skipped': set(), 'regressions': set(), 'fixes': set()}
def fgh(test, result):
""" Helper for updating the fractions and status lists """
fraction[test] = tuple(
[sum(i) for i in zip(fraction[test], result.fraction)])
if result != so.Skip() and status[test] < result:
status[test] = result
for results in self.results:
# Create a set of all of the tset names across all of the runs
self.tests['all'] = set(self.tests['all'] | set(results.tests))
# Create two dictionaries that have a default factory: they return
# a default value instead of a key error.
# This default key must be callable
self.fractions[results.name] = collections.defaultdict(lambda: (0, 0))
self.status[results.name] = collections.defaultdict(so.NotRun)
# short names
fraction = self.fractions[results.name]
status = self.status[results.name]
for key, value in results.tests.iteritems():
# if the first character of key is a / then our while loop will
# become an infinite loop. Beyond that / should never be the
# leading character, if it is then there is a bug in one of the
# test profiles.
assert key[0] != '/'
#FIXME: Add subtest support
# Walk the test name as if it was a path, at each level update
# the tests passed over the total number of tests (fractions),
# and update the status of the current level if the status of
# the previous level was worse, but is not skip
while key != '':
fgh(key, value['result'])
key = path.dirname(key)
# when we hit the root update the 'all' group and stop
fgh('all', value['result'])
# Create the lists of statuses like problems, regressions, fixes,
# changes and skips
for test in self.tests['all']:
status = []
for each in self.results:
try:
status.append(each.tests[test]['result'])
except KeyError:
status.append(so.NotRun())
# Problems include: warn, dmesg-warn, fail, dmesg-fail, and crash.
# Skip does not go on this page, it has the 'skipped' page
if max(status) > so.Pass():
self.tests['problems'].add(test)
# Find all tests with a status of skip
if so.Skip() in status:
self.tests['skipped'].add(test)
# find fixes, regressions, and changes
for i in xrange(len(status) - 1):
first = status[i]
last = status[i + 1]
if max(first, so.Pass()) < last:
self.tests['regressions'].add(test)
if first > max(last, so.Pass()):
self.tests['fixes'].add(test)
# Changes cannot be added in the fixes and regressions passes
# becasue NotRun is a change, but not a regression or fix
if first != last:
self.tests['changes'].add(test)
def main():
parser = argparse.ArgumentParser(sys.argv)
# Either require that a name for the test is passed or that
# resume is requested
excGroup1 = parser.add_mutually_exclusive_group()
excGroup1.add_argument("-n", "--name",
metavar="<test name>",
default=None,
help="Name of this test run")
excGroup1.add_argument("-r", "--resume",
action="store_true",
help="Resume an interupted test run")
# Setting the --dry-run flag is equivalent to env.execute=false
parser.add_argument("-d", "--dry-run",
action="store_false",
dest="execute",
help="Do not execute the tests")
parser.add_argument("-t", "--include-tests",
default=[],
action="append",
metavar="<regex>",
help="Run only matching tests "
"(can be used more than once)")
parser.add_argument("-x", "--exclude-tests",
default=[],
action="append",
metavar="<regex>",
help="Exclude matching tests "
"(can be used more than once)")
parser.add_argument("-1", "--no-concurrency",
action="store_false",
dest="concurrency",
help="Disable concurrent test runs")
parser.add_argument("-p", "--platform",
choices=["glx", "x11_egl", "wayland", "gbm"],
help="Name of windows system passed to waffle")
parser.add_argument("--valgrind",
action="store_true",
help="Run tests in valgrind's memcheck")
parser.add_argument("--dmesg",
action="store_true",
help="Capture a difference in dmesg before and "
"after each test")
parser.add_argument("testProfile",
metavar="<Path to test profile>",
help="Path to testfile to run")
parser.add_argument("resultsPath",
metavar="<Results Path>",
help="Path to results folder")
args = parser.parse_args()
# Set the platform to pass to waffle
if args.platform is not None:
os.environ['PIGLIT_PLATFORM'] = args.platform
# Always Convert Results Path from Relative path to Actual Path.
resultsDir = path.realpath(args.resultsPath)
# If resume is requested attempt to load the results file
# in the specified path
if args.resume is True:
# Load settings from the old results JSON
old_results = core.load_results(resultsDir)
profileFilename = old_results.options['profile']
# Changing the args to the old args allows us to set them
# all in one places down the way
args.exclude_tests = old_results.options['exclude_filter']
args.include_tests = old_results.options['filter']
# Otherwise parse additional settings from the command line
else:
profileFilename = args.testProfile
# Pass arguments into Environment
env = core.Environment(concurrent=args.concurrency,
exclude_filter=args.exclude_tests,
include_filter=args.include_tests,
execute=args.execute,
valgrind=args.valgrind,
dmesg=args.dmesg)
# Change working directory to the root of the piglit directory
piglit_dir = path.dirname(path.realpath(sys.argv[0]))
os.chdir(piglit_dir)
core.checkDir(resultsDir, False)
results = core.TestrunResult()
# Set results.name
if args.name is not None:
results.name = args.name
else:
results.name = path.basename(resultsDir)
# Begin json.
result_filepath = os.path.join(resultsDir, 'main')
result_file = open(result_filepath, 'w')
json_writer = core.JSONWriter(result_file)
json_writer.open_dict()
# Write out command line options for use in resuming.
json_writer.write_dict_key('options')
json_writer.open_dict()
json_writer.write_dict_item('profile', profileFilename)
json_writer.write_dict_item('filter', args.include_tests)
json_writer.write_dict_item('exclude_filter', args.exclude_tests)
json_writer.close_dict()
json_writer.write_dict_item('name', results.name)
for (key, value) in env.collectData().items():
json_writer.write_dict_item(key, value)
profile = core.loadTestProfile(profileFilename)
json_writer.write_dict_key('tests')
json_writer.open_dict()
# If resuming an interrupted test run, re-write all of the existing
# results since we clobbered the results file. Also, exclude them
# from being run again.
if args.resume is True:
for (key, value) in old_results.tests.items():
if os.path.sep != '/':
key = key.replace(os.path.sep, '/', -1)
json_writer.write_dict_item(key, value)
env.exclude_tests.add(key)
time_start = time.time()
profile.run(env, json_writer)
time_end = time.time()
json_writer.close_dict()
results.time_elapsed = time_end - time_start
json_writer.write_dict_item('time_elapsed', results.time_elapsed)
# End json.
json_writer.close_dict()
json_writer.file.close()
print
print 'Thank you for running Piglit!'
print 'Results have been written to ' + result_filepath
def __init__(self, resultfiles):
"""
Create an initial object with all of the result information rolled up
in an easy to process form.
The constructor of the summary class has an attribute for each HTML
summary page, which are fed into the index.mako file to produce HTML
files. resultfiles is a list of paths to JSON results generated by
piglit-run.
"""
# Create a Result object for each piglit result and append it to the
# results list
self.results = [core.load_results(i) for i in resultfiles]
self.status = {}
self.fractions = {}
self.totals = {}
self.tests = {
'all': set(),
'changes': set(),
'problems': set(),
'skipped': set(),
'regressions': set(),
'fixes': set()
}
def fgh(test, result):
""" Helper for updating the fractions and status lists """
fraction[test] = tuple(
[sum(i) for i in zip(fraction[test], result.fraction)])
if result != so.Skip() and status[test] < result:
status[test] = result
for results in self.results:
# Create a set of all of the tset names across all of the runs
self.tests['all'] = set(self.tests['all'] | set(results.tests))
# Create two dictionaries that have a default factory: they return
# a default value instead of a key error.
# This default key must be callable
self.fractions[results.name] = collections.defaultdict(lambda:
(0, 0))
self.status[results.name] = collections.defaultdict(so.NotRun)
# short names
fraction = self.fractions[results.name]
status = self.status[results.name]
for key, value in results.tests.iteritems():
# if the first character of key is a / then our while loop will
# become an infinite loop. Beyond that / should never be the
# leading character, if it is then there is a bug in one of the
# test profiles.
assert key[0] != '/'
#FIXME: Add subtest support
# Walk the test name as if it was a path, at each level update
# the tests passed over the total number of tests (fractions),
# and update the status of the current level if the status of
# the previous level was worse, but is not skip
while key != '':
fgh(key, value['result'])
key = path.dirname(key)
# when we hit the root update the 'all' group and stop
fgh('all', value['result'])
# Create the lists of statuses like problems, regressions, fixes,
# changes and skips
for test in self.tests['all']:
status = []
for each in self.results:
try:
status.append(each.tests[test]['result'])
except KeyError:
status.append(so.NotRun())
# Problems include: warn, dmesg-warn, fail, dmesg-fail, and crash.
# Skip does not go on this page, it has the 'skipped' page
if max(status) > so.Pass():
self.tests['problems'].add(test)
# Find all tests with a status of skip
if so.Skip() in status:
self.tests['skipped'].add(test)
# find fixes, regressions, and changes
for i in xrange(len(status) - 1):
first = status[i]
last = status[i + 1]
if max(first, so.Pass()) < last:
self.tests['regressions'].add(test)
if first > max(last, so.Pass()):
self.tests['fixes'].add(test)
# Changes cannot be added in the fixes and regressions passes
# becasue NotRun is a change, but not a regression or fix
if first != last:
self.tests['changes'].add(test)
def __init__(self, resultfiles):
"""
Create an initial object with all of the result information rolled up
in an easy to process form.
The constructor of the summary class has an attribute for each HTML
summary page, which are fed into the index.mako file to produce HTML
files. resultfiles is a list of paths to JSON results generated by
piglit-run.
"""
# Create a Result object for each piglit result and append it to the
# results list
self.results = [core.load_results(i) for i in resultfiles]
self.status = {}
self.fractions = {}
self.totals = {}
self.tests = {'all': set(), 'changes': set(), 'problems': set(),
'skipped': set(), 'regressions': set(), 'fixes': set(),
'enabled': set(), 'disabled': set()}
def fgh(test, result):
""" Helper for updating the fractions and status lists """
fraction[test] = tuple(
[sum(i) for i in zip(fraction[test], result.fraction)])
# If the new status is worse update it, or if the new status is
# SKIP (which is equivalent to notrun) and the current is NOTRUN
# update it
if (status[test] < result or
(result == so.SKIP and status[test] == so.NOTRUN)):
status[test] = result
for results in self.results:
# Create a set of all of the tset names across all of the runs
self.tests['all'] = set(self.tests['all'] | set(results.tests))
# Create two dictionaries that have a default factory: they return
# a default value instead of a key error.
# This default key must be callable
self.fractions[results.name] = collections.defaultdict(lambda: (0, 0))
self.status[results.name] = collections.defaultdict(lambda: so.NOTRUN)
# short names
fraction = self.fractions[results.name]
status = self.status[results.name]
# store the results to be appeneded to results. Adding them in the
# loop will cause a RuntimeError
temp_results = {}
for key, value in results.tests.iteritems():
# if the first character of key is a / then our while loop will
# become an infinite loop. Beyond that / should never be the
# leading character, if it is then there is a bug in one of the
# test profiles.
assert key[0] != '/'
# Treat a test with subtests as if it is a group, assign the
# subtests' statuses and fractions down to the test, and then
# proceed like normal.
if 'subtest' in value:
for (subt, subv) in value['subtest'].iteritems():
subt = path.join(key, subt)
subv = so.status_lookup(subv)
# Add the subtest to the fractions and status lists
fraction[subt] = subv.fraction
status[subt] = subv
temp_results.update({subt: {'result': subv}})
self.tests['all'].add(subt)
while subt != '':
fgh(subt, subv)
subt = path.dirname(subt)
fgh('all', subv)
# remove the test from the 'all' list, this will cause to
# be treated as a group
self.tests['all'].discard(key)
else:
# Walk the test name as if it was a path, at each level
# update the tests passed over the total number of tests
# (fractions), and update the status of the current level
# if the status of the previous level was worse, but is not
# skip
while key != '':
fgh(key, value['result'])
key = path.dirname(key)
# when we hit the root update the 'all' group and stop
fgh('all', value['result'])
# Update the the results.tests dictionary with the subtests so that
# they are entered into the appropriate pages other than all.
# Updating it in the loop will raise a RuntimeError
for key, value in temp_results.iteritems():
results.tests[key] = value
# Create the lists of statuses like problems, regressions, fixes,
# changes and skips
for test in self.tests['all']:
status = []
for each in self.results:
try:
status.append(each.tests[test]['result'])
except KeyError:
status.append(so.NOTRUN)
# Problems include: warn, dmesg-warn, fail, dmesg-fail, and crash.
# Skip does not go on this page, it has the 'skipped' page
if max(status) > so.PASS:
self.tests['problems'].add(test)
# Find all tests with a status of skip
if so.SKIP in status:
self.tests['skipped'].add(test)
# find fixes, regressions, and changes
for i in xrange(len(status) - 1):
first = status[i]
last = status[i + 1]
if first in [so.SKIP, so.NOTRUN] and last not in [so.SKIP, so.NOTRUN]:
self.tests['enabled'].add(test)
self.tests['changes'].add(test)
elif last in [so.SKIP, so.NOTRUN] and first not in [so.SKIP, so.NOTRUN]:
self.tests['disabled'].add(test)
elif first < last:
self.tests['regressions'].add(test)
self.tests['changes'].add(test)
elif first > last:
self.tests['fixes'].add(test)
self.tests['changes'].add(test)
def test_load_results_file():
""" Test that load_results takes a file """
with utils.resultfile() as tfile:
results = core.load_results(tfile.name)
assert results
