def test_matches_include_caps(self):
"""TestProfile.prepare_test_list: matches capitalized tests."""
env = core.Options(exclude_filter=['test9'])
profile_ = profile.TestProfile()
profile_.test_list = self.data
profile_._prepare_test_list(env)
nt.assert_not_in(grouptools.join('group4', 'Test9'), profile_.test_list)
def test_matches_filter_mar_2(data):
""" Tests 'not env.filter or matches_any_regex() mar is False"""
env = core.Options(include_filter=['test5'])
profile_ = profile.TestProfile()
profile_.test_list = data
profile_._prepare_test_list(env)
baseline = {'group3/test5': 'other'}
nt.assert_dict_equal(profile_.test_list, baseline)
def test_matches_exclude_mar(data):
""" Tests 'not matches_any_regexp() """
env = core.Options(exclude_filter=['test5'])
profile_ = profile.TestProfile()
profile_.test_list = data
profile_._prepare_test_list(env)
baseline = copy.deepcopy(data)
del baseline['group3/test5']
nt.assert_dict_equal(profile_.test_list, baseline)
def test_matches_exclude_mar(self):
"""TestProfile.prepare_test_list: 'not matches_any_regexp()"""
env = core.Options(exclude_filter=['test5'])
profile_ = profile.TestProfile()
profile_.test_list = self.data
profile_._prepare_test_list(env)
baseline = copy.deepcopy(self.data)
del baseline[grouptools.join('group3', 'test5')]
nt.assert_dict_equal(profile_.test_list, baseline)
def resume(input_):
parser = argparse.ArgumentParser()
parser.add_argument("results_path",
type=path.realpath,
metavar="<Results Path>",
help="Path to results folder")
parser.add_argument("-f",
"--config",
dest="config_file",
type=argparse.FileType("r"),
help="Optionally specify a piglit config file to use. "
"Default is piglit.conf")
args = parser.parse_args(input_)
_disable_windows_exception_messages()
results = framework.results.TestrunResult.resume(args.results_path)
opts = core.Options(concurrent=results.options['concurrent'],
exclude_filter=results.options['exclude_filter'],
include_filter=results.options['filter'],
execute=results.options['execute'],
valgrind=results.options['valgrind'],
dmesg=results.options['dmesg'],
sync=results.options['sync'])
core.get_config(args.config_file)
opts.env['PIGLIT_PLATFORM'] = results.options['platform']
results.options['env'] = core.collect_system_info()
results.options['name'] = results.name
# Resume only works with the JSON backend
backend = backends.get_backend('json')(
args.results_path,
file_fsync=opts.sync,
file_start_count=len(results.tests) + 1)
# Specifically do not initialize again, everything initialize does is done.
for name in results.tests.iterkeys():
opts.exclude_tests.add(name)
profile = framework.profile.merge_test_profiles(results.options['profile'])
profile.results_dir = args.results_path
if opts.dmesg:
profile.dmesg = opts.dmesg
# This is resumed, don't bother with time since it wont be accurate anyway
profile.run(opts, results.options['log_level'], backend)
backend.finalize()
print("Thank you for running Piglit!\n"
"Results have been written to {0}".format(args.results_path))
def resume(input_):
parser = argparse.ArgumentParser()
parser.add_argument("results_path",
type=path.realpath,
metavar="<Results Path>",
help="Path to results folder")
parser.add_argument("-f",
"--config",
dest="config_file",
type=argparse.FileType("r"),
help="Optionally specify a piglit config file to use. "
"Default is piglit.conf")
args = parser.parse_args(input_)
results = framework.results.load_results(args.results_path)
opts = core.Options(concurrent=results.options['concurrent'],
exclude_filter=results.options['exclude_filter'],
include_filter=results.options['filter'],
execute=results.options['execute'],
valgrind=results.options['valgrind'],
dmesg=results.options['dmesg'],
verbose=results.options['verbose'])
core.get_config(args.config_file)
if results.options.get('platform'):
opts.env['PIGLIT_PLATFORM'] = results.options['platform']
results_path = path.join(args.results_path, 'results.json')
json_writer = framework.results.JSONWriter(open(results_path, 'w+'))
json_writer.initialize_json(results.options, results.name,
core.collect_system_info())
json_writer.write_dict_key('tests')
json_writer.open_dict()
for key, value in results.tests.iteritems():
json_writer.write_dict_item(key, value)
opts.exclude_tests.add(key)
profile = framework.profile.merge_test_profiles(results.options['profile'])
profile.results_dir = args.results_path
if opts.dmesg:
profile.dmesg = opts.dmesg
# This is resumed, don't bother with time since it wont be accurate anyway
profile.run(opts, json_writer)
json_writer.close_dict()
json_writer.close_json()
print("Thank you for running Piglit!\n"
"Results have ben wrriten to {0}".format(results_path))
def test_matches_filter_mar_1(data):
""" Tests 'not env.filter or matches_any_regex() env.filter is False
Nothing should be filtered.
"""
env = core.Options()
profile_ = profile.TestProfile()
profile_.test_list = data
profile_._prepare_test_list(env)
nt.assert_dict_equal(profile_.test_list, data)
def test_matches_env_exclude(data):
""" Tests 'not path in env.exclude_tests """
env = core.Options()
env.exclude_tests.add('group3/test5')
profile_ = profile.TestProfile()
profile_.test_list = data
profile_._prepare_test_list(env)
baseline = copy.deepcopy(data)
del baseline['group3/test5']
nt.assert_dict_equal(profile_.test_list, baseline)
def test_matches_env_exclude(self):
"""TestProfile.prepare_test_list: 'not path in env.exclude_tests"""
env = core.Options()
env.exclude_tests.add(grouptools.join('group3', 'test5'))
profile_ = profile.TestProfile()
profile_.test_list = self.data
profile_._prepare_test_list(env)
baseline = copy.deepcopy(self.data)
del baseline[grouptools.join('group3', 'test5')]
nt.assert_dict_equal(profile_.test_list, baseline)
def test_matches_filter_mar_2(self):
"""TestProfile.prepare_test_list: 'not env.filter or matches_any_regex()
mar is False
"""
env = core.Options(include_filter=['test5'])
profile_ = profile.TestProfile()
profile_.test_list = self.data
profile_._prepare_test_list(env)
baseline = {grouptools.join('group3', 'test5'): 'other'}
nt.assert_dict_equal(profile_.test_list, baseline)
def setup_class(cls):
super(TestJsonOutput, cls).setup_class()
args = Namespace()
# pylint: disable=attribute-defined-outside-init
args.test_profile = ['fake.py']
args.platform = 'gbm'
args.log_level = 'verbose'
backend = JSONBackend(cls.tdir, file_fsync=True)
backend.initialize(_create_metadata(args, 'test', core.Options()))
backend.write_test('result', {'result': 'pass'})
backend.finalize({'time_elapsed': 1.22})
with open(os.path.join(cls.tdir, 'results.json'), 'r') as f:
cls.json = json.load(f)
def main():
core.get_config()
parser = argparse.ArgumentParser(sys.argv)
parser.add_argument(
"-t",
"--include-tests",
default=[],
action="append",
metavar="<regex>",
help="Run only matching tests (can be used more than once)")
parser.add_argument("-x",
"--exclude-tests",
default=[],
action="append",
metavar="<regex>",
help="Exclude matching tests (can be used more than "
"once)")
parser.add_argument("testProfile",
metavar="<Path to testfile>",
help="Path to results folder")
args = parser.parse_args()
opts = core.Options(exclude_filter=args.exclude_tests,
include_filter=args.include_tests)
# Change to the piglit's path
piglit_dir = path.dirname(path.realpath(sys.argv[0]))
os.chdir(piglit_dir)
profile = framework.profile.load_test_profile(args.testProfile)
def getCommand(test):
command = ''
if isinstance(test, GleanTest):
for var, val in test.env.items():
command += var + "='" + val + "' "
# Make the test command relative to the piglit_dir
testCommand = test.command[:]
testCommand[0] = os.path.relpath(testCommand[0], piglit_dir)
command += ' '.join(testCommand)
return command
profile._prepare_test_list(opts)
for name, test in profile.test_list.items():
assert (isinstance(test, Test))
print(name, ':::', getCommand(test))
def run(input_):
""" Function for piglit run command
This is a function because it allows it to be shared between piglit-run.py
and piglit run
"""
args = _run_parser(input_)
_disable_windows_exception_messages()
# If dmesg is requested we must have serial run, this is becasue dmesg
# isn't reliable with threaded run
if args.dmesg:
args.concurrency = "none"
# Pass arguments into Options
opts = core.Options(concurrent=args.concurrency,
exclude_filter=args.exclude_tests,
include_filter=args.include_tests,
execute=args.execute,
valgrind=args.valgrind,
dmesg=args.dmesg,
sync=args.sync)
# Set the platform to pass to waffle
opts.env['PIGLIT_PLATFORM'] = args.platform
# Change working directory to the root of the piglit directory
piglit_dir = path.dirname(path.realpath(sys.argv[0]))
os.chdir(piglit_dir)
core.checkDir(args.results_path, False)
results = framework.results.TestrunResult()
# Set results.name
if args.name is not None:
results.name = args.name
else:
results.name = path.basename(args.results_path)
backend = backends.get_backend(args.backend)(
args.results_path,
file_fsync=opts.sync,
junit_suffix=args.junit_suffix)
backend.initialize(_create_metadata(args, results.name, opts))
profile = framework.profile.merge_test_profiles(args.test_profile)
profile.results_dir = args.results_path
time_start = time.time()
# Set the dmesg type
if args.dmesg:
profile.dmesg = args.dmesg
profile.run(opts, args.log_level, backend)
time_end = time.time()
results.time_elapsed = time_end - time_start
backend.finalize({'time_elapsed': results.time_elapsed})
print('Thank you for running Piglit!\n'
'Results have been written to ' + args.results_path)
def run(input_):
parser = argparse.ArgumentParser()
parser.add_argument("-n",
"--name",
metavar="<test name>",
default=None,
help="Name of this test run")
parser.add_argument("-d",
"--dry-run",
action="store_false",
dest="execute",
help="Do not execute the tests")
parser.add_argument("-t",
"--include-tests",
default=[],
action="append",
metavar="<regex>",
help="Run only matching tests "
"(can be used more than once)")
parser.add_argument("-x",
"--exclude-tests",
default=[],
action="append",
metavar="<regex>",
help="Exclude matching tests "
"(can be used more than once)")
conc_parser = parser.add_mutually_exclusive_group()
conc_parser.add_argument('-c',
'--all-concurrent',
action="store_const",
default="some",
const="all",
dest="concurrency",
help="Run all tests concurrently")
conc_parser.add_argument("-1",
"--no-concurrency",
action="store_const",
default="some",
const="none",
dest="concurrency",
help="Disable concurrent test runs")
parser.add_argument("-p",
"--platform",
choices=["glx", "x11_egl", "wayland", "gbm"],
help="Name of windows system passed to waffle")
parser.add_argument("-f",
"--config",
dest="config_file",
type=argparse.FileType("r"),
help="Optionally specify a piglit config file to use. "
"Default is piglit.conf")
parser.add_argument("--valgrind",
action="store_true",
help="Run tests in valgrind's memcheck")
parser.add_argument("--dmesg",
action="store_true",
help="Capture a difference in dmesg before and "
"after each test. Implies -1/--no-concurrency")
parser.add_argument("-v",
"--verbose",
action="store_true",
help="Produce a line of output for each test before "
"and after it runs")
parser.add_argument("test_profile",
metavar="<Path to one or more test profile(s)>",
nargs='+',
help="Path to testfile to run")
parser.add_argument("results_path",
type=path.realpath,
metavar="<Results Path>",
help="Path to results folder")
args = parser.parse_args(input_)
# Disable Windows error message boxes for this and all child processes.
if sys.platform == 'win32':
# This disables messages boxes for uncaught exceptions, but it will not
# disable the message boxes for assertion failures or abort(). Those
# are created not by the system but by the CRT itself, and must be
# disabled by the child processes themselves.
import ctypes
SEM_FAILCRITICALERRORS = 0x0001
SEM_NOALIGNMENTFAULTEXCEPT = 0x0004
SEM_NOGPFAULTERRORBOX = 0x0002
SEM_NOOPENFILEERRORBOX = 0x8000
uMode = ctypes.windll.kernel32.SetErrorMode(0)
uMode |= SEM_FAILCRITICALERRORS \
| SEM_NOALIGNMENTFAULTEXCEPT \
| SEM_NOGPFAULTERRORBOX \
| SEM_NOOPENFILEERRORBOX
ctypes.windll.kernel32.SetErrorMode(uMode)
# If dmesg is requested we must have serial run, this is becasue dmesg
# isn't reliable with threaded run
if args.dmesg:
args.concurrency = "none"
# Read the config file
core.get_config(args.config_file)
# Pass arguments into Options
opts = core.Options(concurrent=args.concurrency,
exclude_filter=args.exclude_tests,
include_filter=args.include_tests,
execute=args.execute,
valgrind=args.valgrind,
dmesg=args.dmesg,
verbose=args.verbose)
# Set the platform to pass to waffle
if args.platform:
opts.env['PIGLIT_PLATFORM'] = args.platform
# Change working directory to the root of the piglit directory
piglit_dir = path.dirname(path.realpath(sys.argv[0]))
os.chdir(piglit_dir)
core.checkDir(args.results_path, False)
results = framework.results.TestrunResult()
# Set results.name
if args.name is not None:
results.name = args.name
else:
results.name = path.basename(args.results_path)
# Begin json.
result_filepath = path.join(args.results_path, 'results.json')
result_file = open(result_filepath, 'w')
json_writer = framework.results.JSONWriter(result_file)
# Create a dictionary to pass to initialize json, it needs the contents of
# the env dictionary and profile and platform information
options = {'profile': args.test_profile}
for key, value in opts:
options[key] = value
if args.platform:
options['platform'] = args.platform
json_writer.initialize_json(options, results.name,
core.collect_system_info())
json_writer.write_dict_key('tests')
json_writer.open_dict()
profile = framework.profile.merge_test_profiles(args.test_profile)
profile.results_dir = args.results_path
time_start = time.time()
# Set the dmesg type
if args.dmesg:
profile.dmesg = args.dmesg
profile.run(opts, json_writer)
time_end = time.time()
json_writer.close_dict()
results.time_elapsed = time_end - time_start
json_writer.write_dict_item('time_elapsed', results.time_elapsed)
# End json.
json_writer.close_json()
print('Thank you for running Piglit!\n'
'Results have been written to ' + result_filepath)
try:
import simplejson as json
except ImportError:
import json
import nose.tools as nt
import framework.core as core
import framework.results as results
import framework.backends as backends
import framework.tests.utils as utils
BACKEND_INITIAL_META = {
'name': 'name',
'test_count': 0,
'env': {},
'options': {k: v for k, v in core.Options()},
}
JUNIT_SCHEMA = 'framework/tests/schema/junit-7.xsd'
def test_initialize_jsonbackend():
""" Test that JSONBackend initializes
This needs to be handled separately from the others because it requires
arguments
"""
with utils.tempdir() as tdir:
func = results.JSONBackend(tdir)
assert isinstance(func, results.JSONBackend)
def __init__(self, results, json_file):
with open(json_file) as data:
feature_data = json.load(data)
self.feat_fractions = {}
self.feat_status = {}
self.features = set()
self.results = results
profiles = {}
# we expect all the result sets to be for the same profile
profile_orig = profile.load_test_profile(
results[0].options['profile'][0])
for feature in feature_data:
self.features.add(feature)
incl_str = feature_data[feature]["include_tests"]
excl_str = feature_data[feature]["exclude_tests"]
include_filter = [incl_str
] if incl_str and not incl_str.isspace() else []
exclude_filter = [excl_str
] if excl_str and not excl_str.isspace() else []
opts = core.Options(include_filter=include_filter,
exclude_filter=exclude_filter)
profiles[feature] = copy.deepcopy(profile_orig)
# An empty list will raise PiglitFatalError exception
# But for reporting we need to handle this situation
try:
profiles[feature]._prepare_test_list(opts)
except exceptions.PiglitFatalError:
pass
for results in self.results:
self.feat_fractions[results.name] = {}
self.feat_status[results.name] = {}
for feature in feature_data:
result_set = set(results.tests)
profile_set = set(profiles[feature].test_list)
common_set = profile_set & result_set
passed_list = [
x for x in common_set
if results.tests[x].result == status.PASS
]
total = len(common_set)
passed = len(passed_list)
self.feat_fractions[results.name][feature] = (passed, total)
if total == 0:
self.feat_status[results.name][feature] = status.NOTRUN
else:
if 100 * passed // total >= feature_data[feature][
"target_rate"]:
self.feat_status[results.name][feature] = status.PASS
else:
self.feat_status[results.name][feature] = status.FAIL
def run(input_):
""" Function for piglit run command
This is a function because it allows it to be shared between piglit-run.py
and piglit run
"""
args = _run_parser(input_)
# Disable Windows error message boxes for this and all child processes.
if sys.platform == 'win32':
# This disables messages boxes for uncaught exceptions, but it will not
# disable the message boxes for assertion failures or abort(). Those
# are created not by the system but by the CRT itself, and must be
# disabled by the child processes themselves.
import ctypes
SEM_FAILCRITICALERRORS = 0x0001
SEM_NOALIGNMENTFAULTEXCEPT = 0x0004
SEM_NOGPFAULTERRORBOX = 0x0002
SEM_NOOPENFILEERRORBOX = 0x8000
uMode = ctypes.windll.kernel32.SetErrorMode(0)
uMode |= SEM_FAILCRITICALERRORS \
| SEM_NOALIGNMENTFAULTEXCEPT \
| SEM_NOGPFAULTERRORBOX \
| SEM_NOOPENFILEERRORBOX
ctypes.windll.kernel32.SetErrorMode(uMode)
# If dmesg is requested we must have serial run, this is becasue dmesg
# isn't reliable with threaded run
if args.dmesg:
args.concurrency = "none"
# Pass arguments into Options
opts = core.Options(concurrent=args.concurrency,
exclude_filter=args.exclude_tests,
include_filter=args.include_tests,
execute=args.execute,
valgrind=args.valgrind,
dmesg=args.dmesg,
sync=args.sync)
# Set the platform to pass to waffle
opts.env['PIGLIT_PLATFORM'] = args.platform
# Change working directory to the root of the piglit directory
piglit_dir = path.dirname(path.realpath(sys.argv[0]))
os.chdir(piglit_dir)
core.checkDir(args.results_path, False)
results = framework.results.TestrunResult()
# Set results.name
if args.name is not None:
results.name = args.name
else:
results.name = path.basename(args.results_path)
# Create a dictionary to pass to initialize json, it needs the contents of
# the env dictionary and profile and platform information
options = {'profile': args.test_profile}
for key, value in opts:
options[key] = value
if args.platform:
options['platform'] = args.platform
options['name'] = results.name
options['env'] = core.collect_system_info()
# FIXME: this should be the actual count, but profile needs to be
# refactored to make that possible because of the flattening pass that is
# part of profile.run
options['test_count'] = 0
options['test_suffix'] = args.junit_suffix
options['log_level'] = args.log_level
# Begin json.
backend = framework.results.get_backend(args.backend)(args.results_path,
options,
file_fsync=opts.sync)
profile = framework.profile.merge_test_profiles(args.test_profile)
profile.results_dir = args.results_path
time_start = time.time()
# Set the dmesg type
if args.dmesg:
profile.dmesg = args.dmesg
profile.run(opts, args.log_level, backend)
time_end = time.time()
results.time_elapsed = time_end - time_start
backend.finalize({'time_elapsed': results.time_elapsed})
print('Thank you for running Piglit!\n'
'Results have been written to ' + args.results_path)
