def test_no_such_module(self, tmpdir): """profile.load_test_profile: Trying to load a non-existent module exits. """ # Ensure that the module will not exist by moving to an empty temporary # directory tmpdir.chdir() with pytest.raises(exceptions.PiglitFatalError): profile.load_test_profile('this_module_will_never_ever_exist')
def test_load_test_profile_no_profile(): """ Loading a module with no profile name exits Beacuse load_test_profile uses test.{} to load a module we need a module in tests that doesn't have a profile attribute. The only module that currently meets that requirement is __init__.py """ profile.load_test_profile('__init__')
def test_load_test_profile_no_profile(): """profile.load_test_profile: Loading a module with no profile name exits Because load_test_profile uses test.{} to load a module we need a module in tests that doesn't have a profile attribute. The only module that currently meets that requirement is __init__.py """ profile.load_test_profile('__init__')
def test_no_profile_attribute(self, tmpdir): """profile.load_test_profile: Loading a module with no profile name exits. Because load_test_profile uses test.{} to load a module we need a module in tests that doesn't have a profile attribute. The only module that currently meets that requirement is __init__.py """ p = tmpdir.join('foo.profile') with pytest.raises(exceptions.PiglitFatalError): profile.load_test_profile(six.text_type(p))
def main(): args = parser() OPTIONS.process_isolation = not args.no_process_isolation if args.glsl_arb_compat: os.environ['PIGLIT_FORCE_GLSLPARSER_DESKTOP'] = 'true' profile = load_test_profile(args.input, python=True) serializer(args.name, profile, args.output)
def __init__(self, results, json_file): with open(json_file) as data: feature_data = json.load(data) self.feat_fractions = {} self.feat_status = {} self.features = set() self.results = results profiles = {} # we expect all the result sets to be for the same profile profile_orig = profile.load_test_profile(results[0].options['profile'][0]) for feature in feature_data: self.features.add(feature) incl_str = feature_data[feature]["include_tests"] excl_str = feature_data[feature]["exclude_tests"] include_filter = [incl_str] if incl_str and not incl_str.isspace() else [] exclude_filter = [excl_str] if excl_str and not excl_str.isspace() else [] opts = core.Options(include_filter=include_filter, exclude_filter=exclude_filter) profiles[feature] = copy.deepcopy(profile_orig) # An empty list will raise PiglitFatalError exception # But for reporting we need to handle this situation try: profiles[feature]._prepare_test_list(opts) except exceptions.PiglitFatalError: pass for results in self.results: self.feat_fractions[results.name] = {} self.feat_status[results.name] = {} for feature in feature_data: result_set = set(results.tests) profile_set = set(profiles[feature].test_list) common_set = profile_set & result_set passed_list = [x for x in common_set if results.tests[x].result == status.PASS] total = len(common_set) passed = len(passed_list) self.feat_fractions[results.name][feature] = (passed, total) if total == 0: self.feat_status[results.name][feature] = status.NOTRUN else: if 100 * passed // total >= feature_data[feature]["target_rate"]: self.feat_status[results.name][feature] = status.PASS else: self.feat_status[results.name][feature] = status.FAIL
def __init__(self, results, json_file): with open(json_file) as data: feature_data = json.load(data) self.feat_fractions = {} self.feat_status = {} self.features = set() self.results = results profiles = {} # we expect all the result sets to be for the same profile profile_orig = profile.load_test_profile( results[0].options['profile'][0]) for feature in feature_data: self.features.add(feature) profiles[feature] = profile_orig.copy() incl_str = feature_data[feature]["include_tests"] excl_str = feature_data[feature]["exclude_tests"] profiles[feature].filters.append( profile.RegexFilter( [incl_str] if incl_str and not incl_str.isspace() else [])) profiles[feature].filters.append( profile.RegexFilter( [excl_str] if excl_str and not excl_str.isspace() else [], inverse=True)) for results in self.results: self.feat_fractions[results.name] = {} self.feat_status[results.name] = {} for feature in feature_data: result_set = set(results.tests) profile_set = set(a for a, _ in profiles[feature].itertests()) common_set = profile_set & result_set passed_list = [ x for x in common_set if results.tests[x].result == status.PASS ] total = len(common_set) passed = len(passed_list) self.feat_fractions[results.name][feature] = (passed, total) if total == 0: self.feat_status[results.name][feature] = status.NOTRUN else: if 100 * passed // total >= feature_data[feature][ "target_rate"]: self.feat_status[results.name][feature] = status.PASS else: self.feat_status[results.name][feature] = status.FAIL
def __init__(self, results, json_file): with open(json_file) as data: feature_data = json.load(data) self.feat_fractions = {} self.feat_status = {} self.features = set() self.results = results profiles = {} # we expect all the result sets to be for the same profile profile_orig = profile.load_test_profile(results[0].options['profile'][0]) for feature in feature_data: self.features.add(feature) profiles[feature] = profile_orig.copy() incl_str = feature_data[feature]["include_tests"] excl_str = feature_data[feature]["exclude_tests"] profiles[feature].filters.append( profile.RegexFilter( [incl_str] if incl_str and not incl_str.isspace() else [])) profiles[feature].filters.append( profile.RegexFilter( [excl_str] if excl_str and not excl_str.isspace() else [], inverse=True)) for results in self.results: self.feat_fractions[results.name] = {} self.feat_status[results.name] = {} for feature in feature_data: result_set = set(results.tests) profile_set = set(a for a, _ in profiles[feature].itertests()) common_set = profile_set & result_set passed_list = [x for x in common_set if results.tests[x].result == status.PASS] total = len(common_set) passed = len(passed_list) self.feat_fractions[results.name][feature] = (passed, total) if total == 0: self.feat_status[results.name][feature] = status.NOTRUN else: if 100 * passed // total >= feature_data[feature]["target_rate"]: self.feat_status[results.name][feature] = status.PASS else: self.feat_status[results.name][feature] = status.FAIL
def main(input_): """The main function.""" parser = argparse.ArgumentParser(parents=[parsers.CONFIG]) parser.add_argument("-t", "--include-tests", default=[], action="append", metavar="<regex>", help="Run only matching tests " "(can be used more than once)") parser.add_argument("-x", "--exclude-tests", default=[], action="append", metavar="<regex>", help="Exclude matching tests (can be used more than " "once)") parser.add_argument("--format", dest="format_string", default="{name} ::: {command}", action="store", help="A template string that defines the output " "format. It has two replacement tokens that can " "be provided, along with any arbitrary text, " "which will be printed verbatim. The two tokens " "are '{name}', which will be replaced with the " "name of the test; and '{command}', which will " "be replaced with the command to run the test.") parser.add_argument("testProfile", metavar="<Path to testfile>", help="Path to results folder") args = parser.parse_args(input_) profile_ = profile.load_test_profile(args.testProfile) if args.exclude_tests: profile_.filters.append(profile.RegexFilter(args.exclude_tests)) if args.include_tests: profile_.filters.append(profile.RegexFilter(args.include_tests)) # Change to the piglit's path piglit_dir = os.path.dirname(os.path.realpath(sys.argv[0])) os.chdir(piglit_dir) for name, test in profile_.itertests(): assert isinstance(test, Test) print( args.format_string.format(name=name, command=get_command(test, piglit_dir)))
def main(input_): """The main function.""" parser = argparse.ArgumentParser(parents=[parsers.CONFIG]) parser.add_argument("-t", "--include-tests", default=[], action="append", metavar="<regex>", help="Run only matching tests " "(can be used more than once)") parser.add_argument("-x", "--exclude-tests", default=[], action="append", metavar="<regex>", help="Exclude matching tests (can be used more than " "once)") parser.add_argument("--format", dest="format_string", default="{name} ::: {command}", action="store", help="A template string that defines the output " "format. It has two replacement tokens that can " "be provided, along with any arbitrary text, " "which will be printed verbatim. The two tokens " "are '{name}', which will be replaced with the " "name of the test; and '{command}', which will " "be replaced with the command to run the test.") parser.add_argument("testProfile", metavar="<Path to testfile>", help="Path to results folder") args = parser.parse_args(input_) profile_ = profile.load_test_profile(args.testProfile) if args.exclude_tests: profile_.filters.append(profile.RegexFilter(args.exclude_tests)) if args.include_tests: profile_.filters.append(profile.RegexFilter(args.include_tests)) # Change to the piglit's path piglit_dir = os.path.dirname(os.path.realpath(sys.argv[0])) os.chdir(piglit_dir) for name, test in profile_.itertests(): assert isinstance(test, Test) print(args.format_string.format( name=name, command=get_command(test, piglit_dir)))
def main(): """The main function.""" if six.PY2: input_ = [i.decode('utf-8') for i in sys.argv[1:]] elif six.PY3: input_ = sys.argv[1:] parser = argparse.ArgumentParser(parents=[parsers.CONFIG]) parser.add_argument("-t", "--include-tests", default=[], action="append", metavar="<regex>", help="Run only matching tests " "(can be used more than once)") parser.add_argument("-x", "--exclude-tests", default=[], action="append", metavar="<regex>", help="Exclude matching tests (can be used more than " "once)") parser.add_argument("testProfile", metavar="<Path to testfile>", help="Path to results folder") args = parser.parse_args(input_) options.OPTIONS.exclude_filter = args.exclude_tests options.OPTIONS.include_filter = args.include_tests # Change to the piglit's path piglit_dir = os.path.dirname(os.path.realpath(sys.argv[0])) os.chdir(piglit_dir) profile_ = profile.load_test_profile(args.testProfile) profile_._prepare_test_list() for name, test in six.iteritems(profile_.test_list): assert isinstance(test, Test) print(name, ':::', get_command(test, piglit_dir))
def test_load_test_profile_no_module(): """profile.load_test_profile: Trying to load a non-existant module exits""" profile.load_test_profile('this_module_will_never_ever_exist')
def test_load_test_profile_returns(): """profile.load_test_profile: returns a TestProfile instance""" profile_ = profile.load_test_profile('sanity') nt.ok_(isinstance(profile_, profile.TestProfile))
def resume(input_): unparsed = parsers.parse_config(input_)[1] parser = argparse.ArgumentParser() parser.add_argument("results_path", type=path.realpath, metavar="<Results Path>", help="Path to results folder") parser.add_argument("-f", "--config", dest="config_file", type=argparse.FileType("r"), help="Optionally specify a piglit config file to use. " "Default is piglit.conf") parser.add_argument("-n", "--no-retry", dest="no_retry", action="store_true", help="Do not retry incomplete tests") parser.add_argument( '-j', '--jobs', dest='jobs', action='store', type=int, default=core.PIGLIT_CONFIG.safe_get('core', 'jobs', None), help='Set the maximum number of jobs to run concurrently. ' 'By default, the reported number of CPUs is used.') args = parser.parse_args(unparsed) _disable_windows_exception_messages() results = backends.load(args.results_path) options.OPTIONS.execute = results.options['execute'] options.OPTIONS.valgrind = results.options['valgrind'] options.OPTIONS.sync = results.options['sync'] options.OPTIONS.deqp_mustpass = results.options['deqp_mustpass'] options.OPTIONS.process_isolation = results.options['process_isolation'] options.OPTIONS.jobs = args.jobs options.OPTIONS.no_retry = args.no_retry core.get_config(args.config_file) options.OPTIONS.env['PIGLIT_PLATFORM'] = results.options['platform'] base.Test.timeout = results.options['timeout'] results.options['env'] = core.collect_system_info() results.options['name'] = results.name # Resume only works with the JSON backend backend = backends.get_backend('json')( args.results_path, file_start_count=len(results.tests) + 1) # Specifically do not initialize again, everything initialize does is done. # Don't re-run tests that have already completed, incomplete status tests # have obviously not completed. exclude_tests = set() for name, result in six.iteritems(results.tests): if args.no_retry or result.result != 'incomplete': exclude_tests.add(name) profiles = [ profile.load_test_profile(p) for p in results.options['profile'] ] for p in profiles: p.results_dir = args.results_path if results.options['dmesg']: p.dmesg = dmesg.get_dmesg(results.options['dmesg']) if results.options['monitoring']: p.options['monitor'] = monitoring.Monitoring( results.options['monitoring']) if results.options['ignore_missing']: p.options['ignore_missing'] = results.options['ignore_missing'] if exclude_tests: p.filters.append(lambda n, _: n not in exclude_tests) if results.options['exclude_filter']: p.filters.append( profile.RegexFilter(results.options['exclude_filter'], inverse=True)) if results.options['include_filter']: p.filters.append( profile.RegexFilter(results.options['include_filter'])) if results.options['forced_test_list']: p.forced_test_list = results.options['forced_test_list'] # This is resumed, don't bother with time since it won't be accurate anyway try: profile.run(profiles, results.options['log_level'], backend, results.options['concurrent'], args.jobs) except exceptions.PiglitUserError as e: if str(e) != 'no matching tests': raise backend.finalize() print("Thank you for running Piglit!\n" "Results have been written to {0}".format(args.results_path))
def resume(input_): parser = argparse.ArgumentParser() parser.add_argument("results_path", type=path.realpath, metavar="<Results Path>", help="Path to results folder") parser.add_argument("-f", "--config", dest="config_file", type=argparse.FileType("r"), help="Optionally specify a piglit config file to use. " "Default is piglit.conf") parser.add_argument("-n", "--no-retry", dest="no_retry", action="store_true", help="Do not retry incomplete tests") args = parser.parse_args(input_) _disable_windows_exception_messages() results = backends.load(args.results_path) options.OPTIONS.execute = results.options['execute'] options.OPTIONS.valgrind = results.options['valgrind'] options.OPTIONS.sync = results.options['sync'] options.OPTIONS.deqp_mustpass = results.options['deqp_mustpass'] options.OPTIONS.proces_isolation = results.options['process_isolation'] core.get_config(args.config_file) options.OPTIONS.env['PIGLIT_PLATFORM'] = results.options['platform'] results.options['env'] = core.collect_system_info() results.options['name'] = results.name # Resume only works with the JSON backend backend = backends.get_backend('json')( args.results_path, file_start_count=len(results.tests) + 1) # Specifically do not initialize again, everything initialize does is done. # Don't re-run tests that have already completed, incomplete status tests # have obviously not completed. exclude_tests = set() for name, result in six.iteritems(results.tests): if args.no_retry or result.result != 'incomplete': exclude_tests.add(name) profiles = [profile.load_test_profile(p) for p in results.options['profile']] for p in profiles: p.results_dir = args.results_path if results.options['dmesg']: p.dmesg = dmesg.get_dmesg(results.options['dmesg']) if results.options['monitoring']: p.options['monitor'] = monitoring.Monitoring( results.options['monitoring']) if exclude_tests: p.filters.append(lambda n, _: n not in exclude_tests) if results.options['exclude_filter']: p.filters.append( profile.RegexFilter(results.options['exclude_filter'], inverse=True)) if results.options['include_filter']: p.filters.append( profile.RegexFilter(results.options['include_filter'])) # This is resumed, don't bother with time since it won't be accurate anyway profile.run( profiles, results.options['log_level'], backend, results.options['concurrent']) backend.finalize() print("Thank you for running Piglit!\n" "Results have been written to {0}".format(args.results_path))
def run(input_): """ Function for piglit run command This is a function because it allows it to be shared between piglit-run.py and piglit run """ args = _run_parser(input_) base.Test.timeout = args.timeout _disable_windows_exception_messages() # If dmesg is requested we must have serial run, this is because dmesg # isn't reliable with threaded run if args.dmesg or args.monitored: args.concurrency = "none" # Pass arguments into Options options.OPTIONS.execute = args.execute options.OPTIONS.valgrind = args.valgrind options.OPTIONS.sync = args.sync options.OPTIONS.deqp_mustpass = args.deqp_mustpass options.OPTIONS.process_isolation = args.process_isolation options.OPTIONS.jobs = args.jobs # Set the platform to pass to waffle options.OPTIONS.env['PIGLIT_PLATFORM'] = args.platform # Change working directory to the root of the piglit directory piglit_dir = path.dirname(path.realpath(sys.argv[0])) os.chdir(piglit_dir) # If the results directory already exists and if overwrite was set, then # clear the directory. If it wasn't set, then raise fatal error. try: core.check_dir(args.results_path, failifexists=args.overwrite, handler=_results_handler) except exceptions.PiglitException: raise exceptions.PiglitFatalError( 'Cannot overwrite existing folder without the -o/--overwrite ' 'option being set.') # If a test list is provided then set the forced_test_list value. forced_test_list = None if args.test_list: if len(args.test_profile) != 1: raise exceptions.PiglitFatalError( 'Unable to force a test list with more than one profile') with open(args.test_list) as test_list: # Strip newlines and comments, ignore empty lines stripped = (t.split('#')[0].strip() for t in test_list) forced_test_list = [t for t in stripped if t] time_elapsed = TimeAttribute(start=time.time()) backend = backends.get_backend(args.backend)( args.results_path, junit_suffix=args.junit_suffix, junit_subtests=args.junit_subtests) backend.initialize( _create_metadata(args, args.name or path.basename(args.results_path), forced_test_list)) profiles = [profile.load_test_profile(p) for p in args.test_profile] for p in profiles: p.results_dir = args.results_path # Set the forced_test_list, if applicable if forced_test_list: profiles[0].forced_test_list = forced_test_list # Set the dmesg type if args.dmesg: for p in profiles: p.options['dmesg'] = dmesg.get_dmesg(args.dmesg) if args.monitored: for p in profiles: p.options['monitor'] = monitoring.Monitoring(args.monitored) if args.ignore_missing: for p in profiles: p.options['ignore_missing'] = args.ignore_missing for p in profiles: if args.exclude_tests: p.filters.append( profile.RegexFilter(args.exclude_tests, inverse=True)) if args.include_tests: p.filters.append(profile.RegexFilter(args.include_tests)) profile.run(profiles, args.log_level, backend, args.concurrency, args.jobs) time_elapsed.end = time.time() backend.finalize({'time_elapsed': time_elapsed.to_json()}) print('Thank you for running Piglit!\n' 'Results have been written to ' + args.results_path)
# paragraph) shall be included in all copies or substantial portions of the # Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. from __future__ import (absolute_import, division, print_function, unicode_literals) from framework import core from framework.profile import load_test_profile __all__ = ['profile'] def xts_render_filter(path, test): # Keep any tests that aren't from xts. if 'xts5' not in path: return True # All of Xlib9 is for rendering. return 'xlib9' in path profile = load_test_profile("xts") profile.filter_tests(xts_render_filter)
def test_load_test_profile_returns(): """ load_test_profile returns a TestProfile instance """ profile_ = profile.load_test_profile('sanity') assert isinstance(profile_, profile.TestProfile)
def test_return_type(self): """profile.load_test_profile: returns a TestProfile instance.""" assert isinstance(profile.load_test_profile('sanity'), profile.TestProfile)
# -*- coding: utf-8 -*- # quick.tests minus tests that are known to fail from __future__ import (absolute_import, division, print_function, unicode_literals) from framework.profile import load_test_profile __all__ = ['profile'] profile = load_test_profile('quick') with open("/usr/lib64/piglit/tests/opensuse_qa-skip-tests.txt") as f: to_skip = frozenset(map(lambda line: line[:-1], f)) profile.filters.append(lambda p, _: p not in to_skip)
# # The above copyright notice and this permission notice (including the next # paragraph) shall be included in all copies or substantial portions of the # Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. from framework import core from framework.profile import load_test_profile __all__ = ['profile'] def xts_render_filter(path, test): # Keep any tests that aren't from xts. if 'xts5' not in path: return True # All of Xlib9 is for rendering. return 'xlib9' in path profile = load_test_profile("xts") profile.filter_tests(xts_render_filter)
def resume(input_): parser = argparse.ArgumentParser() parser.add_argument("results_path", type=path.realpath, metavar="<Results Path>", help="Path to results folder") parser.add_argument("-f", "--config", dest="config_file", type=argparse.FileType("r"), help="Optionally specify a piglit config file to use. " "Default is piglit.conf") parser.add_argument("-n", "--no-retry", dest="no_retry", action="store_true", help="Do not retry incomplete tests") args = parser.parse_args(input_) _disable_windows_exception_messages() results = backends.load(args.results_path) options.OPTIONS.execute = results.options['execute'] options.OPTIONS.valgrind = results.options['valgrind'] options.OPTIONS.sync = results.options['sync'] options.OPTIONS.deqp_mustpass = results.options['deqp_mustpass'] options.OPTIONS.proces_isolation = results.options['process_isolation'] core.get_config(args.config_file) options.OPTIONS.env['PIGLIT_PLATFORM'] = results.options['platform'] results.options['env'] = core.collect_system_info() results.options['name'] = results.name # Resume only works with the JSON backend backend = backends.get_backend('json')( args.results_path, file_start_count=len(results.tests) + 1) # Specifically do not initialize again, everything initialize does is done. # Don't re-run tests that have already completed, incomplete status tests # have obviously not completed. exclude_tests = set() for name, result in six.iteritems(results.tests): if args.no_retry or result.result != 'incomplete': exclude_tests.add(name) profiles = [profile.load_test_profile(p) for p in results.options['profile']] for p in profiles: p.results_dir = args.results_path if results.options['dmesg']: p.dmesg = dmesg.get_dmesg(results.options['dmesg']) if results.options['monitoring']: p.options['monitor'] = monitoring.Monitoring( results.options['monitoring']) if exclude_tests: p.filters.append(lambda n, _: n not in exclude_tests) if results.options['exclude_filter']: p.filters.append( profile.RegexFilter(results.options['exclude_filter'], inverse=True)) if results.options['include_filter']: p.filters.append( profile.RegexFilter(results.options['include_filter'])) if results.options['forced_test_list']: p.forced_test_list = results.options['forced_test_list'] # This is resumed, don't bother with time since it won't be accurate anyway profile.run( profiles, results.options['log_level'], backend, results.options['concurrent']) backend.finalize() print("Thank you for running Piglit!\n" "Results have been written to {0}".format(args.results_path))
def run(input_): """ Function for piglit run command This is a function because it allows it to be shared between piglit-run.py and piglit run """ args = _run_parser(input_) _disable_windows_exception_messages() # If dmesg is requested we must have serial run, this is because dmesg # isn't reliable with threaded run if args.dmesg or args.monitored: args.concurrency = "none" # Pass arguments into Options options.OPTIONS.execute = args.execute options.OPTIONS.valgrind = args.valgrind options.OPTIONS.sync = args.sync options.OPTIONS.deqp_mustpass = args.deqp_mustpass options.OPTIONS.process_isolation = args.process_isolation # Set the platform to pass to waffle options.OPTIONS.env['PIGLIT_PLATFORM'] = args.platform # Change working directory to the root of the piglit directory piglit_dir = path.dirname(path.realpath(sys.argv[0])) os.chdir(piglit_dir) # If the results directory already exists and if overwrite was set, then # clear the directory. If it wasn't set, then raise fatal error. try: core.check_dir(args.results_path, failifexists=args.overwrite, handler=_results_handler) except exceptions.PiglitException: raise exceptions.PiglitFatalError( 'Cannot overwrite existing folder without the -o/--overwrite ' 'option being set.') # If a test list is provided then set the forced_test_list value. forced_test_list = None if args.test_list: if len(args.test_profile) != 1: raise exceptions.PiglitFatalError( 'Unable to force a test list with more than one profile') with open(args.test_list) as test_list: # Strip newlines and comments, ignore empty lines stripped = (t.split('#')[0].strip() for t in test_list) forced_test_list = [t for t in stripped if t] backend = backends.get_backend(args.backend)( args.results_path, junit_suffix=args.junit_suffix, junit_subtests=args.junit_subtests) backend.initialize(_create_metadata( args, args.name or path.basename(args.results_path), forced_test_list)) profiles = [profile.load_test_profile(p) for p in args.test_profile] for p in profiles: p.results_dir = args.results_path # Set the forced_test_list, if applicable if forced_test_list: profiles[0].forced_test_list = forced_test_list # Set the dmesg type if args.dmesg: for p in profiles: p.options['dmesg'] = dmesg.get_dmesg(args.dmesg) if args.monitored: for p in profiles: p.options['monitor'] = monitoring.Monitoring(args.monitored) for p in profiles: if args.exclude_tests: p.filters.append(profile.RegexFilter(args.exclude_tests, inverse=True)) if args.include_tests: p.filters.append(profile.RegexFilter(args.include_tests)) time_elapsed = TimeAttribute(start=time.time()) profile.run(profiles, args.log_level, backend, args.concurrency) time_elapsed.end = time.time() backend.finalize({'time_elapsed': time_elapsed.to_json()}) print('Thank you for running Piglit!\n' 'Results have been written to ' + args.results_path)
def __init__(self, results, json_file): with open(json_file) as data: feature_data = json.load(data) self.feat_fractions = {} self.feat_status = {} self.features = set() self.results = results profiles = {} # we expect all the result sets to be for the same profile profile_orig = profile.load_test_profile( results[0].options['profile'][0]) for feature in feature_data: self.features.add(feature) incl_str = feature_data[feature]["include_tests"] excl_str = feature_data[feature]["exclude_tests"] include_filter = [incl_str ] if incl_str and not incl_str.isspace() else [] exclude_filter = [excl_str ] if excl_str and not excl_str.isspace() else [] options.OPTIONS.exclude_filter = exclude_filter options.OPTIONS.include_filter = include_filter profiles[feature] = profile.TestProfile() profiles[feature].update(profile_orig) # An empty list will raise PiglitFatalError exception # But for reporting we need to handle this situation try: profiles[feature]._prepare_test_list() except exceptions.PiglitFatalError: pass for results in self.results: self.feat_fractions[results.name] = {} self.feat_status[results.name] = {} for feature in feature_data: result_set = set(results.tests) profile_set = set(profiles[feature].test_list) common_set = profile_set & result_set passed_list = [ x for x in common_set if results.tests[x].result == status.PASS ] total = len(common_set) passed = len(passed_list) self.feat_fractions[results.name][feature] = (passed, total) if total == 0: self.feat_status[results.name][feature] = status.NOTRUN else: if 100 * passed // total >= feature_data[feature][ "target_rate"]: self.feat_status[results.name][feature] = status.PASS else: self.feat_status[results.name][feature] = status.FAIL