def gather_tests(include=[], exclude=[], benchmarker=None): ''' Given test names as strings, returns a list of FrameworkTest objects. For example, 'aspnet-mysql-raw' turns into a FrameworkTest object with variables for checking the test directory, the test database os, and other useful items. With no arguments, every test in this framework will be returned. With include, only tests with this exact name will be returned. With exclude, all tests but those excluded will be returned. A benchmarker is needed to construct full FrameworkTest objects. If one is not provided, a default Benchmarker will be created. ''' # Avoid setting up a circular import from benchmark import framework_test from benchmark.benchmarker import Benchmarker from setup.linux import setup_util # Help callers out a bit if include is None: include = [] if exclude is None: exclude = [] # Old, hacky method to exclude all tests was to # request a test known to not exist, such as ''. # If test '' was requested, short-circuit and return # nothing immediately if len(include) == 1 and '' in include: return [] # Setup default Benchmarker using example configuration if benchmarker is None: default_config = setup_util.get_fwroot() + "/benchmark.cfg" config = ConfigParser.SafeConfigParser() config.readfp(open(default_config)) defaults = dict(config.items("Defaults")) # Convert strings into proper python types for k, v in defaults.iteritems(): try: defaults[k] = literal_eval(v) except Exception: pass # Ensure we only run the __init__ method of Benchmarker defaults['install'] = None defaults[ 'results_name'] = "(unspecified, datetime = %Y-%m-%d %H:%M:%S)" defaults['results_environment'] = "My Server Environment" defaults['test_dir'] = None defaults['test_lang'] = None defaults['quiet'] = True benchmarker = Benchmarker(defaults) # Search for configuration files fwroot = setup_util.get_fwroot() config_files = [] if benchmarker.test_lang: benchmarker.test_dir = [] for lang in benchmarker.test_lang: if os.path.exists("{!s}/frameworks/{!s}".format(fwroot, lang)): for test_dir in os.listdir("{!s}/frameworks/{!s}".format( fwroot, lang)): benchmarker.test_dir.append("{!s}/{!s}".format( lang, test_dir)) else: raise Exception( "Unable to locate language directory: {!s}".format(lang)) if benchmarker.test_dir: for test_dir in benchmarker.test_dir: dir_config_files = glob.glob( "{!s}/frameworks/{!s}/benchmark_config.json".format( fwroot, test_dir)) if len(dir_config_files): config_files.extend(dir_config_files) else: raise Exception( "Unable to locate tests in test-dir: {!s}".format( test_dir)) else: config_files.extend( glob.glob( "{!s}/frameworks/*/*/benchmark_config.json".format(fwroot))) tests = [] for config_file_name in config_files: config = None with open(config_file_name, 'r') as config_file: try: config = json.load(config_file) except ValueError: # User-friendly errors print("Error loading '{!s}'.".format(config_file_name)) raise # Find all tests in the config file config_tests = framework_test.parse_config( config, os.path.dirname(config_file_name), benchmarker) # Filter for test in config_tests: if len(include) is 0 and len(exclude) is 0: # No filters, we are running everything tests.append(test) elif test.name in exclude: continue elif test.name in include: tests.append(test) else: # An include list exists, but this test is # not listed there, so we ignore it pass # Ensure we were able to locate everything that was # explicitly included if 0 != len(include): names = {test.name for test in tests} if 0 != len(set(include) - set(names)): missing = list(set(include) - set(names)) raise Exception("Unable to locate tests %s" % missing) tests.sort(key=lambda x: x.name) return tests