def makeTest(args): Project.findAndLoadProject() cls = Project.getInstance().makerClassname.split('.') module = importlib.import_module('.'.join(cls[:-1])) maker = getattr(module, cls[-1])("make") try: maker.parseArgs(args) maker.makeTest() except UserError as e: sys.stdout.flush() sys.stderr.write("ERROR: %s\n" % e) sys.exit(10)
def getProjectHelp(self): help = Project.getInstance().projectHelp if not help: return None lines = help.rstrip().split('\n') # delete blank lines at start while lines and not lines[0].strip(): del lines[0] if not lines: return None # strip initial indentation and convert tabs to spaces for i in range(len(lines)): l = lines[i].replace('\t', ' '*3) # this is the amount of indentation we use for the standard help message if i == 0: indenttostrip = len(l) - len(l.lstrip()) indenttoadd = '' if l.strip().startswith('-'): # align everything with existing content indenttoadd = ' '*3 if len(l)-len(l.lstrip()) >= indenttostrip: l = indenttoadd+l[indenttostrip:] lines[i] = l # if user hasn't provided their own heading, add ours if '----' not in help: lines = ['Project help', '-'*12]+lines return '\n'.join(lines)
def loadDescriptors(dir=None): """Load descriptor objects representing a set of tests to run for the current project, returning the list. :meta private: Deprecated and since 1.5.1 also hidden; use `pysys.config.descriptor.DescriptorLoader` instead. :param dir: The parent directory to search for runnable tests :return: List of L{pysys.config.descriptor.TestDescriptor} objects. Caller must sort this list to ensure deterministic behaviour. :rtype: list :raises UserError: Raised if no testcases can be found. """ if dir is None: dir = os.getcwd() loader = Project.getInstance().descriptorLoaderClass(Project.getInstance()) return loader.loadDescriptors(dir)
def runTest(args): try: launcher = ConsoleLaunchHelper(os.getcwd(), "run") args = launcher.parseArgs(args) cls = Project.getInstance().runnerClassname.split('.') module = importlib.import_module('.'.join(cls[:-1])) runner = getattr(module, cls[-1])(*args) runner.start() for cycledict in runner.results.values(): for outcome in OUTCOMES: if outcome.isFailure() and cycledict.get(outcome, None): sys.exit(2) sys.exit(0) except Exception as e: sys.stderr.write('\nPYSYS FATAL ERROR: %s\n' % e) if not isinstance(e, UserError): traceback.print_exc() sys.exit(10)
def __init__(self, name="make", **kwargs): self.name = name self.parentDir = os.getcwd() self.project = Project.getInstance() self.skipValidation = False
def perfReportsToolMain(args): USAGE = """ perfreportstool.py aggregate PATH1 PATH2... > aggregated.csv perfreportstool.py compare PATH_GLOB1 PATH_GLOB2... where PATH is a .csv file or directory of .csv files GLOB_PATH is a path to a file or files, optionally containing by * and ** globs The aggregate command combines the specifies CSVfile(s) to form a single file with one row for each resultKey, with the 'value' equal to the mean of all values for that resultKey and the 'stdDev' updated with the standard deviation. This can also be used with one or more .csv file to aggregate results from multiple cycles. The compare command prints a comparison from each listed performance file to the final one in the list. Note that the format of the output may change at any time, and it is not intended for machine parsing. """ # could later add support for automatically comparing files if '-h' in sys.argv or '--help' in args or len( args) < 2 or args[0] not in ['aggregate', 'compare']: sys.stderr.write(USAGE) sys.exit(1) cmd = args[0] # send log output to stderr to avoid interfering with output we might be redirecting to a file logging.basicConfig(format='%(levelname)s: %(message)s', stream=sys.stderr, level=getattr( logging, os.getenv('PYSYS_LOG_LEVEL', 'INFO').upper())) if cmd == 'aggregate': paths = [] for p in args[1:]: if os.path.isfile(p): paths.append(p) elif os.path.isdir(p): for (dirpath, dirnames, filenames) in os.walk(p): for f in sorted(filenames): if f.endswith('.csv'): paths.append(dirpath + '/' + f) else: raise Exception('Cannot find file: %s' % p) if not paths: raise Exception('No .csv files found') files = [] for p in paths: with io.open(toLongPathSafe(os.path.abspath(p)), encoding='utf-8') as f: files.append(CSVPerformanceFile(f.read())) f = CSVPerformanceFile.aggregate(files) f.dump(sys.stdout) elif cmd == 'compare': paths = args[1:] from pysys.config.project import Project project = Project.findAndLoadProject() # Can't easily get these classes from project without replicating the logic to instantiate them, which would # be error prone performanceReporterClasses = [ CSVPerformanceReporter, JSONPerformanceReporter ] gen = PerformanceComparisonGenerator(performanceReporterClasses) files = gen.loadFiles(baselineBaseDir=project.testRootDir, paths=paths) gen.logComparisons(files)
def printTests(self): Project.findAndLoadProject() # nb: mode filtering happens later descriptors = createDescriptors( self.arguments, self.type, self.includes, self.excludes, self.trace, self.workingDir, expandmodes=True if self.modefilter else False, modeincludes=self.modefilter) if self.grep: regex = re.compile(self.grep, flags=re.IGNORECASE) descriptors = [ d for d in descriptors if (regex.search(d.id) or regex.search(d.title)) ] # nb: do this case insensitively since if someone uses inconsistent capitaization they still probably # want related items to show up next to each other if not self.sort: descriptors.sort(key=lambda d: d._defaultSortKey.lower()) elif (self.sort.lower() == 'id'): descriptors.sort(key=lambda d: d.id.lower()) elif self.sort.lower().replace( '-', '') in ['executionorderhint', 'orderhint', 'order']: descriptors.sort(key=lambda d: [d.executionOrderHint, d._defaultSortKey.lower()]) elif self.sort.lower() == 'title': descriptors.sort( key=lambda d: [d.title.lower(), d.title, d._defaultSortKey.lower()]) else: raise UserError('Unknown sort key: %s' % self.sort) if self.json: print( json.dumps([d.toDict() for d in descriptors], indent=3, sort_keys=False)) return exit = 0 if self.groups == True: groups = [] for descriptor in descriptors: for group in descriptor.groups: if group not in groups: groups.append(group) print("\nGroups defined: ") for group in groups: print(" %s" % (group)) exit = 1 if self.modes == True: modes = [] for descriptor in descriptors: for mode in descriptor.modes: if mode not in modes: modes.append(mode) print("\nModes defined: ") for mode in modes: print(" %s" % (mode)) exit = 1 if self.requirements == True: requirements = [] for descriptor in descriptors: for requirement in descriptor.traceability: if requirement not in requirements: requirements.append(requirement) print("\nTraceability requirement ids covered: ") for requirement in requirements: print(" %s" % (requirement)) exit = 1 if exit: return maxsize = 0 for descriptor in descriptors: if len(descriptor.id) > maxsize: maxsize = len(descriptor.id) maxsize = maxsize + 2 for descriptor in descriptors: padding = " " * (maxsize - len(descriptor.id)) if not self.full: print("%s%s| %s" % (descriptor.id, padding, descriptor.title)) else: print("=" * 80) print(descriptor)
def printUsage(self, printXOptions): """ Print the pysys run usage. """ # printXOptions is not documented so probably little-used _PYSYS_SCRIPT_NAME = os.path.basename(sys.argv[0]) if '__main__' not in sys.argv[0] else 'pysys.py' print("\nPySys System Test Framework (version %s)" % __version__) print("\nUsage: %s %s [TESTIDS | OPTIONS]*" % (_PYSYS_SCRIPT_NAME, self.name)) # chars |80 | 120 print(""" Execution options ----------------- -c, --cycle NUM run each test the specified number of times -o, --outdir STRING set the directory to use for each test's output (a relative or absolute path); setting this is helpful for tagging/naming test output for different invocations of PySys as you try out various changes to the application under test; tests can access the final dir name of the outdir using the ${outDirName} project property -j, --threads NUM | xNUM set the number of jobs (threads) to run tests in parallel (defaults to 1); specify either an absolute number, or a multiplier on the number of CPUs e.g. "x1.5"; auto | 0 equivalent to x1.0 (or the PYSYS_DEFAULT_THREADS env var if set) --ci set optimal options for automated/non-interactive test execution in a CI job: --purge --record -j0 --type=auto --mode=ALL --printLogs=FAILURES -XcodeCoverage -v, --verbosity LEVEL set the verbosity for most pysys logging (CRIT, WARN, INFO, DEBUG) CAT=LEVEL set the verbosity for a PySys/Python logging category e.g. -vassertions=, -vprocess= -y, --validateOnly test the validate() method without re-running execute() -h, --help print this message -Xkey[=value] set user-defined override attributes to be set on the testcase and runner instances. The value is available to Python as "self.key". Value is True if not explicitly provided. """.rstrip()) if printXOptions: printXOptions() print(""" Advanced: -g, --progress print progress updates after completion of each test -r, --record use configured 'writers' to record the test results (e.g. XML, JUnit, etc) -p, --purge purge files except run.log from the output directory to save space (unless test fails) --printLogs STRING indicates for which outcome types the run.log output will be printed to the stdout console; options are: all|none|failures (default is all). -s, --sort STRING sort by: random (useful for performance testing and and reproducing test races) -b, --abort STRING set the default abort on error property (true|false, overrides that specified in the project properties) -XcodeCoverage enable collecting and reporting on code coverage with all coverage writers in the project -XautoUpdateAssertDiffReferences this is a special command for automatically updating the reference files when an assertDiff fails The PYSYS_DEFAULT_ARGS environment variable can be used to specify any pysys run arguments that you always wish to use, for example PYSYS_DEFAULT_ARGS=--progress --outdir __pysys_output. Selection and filtering options ------------------------------- -i, --include STRING set the test groups to include (can be specified multiple times) -e, --exclude STRING set the test groups to exclude (can be specified multiple times) -G, --grep STRING run only tests whose title or id contains the specified regex (case insensitive) -m, --mode, --modeinclude ALL,PRIMARY,!PRIMARY,MyMode1,!MyMode2,... run tests in the specifies mode(s): - use PRIMARY to select the test's first/main mode(s) (this is the default) - use ALL to select all modes - use !MODE as an alias for modeexclude - regular expressions can be used --modeexclude MyMode1,MyMode2,... run tests excluding specified mode(s); excludes take precedence over includes -a, --type STRING set the test type to run (auto or manual, default is both)" -t, --trace STRING set the requirement id for the test run Test identifiers ---------------- By default, PySys executes all available tests under the current directory will be run. Alternatively to run just a subset, one or more tests or sequences of tests can be specified on the command line. In both cases, tests are filtered based on the selection options listed above (e.g. --include/--exclude). Tests should contain only alphanumeric and the underscore characters. The following syntax is used to select an individual test, or a sequence of numbered tests: Test_001 - a single testcase with id equal to or ending with Test_001 _001 - a single testcase with id equal to or ending with _001 1 - a single testcase ending with number 1 (but not ending '11') (if it has multiple modes, runs the primary mode(s), or uses --mode) Test_001~ModeA - run testcase with id Test_001 in ModeA :Test_002 - all tests up to and including the testcase with id Test_002 Test_001: - all tests from Test_001 onwards Test_001:Test_002 - all tests between tests with ids Test_001 and Test_002 (inclusive) 2 Test_001 - Test_001 and Test_002 ^Test.* - All tests matching the specified regex e.g. {scriptname} run -c2 -w4 -u -j=x1.5 Test_007 Test_001: 3:5 {scriptname} run -vDEBUG --include MYTESTS -Xhost=localhost """.format(scriptname=_PYSYS_SCRIPT_NAME)) # show project help at the end so it's more prominent Project.findAndLoadProject() help = self.getProjectHelp() if help: print(help) sys.exit()
def parseArgs(self, args, printXOptions=None): # add any default args first; shlex.split does a great job of providing consistent parsing from str->list, # but need to avoid mangling \'s on windows; since this env var will be different for each OS no need for consistent win+unix behaviour if os.getenv('PYSYS_DEFAULT_ARGS',''): log.info('Using PYSYS_DEFAULT_ARGS = %s'%os.environ['PYSYS_DEFAULT_ARGS']) args = shlex.split(os.environ['PYSYS_DEFAULT_ARGS'].replace(os.sep, os.sep*2 if os.sep=='\\' else os.sep)) + args printLogsDefault = PrintLogs.ALL ci ='--ci' in args if ci: # to ensure identical behaviour, set these as if on the command line # (printLogs we don't set here since we use the printLogsDefault mechanism to allow it to be overridden # by CI writers and/or the command line) # Also we don't set --modes=ALL here to allow for it to be overridden explicitly if needed args = ['--purge', '--record', '-j0', '--type=auto', '--exclude=manual', '-XcodeCoverage']+args printLogsDefault = PrintLogs.FAILURES try: optlist, self.arguments = getopt.gnu_getopt(args, self.optionString, self.optionList) except Exception: log.warning("Error parsing command line arguments: %s" % (sys.exc_info()[1])) sys.exit(1) log.debug('PySys arguments: tests=%s options=%s', self.arguments, optlist) EXPR1 = re.compile(r"^[\w\.]*=.*$") EXPR2 = re.compile(r"^[\w\.]*$") printLogs = None defaultAbortOnError = None logging.getLogger('pysys').setLevel(logging.INFO) # as a special case, set a non-DEBUG log level for the implementation of assertions # so that it doesn't get enabled with -vDEBUG only -vassertions=DEBUG # as it is incredibly verbose and slow and not often useful logging.getLogger('pysys.assertions').setLevel(logging.INFO) for option, value in optlist: if option in ("-h", "--help"): self.printUsage(printXOptions) elif option in ['--ci']: continue # handled above elif option in ("-r", "--record"): self.record = True elif option in ("-p", "--purge"): self.purge = True elif option in ("-v", "--verbosity"): verbosity = value if '=' in verbosity: loggername, verbosity = value.split('=') if loggername.startswith('python:'): # this is weird but was documented, so leave it in place just in case someone is using it loggername = loggername[len('python:'):] assert not loggername.startswith('pysys'), 'Cannot use python: with pysys.*' # would produce a duplicate log handler else: loggername = None if verbosity.upper() == "DEBUG": verbosity = logging.DEBUG elif verbosity.upper() == "INFO": verbosity = logging.INFO elif verbosity.upper() == "WARN": verbosity = logging.WARN elif verbosity.upper() == "CRIT": verbosity = logging.CRITICAL else: log.warning('Invalid log level "%s"'%verbosity) sys.exit(1) if loggername is None: # when setting global log level to a higher level like WARN etc we want to affect stdout but # not necessarily downgrade the root level (would make run.log less useful and break # some PrintLogs behaviour) stdoutHandler.setLevel(verbosity) if verbosity == logging.DEBUG: logging.getLogger('pysys').setLevel(logging.DEBUG) logging.getLogger().setLevel(logging.DEBUG) else: # for specific level setting we need the opposite - only change stdoutHandler if we're # turning up the logging (since otherwise it wouldn't be seen) but also change the specified level # make the user of "pysys." prefix optional logging.getLogger(loggername).setLevel(verbosity) logging.getLogger('pysys.'+loggername).setLevel(verbosity) elif option in ("-a", "--type"): self.type = value if self.type not in ["auto", "manual"]: log.warning("Unsupported test type - valid types are auto and manual") sys.exit(1) elif option in ("-t", "--trace"): self.trace = value elif option in ("-i", "--include"): self.includes.append(value) elif option in ("-e", "--exclude"): self.excludes.append(value) elif option in ("-c", "--cycle"): try: self.cycle = int(value) except Exception: print("Error parsing command line arguments: A valid integer for the number of cycles must be supplied") sys.exit(1) elif option in ("-o", "--outdir"): value = os.path.normpath(value) if os.path.isabs(value) and not value.startswith('\\\\?\\'): value = fromLongPathSafe(toLongPathSafe(value)) self.outsubdir = value elif option in ("-m", "--mode", "--modeinclude"): self.modeinclude = self.modeinclude+[x.strip() for x in value.split(',')] elif option in ["--modeexclude"]: self.modeexclude = self.modeexclude+[x.strip() for x in value.split(',')] elif option in ["-n", "-j", "--threads"]: try: N_CPUS = len(os.sched_getaffinity(0)) # as recommended in Python docs, use the allocated CPUs for current process multiprocessing.cpu_count() except Exception: # no always available, e.g. on Windows N_CPUS = os.cpu_count() if value.lower()=='auto': value='0' if value.lower().startswith('x'): self.threads = max(1, int(float(value[1:])*N_CPUS)) else: self.threads = int(value) if self.threads <= 0: self.threads = int(os.getenv('PYSYS_DEFAULT_THREADS', N_CPUS)) elif option in ("-b", "--abort"): defaultAbortOnError = str(value.lower()=='true') elif option in ["-g", "--progress"]: self.progress = True elif option in ["--printLogs"]: printLogs = getattr(PrintLogs, value.upper(), None) if printLogs is None: print("Error parsing command line arguments: Unsupported --printLogs value '%s'"%value) sys.exit(1) elif option in ["-X"]: if '=' in value: key, value = value.split('=', 1) else: key, value = value, 'true' # best not to risk unintended consequences with matching of other types, but for boolean # it's worth it to resolve the inconsistent behaviour of -Xkey=true and -Xkey that existed until 1.6.0, # and because getting a bool where you expected a string is a bit more likely to give an exception # and be noticed that getting a string where you expected a boolean (e.g. the danger of if "false":) if value.lower() == 'true': value = True elif value.lower() == 'false': value = False self.userOptions[key] = value elif option in ("-y", "--validateOnly"): self.userOptions['validateOnly'] = True elif option in ("-G", "--grep"): self.grep = value elif option in ("-s", "--sort"): self.sort = value if value not in ['random']: print("The only supported sort type for pysys run is currently 'random'") sys.exit(10) else: print("Unknown option: %s"%option) sys.exit(1) if ci and not self.modeinclude: # only set this if there is no explicit --mode arguments self.modeinclude = ['ALL'] # log this once we've got the log levels setup log.debug('PySys is installed at: %s; python from %s', os.path.dirname(pysys.__file__), sys.executable) log.debug('Working dir=%s, args=%s', os.getcwd(), " ".join(sys.argv)) log.debug('Main environment vars: %s', '\n '.join(f'{envvar}={os.environ[envvar]}' for envvar in sorted(os.environ.keys()) if envvar.endswith('PATH') or envvar.startswith(('PYTHON', 'PYSYS')))) # retained for compatibility, but PYSYS_DEFAULT_ARGS is a better way to achieve the same thing if os.getenv('PYSYS_PROGRESS','').lower()=='true': self.progress = True # special hidden dict of extra values to pass to the runner, since we can't change # the public API now self.userOptions['__extraRunnerOptions'] = { 'progressWritersEnabled':self.progress, 'printLogs': printLogs, 'printLogsDefault': printLogsDefault, # to use if not provided by a CI writer or cmdline 'sort': self.sort } # load project AFTER we've parsed the arguments, which opens the possibility of using cmd line config in # project properties if needed Project.findAndLoadProject(outdir=self.outsubdir) if defaultAbortOnError is not None: setattr(Project.getInstance(), 'defaultAbortOnError', defaultAbortOnError) descriptors = createDescriptors(self.arguments, self.type, self.includes, self.excludes, self.trace, self.workingDir, modeincludes=self.modeinclude, modeexcludes=self.modeexclude, expandmodes=True) descriptors.sort(key=lambda d: [d.executionOrderHint, d._defaultSortKey]) # No exception handler above, as any createDescriptors failure is really a fatal problem that should cause us to # terminate with a non-zero exit code; we don't want to run no tests without realizing it and return success if self.grep: regex = re.compile(self.grep, flags=re.IGNORECASE) descriptors = [d for d in descriptors if (regex.search(d.id) or regex.search(d.title))] return self.record, self.purge, self.cycle, None, self.threads, self.outsubdir, descriptors, self.userOptions
def createDescriptors(testIdSpecs, type, includes, excludes, trace, dir=None, modeincludes=[], modeexcludes=[], expandmodes=True): """Create a list of descriptor objects representing a set of tests to run, filtering by various parameters, returning the list. :meta private: Not for use outside the framwork. :param testIdSpecs: A list of strings specifying the set of testcase identifiers :param type: The type of the tests to run (manual | auto) :param includes: A list of test groups to include in the returned set :param excludes: A list of test groups to exclude in the returned set :param trace: A list of requirements to indicate tests to include in the returned set :param dir: The parent directory to search for runnable tests :param modeincludes: A list specifying the modes to be included. :param modeexcludes: A list specifying the modes to be excluded. :param expandmodes: Set to False to disable expanding a test with multiple modes into separate descriptors for each one (used for pysys print). :return: List of L{pysys.config.descriptor.TestDescriptor} objects :rtype: list :raises UserError: Raised if no testcases can be found or are returned by the requested input parameters """ project = Project.getInstance() descriptors = loadDescriptors(dir=dir) # must sort by id for range matching and dup detection to work deterministically descriptors.sort(key=lambda d: [d.id, d.file]) # as a convenience support !mode syntax in the includes modeexcludes = modeexcludes+[x[1:] for x in modeincludes if x.startswith('!')] modeincludes = [x for x in modeincludes if not x.startswith('!')] for x in modeexcludes: if x.startswith('!'): raise UserError('Cannot use ! in a mode exclusion: "%s"'%x) # populate modedescriptors data structure MODES_ALL = 'ALL' MODES_PRIMARY = 'PRIMARY' assert MODES_ALL not in modeexcludes, "Cannot exclude all modes, that doesn't make sense" if not modeincludes: # pick a useful default if modeexcludes: modeincludes = [MODES_ALL] else: modeincludes = [MODES_PRIMARY] modedescriptors = {} # populate this with testid:[descriptors list] allmodes = {} # populate this as we go; could have used a set, but instead use a dict so we can check or capitalization mismatches easily at the same time; #the key is a lowercase version of mode name, value is the canonical capitalized name modeincludesnone = ((MODES_ALL in modeincludes or MODES_PRIMARY in modeincludes or '' in modeincludes) and (MODES_PRIMARY not in modeexcludes and '' not in modeexcludes)) NON_MODE_CHARS = '[^'+MODE_CHARS+']' def isregex(m): return re.search(NON_MODE_CHARS, m) regexmodeincludes = [re.compile(m, flags=re.IGNORECASE) for m in modeincludes if isregex(m)] regexmodeexcludes = [re.compile(m, flags=re.IGNORECASE) for m in modeexcludes if isregex(m)] for d in descriptors: if not d.modes: # for tests that have no modes, there is only one descriptor and it's treated as the primary mode; # user can also specify '' to indicate no mode if modeincludesnone: if expandmodes: d.mode = None modedescriptors[d.id] = [d] else: modedescriptors[d.id] = [] else: thisdescriptorlist = [] modedescriptors[d.id] = thisdescriptorlist # even if it ends up being empty # create a copy of the descriptor for each selected mode for m in d.modes: try: canonicalmodecapitalization = allmodes[m.lower()] except KeyError: allmodes[m.lower()] = m else: if m != canonicalmodecapitalization: # this is useful to detect early; it's almost certain to lead to buggy tests # since people would be comparing self.mode to a string that might have different capitalization raise UserError('Cannot have multiple modes with same name but different capitalization: "%s" and "%s"'%(m, canonicalmodecapitalization)) # apply modes filter isprimary = getattr(m, 'isPrimary', False) # use getattr in case a pre-2.0 str has crept in from a custom DescriptorLoader # excludes if isprimary and MODES_PRIMARY in modeexcludes: continue if m in modeexcludes or any(regex.match(m) for regex in regexmodeexcludes): continue # includes if not (MODES_ALL in modeincludes or m in modeincludes or any(regex.match(m) for regex in regexmodeincludes) or (isprimary and MODES_PRIMARY in modeincludes) ): continue thisdescriptorlist.append(d._createDescriptorForMode(m)) for m in [MODES_ALL, MODES_PRIMARY]: if m.lower() in allmodes: raise UserError('The mode name "%s" is reserved, please select another mode name'%m) # don't permit the user to specify a non existent mode by mistake for m in modeincludes+modeexcludes: if (not m) or m.upper() in [MODES_ALL, MODES_PRIMARY]: continue if isregex(m): if any(re.search(m, x, flags=re.IGNORECASE) for x in allmodes): continue else: if allmodes.get(m.lower(),None) == m: continue raise UserError('Unknown mode (or mode regex) "%s"; the available modes for descriptors in this directory are: %s'%( m, ', '.join(sorted(allmodes.values() or ['<none>'])))) # first check for duplicate ids ids = {} dups = [] d = None for d in descriptors: if d.id in ids: dups.append('%s - in %s and %s'%(d.id, ids[d.id], d.file)) else: ids[d.id] = d.file if dups: dupmsg = 'Found %d duplicate descriptor ids: %s'%(len(dups), '\n'.join(dups)) if os.getenv('PYSYS_ALLOW_DUPLICATE_IDS','').lower()=='true': logging.getLogger('pysys').warning(dupmsg) # undocumented option just in case anyone complains else: raise UserError(dupmsg) # trim down the list for those tests in the test specifiers # unless user the testspec includes a mode suffix, this stage ignores modes, # and then we expand the modes out afterwards tests = [] if testIdSpecs == []: tests = descriptors else: testids = {d.id: index for index, d in enumerate(descriptors)} def findMatchingIndex(specId): # change this to return the match, rather than whether it matches # optimize the case where we specify the full id; no need to iterate index = testids.get(specId, None) if index is not None: return index if specId.isdigit(): regex = re.compile('.+_0*'+(specId.lstrip('0') if (len(specId)>0) else specId)+'$') matches = [index for index, d in enumerate(descriptors) if regex.match(d.id)] else: # permit specifying suffix at end of testcase, which is # important to allow shell directory completion to be used if an id-prefix is # being added onto the directory id; but only do this if spec is non-numeric # since we don't want to match test_104 against spec 04 matches = [index for index, d in enumerate(descriptors) if d.id.endswith(specId)] if len(matches) == 1: return matches[0] if len(matches) == 0: raise UserError('No tests found matching id: "%s"'%specId) # as a special-case, see if there's an exact match with the dirname dirnameMatches = [index for index, d in enumerate(descriptors) if os.path.basename(d.testDir)==specId] if len(dirnameMatches)==1: return dirnameMatches[0] # nb: use space not comma as the delimiter so it's easy to copy paste it raise UserError('Multiple tests found matching "%s"; please specify which one you want: %s'%(specId, ' '.join([descriptors[index].id for index in matches[:20] ]))) for t in testIdSpecs: try: matches = None index = index1 = index2 = None t = t.rstrip('/\\') if re.search(r'^[%s]*$'%MODE_CHARS, t): # single test id (not a range or regex) if '~' in t: testspecid, testspecmode = t.split('~') index = findMatchingIndex(testspecid) # first match the id, then the mode matchingmode = next((m for m in descriptors[index].modes if m == testspecmode), None) if matchingmode is None: raise UserError('Unknown mode "%s": the available modes for this test are: %s'%( testspecmode, ', '.join(sorted(descriptors[index].modes or ['<none>'])))) matches = [descriptors[index]._createDescriptorForMode(matchingmode)] # note test id+mode combinations selected explicitly like this way are included regardless of what modes are enabled/disabled else: # normal case where it's not a mode index = findMatchingIndex(t) matches = descriptors[index:index+1] if not modedescriptors[matches[0].id]: # if user explicitly specified an individual test and excluded all modes it can run in, # we shouldn't silently skip/exclude it as they clearly made a mistake raise UserError('Test "%s" cannot be selected with the specified mode(s).'%matches[0].id) elif '~' in t: # The utility of this would be close to zero and lots more to implement/test, so not worth it raise UserError('A ~MODE test mode selector can only be use with a test id, not a range or regular expression') elif re.search('^:[%s]*'%TEST_ID_CHARS, t): index = findMatchingIndex(t.split(':')[1]) matches = descriptors[:index+1] elif re.search('^[%s]*:$'%TEST_ID_CHARS, t): index = findMatchingIndex(t.split(':')[0]) matches = descriptors[index:] elif re.search('^[%s]*:[%s]*$'%(TEST_ID_CHARS,TEST_ID_CHARS), t): index1 = findMatchingIndex(t.split(':')[0]) index2 = findMatchingIndex(t.split(':')[1]) if index1 > index2: index1, index2 = index2, index1 matches = descriptors[index1:index2+1] else: # regex match try: matches = [descriptors[i] for i in range(0,len(descriptors)) if re.search(t, descriptors[i].id)] except Exception as ex: raise UserError('"%s" contains characters not valid in a test id, but isn\'t a valid regular expression either: %s'%(t, ex)) if not matches: raise UserError("No test ids found matching regular expression: \"%s\""%t) if not matches: raise UserError("No test ids found matching: \"%s\""%st) tests.extend(matches) except UserError: raise except Exception: raise # this shouldn't be possible so no need to sugar coat the error message # trim down the list based on the type if type: index = 0 while index != len(tests): if type != tests[index].type: tests.pop(index) else: index = index + 1 # trim down the list based on the include and exclude groups if len(excludes) != 0: index = 0 while index != len(tests): remove = False for exclude in excludes: if exclude in tests[index].groups: remove = True break if remove: tests.pop(index) else: index = index +1 if includes != []: index = 0 while index != len(tests): keep = False for include in includes: if include in tests[index].groups: keep = True break if not keep: tests.pop(index) else: index = index +1 # trim down the list based on the traceability if trace: index = 0 while index != len(tests): if trace not in tests[index].traceability : tests.pop(index) else: index = index + 1 # expand based on modes (unless we're printing without any mode filters in which case expandmodes=False) if expandmodes: expandedtests = [] for t in tests: if hasattr(t, 'mode'): # if mode if set it has no modes or has a test id~mode that was explicitly specified in a testspec, so does not need expanding expandedtests.append(t) else: expandedtests.extend(modedescriptors[t.id]) tests = expandedtests # combine execution order hints from descriptors with global project configuration; # we only need to do this if there are any executionOrderHints defined (may be pure group hints not for modes) # or if there are any tests with multiple modes (since then the secondary hint mode delta applies) if (len(allmodes)>0) or project.executionOrderHints: def calculateNewHint(d, mode): hint = d.executionOrderHint for hintdelta, hintmatcher in project.executionOrderHints: if hintmatcher(d.groups, mode): hint += hintdelta if mode and not getattr(mode, 'isPrimary', False): # bit of a fudge in the case of isPrimary!=(index==0) but good enough hint += project.executionOrderSecondaryModesHintDelta * (d.modes.index(mode)) return hint for d in tests: hintspermode = [] if expandmodes: # used for pysys run; d.mode will have been set d.executionOrderHint = calculateNewHint(d, d.mode) else: modes = [None] if len(d.modes)==0 else d.modes # set this for the benefit of pysys print d.executionOrderHintsByMode = [calculateNewHint(d, m) for m in modes] d.executionOrderHint = d.executionOrderHintsByMode[0] if len(tests) == 0: raise UserError("The supplied options did not result in the selection of any tests") else: return tests