def getProjectHelp(self): help = Project.getInstance().projectHelp if not help: return None lines = help.rstrip().split('\n') # delete blank lines at start while lines and not lines[0].strip(): del lines[0] if not lines: return None # strip initial indentation and convert tabs to spaces for i in range(len(lines)): l = lines[i].replace('\t', ' '*3) # this is the amount of indentation we use for the standard help message if i == 0: indenttostrip = len(l) - len(l.lstrip()) indenttoadd = '' if l.strip().startswith('-'): # align everything with existing content indenttoadd = ' '*3 if len(l)-len(l.lstrip()) >= indenttostrip: l = indenttoadd+l[indenttostrip:] lines[i] = l # if user hasn't provided their own heading, add ours if '----' not in help: lines = ['Project help', '-'*12]+lines return '\n'.join(lines)
def loadDescriptors(dir=None): """Load descriptor objects representing a set of tests to run for the current project, returning the list. :meta private: Deprecated and since 1.5.1 also hidden; use `pysys.config.descriptor.DescriptorLoader` instead. :param dir: The parent directory to search for runnable tests :return: List of L{pysys.config.descriptor.TestDescriptor} objects. Caller must sort this list to ensure deterministic behaviour. :rtype: list :raises UserError: Raised if no testcases can be found. """ if dir is None: dir = os.getcwd() loader = Project.getInstance().descriptorLoaderClass(Project.getInstance()) return loader.loadDescriptors(dir)
def makeTest(args): Project.findAndLoadProject() cls = Project.getInstance().makerClassname.split('.') module = importlib.import_module('.'.join(cls[:-1])) maker = getattr(module, cls[-1])("make") try: maker.parseArgs(args) maker.makeTest() except UserError as e: sys.stdout.flush() sys.stderr.write("ERROR: %s\n" % e) sys.exit(10)
def runTest(args): try: launcher = ConsoleLaunchHelper(os.getcwd(), "run") args = launcher.parseArgs(args) cls = Project.getInstance().runnerClassname.split('.') module = importlib.import_module('.'.join(cls[:-1])) runner = getattr(module, cls[-1])(*args) runner.start() for cycledict in runner.results.values(): for outcome in OUTCOMES: if outcome.isFailure() and cycledict.get(outcome, None): sys.exit(2) sys.exit(0) except Exception as e: sys.stderr.write('\nPYSYS FATAL ERROR: %s\n' % e) if not isinstance(e, UserError): traceback.print_exc() sys.exit(10)
def __init__(self, name="make", **kwargs): self.name = name self.parentDir = os.getcwd() self.project = Project.getInstance() self.skipValidation = False
def parseArgs(self, args, printXOptions=None): # add any default args first; shlex.split does a great job of providing consistent parsing from str->list, # but need to avoid mangling \'s on windows; since this env var will be different for each OS no need for consistent win+unix behaviour if os.getenv('PYSYS_DEFAULT_ARGS',''): log.info('Using PYSYS_DEFAULT_ARGS = %s'%os.environ['PYSYS_DEFAULT_ARGS']) args = shlex.split(os.environ['PYSYS_DEFAULT_ARGS'].replace(os.sep, os.sep*2 if os.sep=='\\' else os.sep)) + args printLogsDefault = PrintLogs.ALL ci ='--ci' in args if ci: # to ensure identical behaviour, set these as if on the command line # (printLogs we don't set here since we use the printLogsDefault mechanism to allow it to be overridden # by CI writers and/or the command line) # Also we don't set --modes=ALL here to allow for it to be overridden explicitly if needed args = ['--purge', '--record', '-j0', '--type=auto', '--exclude=manual', '-XcodeCoverage']+args printLogsDefault = PrintLogs.FAILURES try: optlist, self.arguments = getopt.gnu_getopt(args, self.optionString, self.optionList) except Exception: log.warning("Error parsing command line arguments: %s" % (sys.exc_info()[1])) sys.exit(1) log.debug('PySys arguments: tests=%s options=%s', self.arguments, optlist) EXPR1 = re.compile(r"^[\w\.]*=.*$") EXPR2 = re.compile(r"^[\w\.]*$") printLogs = None defaultAbortOnError = None logging.getLogger('pysys').setLevel(logging.INFO) # as a special case, set a non-DEBUG log level for the implementation of assertions # so that it doesn't get enabled with -vDEBUG only -vassertions=DEBUG # as it is incredibly verbose and slow and not often useful logging.getLogger('pysys.assertions').setLevel(logging.INFO) for option, value in optlist: if option in ("-h", "--help"): self.printUsage(printXOptions) elif option in ['--ci']: continue # handled above elif option in ("-r", "--record"): self.record = True elif option in ("-p", "--purge"): self.purge = True elif option in ("-v", "--verbosity"): verbosity = value if '=' in verbosity: loggername, verbosity = value.split('=') if loggername.startswith('python:'): # this is weird but was documented, so leave it in place just in case someone is using it loggername = loggername[len('python:'):] assert not loggername.startswith('pysys'), 'Cannot use python: with pysys.*' # would produce a duplicate log handler else: loggername = None if verbosity.upper() == "DEBUG": verbosity = logging.DEBUG elif verbosity.upper() == "INFO": verbosity = logging.INFO elif verbosity.upper() == "WARN": verbosity = logging.WARN elif verbosity.upper() == "CRIT": verbosity = logging.CRITICAL else: log.warning('Invalid log level "%s"'%verbosity) sys.exit(1) if loggername is None: # when setting global log level to a higher level like WARN etc we want to affect stdout but # not necessarily downgrade the root level (would make run.log less useful and break # some PrintLogs behaviour) stdoutHandler.setLevel(verbosity) if verbosity == logging.DEBUG: logging.getLogger('pysys').setLevel(logging.DEBUG) logging.getLogger().setLevel(logging.DEBUG) else: # for specific level setting we need the opposite - only change stdoutHandler if we're # turning up the logging (since otherwise it wouldn't be seen) but also change the specified level # make the user of "pysys." prefix optional logging.getLogger(loggername).setLevel(verbosity) logging.getLogger('pysys.'+loggername).setLevel(verbosity) elif option in ("-a", "--type"): self.type = value if self.type not in ["auto", "manual"]: log.warning("Unsupported test type - valid types are auto and manual") sys.exit(1) elif option in ("-t", "--trace"): self.trace = value elif option in ("-i", "--include"): self.includes.append(value) elif option in ("-e", "--exclude"): self.excludes.append(value) elif option in ("-c", "--cycle"): try: self.cycle = int(value) except Exception: print("Error parsing command line arguments: A valid integer for the number of cycles must be supplied") sys.exit(1) elif option in ("-o", "--outdir"): value = os.path.normpath(value) if os.path.isabs(value) and not value.startswith('\\\\?\\'): value = fromLongPathSafe(toLongPathSafe(value)) self.outsubdir = value elif option in ("-m", "--mode", "--modeinclude"): self.modeinclude = self.modeinclude+[x.strip() for x in value.split(',')] elif option in ["--modeexclude"]: self.modeexclude = self.modeexclude+[x.strip() for x in value.split(',')] elif option in ["-n", "-j", "--threads"]: try: N_CPUS = len(os.sched_getaffinity(0)) # as recommended in Python docs, use the allocated CPUs for current process multiprocessing.cpu_count() except Exception: # no always available, e.g. on Windows N_CPUS = os.cpu_count() if value.lower()=='auto': value='0' if value.lower().startswith('x'): self.threads = max(1, int(float(value[1:])*N_CPUS)) else: self.threads = int(value) if self.threads <= 0: self.threads = int(os.getenv('PYSYS_DEFAULT_THREADS', N_CPUS)) elif option in ("-b", "--abort"): defaultAbortOnError = str(value.lower()=='true') elif option in ["-g", "--progress"]: self.progress = True elif option in ["--printLogs"]: printLogs = getattr(PrintLogs, value.upper(), None) if printLogs is None: print("Error parsing command line arguments: Unsupported --printLogs value '%s'"%value) sys.exit(1) elif option in ["-X"]: if '=' in value: key, value = value.split('=', 1) else: key, value = value, 'true' # best not to risk unintended consequences with matching of other types, but for boolean # it's worth it to resolve the inconsistent behaviour of -Xkey=true and -Xkey that existed until 1.6.0, # and because getting a bool where you expected a string is a bit more likely to give an exception # and be noticed that getting a string where you expected a boolean (e.g. the danger of if "false":) if value.lower() == 'true': value = True elif value.lower() == 'false': value = False self.userOptions[key] = value elif option in ("-y", "--validateOnly"): self.userOptions['validateOnly'] = True elif option in ("-G", "--grep"): self.grep = value elif option in ("-s", "--sort"): self.sort = value if value not in ['random']: print("The only supported sort type for pysys run is currently 'random'") sys.exit(10) else: print("Unknown option: %s"%option) sys.exit(1) if ci and not self.modeinclude: # only set this if there is no explicit --mode arguments self.modeinclude = ['ALL'] # log this once we've got the log levels setup log.debug('PySys is installed at: %s; python from %s', os.path.dirname(pysys.__file__), sys.executable) log.debug('Working dir=%s, args=%s', os.getcwd(), " ".join(sys.argv)) log.debug('Main environment vars: %s', '\n '.join(f'{envvar}={os.environ[envvar]}' for envvar in sorted(os.environ.keys()) if envvar.endswith('PATH') or envvar.startswith(('PYTHON', 'PYSYS')))) # retained for compatibility, but PYSYS_DEFAULT_ARGS is a better way to achieve the same thing if os.getenv('PYSYS_PROGRESS','').lower()=='true': self.progress = True # special hidden dict of extra values to pass to the runner, since we can't change # the public API now self.userOptions['__extraRunnerOptions'] = { 'progressWritersEnabled':self.progress, 'printLogs': printLogs, 'printLogsDefault': printLogsDefault, # to use if not provided by a CI writer or cmdline 'sort': self.sort } # load project AFTER we've parsed the arguments, which opens the possibility of using cmd line config in # project properties if needed Project.findAndLoadProject(outdir=self.outsubdir) if defaultAbortOnError is not None: setattr(Project.getInstance(), 'defaultAbortOnError', defaultAbortOnError) descriptors = createDescriptors(self.arguments, self.type, self.includes, self.excludes, self.trace, self.workingDir, modeincludes=self.modeinclude, modeexcludes=self.modeexclude, expandmodes=True) descriptors.sort(key=lambda d: [d.executionOrderHint, d._defaultSortKey]) # No exception handler above, as any createDescriptors failure is really a fatal problem that should cause us to # terminate with a non-zero exit code; we don't want to run no tests without realizing it and return success if self.grep: regex = re.compile(self.grep, flags=re.IGNORECASE) descriptors = [d for d in descriptors if (regex.search(d.id) or regex.search(d.title))] return self.record, self.purge, self.cycle, None, self.threads, self.outsubdir, descriptors, self.userOptions
def createDescriptors(testIdSpecs, type, includes, excludes, trace, dir=None, modeincludes=[], modeexcludes=[], expandmodes=True): """Create a list of descriptor objects representing a set of tests to run, filtering by various parameters, returning the list. :meta private: Not for use outside the framwork. :param testIdSpecs: A list of strings specifying the set of testcase identifiers :param type: The type of the tests to run (manual | auto) :param includes: A list of test groups to include in the returned set :param excludes: A list of test groups to exclude in the returned set :param trace: A list of requirements to indicate tests to include in the returned set :param dir: The parent directory to search for runnable tests :param modeincludes: A list specifying the modes to be included. :param modeexcludes: A list specifying the modes to be excluded. :param expandmodes: Set to False to disable expanding a test with multiple modes into separate descriptors for each one (used for pysys print). :return: List of L{pysys.config.descriptor.TestDescriptor} objects :rtype: list :raises UserError: Raised if no testcases can be found or are returned by the requested input parameters """ project = Project.getInstance() descriptors = loadDescriptors(dir=dir) # must sort by id for range matching and dup detection to work deterministically descriptors.sort(key=lambda d: [d.id, d.file]) # as a convenience support !mode syntax in the includes modeexcludes = modeexcludes+[x[1:] for x in modeincludes if x.startswith('!')] modeincludes = [x for x in modeincludes if not x.startswith('!')] for x in modeexcludes: if x.startswith('!'): raise UserError('Cannot use ! in a mode exclusion: "%s"'%x) # populate modedescriptors data structure MODES_ALL = 'ALL' MODES_PRIMARY = 'PRIMARY' assert MODES_ALL not in modeexcludes, "Cannot exclude all modes, that doesn't make sense" if not modeincludes: # pick a useful default if modeexcludes: modeincludes = [MODES_ALL] else: modeincludes = [MODES_PRIMARY] modedescriptors = {} # populate this with testid:[descriptors list] allmodes = {} # populate this as we go; could have used a set, but instead use a dict so we can check or capitalization mismatches easily at the same time; #the key is a lowercase version of mode name, value is the canonical capitalized name modeincludesnone = ((MODES_ALL in modeincludes or MODES_PRIMARY in modeincludes or '' in modeincludes) and (MODES_PRIMARY not in modeexcludes and '' not in modeexcludes)) NON_MODE_CHARS = '[^'+MODE_CHARS+']' def isregex(m): return re.search(NON_MODE_CHARS, m) regexmodeincludes = [re.compile(m, flags=re.IGNORECASE) for m in modeincludes if isregex(m)] regexmodeexcludes = [re.compile(m, flags=re.IGNORECASE) for m in modeexcludes if isregex(m)] for d in descriptors: if not d.modes: # for tests that have no modes, there is only one descriptor and it's treated as the primary mode; # user can also specify '' to indicate no mode if modeincludesnone: if expandmodes: d.mode = None modedescriptors[d.id] = [d] else: modedescriptors[d.id] = [] else: thisdescriptorlist = [] modedescriptors[d.id] = thisdescriptorlist # even if it ends up being empty # create a copy of the descriptor for each selected mode for m in d.modes: try: canonicalmodecapitalization = allmodes[m.lower()] except KeyError: allmodes[m.lower()] = m else: if m != canonicalmodecapitalization: # this is useful to detect early; it's almost certain to lead to buggy tests # since people would be comparing self.mode to a string that might have different capitalization raise UserError('Cannot have multiple modes with same name but different capitalization: "%s" and "%s"'%(m, canonicalmodecapitalization)) # apply modes filter isprimary = getattr(m, 'isPrimary', False) # use getattr in case a pre-2.0 str has crept in from a custom DescriptorLoader # excludes if isprimary and MODES_PRIMARY in modeexcludes: continue if m in modeexcludes or any(regex.match(m) for regex in regexmodeexcludes): continue # includes if not (MODES_ALL in modeincludes or m in modeincludes or any(regex.match(m) for regex in regexmodeincludes) or (isprimary and MODES_PRIMARY in modeincludes) ): continue thisdescriptorlist.append(d._createDescriptorForMode(m)) for m in [MODES_ALL, MODES_PRIMARY]: if m.lower() in allmodes: raise UserError('The mode name "%s" is reserved, please select another mode name'%m) # don't permit the user to specify a non existent mode by mistake for m in modeincludes+modeexcludes: if (not m) or m.upper() in [MODES_ALL, MODES_PRIMARY]: continue if isregex(m): if any(re.search(m, x, flags=re.IGNORECASE) for x in allmodes): continue else: if allmodes.get(m.lower(),None) == m: continue raise UserError('Unknown mode (or mode regex) "%s"; the available modes for descriptors in this directory are: %s'%( m, ', '.join(sorted(allmodes.values() or ['<none>'])))) # first check for duplicate ids ids = {} dups = [] d = None for d in descriptors: if d.id in ids: dups.append('%s - in %s and %s'%(d.id, ids[d.id], d.file)) else: ids[d.id] = d.file if dups: dupmsg = 'Found %d duplicate descriptor ids: %s'%(len(dups), '\n'.join(dups)) if os.getenv('PYSYS_ALLOW_DUPLICATE_IDS','').lower()=='true': logging.getLogger('pysys').warning(dupmsg) # undocumented option just in case anyone complains else: raise UserError(dupmsg) # trim down the list for those tests in the test specifiers # unless user the testspec includes a mode suffix, this stage ignores modes, # and then we expand the modes out afterwards tests = [] if testIdSpecs == []: tests = descriptors else: testids = {d.id: index for index, d in enumerate(descriptors)} def findMatchingIndex(specId): # change this to return the match, rather than whether it matches # optimize the case where we specify the full id; no need to iterate index = testids.get(specId, None) if index is not None: return index if specId.isdigit(): regex = re.compile('.+_0*'+(specId.lstrip('0') if (len(specId)>0) else specId)+'$') matches = [index for index, d in enumerate(descriptors) if regex.match(d.id)] else: # permit specifying suffix at end of testcase, which is # important to allow shell directory completion to be used if an id-prefix is # being added onto the directory id; but only do this if spec is non-numeric # since we don't want to match test_104 against spec 04 matches = [index for index, d in enumerate(descriptors) if d.id.endswith(specId)] if len(matches) == 1: return matches[0] if len(matches) == 0: raise UserError('No tests found matching id: "%s"'%specId) # as a special-case, see if there's an exact match with the dirname dirnameMatches = [index for index, d in enumerate(descriptors) if os.path.basename(d.testDir)==specId] if len(dirnameMatches)==1: return dirnameMatches[0] # nb: use space not comma as the delimiter so it's easy to copy paste it raise UserError('Multiple tests found matching "%s"; please specify which one you want: %s'%(specId, ' '.join([descriptors[index].id for index in matches[:20] ]))) for t in testIdSpecs: try: matches = None index = index1 = index2 = None t = t.rstrip('/\\') if re.search(r'^[%s]*$'%MODE_CHARS, t): # single test id (not a range or regex) if '~' in t: testspecid, testspecmode = t.split('~') index = findMatchingIndex(testspecid) # first match the id, then the mode matchingmode = next((m for m in descriptors[index].modes if m == testspecmode), None) if matchingmode is None: raise UserError('Unknown mode "%s": the available modes for this test are: %s'%( testspecmode, ', '.join(sorted(descriptors[index].modes or ['<none>'])))) matches = [descriptors[index]._createDescriptorForMode(matchingmode)] # note test id+mode combinations selected explicitly like this way are included regardless of what modes are enabled/disabled else: # normal case where it's not a mode index = findMatchingIndex(t) matches = descriptors[index:index+1] if not modedescriptors[matches[0].id]: # if user explicitly specified an individual test and excluded all modes it can run in, # we shouldn't silently skip/exclude it as they clearly made a mistake raise UserError('Test "%s" cannot be selected with the specified mode(s).'%matches[0].id) elif '~' in t: # The utility of this would be close to zero and lots more to implement/test, so not worth it raise UserError('A ~MODE test mode selector can only be use with a test id, not a range or regular expression') elif re.search('^:[%s]*'%TEST_ID_CHARS, t): index = findMatchingIndex(t.split(':')[1]) matches = descriptors[:index+1] elif re.search('^[%s]*:$'%TEST_ID_CHARS, t): index = findMatchingIndex(t.split(':')[0]) matches = descriptors[index:] elif re.search('^[%s]*:[%s]*$'%(TEST_ID_CHARS,TEST_ID_CHARS), t): index1 = findMatchingIndex(t.split(':')[0]) index2 = findMatchingIndex(t.split(':')[1]) if index1 > index2: index1, index2 = index2, index1 matches = descriptors[index1:index2+1] else: # regex match try: matches = [descriptors[i] for i in range(0,len(descriptors)) if re.search(t, descriptors[i].id)] except Exception as ex: raise UserError('"%s" contains characters not valid in a test id, but isn\'t a valid regular expression either: %s'%(t, ex)) if not matches: raise UserError("No test ids found matching regular expression: \"%s\""%t) if not matches: raise UserError("No test ids found matching: \"%s\""%st) tests.extend(matches) except UserError: raise except Exception: raise # this shouldn't be possible so no need to sugar coat the error message # trim down the list based on the type if type: index = 0 while index != len(tests): if type != tests[index].type: tests.pop(index) else: index = index + 1 # trim down the list based on the include and exclude groups if len(excludes) != 0: index = 0 while index != len(tests): remove = False for exclude in excludes: if exclude in tests[index].groups: remove = True break if remove: tests.pop(index) else: index = index +1 if includes != []: index = 0 while index != len(tests): keep = False for include in includes: if include in tests[index].groups: keep = True break if not keep: tests.pop(index) else: index = index +1 # trim down the list based on the traceability if trace: index = 0 while index != len(tests): if trace not in tests[index].traceability : tests.pop(index) else: index = index + 1 # expand based on modes (unless we're printing without any mode filters in which case expandmodes=False) if expandmodes: expandedtests = [] for t in tests: if hasattr(t, 'mode'): # if mode if set it has no modes or has a test id~mode that was explicitly specified in a testspec, so does not need expanding expandedtests.append(t) else: expandedtests.extend(modedescriptors[t.id]) tests = expandedtests # combine execution order hints from descriptors with global project configuration; # we only need to do this if there are any executionOrderHints defined (may be pure group hints not for modes) # or if there are any tests with multiple modes (since then the secondary hint mode delta applies) if (len(allmodes)>0) or project.executionOrderHints: def calculateNewHint(d, mode): hint = d.executionOrderHint for hintdelta, hintmatcher in project.executionOrderHints: if hintmatcher(d.groups, mode): hint += hintdelta if mode and not getattr(mode, 'isPrimary', False): # bit of a fudge in the case of isPrimary!=(index==0) but good enough hint += project.executionOrderSecondaryModesHintDelta * (d.modes.index(mode)) return hint for d in tests: hintspermode = [] if expandmodes: # used for pysys run; d.mode will have been set d.executionOrderHint = calculateNewHint(d, d.mode) else: modes = [None] if len(d.modes)==0 else d.modes # set this for the benefit of pysys print d.executionOrderHintsByMode = [calculateNewHint(d, m) for m in modes] d.executionOrderHint = d.executionOrderHintsByMode[0] if len(tests) == 0: raise UserError("The supplied options did not result in the selection of any tests") else: return tests