def getProjectHelp(self):
help = Project.getInstance().projectHelp
if not help: return None
lines = help.rstrip().split('\n')
# delete blank lines at start
while lines and not lines[0].strip():
del lines[0]
if not lines: return None
# strip initial indentation and convert tabs to spaces
for i in range(len(lines)):
l = lines[i].replace('\t', ' '*3) # this is the amount of indentation we use for the standard help message
if i == 0:
indenttostrip = len(l) - len(l.lstrip())
indenttoadd = ''
if l.strip().startswith('-'):
# align everything with existing content
indenttoadd = ' '*3
if len(l)-len(l.lstrip()) >= indenttostrip:
l = indenttoadd+l[indenttostrip:]
lines[i] = l
# if user hasn't provided their own heading, add ours
if '----' not in help:
lines = ['Project help', '-'*12]+lines
return '\n'.join(lines)
def loadDescriptors(dir=None):
"""Load descriptor objects representing a set of tests to run for
the current project, returning the list.
:meta private: Deprecated and since 1.5.1 also hidden; use `pysys.config.descriptor.DescriptorLoader` instead.
:param dir: The parent directory to search for runnable tests
:return: List of L{pysys.config.descriptor.TestDescriptor} objects.
Caller must sort this list to ensure deterministic behaviour.
:rtype: list
:raises UserError: Raised if no testcases can be found.
"""
if dir is None: dir = os.getcwd()
loader = Project.getInstance().descriptorLoaderClass(Project.getInstance())
return loader.loadDescriptors(dir)
def makeTest(args):
Project.findAndLoadProject()
cls = Project.getInstance().makerClassname.split('.')
module = importlib.import_module('.'.join(cls[:-1]))
maker = getattr(module, cls[-1])("make")
try:
maker.parseArgs(args)
maker.makeTest()
except UserError as e:
sys.stdout.flush()
sys.stderr.write("ERROR: %s\n" % e)
sys.exit(10)
def runTest(args):
try:
launcher = ConsoleLaunchHelper(os.getcwd(), "run")
args = launcher.parseArgs(args)
cls = Project.getInstance().runnerClassname.split('.')
module = importlib.import_module('.'.join(cls[:-1]))
runner = getattr(module, cls[-1])(*args)
runner.start()
for cycledict in runner.results.values():
for outcome in OUTCOMES:
if outcome.isFailure() and cycledict.get(outcome, None): sys.exit(2)
sys.exit(0)
except Exception as e:
sys.stderr.write('\nPYSYS FATAL ERROR: %s\n' % e)
if not isinstance(e, UserError): traceback.print_exc()
sys.exit(10)
def __init__(self, name="make", **kwargs):
self.name = name
self.parentDir = os.getcwd()
self.project = Project.getInstance()
self.skipValidation = False
def parseArgs(self, args, printXOptions=None):
# add any default args first; shlex.split does a great job of providing consistent parsing from str->list,
# but need to avoid mangling \'s on windows; since this env var will be different for each OS no need for consistent win+unix behaviour
if os.getenv('PYSYS_DEFAULT_ARGS',''):
log.info('Using PYSYS_DEFAULT_ARGS = %s'%os.environ['PYSYS_DEFAULT_ARGS'])
args = shlex.split(os.environ['PYSYS_DEFAULT_ARGS'].replace(os.sep, os.sep*2 if os.sep=='\\' else os.sep)) + args
printLogsDefault = PrintLogs.ALL
ci ='--ci' in args
if ci:
# to ensure identical behaviour, set these as if on the command line
# (printLogs we don't set here since we use the printLogsDefault mechanism to allow it to be overridden
# by CI writers and/or the command line)
# Also we don't set --modes=ALL here to allow for it to be overridden explicitly if needed
args = ['--purge', '--record', '-j0', '--type=auto', '--exclude=manual', '-XcodeCoverage']+args
printLogsDefault = PrintLogs.FAILURES
try:
optlist, self.arguments = getopt.gnu_getopt(args, self.optionString, self.optionList)
except Exception:
log.warning("Error parsing command line arguments: %s" % (sys.exc_info()[1]))
sys.exit(1)
log.debug('PySys arguments: tests=%s options=%s', self.arguments, optlist)
EXPR1 = re.compile(r"^[\w\.]*=.*$")
EXPR2 = re.compile(r"^[\w\.]*$")
printLogs = None
defaultAbortOnError = None
logging.getLogger('pysys').setLevel(logging.INFO)
# as a special case, set a non-DEBUG log level for the implementation of assertions
# so that it doesn't get enabled with -vDEBUG only -vassertions=DEBUG
# as it is incredibly verbose and slow and not often useful
logging.getLogger('pysys.assertions').setLevel(logging.INFO)
for option, value in optlist:
if option in ("-h", "--help"):
self.printUsage(printXOptions)
elif option in ['--ci']:
continue # handled above
elif option in ("-r", "--record"):
self.record = True
elif option in ("-p", "--purge"):
self.purge = True
elif option in ("-v", "--verbosity"):
verbosity = value
if '=' in verbosity:
loggername, verbosity = value.split('=')
if loggername.startswith('python:'): # this is weird but was documented, so leave it in place just in case someone is using it
loggername = loggername[len('python:'):]
assert not loggername.startswith('pysys'), 'Cannot use python: with pysys.*' # would produce a duplicate log handler
else:
loggername = None
if verbosity.upper() == "DEBUG":
verbosity = logging.DEBUG
elif verbosity.upper() == "INFO":
verbosity = logging.INFO
elif verbosity.upper() == "WARN":
verbosity = logging.WARN
elif verbosity.upper() == "CRIT":
verbosity = logging.CRITICAL
else:
log.warning('Invalid log level "%s"'%verbosity)
sys.exit(1)
if loggername is None:
# when setting global log level to a higher level like WARN etc we want to affect stdout but
# not necessarily downgrade the root level (would make run.log less useful and break
# some PrintLogs behaviour)
stdoutHandler.setLevel(verbosity)
if verbosity == logging.DEBUG:
logging.getLogger('pysys').setLevel(logging.DEBUG)
logging.getLogger().setLevel(logging.DEBUG)
else:
# for specific level setting we need the opposite - only change stdoutHandler if we're
# turning up the logging (since otherwise it wouldn't be seen) but also change the specified level
# make the user of "pysys." prefix optional
logging.getLogger(loggername).setLevel(verbosity)
logging.getLogger('pysys.'+loggername).setLevel(verbosity)
elif option in ("-a", "--type"):
self.type = value
if self.type not in ["auto", "manual"]:
log.warning("Unsupported test type - valid types are auto and manual")
sys.exit(1)
elif option in ("-t", "--trace"):
self.trace = value
elif option in ("-i", "--include"):
self.includes.append(value)
elif option in ("-e", "--exclude"):
self.excludes.append(value)
elif option in ("-c", "--cycle"):
try:
self.cycle = int(value)
except Exception:
print("Error parsing command line arguments: A valid integer for the number of cycles must be supplied")
sys.exit(1)
elif option in ("-o", "--outdir"):
value = os.path.normpath(value)
if os.path.isabs(value) and not value.startswith('\\\\?\\'): value = fromLongPathSafe(toLongPathSafe(value))
self.outsubdir = value
elif option in ("-m", "--mode", "--modeinclude"):
self.modeinclude = self.modeinclude+[x.strip() for x in value.split(',')]
elif option in ["--modeexclude"]:
self.modeexclude = self.modeexclude+[x.strip() for x in value.split(',')]
elif option in ["-n", "-j", "--threads"]:
try:
N_CPUS = len(os.sched_getaffinity(0)) # as recommended in Python docs, use the allocated CPUs for current process multiprocessing.cpu_count()
except Exception: # no always available, e.g. on Windows
N_CPUS = os.cpu_count()
if value.lower()=='auto': value='0'
if value.lower().startswith('x'):
self.threads = max(1, int(float(value[1:])*N_CPUS))
else:
self.threads = int(value)
if self.threads <= 0: self.threads = int(os.getenv('PYSYS_DEFAULT_THREADS', N_CPUS))
elif option in ("-b", "--abort"):
defaultAbortOnError = str(value.lower()=='true')
elif option in ["-g", "--progress"]:
self.progress = True
elif option in ["--printLogs"]:
printLogs = getattr(PrintLogs, value.upper(), None)
if printLogs is None:
print("Error parsing command line arguments: Unsupported --printLogs value '%s'"%value)
sys.exit(1)
elif option in ["-X"]:
if '=' in value:
key, value = value.split('=', 1)
else:
key, value = value, 'true'
# best not to risk unintended consequences with matching of other types, but for boolean
# it's worth it to resolve the inconsistent behaviour of -Xkey=true and -Xkey that existed until 1.6.0,
# and because getting a bool where you expected a string is a bit more likely to give an exception
# and be noticed that getting a string where you expected a boolean (e.g. the danger of if "false":)
if value.lower() == 'true':
value = True
elif value.lower() == 'false':
value = False
self.userOptions[key] = value
elif option in ("-y", "--validateOnly"):
self.userOptions['validateOnly'] = True
elif option in ("-G", "--grep"):
self.grep = value
elif option in ("-s", "--sort"):
self.sort = value
if value not in ['random']:
print("The only supported sort type for pysys run is currently 'random'")
sys.exit(10)
else:
print("Unknown option: %s"%option)
sys.exit(1)
if ci and not self.modeinclude: # only set this if there is no explicit --mode arguments
self.modeinclude = ['ALL']
# log this once we've got the log levels setup
log.debug('PySys is installed at: %s; python from %s', os.path.dirname(pysys.__file__), sys.executable)
log.debug('Working dir=%s, args=%s', os.getcwd(), " ".join(sys.argv))
log.debug('Main environment vars: %s', '\n '.join(f'{envvar}={os.environ[envvar]}' for envvar in sorted(os.environ.keys())
if envvar.endswith('PATH') or envvar.startswith(('PYTHON', 'PYSYS'))))
# retained for compatibility, but PYSYS_DEFAULT_ARGS is a better way to achieve the same thing
if os.getenv('PYSYS_PROGRESS','').lower()=='true': self.progress = True
# special hidden dict of extra values to pass to the runner, since we can't change
# the public API now
self.userOptions['__extraRunnerOptions'] = {
'progressWritersEnabled':self.progress,
'printLogs': printLogs,
'printLogsDefault': printLogsDefault, # to use if not provided by a CI writer or cmdline
'sort': self.sort
}
# load project AFTER we've parsed the arguments, which opens the possibility of using cmd line config in
# project properties if needed
Project.findAndLoadProject(outdir=self.outsubdir)
if defaultAbortOnError is not None: setattr(Project.getInstance(), 'defaultAbortOnError', defaultAbortOnError)
descriptors = createDescriptors(self.arguments, self.type, self.includes, self.excludes, self.trace, self.workingDir,
modeincludes=self.modeinclude, modeexcludes=self.modeexclude, expandmodes=True)
descriptors.sort(key=lambda d: [d.executionOrderHint, d._defaultSortKey])
# No exception handler above, as any createDescriptors failure is really a fatal problem that should cause us to
# terminate with a non-zero exit code; we don't want to run no tests without realizing it and return success
if self.grep:
regex = re.compile(self.grep, flags=re.IGNORECASE)
descriptors = [d for d in descriptors if (regex.search(d.id) or regex.search(d.title))]
return self.record, self.purge, self.cycle, None, self.threads, self.outsubdir, descriptors, self.userOptions
def createDescriptors(testIdSpecs, type, includes, excludes, trace, dir=None, modeincludes=[], modeexcludes=[], expandmodes=True):
"""Create a list of descriptor objects representing a set of tests to run, filtering by various parameters, returning the list.
:meta private: Not for use outside the framwork.
:param testIdSpecs: A list of strings specifying the set of testcase identifiers
:param type: The type of the tests to run (manual | auto)
:param includes: A list of test groups to include in the returned set
:param excludes: A list of test groups to exclude in the returned set
:param trace: A list of requirements to indicate tests to include in the returned set
:param dir: The parent directory to search for runnable tests
:param modeincludes: A list specifying the modes to be included.
:param modeexcludes: A list specifying the modes to be excluded.
:param expandmodes: Set to False to disable expanding a test with multiple
modes into separate descriptors for each one (used for pysys print).
:return: List of L{pysys.config.descriptor.TestDescriptor} objects
:rtype: list
:raises UserError: Raised if no testcases can be found or are returned by the requested input parameters
"""
project = Project.getInstance()
descriptors = loadDescriptors(dir=dir)
# must sort by id for range matching and dup detection to work deterministically
descriptors.sort(key=lambda d: [d.id, d.file])
# as a convenience support !mode syntax in the includes
modeexcludes = modeexcludes+[x[1:] for x in modeincludes if x.startswith('!')]
modeincludes = [x for x in modeincludes if not x.startswith('!')]
for x in modeexcludes:
if x.startswith('!'): raise UserError('Cannot use ! in a mode exclusion: "%s"'%x)
# populate modedescriptors data structure
MODES_ALL = 'ALL'
MODES_PRIMARY = 'PRIMARY'
assert MODES_ALL not in modeexcludes, "Cannot exclude all modes, that doesn't make sense"
if not modeincludes: # pick a useful default
if modeexcludes:
modeincludes = [MODES_ALL]
else:
modeincludes = [MODES_PRIMARY]
modedescriptors = {} # populate this with testid:[descriptors list]
allmodes = {} # populate this as we go; could have used a set, but instead use a dict so we can check or capitalization mismatches easily at the same time;
#the key is a lowercase version of mode name, value is the canonical capitalized name
modeincludesnone = ((MODES_ALL in modeincludes or MODES_PRIMARY in modeincludes or '' in modeincludes) and
(MODES_PRIMARY not in modeexcludes and '' not in modeexcludes))
NON_MODE_CHARS = '[^'+MODE_CHARS+']'
def isregex(m): return re.search(NON_MODE_CHARS, m)
regexmodeincludes = [re.compile(m, flags=re.IGNORECASE) for m in modeincludes if isregex(m)]
regexmodeexcludes = [re.compile(m, flags=re.IGNORECASE) for m in modeexcludes if isregex(m)]
for d in descriptors:
if not d.modes:
# for tests that have no modes, there is only one descriptor and it's treated as the primary mode;
# user can also specify '' to indicate no mode
if modeincludesnone:
if expandmodes: d.mode = None
modedescriptors[d.id] = [d]
else:
modedescriptors[d.id] = []
else:
thisdescriptorlist = []
modedescriptors[d.id] = thisdescriptorlist # even if it ends up being empty
# create a copy of the descriptor for each selected mode
for m in d.modes:
try:
canonicalmodecapitalization = allmodes[m.lower()]
except KeyError:
allmodes[m.lower()] = m
else:
if m != canonicalmodecapitalization:
# this is useful to detect early; it's almost certain to lead to buggy tests
# since people would be comparing self.mode to a string that might have different capitalization
raise UserError('Cannot have multiple modes with same name but different capitalization: "%s" and "%s"'%(m, canonicalmodecapitalization))
# apply modes filter
isprimary = getattr(m, 'isPrimary', False) # use getattr in case a pre-2.0 str has crept in from a custom DescriptorLoader
# excludes
if isprimary and MODES_PRIMARY in modeexcludes: continue
if m in modeexcludes or any(regex.match(m) for regex in regexmodeexcludes): continue
# includes
if not (MODES_ALL in modeincludes or
m in modeincludes or
any(regex.match(m) for regex in regexmodeincludes) or
(isprimary and MODES_PRIMARY in modeincludes)
):
continue
thisdescriptorlist.append(d._createDescriptorForMode(m))
for m in [MODES_ALL, MODES_PRIMARY]:
if m.lower() in allmodes: raise UserError('The mode name "%s" is reserved, please select another mode name'%m)
# don't permit the user to specify a non existent mode by mistake
for m in modeincludes+modeexcludes:
if (not m) or m.upper() in [MODES_ALL, MODES_PRIMARY]: continue
if isregex(m):
if any(re.search(m, x, flags=re.IGNORECASE) for x in allmodes): continue
else:
if allmodes.get(m.lower(),None) == m: continue
raise UserError('Unknown mode (or mode regex) "%s"; the available modes for descriptors in this directory are: %s'%(
m, ', '.join(sorted(allmodes.values() or ['<none>']))))
# first check for duplicate ids
ids = {}
dups = []
d = None
for d in descriptors:
if d.id in ids:
dups.append('%s - in %s and %s'%(d.id, ids[d.id], d.file))
else:
ids[d.id] = d.file
if dups:
dupmsg = 'Found %d duplicate descriptor ids: %s'%(len(dups), '\n'.join(dups))
if os.getenv('PYSYS_ALLOW_DUPLICATE_IDS','').lower()=='true':
logging.getLogger('pysys').warning(dupmsg) # undocumented option just in case anyone complains
else:
raise UserError(dupmsg)
# trim down the list for those tests in the test specifiers
# unless user the testspec includes a mode suffix, this stage ignores modes,
# and then we expand the modes out afterwards
tests = []
if testIdSpecs == []:
tests = descriptors
else:
testids = {d.id: index for index, d in enumerate(descriptors)}
def findMatchingIndex(specId): # change this to return the match, rather than whether it matches
# optimize the case where we specify the full id; no need to iterate
index = testids.get(specId, None)
if index is not None: return index
if specId.isdigit():
regex = re.compile('.+_0*'+(specId.lstrip('0') if (len(specId)>0) else specId)+'$')
matches = [index for index, d in enumerate(descriptors) if regex.match(d.id)]
else:
# permit specifying suffix at end of testcase, which is
# important to allow shell directory completion to be used if an id-prefix is
# being added onto the directory id; but only do this if spec is non-numeric
# since we don't want to match test_104 against spec 04
matches = [index for index, d in enumerate(descriptors) if d.id.endswith(specId)]
if len(matches) == 1: return matches[0]
if len(matches) == 0: raise UserError('No tests found matching id: "%s"'%specId)
# as a special-case, see if there's an exact match with the dirname
dirnameMatches = [index for index, d in enumerate(descriptors) if os.path.basename(d.testDir)==specId]
if len(dirnameMatches)==1: return dirnameMatches[0]
# nb: use space not comma as the delimiter so it's easy to copy paste it
raise UserError('Multiple tests found matching "%s"; please specify which one you want: %s'%(specId,
' '.join([descriptors[index].id for index in matches[:20] ])))
for t in testIdSpecs:
try:
matches = None
index = index1 = index2 = None
t = t.rstrip('/\\')
if re.search(r'^[%s]*$'%MODE_CHARS, t): # single test id (not a range or regex)
if '~' in t:
testspecid, testspecmode = t.split('~')
index = findMatchingIndex(testspecid)
# first match the id, then the mode
matchingmode = next((m for m in descriptors[index].modes if m == testspecmode), None)
if matchingmode is None:
raise UserError('Unknown mode "%s": the available modes for this test are: %s'%(
testspecmode, ', '.join(sorted(descriptors[index].modes or ['<none>']))))
matches = [descriptors[index]._createDescriptorForMode(matchingmode)]
# note test id+mode combinations selected explicitly like this way are included regardless of what modes are enabled/disabled
else: # normal case where it's not a mode
index = findMatchingIndex(t)
matches = descriptors[index:index+1]
if not modedescriptors[matches[0].id]:
# if user explicitly specified an individual test and excluded all modes it can run in,
# we shouldn't silently skip/exclude it as they clearly made a mistake
raise UserError('Test "%s" cannot be selected with the specified mode(s).'%matches[0].id)
elif '~' in t:
# The utility of this would be close to zero and lots more to implement/test, so not worth it
raise UserError('A ~MODE test mode selector can only be use with a test id, not a range or regular expression')
elif re.search('^:[%s]*'%TEST_ID_CHARS, t):
index = findMatchingIndex(t.split(':')[1])
matches = descriptors[:index+1]
elif re.search('^[%s]*:$'%TEST_ID_CHARS, t):
index = findMatchingIndex(t.split(':')[0])
matches = descriptors[index:]
elif re.search('^[%s]*:[%s]*$'%(TEST_ID_CHARS,TEST_ID_CHARS), t):
index1 = findMatchingIndex(t.split(':')[0])
index2 = findMatchingIndex(t.split(':')[1])
if index1 > index2:
index1, index2 = index2, index1
matches = descriptors[index1:index2+1]
else:
# regex match
try:
matches = [descriptors[i] for i in range(0,len(descriptors)) if re.search(t, descriptors[i].id)]
except Exception as ex:
raise UserError('"%s" contains characters not valid in a test id, but isn\'t a valid regular expression either: %s'%(t, ex))
if not matches: raise UserError("No test ids found matching regular expression: \"%s\""%t)
if not matches: raise UserError("No test ids found matching: \"%s\""%st)
tests.extend(matches)
except UserError:
raise
except Exception:
raise # this shouldn't be possible so no need to sugar coat the error message
# trim down the list based on the type
if type:
index = 0
while index != len(tests):
if type != tests[index].type:
tests.pop(index)
else:
index = index + 1
# trim down the list based on the include and exclude groups
if len(excludes) != 0:
index = 0
while index != len(tests):
remove = False
for exclude in excludes:
if exclude in tests[index].groups:
remove = True
break
if remove:
tests.pop(index)
else:
index = index +1
if includes != []:
index = 0
while index != len(tests):
keep = False
for include in includes:
if include in tests[index].groups:
keep = True
break
if not keep:
tests.pop(index)
else:
index = index +1
# trim down the list based on the traceability
if trace:
index = 0
while index != len(tests):
if trace not in tests[index].traceability :
tests.pop(index)
else:
index = index + 1
# expand based on modes (unless we're printing without any mode filters in which case expandmodes=False)
if expandmodes:
expandedtests = []
for t in tests:
if hasattr(t, 'mode'):
# if mode if set it has no modes or has a test id~mode that was explicitly specified in a testspec, so does not need expanding
expandedtests.append(t)
else:
expandedtests.extend(modedescriptors[t.id])
tests = expandedtests
# combine execution order hints from descriptors with global project configuration;
# we only need to do this if there are any executionOrderHints defined (may be pure group hints not for modes)
# or if there are any tests with multiple modes (since then the secondary hint mode delta applies)
if (len(allmodes)>0) or project.executionOrderHints:
def calculateNewHint(d, mode):
hint = d.executionOrderHint
for hintdelta, hintmatcher in project.executionOrderHints:
if hintmatcher(d.groups, mode): hint += hintdelta
if mode and not getattr(mode, 'isPrimary', False): # bit of a fudge in the case of isPrimary!=(index==0) but good enough
hint += project.executionOrderSecondaryModesHintDelta * (d.modes.index(mode))
return hint
for d in tests:
hintspermode = []
if expandmodes: # used for pysys run; d.mode will have been set
d.executionOrderHint = calculateNewHint(d, d.mode)
else:
modes = [None] if len(d.modes)==0 else d.modes
# set this for the benefit of pysys print
d.executionOrderHintsByMode = [calculateNewHint(d, m) for m in modes]
d.executionOrderHint = d.executionOrderHintsByMode[0]
if len(tests) == 0:
raise UserError("The supplied options did not result in the selection of any tests")
else:
return tests