def walkDirTree(dir, dirIgnores=None, followlinks=False):
"""
:meta private: Not public API.
Walks the specified directory tree, yielding an item for each directory in the tree
(similar to ``os.walk`` but better).
Compared to ``os.walk`` this method is more efficient and easier to use, correctly handles nested paths that
exceed the windows MAX_PATH limit, and has a simpler mechanism for skipping directories from the search tree.
Entries are sorted before being returned to ensure deterministic results.
For example::
for dirpath, entries in walkDirTree(i, dirIgnores=pysys.constants.OSWALK_IGNORES):
for entry in entries:
if entry.is_file() and entry.name.endswith('.java'):
inputfiles.append(entry.path)
:param list[str], callable(DirEntry)->bool dirIgnores: Either a callable that returns True if the specified
directory should be skipped from traversal and from being returned, or a string or list of strings which are
glob patterns to be evaluated against the basename, e.g. ``dirIgnores=['.svn', '.git*']``.
:param bool followlinks: If True, directory symbolic links will be traversed as directories. Be careful of loops
when using this option.
:return (str,list[DirEntry]): Returns a generator that yields a tuple ``(dirpath: str, contents: list[DirEntry])``
for each (non-ignored) directory in the tree. If running on Windows, the returned paths will have the long path
prefix ``\\?\`` which can be removed using `pysys.utils.fileutils.fromLongPathSafe` if needed.
"""
if not callable(dirIgnores):
if dirIgnores is None:
dirIgnores = lambda e: False
else:
if isstring(dirIgnores): dirIgnores = [dirIgnores]
dirIgnores = lambda e, dirIgnores=dirIgnores: any(
fnmatch.fnmatch(e.name, ignore) for ignore in dirIgnores)
dir = pysys.utils.fileutils.toLongPathSafe(dir)
with os.scandir(dir) as it:
contents = []
sortedEntries = list(it)
sortedEntries.sort(key=lambda entry: entry.name)
for entry in sortedEntries:
if not entry.is_dir(follow_symlinks=followlinks):
contents.append(entry)
continue
if dirIgnores(entry):
continue
contents.append(entry)
yield from walkDirTree(entry.path,
dirIgnores=dirIgnores,
followlinks=followlinks)
yield dir, contents
def __str__(self):
"""Return an informal string representation of the xml descriptor container object
:return: The string represention
:rtype: string
"""
str = "Test id: %s\n" % self.id
reltestdir = self.testDir if not self.isDirConfig else '' # relative to current dir is most useful
if reltestdir.lower().replace('\\', '/').startswith(
os.getcwd().lower().replace('\\', '/')):
reltestdir = reltestdir[len(os.getcwd()) + 1:]
str = str + "Test directory: %s\n" % reltestdir # use OS slashes to facilitate copy+paste
str = str + "Test type: %s\n" % self.type
str = str + "Test state: %s\n" % self.state
if self.skippedReason:
str = str + "Test skip reason: %s\n" % self.skippedReason
str = str + "Test title: %s\n" % self.title
str = str + "Test purpose: "
purpose = self.purpose.split('\n') if self.purpose is not None else [
''
]
for index in range(0, len(purpose)):
if index == 0: str = str + "%s\n" % purpose[index]
if index != 0:
str = str + " %s\n" % purpose[index]
str = str + "Test order hint: %s\n" % (
u', '.join('%s' % hint for hint in
self.executionOrderHintsByMode) # for multi-mode tests
if hasattr(self, 'executionOrderHintsByMode') else
self.executionOrderHint)
str = str + "Test groups: %s\n" % (u', '.join(
(u"'%s'" % x if u' ' in x else x)
for x in self.groups) or u'<none>')
if getattr(self, 'mode', None): # multi mode per run
str = str + "Test mode: %s\n" % self.mode
else: # print available modes instead
str = str + "Test modes: %s\n" % (u', '.join(
(u"'%s'" % x if u' ' in x else x)
for x in self.modes) or u'<none>')
str = str + "Test classname: %s\n" % self.classname
str = str + "Test module: %s\n" % self.module
str = str + "Test input: %s\n" % self.input
str = str + "Test output: %s\n" % self.output
str = str + "Test reference: %s\n" % self.reference
str = str + "Test traceability: %s\n" % (u', '.join(
(u"'%s'" % x if u' ' in x else x)
for x in self.traceability) or u'<none>')
if self.userData:
str = str + "Test user data: %s\n" % ', '.join(
'%s=%s' % (k, repr(v).lstrip('u') if isstring(v) else str(v))
for k, v in (self.userData.items()))
str = str + ""
return str
def _argsOrArgsFile(self, args, stdouterr):
# If the length of the command line looks to be on the long side, put it into a separate @args file
# Windows allows approx 32,000 chars; POSIX limit can be as low as 4096. 3000 seems safe.
if len(' '.join(args)) <= 3000: return args
# Create a file using the default encoding for the java arguments
argsFilename = stdouterr if isstring(stdouterr) else stdouterr[0][:-4]
argsFilename = os.path.join(self.owner.output,
argsFilename + '.args.txt')
with openfile(argsFilename, 'w') as f:
for a in args:
f.write('"%s"' % a.replace('\\', '\\\\') + '\n')
return ['@' + argsFilename]
def replace(list, replacementList, flags=0):
"""Replace all occurrences of keyword values in a list of strings with a set value, returning the processed list.
The replacementList parameter should contain a list of tuples to use in the replacement, e.g.
[('foo', 'bar'), ('swim', 'swam')].
:param list: The input list of strings to performed the replacement on
:param replacementList: A list of tuples (key, value) where matches to key are replaced with value
:return: The processed list
:rtype: list
"""
for pair in replacementList:
assert not isstring(
pair
), 'Each item in the replacement list must be a tuple of (string,string)'
x, y = pair
regexp = re.compile(pair[0], flags=flags)
for j in range(0, len(list)):
list[j] = re.sub(regexp, pair[1], list[j])
return list
def toClasspathList(self, classpath):
"""
Converts the specified classpath string to a list of classpath entries (or just returns the input if it's
already a list). Glob expressions such as ``*.jar`` will be expanded if the parent directory exists and there
is at least one match.
If None is specified, the `defaultClasspath` is used (either from the test/dir descriptor's ``classpath``
user-data or the ``defaultClasspath`` plugin property).
In this PySys plugin classpath strings can be delimited with the usual OS separator ``os.pathsep``
(e.g. ``:`` or ``;``), but to allow for platform-independence (given Windows uses ``:`` for drive letters),
if the string contains ``;`` or a newline separator those will be used for splitting instead. Any whitespace or empty
elements will be deleted.
It is recommended to use absolute not relative paths for classpath entries.
>>> plugin = JavaPlugin()
>>> plugin.toClasspathList(['a.jar', 'b.jar'])
['a.jar', 'b.jar']
>>> plugin.toClasspathList(' a.jar ; b.jar ; c:/foo '])
['a.jar', 'b.jar', 'c:/foo']
>>> plugin.toClasspathList(os.pathsep.join(['a.jar', 'b.jar'])
['a.jar', 'b.jar']
>>> plugin.toClasspathList(None) is None
True
"""
if classpath is None:
assert self.defaultClasspath is not None
return self.toClasspathList(
self.defaultClasspath
) # call it again in case user has messed with it
if isstring(
classpath): # assume it's already split unless it's a string
if ';' in classpath:
classpath = classpath.replace('\n', ';').split(';')
else:
classpath = classpath.replace('\n',
os.pathsep).split(os.pathsep)
classpath = [c.strip() for c in classpath if len(c.strip()) > 0]
# glob expansion
if '*' not in ''.join(classpath): return classpath
expanded = []
for c in classpath:
if '*' not in c:
expanded.append(c)
continue
globbed = sorted(glob.glob(c))
if len(globbed) == 0: # Fail in an obvious way in this case
raise Exception('Classpath glob entry has no matches: "%s"', c)
else:
expanded.extend(globbed)
return expanded
def compile(self,
input=None,
output='javaclasses',
classpath=None,
arguments=None,
**kwargs):
"""Compile Java source files into classes. By default we compile Java files from the test's input directory to
``self.output/javaclasses``.
For example::
self.java.compile(self.input, arguments=['--Werror'])
:param input: Typically a directory (relative to the test Input dir) in which to search for
classpaths; alternatively a list of Java source files. By default we compile source files under the
test Input directory.
:type input: str or list[str]
:param str output: The path (relative to the test Output directory) where the output will be written.
:param classpath: The classpath to use, or None if the ``self.defaultClasspath`` should
be used (which by default is empty). The classpath can be specified as a list or a single string delimited
by ``;``, newline or ``os.pathsep``; see `toClasspathList()` for details.
:type classpath: str or list[str]
:param list[str] arguments: List of compiler arguments such as ``--Werror`` or ``-nowarn`` (to control warn
behaviour). If not specified, the ``defaultCompilerArgs`` plugin property is used.
:param kwargs: Additional keyword arguments such as ``timeout=`` will be passed to
`pysys.basetest.BaseTest.startProcess`.
:return: The process object, with the full path to the output dir in the ``info`` dictionary.
:rtype: pysys.process.Process
"""
# need to escape windows \ else it gets removed; do this the same on all platforms for consistency)
if arguments is None:
arguments = self._splitShellArgs(self.defaultCompilerArgs)
if input is None: input = self.owner.input
if isstring(input): input = [input]
assert input, 'At least one input must be specified'
inputfiles = []
for i in input:
i = os.path.join(self.owner.input, i)
if os.path.isfile(i):
inputfiles.append(i)
elif os.path.isdir(i):
for entry in walkDirTreeContents(i, dirIgnores=OSWALK_IGNORES):
if entry.is_file() and entry.name.endswith('.java'):
inputfiles.append(
fromLongPathSafe(entry.path)
if len(entry.path) < 256 else entry.path)
else:
assert False, 'Compilation input path does not exist: %s' % i
assert inputfiles, 'No .java files found to compile in %s' % input
displayName = kwargs.pop(
'displayName', 'javac<%s => %s>' %
(os.path.basename(input[0]), os.path.basename(output)))
classpath = self.toClasspathList(classpath)
args = list(arguments)
output = mkdir(os.path.join(self.owner.output, output))
if os.listdir(output):
self.log.warn(
'Compiling Java to an output directory that already contains some files: %s',
output)
args.extend(['-d', output])
args.extend(inputfiles)
self.log.debug('Javac compiler classpath is: \n%s' % '\n'.join(
" cp #%-2d : %s%s" %
(i + 1, pathelement,
'' if os.path.exists(pathelement) else ' (does not exist!)')
for i, pathelement in enumerate(classpath)) or '(none)')
if classpath: args = ['-classpath', os.pathsep.join(classpath)] + args
stdouterr = kwargs.pop(
'stdouterr',
self.owner.allocateUniqueStdOutErr('javac.%s' %
os.path.basename(output)))
process = self.owner.startProcess(
self.compilerExecutable,
self._argsOrArgsFile(args, stdouterr),
stdouterr=stdouterr,
displayName=displayName,
onError=lambda process: [
self.owner.logFileContents(
process.stderr,
maxLines=0,
logFunction=lambda
line: # colouring the main lines in red makes this a lot easier to read
self.log.info(u' %s',
line,
extra=pysys.utils.logutils.BaseLogFormatter.
tag(LOG_ERROR
if ': error:' in line else LOG_WARN
if ': warning:' in line else
LOG_FILE_CONTENTS))),
self.owner.getExprFromFile(process.stderr,
'(.*(error|invalid).*)')
][-1],
info={'output': output},
**kwargs)
# log stderr even when it works so we see warnings
self.owner.logFileContents(process.stderr, maxLines=0)
return process
def __init__(self, regex, replacement): self.__str = 'RegexReplace(%s, %s)'%(regex, replacement) self.regex = re.compile(regex) if isstring(regex) else regex self.repl = replacement
def __init__(self, regex): self.__str = 'ExcludeLinesMatching(%s)'%(regex) self.regex = re.compile(regex) if isstring(regex) else regex
