def walkDirTree(dir, dirIgnores=None, followlinks=False): """ :meta private: Not public API. Walks the specified directory tree, yielding an item for each directory in the tree (similar to ``os.walk`` but better). Compared to ``os.walk`` this method is more efficient and easier to use, correctly handles nested paths that exceed the windows MAX_PATH limit, and has a simpler mechanism for skipping directories from the search tree. Entries are sorted before being returned to ensure deterministic results. For example:: for dirpath, entries in walkDirTree(i, dirIgnores=pysys.constants.OSWALK_IGNORES): for entry in entries: if entry.is_file() and entry.name.endswith('.java'): inputfiles.append(entry.path) :param list[str], callable(DirEntry)->bool dirIgnores: Either a callable that returns True if the specified directory should be skipped from traversal and from being returned, or a string or list of strings which are glob patterns to be evaluated against the basename, e.g. ``dirIgnores=['.svn', '.git*']``. :param bool followlinks: If True, directory symbolic links will be traversed as directories. Be careful of loops when using this option. :return (str,list[DirEntry]): Returns a generator that yields a tuple ``(dirpath: str, contents: list[DirEntry])`` for each (non-ignored) directory in the tree. If running on Windows, the returned paths will have the long path prefix ``\\?\`` which can be removed using `pysys.utils.fileutils.fromLongPathSafe` if needed. """ if not callable(dirIgnores): if dirIgnores is None: dirIgnores = lambda e: False else: if isstring(dirIgnores): dirIgnores = [dirIgnores] dirIgnores = lambda e, dirIgnores=dirIgnores: any( fnmatch.fnmatch(e.name, ignore) for ignore in dirIgnores) dir = pysys.utils.fileutils.toLongPathSafe(dir) with os.scandir(dir) as it: contents = [] sortedEntries = list(it) sortedEntries.sort(key=lambda entry: entry.name) for entry in sortedEntries: if not entry.is_dir(follow_symlinks=followlinks): contents.append(entry) continue if dirIgnores(entry): continue contents.append(entry) yield from walkDirTree(entry.path, dirIgnores=dirIgnores, followlinks=followlinks) yield dir, contents
def __str__(self): """Return an informal string representation of the xml descriptor container object :return: The string represention :rtype: string """ str = "Test id: %s\n" % self.id reltestdir = self.testDir if not self.isDirConfig else '' # relative to current dir is most useful if reltestdir.lower().replace('\\', '/').startswith( os.getcwd().lower().replace('\\', '/')): reltestdir = reltestdir[len(os.getcwd()) + 1:] str = str + "Test directory: %s\n" % reltestdir # use OS slashes to facilitate copy+paste str = str + "Test type: %s\n" % self.type str = str + "Test state: %s\n" % self.state if self.skippedReason: str = str + "Test skip reason: %s\n" % self.skippedReason str = str + "Test title: %s\n" % self.title str = str + "Test purpose: " purpose = self.purpose.split('\n') if self.purpose is not None else [ '' ] for index in range(0, len(purpose)): if index == 0: str = str + "%s\n" % purpose[index] if index != 0: str = str + " %s\n" % purpose[index] str = str + "Test order hint: %s\n" % ( u', '.join('%s' % hint for hint in self.executionOrderHintsByMode) # for multi-mode tests if hasattr(self, 'executionOrderHintsByMode') else self.executionOrderHint) str = str + "Test groups: %s\n" % (u', '.join( (u"'%s'" % x if u' ' in x else x) for x in self.groups) or u'<none>') if getattr(self, 'mode', None): # multi mode per run str = str + "Test mode: %s\n" % self.mode else: # print available modes instead str = str + "Test modes: %s\n" % (u', '.join( (u"'%s'" % x if u' ' in x else x) for x in self.modes) or u'<none>') str = str + "Test classname: %s\n" % self.classname str = str + "Test module: %s\n" % self.module str = str + "Test input: %s\n" % self.input str = str + "Test output: %s\n" % self.output str = str + "Test reference: %s\n" % self.reference str = str + "Test traceability: %s\n" % (u', '.join( (u"'%s'" % x if u' ' in x else x) for x in self.traceability) or u'<none>') if self.userData: str = str + "Test user data: %s\n" % ', '.join( '%s=%s' % (k, repr(v).lstrip('u') if isstring(v) else str(v)) for k, v in (self.userData.items())) str = str + "" return str
def _argsOrArgsFile(self, args, stdouterr): # If the length of the command line looks to be on the long side, put it into a separate @args file # Windows allows approx 32,000 chars; POSIX limit can be as low as 4096. 3000 seems safe. if len(' '.join(args)) <= 3000: return args # Create a file using the default encoding for the java arguments argsFilename = stdouterr if isstring(stdouterr) else stdouterr[0][:-4] argsFilename = os.path.join(self.owner.output, argsFilename + '.args.txt') with openfile(argsFilename, 'w') as f: for a in args: f.write('"%s"' % a.replace('\\', '\\\\') + '\n') return ['@' + argsFilename]
def replace(list, replacementList, flags=0): """Replace all occurrences of keyword values in a list of strings with a set value, returning the processed list. The replacementList parameter should contain a list of tuples to use in the replacement, e.g. [('foo', 'bar'), ('swim', 'swam')]. :param list: The input list of strings to performed the replacement on :param replacementList: A list of tuples (key, value) where matches to key are replaced with value :return: The processed list :rtype: list """ for pair in replacementList: assert not isstring( pair ), 'Each item in the replacement list must be a tuple of (string,string)' x, y = pair regexp = re.compile(pair[0], flags=flags) for j in range(0, len(list)): list[j] = re.sub(regexp, pair[1], list[j]) return list
def toClasspathList(self, classpath): """ Converts the specified classpath string to a list of classpath entries (or just returns the input if it's already a list). Glob expressions such as ``*.jar`` will be expanded if the parent directory exists and there is at least one match. If None is specified, the `defaultClasspath` is used (either from the test/dir descriptor's ``classpath`` user-data or the ``defaultClasspath`` plugin property). In this PySys plugin classpath strings can be delimited with the usual OS separator ``os.pathsep`` (e.g. ``:`` or ``;``), but to allow for platform-independence (given Windows uses ``:`` for drive letters), if the string contains ``;`` or a newline separator those will be used for splitting instead. Any whitespace or empty elements will be deleted. It is recommended to use absolute not relative paths for classpath entries. >>> plugin = JavaPlugin() >>> plugin.toClasspathList(['a.jar', 'b.jar']) ['a.jar', 'b.jar'] >>> plugin.toClasspathList(' a.jar ; b.jar ; c:/foo ']) ['a.jar', 'b.jar', 'c:/foo'] >>> plugin.toClasspathList(os.pathsep.join(['a.jar', 'b.jar']) ['a.jar', 'b.jar'] >>> plugin.toClasspathList(None) is None True """ if classpath is None: assert self.defaultClasspath is not None return self.toClasspathList( self.defaultClasspath ) # call it again in case user has messed with it if isstring( classpath): # assume it's already split unless it's a string if ';' in classpath: classpath = classpath.replace('\n', ';').split(';') else: classpath = classpath.replace('\n', os.pathsep).split(os.pathsep) classpath = [c.strip() for c in classpath if len(c.strip()) > 0] # glob expansion if '*' not in ''.join(classpath): return classpath expanded = [] for c in classpath: if '*' not in c: expanded.append(c) continue globbed = sorted(glob.glob(c)) if len(globbed) == 0: # Fail in an obvious way in this case raise Exception('Classpath glob entry has no matches: "%s"', c) else: expanded.extend(globbed) return expanded
def compile(self, input=None, output='javaclasses', classpath=None, arguments=None, **kwargs): """Compile Java source files into classes. By default we compile Java files from the test's input directory to ``self.output/javaclasses``. For example:: self.java.compile(self.input, arguments=['--Werror']) :param input: Typically a directory (relative to the test Input dir) in which to search for classpaths; alternatively a list of Java source files. By default we compile source files under the test Input directory. :type input: str or list[str] :param str output: The path (relative to the test Output directory) where the output will be written. :param classpath: The classpath to use, or None if the ``self.defaultClasspath`` should be used (which by default is empty). The classpath can be specified as a list or a single string delimited by ``;``, newline or ``os.pathsep``; see `toClasspathList()` for details. :type classpath: str or list[str] :param list[str] arguments: List of compiler arguments such as ``--Werror`` or ``-nowarn`` (to control warn behaviour). If not specified, the ``defaultCompilerArgs`` plugin property is used. :param kwargs: Additional keyword arguments such as ``timeout=`` will be passed to `pysys.basetest.BaseTest.startProcess`. :return: The process object, with the full path to the output dir in the ``info`` dictionary. :rtype: pysys.process.Process """ # need to escape windows \ else it gets removed; do this the same on all platforms for consistency) if arguments is None: arguments = self._splitShellArgs(self.defaultCompilerArgs) if input is None: input = self.owner.input if isstring(input): input = [input] assert input, 'At least one input must be specified' inputfiles = [] for i in input: i = os.path.join(self.owner.input, i) if os.path.isfile(i): inputfiles.append(i) elif os.path.isdir(i): for entry in walkDirTreeContents(i, dirIgnores=OSWALK_IGNORES): if entry.is_file() and entry.name.endswith('.java'): inputfiles.append( fromLongPathSafe(entry.path) if len(entry.path) < 256 else entry.path) else: assert False, 'Compilation input path does not exist: %s' % i assert inputfiles, 'No .java files found to compile in %s' % input displayName = kwargs.pop( 'displayName', 'javac<%s => %s>' % (os.path.basename(input[0]), os.path.basename(output))) classpath = self.toClasspathList(classpath) args = list(arguments) output = mkdir(os.path.join(self.owner.output, output)) if os.listdir(output): self.log.warn( 'Compiling Java to an output directory that already contains some files: %s', output) args.extend(['-d', output]) args.extend(inputfiles) self.log.debug('Javac compiler classpath is: \n%s' % '\n'.join( " cp #%-2d : %s%s" % (i + 1, pathelement, '' if os.path.exists(pathelement) else ' (does not exist!)') for i, pathelement in enumerate(classpath)) or '(none)') if classpath: args = ['-classpath', os.pathsep.join(classpath)] + args stdouterr = kwargs.pop( 'stdouterr', self.owner.allocateUniqueStdOutErr('javac.%s' % os.path.basename(output))) process = self.owner.startProcess( self.compilerExecutable, self._argsOrArgsFile(args, stdouterr), stdouterr=stdouterr, displayName=displayName, onError=lambda process: [ self.owner.logFileContents( process.stderr, maxLines=0, logFunction=lambda line: # colouring the main lines in red makes this a lot easier to read self.log.info(u' %s', line, extra=pysys.utils.logutils.BaseLogFormatter. tag(LOG_ERROR if ': error:' in line else LOG_WARN if ': warning:' in line else LOG_FILE_CONTENTS))), self.owner.getExprFromFile(process.stderr, '(.*(error|invalid).*)') ][-1], info={'output': output}, **kwargs) # log stderr even when it works so we see warnings self.owner.logFileContents(process.stderr, maxLines=0) return process
def __init__(self, regex, replacement): self.__str = 'RegexReplace(%s, %s)'%(regex, replacement) self.regex = re.compile(regex) if isstring(regex) else regex self.repl = replacement
def __init__(self, regex): self.__str = 'ExcludeLinesMatching(%s)'%(regex) self.regex = re.compile(regex) if isstring(regex) else regex