def _add_spec_excludes_to_build_ignore_patterns(build_root, build_ignore_patterns=None, spec_excludes=None): if not build_ignore_patterns: build_ignore_patterns = PathSpec.from_lines(GitIgnorePattern, []) patterns = list(build_ignore_patterns.patterns) patterns.extend(PathSpec.from_lines( GitIgnorePattern, BuildFile._spec_excludes_to_gitignore_syntax(build_root, spec_excludes)).patterns) return PathSpec(patterns)
def reload_ignore_config(): global spec with reload_lock: try: with open('.reloadignore', 'r') as fh: spec = PathSpec.from_lines('gitwildmatch', fh) except IOError as e: if e.errno == 2: logger.info("'.reloadignore' not found. Using default spec.") spec = PathSpec.from_lines('gitwildmatch', DEFAULT_RELOADIGNORE.splitlines()) else: raise
def reload_reloadignore(self): try: # have to specify full path here, otherwise file is not found with open(self.launch_params.working_directory + '/.reloadignore', 'r') as fh: self.spec = PathSpec.from_lines('gitwildmatch', fh) except IOError as e: if e.errno == 2: # may happen if file is deleted and inotifyevent is triggered for that # print("'.reloadignore' not found. Using default spec.") self.spec = PathSpec.from_lines('gitwildmatch', DEFAULT_RELOADIGNORE.splitlines()) else: raise
def _add_spec_excludes_to_build_ignore_patterns(build_root, build_ignore_patterns=None, spec_excludes=None): if not build_ignore_patterns: build_ignore_patterns = PathSpec.from_lines(GitIgnorePattern, []) patterns = list(build_ignore_patterns.patterns) patterns.extend( PathSpec.from_lines( GitIgnorePattern, BuildFile._spec_excludes_to_gitignore_syntax( build_root, spec_excludes)).patterns) return PathSpec(patterns)
def walk_not_git_ignored(path, keep, extra_ignore): spec = PathSpec.from_lines("gitwildmatch", [".git"] + extra_ignore) ignores = {} # detect git folder, collect ignores up to root at = path while True: git_exist = (at / ".git").exists() if git_exist: ignores[at.parent] = spec # go down back and load all ignores for part in (".", ) + path.relative_to(at).parts: at = at / part spec = _load_ignore(at, spec, ignores) break if at == at.parent: ignores[path] = spec break at = at.parent # now walk from root, collect new ignores and evaluate for root, dirs, files in os.walk(str(path)): root_path = Path(root).relative_to(path) # current path current_path = path / root parent_spec = ignores.get(current_path) or next( ignores[p] for p in current_path.parents if p in ignores) spec = _load_ignore(Path(root), parent_spec, ignores) for file_name in files: result = root_path / file_name if (file_name != ".gitignore" and keep(result) and not spec.match_file(str(result))): yield path / result for cur_dir in list(dirs): if spec.match_file(str(root_path / cur_dir)): dirs.remove(cur_dir)
def __init__(self, symbol_table_cls, parser_cls, build_patterns=None, build_ignore_patterns=None, exclude_target_regexps=None, subproject_roots=None): """Create an AddressMapper. Both the set of files that define a mappable BUILD files and the parser used to parse those files can be customized. See the `pants.engine.parsers` module for example parsers. :param symbol_table_cls: The symbol table cls to expose a symbol table dict. :type symbol_table_cls: A :class:`pants.engine.parser.SymbolTable`. :param parser_cls: The BUILD file parser cls to use. :type parser_cls: A :class:`pants.engine.parser.Parser`. :param tuple build_patterns: A tuple of fnmatch-compatible patterns for identifying BUILD files used to resolve addresses. :param list build_ignore_patterns: A list of path ignore patterns used when searching for BUILD files. :param list exclude_target_regexps: A list of regular expressions for excluding targets. """ self.symbol_table_cls = symbol_table_cls self.parser_cls = parser_cls self.build_patterns = build_patterns or (b'BUILD', b'BUILD.*') self.build_ignore_patterns = PathSpec.from_lines(GitWildMatchPattern, build_ignore_patterns or []) self._exclude_target_regexps = exclude_target_regexps or [] self.exclude_patterns = [re.compile(pattern) for pattern in self._exclude_target_regexps] self.subproject_roots = subproject_roots or []
def configure_project(self, targets, debug_port): jvm_targets = [t for t in targets if t.has_label('jvm') or t.has_label('java') or isinstance(t, Resources)] if self.intransitive: jvm_targets = set(self.context.target_roots).intersection(jvm_targets) build_ignore_patterns = self.context.options.for_global_scope().ignore_patterns or [] build_ignore_patterns.extend(BuildFile._spec_excludes_to_gitignore_syntax( os.path.realpath(get_buildroot()), self.context.options.for_global_scope().spec_excludes)) project = Project(self.project_name, self.python, self.skip_java, self.skip_scala, self.use_source_root, get_buildroot(), debug_port, self.context, jvm_targets, not self.intransitive, self.TargetUtil(self.context), None, PathSpec.from_lines(GitIgnorePattern, build_ignore_patterns)) if self.python: python_source_paths = self.get_options().python_source_paths python_test_paths = self.get_options().python_test_paths python_lib_paths = self.get_options().python_lib_paths project.configure_python(python_source_paths, python_test_paths, python_lib_paths) extra_source_paths = self.get_options().extra_jvm_source_paths extra_test_paths = self.get_options().extra_jvm_test_paths all_targets = project.configure_jvm(extra_source_paths, extra_test_paths) return all_targets, project
def file_indexer(queue, path, ignore_patterns, max_files): from pathspec import PathSpec def _list_files(dirname): files = [] try: basenames = os.listdir(dirname) except Exception: pass else: for basename in basenames: path = os.path.join(dirname, basename) if ignore_spec.match_file(path): continue if os.path.isdir(path): files += _list_files(path) else: files.append(path) if len(files) > max_files: raise ValueError(u'Too many files') return files ignore_spec = PathSpec.from_lines('gitwildmatch', ignore_patterns) try: files = _list_files(path) except ValueError: files = None queue.put(files)
def get_pathspec(directory): ignore_file = get_gitignore_file(directory) if not ignore_file: return _null_spec with open(ignore_file) as f: spec = PathSpec.from_lines('gitwildmatch', f.readlines()) return spec
def configure_project(self, targets, debug_port): jvm_targets = [t for t in targets if t.has_label('jvm') or t.has_label('java') or isinstance(t, Resources)] if self.intransitive: jvm_targets = set(self.context.target_roots).intersection(jvm_targets) build_ignore_patterns = self.context.options.for_global_scope().build_ignore or [] project = Project(self.project_name, self.python, self.skip_java, self.skip_scala, self.use_source_root, get_buildroot(), debug_port, self.context, jvm_targets, not self.intransitive, self.TargetUtil(self.context), PathSpec.from_lines(GitIgnorePattern, build_ignore_patterns)) if self.python: python_source_paths = self.get_options().python_source_paths python_test_paths = self.get_options().python_test_paths python_lib_paths = self.get_options().python_lib_paths project.configure_python(python_source_paths, python_test_paths, python_lib_paths) extra_source_paths = self.get_options().extra_jvm_source_paths extra_test_paths = self.get_options().extra_jvm_test_paths all_targets = project.configure_jvm(extra_source_paths, extra_test_paths) return all_targets, project
def __init__(self, parser, build_patterns=None, build_ignore_patterns=None, exclude_target_regexps=None, subproject_roots=None): """Create an AddressMapper. Both the set of files that define a mappable BUILD files and the parser used to parse those files can be customized. See the `pants.engine.parsers` module for example parsers. :param parser: The BUILD file parser to use. :type parser: An instance of :class:`pants.engine.parser.Parser`. :param tuple build_patterns: A tuple of fnmatch-compatible patterns for identifying BUILD files used to resolve addresses. :param list build_ignore_patterns: A list of path ignore patterns used when searching for BUILD files. :param list exclude_target_regexps: A list of regular expressions for excluding targets. """ self.parser = parser self.build_patterns = build_patterns or (b'BUILD', b'BUILD.*') self.build_ignore_patterns = PathSpec.from_lines( GitWildMatchPattern, build_ignore_patterns or []) self._exclude_target_regexps = exclude_target_regexps or [] self.exclude_patterns = [ re.compile(pattern) for pattern in self._exclude_target_regexps ] self.subproject_roots = subproject_roots or []
def __init__(self, symbol_table_cls, parser_cls, build_pattern=None, build_ignore_patterns=None, exclude_target_regexps=None): """Create an AddressMapper. Both the set of files that define a mappable BUILD files and the parser used to parse those files can be customized. See the `pants.engine.parsers` module for example parsers. :param symbol_table_cls: The symbol table cls to expose a symbol table dict. :type symbol_table_cls: A :class:`pants.engine.parser.SymbolTable`. :param parser_cls: The BUILD file parser cls to use. :type parser_cls: A :class:`pants.engine.parser.Parser`. :param string build_pattern: A fnmatch-compatible pattern for identifying BUILD files used to resolve addresses; by default looks for `BUILD*` files. :param list build_ignore_patterns: A list of path ignore patterns used when searching for BUILD files. :param list exclude_target_regexps: A list of regular expressions for excluding targets. """ self.symbol_table_cls = symbol_table_cls self.parser_cls = parser_cls self.build_pattern = build_pattern or 'BUILD*' self.build_ignore_patterns = PathSpec.from_lines( GitIgnorePattern, build_ignore_patterns or []) self._exclude_target_regexps = exclude_target_regexps or [] self.exclude_patterns = [ re.compile(pattern) for pattern in self._exclude_target_regexps ]
def configure_project(self, targets, debug_port): jvm_targets = [ t for t in targets if isinstance(t, (JvmTarget, JarLibrary, Resources)) ] if self.intransitive: jvm_targets = set( self.context.target_roots).intersection(jvm_targets) build_ignore_patterns = self.context.options.for_global_scope( ).build_ignore or [] project = Project( self.project_name, self.python, self.skip_java, self.skip_scala, self.use_source_root, get_buildroot(), debug_port, self.context, jvm_targets, not self.intransitive, self.TargetUtil(self.context), PathSpec.from_lines(GitWildMatchPattern, build_ignore_patterns)) if self.python: python_source_paths = self.get_options().python_source_paths python_test_paths = self.get_options().python_test_paths python_lib_paths = self.get_options().python_lib_paths project.configure_python(python_source_paths, python_test_paths, python_lib_paths) extra_source_paths = self.get_options().extra_jvm_source_paths extra_test_paths = self.get_options().extra_jvm_test_paths all_targets = project.configure_jvm(extra_source_paths, extra_test_paths) return all_targets, project
class DirectoryWatcher: """Iterator that detects and yields file changes by polling the filesystem.""" path: FileSystemPath interval: float = 0.6 ignore: PathSpec = field(init=False) ignore_file: Optional[FileSystemPath] = extra_field(default=None) ignore_patterns: Sequence[str] = extra_field(default=()) files: Dict[str, float] = extra_field(init=False, default_factory=dict) def __post_init__(self): self.ignore_patterns = list(self.ignore_patterns) if self.ignore_file: ignore_file = Path(self.ignore_file) if ignore_file.parts == (ignore_file.name,): for directory in Path(self.path, ignore_file).parents: if (path := (directory / ignore_file)).is_file(): ignore_file = path break if ignore_file.is_file(): self.ignore_patterns += [ pattern for line in ignore_file.read_text().splitlines() if not line.startswith("#") and (pattern := line.strip()) ] self.ignore = PathSpec.from_lines("gitwildmatch", self.ignore_patterns)
def get_gitignore(root): """Return a PathSpec matching gitignore content if present.""" gitignore = root / ".gitignore" lines = [] if gitignore.is_file(): with gitignore.open(encoding="utf-8") as gf: lines = gf.readlines() return PathSpec.from_lines("gitwildmatch", lines)
def get_gitignore(root: Path) -> PathSpec: """ Return a PathSpec matching gitignore content if present.""" gitignore = root / ".gitignore" lines: List[str] = [] if gitignore.is_file(): with gitignore.open() as gf: lines = gf.readlines() return PathSpec.from_lines("gitwildmatch", lines)
def __init__(self, ignore_file_path, tree): assert os.path.isabs(ignore_file_path) self.ignore_file_path = ignore_file_path self.dirname = os.path.normpath(os.path.dirname(ignore_file_path)) with tree.open(ignore_file_path, encoding="utf-8") as fobj: self.ignore_spec = PathSpec.from_lines(GitWildMatchPattern, fobj)
def _load_ignore(at_path, parent_spec, ignores): ignore_file = at_path / ".gitignore" if not ignore_file.exists(): return parent_spec lines = ignore_file.read_text().split(os.linesep) spec = PathSpec.from_lines("gitwildmatch", lines) spec = PathSpec(parent_spec.patterns + spec.patterns) ignores[at_path] = spec return spec
def _get_match_filter(dir_abspath, ignore, **kwargs): """Helper to construct a function for filtering of paths. """ ignore = [] if ignore is None else list(ignore) ignore = _parse_ignorefile(dir_abspath) + ignore match_spec = _get_match_spec(ignore=ignore, **kwargs) path_spec = PathSpec.from_lines(GitWildMatchPattern, match_spec) return path_spec.match_files
def mock_dvcignore(dvcignore_path, patterns): mock_ignore_file_handler = Mock() with patch.object( mock_ignore_file_handler, "read_patterns", return_value=PathSpec.from_lines(GitWildMatchPattern, patterns), ): ignore_file = DvcIgnoreFromFile(dvcignore_path, mock_ignore_file_handler) return ignore_file
def get_gitignore(root): """ Return a PathSpec matching gitignore content if present. This function is a modified version of the one present in Black (https://github.com/psf/black) available under MIT License. """ gitignore = root / ".gitignore" lines = [] if gitignore.is_file(): with gitignore.open() as gi_file: lines = gi_file.readlines() return PathSpec.from_lines("gitwildmatch", lines)
def gitignore(self): """Load a repo's .gitignore and .git/info/exclude files for path matching.""" patterns = [] for path in ('.gitignore', '.git/info/exclude'): try: with open(pjoin(self.options.target_repo.location, path)) as f: patterns.extend(f) except FileNotFoundError: pass except IOError as e: logger.warning(f'failed reading {path!r}: {e}') return PathSpec.from_lines('gitwildmatch', patterns)
def get_gitignore() -> PathSpec: """ Return a PathSpec matching gitignore content if present. This function has been copied from Black (https://github.com/psf/black). """ gitignore = ROOT_PATH / ".gitignore" lines: List[str] = [] if gitignore.is_file(): with gitignore.open() as gf: lines = gf.readlines() return PathSpec.from_lines("gitwildmatch", lines)
def gitignore(self): """Load a repo's .gitignore file for path matching usage.""" path = pjoin(self.options.target_repo.location, '.gitignore') patterns = [] try: with open(path) as f: patterns = f.readlines() except FileNotFoundError: pass except IOError as e: logger.warning(f'failed reading {path!r}: {e}') return PathSpec.from_lines('gitwildmatch', patterns)
def get_backup_filepaths(self, dpath): """Return filtered list of file paths.""" fname = os.path.join(self.BASE_DIR, dpath, 'backup.paths') if not os.path.exists(fname): return backup_spec = open(fname).read() spec = PathSpec.from_lines(GitWildMatchPattern, backup_spec.splitlines()) return [ os.path.join(dpath, p) for p in spec.match_tree(os.path.join(self.BASE_DIR, dpath)) ]
def get_gitignore(root: Path) -> PathSpec: """Return a PathSpec matching gitignore content if present.""" gitignore = root / ".gitignore" lines: List[str] = [] if gitignore.is_file(): with gitignore.open(encoding="utf-8") as gf: lines = gf.readlines() try: return PathSpec.from_lines("gitwildmatch", lines) except GitWildMatchPatternError as e: err(f"Could not parse {gitignore}: {e}") raise
def _apply_exclude_patterns(names, exclude_filter): """Exclude matched patterns from passed names.""" included = set(names) # Assume old way for easier testing if hasattr(exclude_filter, '__iter__'): exclude_filter = PathSpec.from_lines('gitwildmatch', exclude_filter) for excluded in exclude_filter.match_files(names): included.discard(excluded) return sorted(included)
def __init__(self, build_file_parser, project_tree, build_ignore_patterns=None, exclude_target_regexps=None): """Create a BuildFileAddressMapper. :param build_file_parser: An instance of BuildFileParser :param build_file_type: A subclass of BuildFile used to construct and cache BuildFile objects """ self._build_file_parser = build_file_parser self._spec_path_to_address_map_map = {} # {spec_path: {address: addressable}} mapping self._project_tree = project_tree self._build_ignore_patterns = PathSpec.from_lines(GitWildMatchPattern, build_ignore_patterns or []) self._exclude_target_regexps = exclude_target_regexps or [] self._exclude_patterns = [re.compile(pattern) for pattern in self._exclude_target_regexps]
def get_ebignore_list(): location = get_ebignore_location() if not os.path.isfile(location): return None with codecs.open(location, 'r', encoding='utf-8') as f: spec = PathSpec.from_lines('gitwildmatch', f) ignore_list = [f for f in spec.match_tree(get_project_root())] ignore_list.append('.ebignore') return ignore_list
def __init__(self, build_file_parser, project_tree, build_ignore_patterns=None, exclude_target_regexps=None): """Create a BuildFileAddressMapper. :param build_file_parser: An instance of BuildFileParser :param build_file_type: A subclass of BuildFile used to construct and cache BuildFile objects """ self._build_file_parser = build_file_parser self._spec_path_to_address_map_map = {} # {spec_path: {address: addressable}} mapping self._project_tree = project_tree self._build_ignore_patterns = PathSpec.from_lines(GitIgnorePattern, build_ignore_patterns or []) self._exclude_target_regexps = exclude_target_regexps or [] self._exclude_patterns = [re.compile(pattern) for pattern in self._exclude_target_regexps]
def find_files_python( path: str, fdignore_path: Optional[str] = None, pathspec: Optional[PathSpec] = None, ) -> Iterator[TDirEntry]: try: entries = os.scandir(path) except FileNotFoundError: logger.info('Directory not found "%s"', path) return except PermissionError: logger.info('No permissions to read directory "%s"', path) return if pathspec is None: if fdignore_path is not None: with open(fdignore_path, 'rt') as f: pathspec = PathSpec.from_lines('gitwildmatch', f) else: logger.info('Loading default fdignore') fdignore_bytes = pkgutil.get_data('human_activities.etc', 'human-activities.fdignore') if fdignore_bytes: pathspec = PathSpec.from_lines( 'gitwildmatch', fdignore_bytes.decode().splitlines()) else: logger.error('Failed to load default fdignoe') pathspec = PathSpec.from_lines('gitwildmatch', []) for entry in entries: if entry.is_symlink(): continue if is_hidden(entry) or pathspec.match_file(entry.path): continue if entry.is_file(): yield entry elif entry.is_dir(): yield from find_files_python(entry.path, fdignore_path, pathspec=pathspec)
def get_ebignore_list(): location = get_ebignore_location() if not os.path.isfile(location): return None with codecs.open(location, 'r', encoding='utf-8') as f: spec = PathSpec.from_lines('gitwildmatch', f) ignore_list = {f for f in spec.match_tree(get_project_root())} ignore_list.add('.ebignore') return ignore_list
def __init__( self, linked_dirs=True, linked_files=True, match=None, ): self.linked_dirs = linked_dirs self.linked_files = linked_files self._match_patterns = tuple('*') if match is None else tuple(match) if self._match_patterns != tuple('*'): self._path_spec = PathSpec.from_lines(GitWildMatchPattern, self.match_patterns) else: self._path_spec = None
def __init__(self, build_file_parser, project_tree, build_ignore_patterns=None): """Create a BuildFileAddressMapper. :param build_file_parser: An instance of BuildFileParser :param build_file_type: A subclass of BuildFile used to construct and cache BuildFile objects """ self._build_file_parser = build_file_parser self._spec_path_to_address_map_map = {} # {spec_path: {address: addressable}} mapping if isinstance(project_tree, ProjectTree): self._project_tree = project_tree else: # If project_tree is BuildFile class actually. # TODO(tabishev): Remove after transition period. self._project_tree = project_tree._get_project_tree(self.root_dir) self._build_ignore_patterns = PathSpec.from_lines(GitIgnorePattern, build_ignore_patterns or [])
def get_gitignore(root: Path, no_gitignore: bool = False) -> PathSpec: """Return a PathSpec matching gitignore content, if present. :param root: root path to search for `.gitignore`. :param no_gitignore: `config.no_gitignore` value (default=False). :returns: PathSpec matching gitignore content, if present. """ lines: List[str] = [] if not no_gitignore: path = os.path.join(root, GITIGNORE) if os.path.isfile(path): if os.access(path, os.R_OK): with tokenize.open(path) as ignore_file: lines = ignore_file.readlines() return PathSpec.from_lines(GitWildMatchPattern, lines)
def find_matches( path: str, include_patterns: Optional[Union[str, Iterable[str]]] = None, output_absolute_paths=False, ) -> Iterable: _include_patterns = resolve_include_patterns(include_patterns) path_spec = PathSpec.from_lines(patterns.GitWildMatchPattern, _include_patterns) matches = path_spec.match_tree(path) if output_absolute_paths: matches = (os.path.join(path, m) for m in matches) return matches
def patch_files_walk(repo_top, path, ignored): """Yields string for every valid patch file in [path] whose file name does not match the wildmatch patterns in [ignored], treated relative to [repo_top]. If another `thcrap_ignore.txt` is found along the directory hierarchy, its contents are added to a copy of [ignored], which is then used for this directory and its subdirectories.""" local_ignore = thcrap_ignore_get(path) if len(local_ignore) >= 1: ignored = set(ignored).union(local_ignore) spec = PathSpec.from_lines('gitwildmatch', ignored) for i in os.scandir(path): if spec.match_file(os.path.relpath(i.path, repo_top)) == False: if i.is_dir(): yield from patch_files_walk(repo_top, i.path, ignored) else: yield i.path
def configure_project(self, targets, debug_port): jvm_targets = [t for t in targets if t.has_label("jvm") or t.has_label("java") or isinstance(t, Resources)] if self.intransitive: jvm_targets = set(self.context.target_roots).intersection(jvm_targets) build_ignore_patterns = self.context.options.for_global_scope().ignore_patterns or [] build_ignore_patterns.extend( BuildFile._spec_excludes_to_gitignore_syntax( os.path.realpath(get_buildroot()), self.context.options.for_global_scope().spec_excludes ) ) project = Project( self.project_name, self.python, self.skip_java, self.skip_scala, self.use_source_root, get_buildroot(), debug_port, self.context, jvm_targets, not self.intransitive, self.TargetUtil(self.context), None, PathSpec.from_lines(GitIgnorePattern, build_ignore_patterns), ) if self.python: python_source_paths = self.get_options().python_source_paths python_test_paths = self.get_options().python_test_paths python_lib_paths = self.get_options().python_lib_paths project.configure_python(python_source_paths, python_test_paths, python_lib_paths) extra_source_paths = self.get_options().extra_jvm_source_paths extra_test_paths = self.get_options().extra_jvm_test_paths all_targets = project.configure_jvm(extra_source_paths, extra_test_paths) return all_targets, project
def ignore(*args): return PathSpec.from_lines('gitwildmatch', args)
def _create_ignore_spec(self, build_ignore_patterns): return PathSpec.from_lines(GitIgnorePattern, build_ignore_patterns or [])