def _add_spec_excludes_to_build_ignore_patterns(build_root, build_ignore_patterns=None, spec_excludes=None):
if not build_ignore_patterns:
build_ignore_patterns = PathSpec.from_lines(GitIgnorePattern, [])
patterns = list(build_ignore_patterns.patterns)
patterns.extend(PathSpec.from_lines(
GitIgnorePattern,
BuildFile._spec_excludes_to_gitignore_syntax(build_root, spec_excludes)).patterns)
return PathSpec(patterns)
def reload_ignore_config():
global spec
with reload_lock:
try:
with open('.reloadignore', 'r') as fh:
spec = PathSpec.from_lines('gitwildmatch', fh)
except IOError as e:
if e.errno == 2:
logger.info("'.reloadignore' not found. Using default spec.")
spec = PathSpec.from_lines('gitwildmatch', DEFAULT_RELOADIGNORE.splitlines())
else:
raise
def reload_reloadignore(self):
try:
# have to specify full path here, otherwise file is not found
with open(self.launch_params.working_directory + '/.reloadignore', 'r') as fh:
self.spec = PathSpec.from_lines('gitwildmatch', fh)
except IOError as e:
if e.errno == 2:
# may happen if file is deleted and inotifyevent is triggered for that
# print("'.reloadignore' not found. Using default spec.")
self.spec = PathSpec.from_lines('gitwildmatch', DEFAULT_RELOADIGNORE.splitlines())
else:
raise
def _add_spec_excludes_to_build_ignore_patterns(build_root,
build_ignore_patterns=None,
spec_excludes=None):
if not build_ignore_patterns:
build_ignore_patterns = PathSpec.from_lines(GitIgnorePattern, [])
patterns = list(build_ignore_patterns.patterns)
patterns.extend(
PathSpec.from_lines(
GitIgnorePattern,
BuildFile._spec_excludes_to_gitignore_syntax(
build_root, spec_excludes)).patterns)
return PathSpec(patterns)
def walk_not_git_ignored(path, keep, extra_ignore):
spec = PathSpec.from_lines("gitwildmatch", [".git"] + extra_ignore)
ignores = {}
# detect git folder, collect ignores up to root
at = path
while True:
git_exist = (at / ".git").exists()
if git_exist:
ignores[at.parent] = spec
# go down back and load all ignores
for part in (".", ) + path.relative_to(at).parts:
at = at / part
spec = _load_ignore(at, spec, ignores)
break
if at == at.parent:
ignores[path] = spec
break
at = at.parent
# now walk from root, collect new ignores and evaluate
for root, dirs, files in os.walk(str(path)):
root_path = Path(root).relative_to(path) # current path
current_path = path / root
parent_spec = ignores.get(current_path) or next(
ignores[p] for p in current_path.parents if p in ignores)
spec = _load_ignore(Path(root), parent_spec, ignores)
for file_name in files:
result = root_path / file_name
if (file_name != ".gitignore" and keep(result)
and not spec.match_file(str(result))):
yield path / result
for cur_dir in list(dirs):
if spec.match_file(str(root_path / cur_dir)):
dirs.remove(cur_dir)
def __init__(self,
symbol_table_cls,
parser_cls,
build_patterns=None,
build_ignore_patterns=None,
exclude_target_regexps=None,
subproject_roots=None):
"""Create an AddressMapper.
Both the set of files that define a mappable BUILD files and the parser used to parse those
files can be customized. See the `pants.engine.parsers` module for example parsers.
:param symbol_table_cls: The symbol table cls to expose a symbol table dict.
:type symbol_table_cls: A :class:`pants.engine.parser.SymbolTable`.
:param parser_cls: The BUILD file parser cls to use.
:type parser_cls: A :class:`pants.engine.parser.Parser`.
:param tuple build_patterns: A tuple of fnmatch-compatible patterns for identifying BUILD files
used to resolve addresses.
:param list build_ignore_patterns: A list of path ignore patterns used when searching for BUILD files.
:param list exclude_target_regexps: A list of regular expressions for excluding targets.
"""
self.symbol_table_cls = symbol_table_cls
self.parser_cls = parser_cls
self.build_patterns = build_patterns or (b'BUILD', b'BUILD.*')
self.build_ignore_patterns = PathSpec.from_lines(GitWildMatchPattern, build_ignore_patterns or [])
self._exclude_target_regexps = exclude_target_regexps or []
self.exclude_patterns = [re.compile(pattern) for pattern in self._exclude_target_regexps]
self.subproject_roots = subproject_roots or []
def configure_project(self, targets, debug_port):
jvm_targets = [t for t in targets if t.has_label('jvm') or t.has_label('java') or
isinstance(t, Resources)]
if self.intransitive:
jvm_targets = set(self.context.target_roots).intersection(jvm_targets)
build_ignore_patterns = self.context.options.for_global_scope().ignore_patterns or []
build_ignore_patterns.extend(BuildFile._spec_excludes_to_gitignore_syntax(
os.path.realpath(get_buildroot()), self.context.options.for_global_scope().spec_excludes))
project = Project(self.project_name,
self.python,
self.skip_java,
self.skip_scala,
self.use_source_root,
get_buildroot(),
debug_port,
self.context,
jvm_targets,
not self.intransitive,
self.TargetUtil(self.context),
None,
PathSpec.from_lines(GitIgnorePattern, build_ignore_patterns))
if self.python:
python_source_paths = self.get_options().python_source_paths
python_test_paths = self.get_options().python_test_paths
python_lib_paths = self.get_options().python_lib_paths
project.configure_python(python_source_paths, python_test_paths, python_lib_paths)
extra_source_paths = self.get_options().extra_jvm_source_paths
extra_test_paths = self.get_options().extra_jvm_test_paths
all_targets = project.configure_jvm(extra_source_paths, extra_test_paths)
return all_targets, project
def file_indexer(queue, path, ignore_patterns, max_files):
from pathspec import PathSpec
def _list_files(dirname):
files = []
try:
basenames = os.listdir(dirname)
except Exception:
pass
else:
for basename in basenames:
path = os.path.join(dirname, basename)
if ignore_spec.match_file(path):
continue
if os.path.isdir(path):
files += _list_files(path)
else:
files.append(path)
if len(files) > max_files:
raise ValueError(u'Too many files')
return files
ignore_spec = PathSpec.from_lines('gitwildmatch', ignore_patterns)
try:
files = _list_files(path)
except ValueError:
files = None
queue.put(files)
def get_pathspec(directory):
ignore_file = get_gitignore_file(directory)
if not ignore_file:
return _null_spec
with open(ignore_file) as f:
spec = PathSpec.from_lines('gitwildmatch', f.readlines())
return spec
def configure_project(self, targets, debug_port):
jvm_targets = [t for t in targets if t.has_label('jvm') or t.has_label('java') or
isinstance(t, Resources)]
if self.intransitive:
jvm_targets = set(self.context.target_roots).intersection(jvm_targets)
build_ignore_patterns = self.context.options.for_global_scope().build_ignore or []
project = Project(self.project_name,
self.python,
self.skip_java,
self.skip_scala,
self.use_source_root,
get_buildroot(),
debug_port,
self.context,
jvm_targets,
not self.intransitive,
self.TargetUtil(self.context),
PathSpec.from_lines(GitIgnorePattern, build_ignore_patterns))
if self.python:
python_source_paths = self.get_options().python_source_paths
python_test_paths = self.get_options().python_test_paths
python_lib_paths = self.get_options().python_lib_paths
project.configure_python(python_source_paths, python_test_paths, python_lib_paths)
extra_source_paths = self.get_options().extra_jvm_source_paths
extra_test_paths = self.get_options().extra_jvm_test_paths
all_targets = project.configure_jvm(extra_source_paths, extra_test_paths)
return all_targets, project
def __init__(self,
parser,
build_patterns=None,
build_ignore_patterns=None,
exclude_target_regexps=None,
subproject_roots=None):
"""Create an AddressMapper.
Both the set of files that define a mappable BUILD files and the parser used to parse those
files can be customized. See the `pants.engine.parsers` module for example parsers.
:param parser: The BUILD file parser to use.
:type parser: An instance of :class:`pants.engine.parser.Parser`.
:param tuple build_patterns: A tuple of fnmatch-compatible patterns for identifying BUILD files
used to resolve addresses.
:param list build_ignore_patterns: A list of path ignore patterns used when searching for BUILD files.
:param list exclude_target_regexps: A list of regular expressions for excluding targets.
"""
self.parser = parser
self.build_patterns = build_patterns or (b'BUILD', b'BUILD.*')
self.build_ignore_patterns = PathSpec.from_lines(
GitWildMatchPattern, build_ignore_patterns or [])
self._exclude_target_regexps = exclude_target_regexps or []
self.exclude_patterns = [
re.compile(pattern) for pattern in self._exclude_target_regexps
]
self.subproject_roots = subproject_roots or []
def __init__(self,
symbol_table_cls,
parser_cls,
build_pattern=None,
build_ignore_patterns=None,
exclude_target_regexps=None):
"""Create an AddressMapper.
Both the set of files that define a mappable BUILD files and the parser used to parse those
files can be customized. See the `pants.engine.parsers` module for example parsers.
:param symbol_table_cls: The symbol table cls to expose a symbol table dict.
:type symbol_table_cls: A :class:`pants.engine.parser.SymbolTable`.
:param parser_cls: The BUILD file parser cls to use.
:type parser_cls: A :class:`pants.engine.parser.Parser`.
:param string build_pattern: A fnmatch-compatible pattern for identifying BUILD files used
to resolve addresses; by default looks for `BUILD*` files.
:param list build_ignore_patterns: A list of path ignore patterns used when searching for BUILD files.
:param list exclude_target_regexps: A list of regular expressions for excluding targets.
"""
self.symbol_table_cls = symbol_table_cls
self.parser_cls = parser_cls
self.build_pattern = build_pattern or 'BUILD*'
self.build_ignore_patterns = PathSpec.from_lines(
GitIgnorePattern, build_ignore_patterns or [])
self._exclude_target_regexps = exclude_target_regexps or []
self.exclude_patterns = [
re.compile(pattern) for pattern in self._exclude_target_regexps
]
def configure_project(self, targets, debug_port):
jvm_targets = [
t for t in targets
if isinstance(t, (JvmTarget, JarLibrary, Resources))
]
if self.intransitive:
jvm_targets = set(
self.context.target_roots).intersection(jvm_targets)
build_ignore_patterns = self.context.options.for_global_scope(
).build_ignore or []
project = Project(
self.project_name, self.python,
self.skip_java, self.skip_scala, self.use_source_root,
get_buildroot(), debug_port, self.context, jvm_targets,
not self.intransitive, self.TargetUtil(self.context),
PathSpec.from_lines(GitWildMatchPattern, build_ignore_patterns))
if self.python:
python_source_paths = self.get_options().python_source_paths
python_test_paths = self.get_options().python_test_paths
python_lib_paths = self.get_options().python_lib_paths
project.configure_python(python_source_paths, python_test_paths,
python_lib_paths)
extra_source_paths = self.get_options().extra_jvm_source_paths
extra_test_paths = self.get_options().extra_jvm_test_paths
all_targets = project.configure_jvm(extra_source_paths,
extra_test_paths)
return all_targets, project
class DirectoryWatcher:
"""Iterator that detects and yields file changes by polling the filesystem."""
path: FileSystemPath
interval: float = 0.6
ignore: PathSpec = field(init=False)
ignore_file: Optional[FileSystemPath] = extra_field(default=None)
ignore_patterns: Sequence[str] = extra_field(default=())
files: Dict[str, float] = extra_field(init=False, default_factory=dict)
def __post_init__(self):
self.ignore_patterns = list(self.ignore_patterns)
if self.ignore_file:
ignore_file = Path(self.ignore_file)
if ignore_file.parts == (ignore_file.name,):
for directory in Path(self.path, ignore_file).parents:
if (path := (directory / ignore_file)).is_file():
ignore_file = path
break
if ignore_file.is_file():
self.ignore_patterns += [
pattern
for line in ignore_file.read_text().splitlines()
if not line.startswith("#") and (pattern := line.strip())
]
self.ignore = PathSpec.from_lines("gitwildmatch", self.ignore_patterns)
def get_gitignore(root):
"""Return a PathSpec matching gitignore content if present."""
gitignore = root / ".gitignore"
lines = []
if gitignore.is_file():
with gitignore.open(encoding="utf-8") as gf:
lines = gf.readlines()
return PathSpec.from_lines("gitwildmatch", lines)
def get_gitignore(root: Path) -> PathSpec:
""" Return a PathSpec matching gitignore content if present."""
gitignore = root / ".gitignore"
lines: List[str] = []
if gitignore.is_file():
with gitignore.open() as gf:
lines = gf.readlines()
return PathSpec.from_lines("gitwildmatch", lines)
def __init__(self, ignore_file_path, tree):
assert os.path.isabs(ignore_file_path)
self.ignore_file_path = ignore_file_path
self.dirname = os.path.normpath(os.path.dirname(ignore_file_path))
with tree.open(ignore_file_path, encoding="utf-8") as fobj:
self.ignore_spec = PathSpec.from_lines(GitWildMatchPattern, fobj)
def _load_ignore(at_path, parent_spec, ignores):
ignore_file = at_path / ".gitignore"
if not ignore_file.exists():
return parent_spec
lines = ignore_file.read_text().split(os.linesep)
spec = PathSpec.from_lines("gitwildmatch", lines)
spec = PathSpec(parent_spec.patterns + spec.patterns)
ignores[at_path] = spec
return spec
def _get_match_filter(dir_abspath, ignore, **kwargs):
"""Helper to construct a function for filtering of paths.
"""
ignore = [] if ignore is None else list(ignore)
ignore = _parse_ignorefile(dir_abspath) + ignore
match_spec = _get_match_spec(ignore=ignore, **kwargs)
path_spec = PathSpec.from_lines(GitWildMatchPattern, match_spec)
return path_spec.match_files
def mock_dvcignore(dvcignore_path, patterns):
mock_ignore_file_handler = Mock()
with patch.object(
mock_ignore_file_handler,
"read_patterns",
return_value=PathSpec.from_lines(GitWildMatchPattern, patterns),
):
ignore_file = DvcIgnoreFromFile(dvcignore_path,
mock_ignore_file_handler)
return ignore_file
def get_gitignore(root):
"""
Return a PathSpec matching gitignore content if present.
This function is a modified version of the one present in Black
(https://github.com/psf/black) available under MIT License.
"""
gitignore = root / ".gitignore"
lines = []
if gitignore.is_file():
with gitignore.open() as gi_file:
lines = gi_file.readlines()
return PathSpec.from_lines("gitwildmatch", lines)
def gitignore(self):
"""Load a repo's .gitignore and .git/info/exclude files for path matching."""
patterns = []
for path in ('.gitignore', '.git/info/exclude'):
try:
with open(pjoin(self.options.target_repo.location, path)) as f:
patterns.extend(f)
except FileNotFoundError:
pass
except IOError as e:
logger.warning(f'failed reading {path!r}: {e}')
return PathSpec.from_lines('gitwildmatch', patterns)
def get_gitignore() -> PathSpec:
"""
Return a PathSpec matching gitignore content if present.
This function has been copied from Black (https://github.com/psf/black).
"""
gitignore = ROOT_PATH / ".gitignore"
lines: List[str] = []
if gitignore.is_file():
with gitignore.open() as gf:
lines = gf.readlines()
return PathSpec.from_lines("gitwildmatch", lines)
def gitignore(self):
"""Load a repo's .gitignore file for path matching usage."""
path = pjoin(self.options.target_repo.location, '.gitignore')
patterns = []
try:
with open(path) as f:
patterns = f.readlines()
except FileNotFoundError:
pass
except IOError as e:
logger.warning(f'failed reading {path!r}: {e}')
return PathSpec.from_lines('gitwildmatch', patterns)
def get_backup_filepaths(self, dpath):
"""Return filtered list of file paths."""
fname = os.path.join(self.BASE_DIR, dpath, 'backup.paths')
if not os.path.exists(fname):
return
backup_spec = open(fname).read()
spec = PathSpec.from_lines(GitWildMatchPattern,
backup_spec.splitlines())
return [
os.path.join(dpath, p)
for p in spec.match_tree(os.path.join(self.BASE_DIR, dpath))
]
def get_gitignore(root: Path) -> PathSpec:
"""Return a PathSpec matching gitignore content if present."""
gitignore = root / ".gitignore"
lines: List[str] = []
if gitignore.is_file():
with gitignore.open(encoding="utf-8") as gf:
lines = gf.readlines()
try:
return PathSpec.from_lines("gitwildmatch", lines)
except GitWildMatchPatternError as e:
err(f"Could not parse {gitignore}: {e}")
raise
def _apply_exclude_patterns(names, exclude_filter):
"""Exclude matched patterns from passed names."""
included = set(names)
# Assume old way for easier testing
if hasattr(exclude_filter, '__iter__'):
exclude_filter = PathSpec.from_lines('gitwildmatch', exclude_filter)
for excluded in exclude_filter.match_files(names):
included.discard(excluded)
return sorted(included)
def __init__(self, build_file_parser, project_tree, build_ignore_patterns=None, exclude_target_regexps=None):
"""Create a BuildFileAddressMapper.
:param build_file_parser: An instance of BuildFileParser
:param build_file_type: A subclass of BuildFile used to construct and cache BuildFile objects
"""
self._build_file_parser = build_file_parser
self._spec_path_to_address_map_map = {} # {spec_path: {address: addressable}} mapping
self._project_tree = project_tree
self._build_ignore_patterns = PathSpec.from_lines(GitWildMatchPattern, build_ignore_patterns or [])
self._exclude_target_regexps = exclude_target_regexps or []
self._exclude_patterns = [re.compile(pattern) for pattern in self._exclude_target_regexps]
def get_ebignore_list():
location = get_ebignore_location()
if not os.path.isfile(location):
return None
with codecs.open(location, 'r', encoding='utf-8') as f:
spec = PathSpec.from_lines('gitwildmatch', f)
ignore_list = [f for f in spec.match_tree(get_project_root())]
ignore_list.append('.ebignore')
return ignore_list
def __init__(self, build_file_parser, project_tree, build_ignore_patterns=None, exclude_target_regexps=None):
"""Create a BuildFileAddressMapper.
:param build_file_parser: An instance of BuildFileParser
:param build_file_type: A subclass of BuildFile used to construct and cache BuildFile objects
"""
self._build_file_parser = build_file_parser
self._spec_path_to_address_map_map = {} # {spec_path: {address: addressable}} mapping
self._project_tree = project_tree
self._build_ignore_patterns = PathSpec.from_lines(GitIgnorePattern, build_ignore_patterns or [])
self._exclude_target_regexps = exclude_target_regexps or []
self._exclude_patterns = [re.compile(pattern) for pattern in self._exclude_target_regexps]
def find_files_python(
path: str,
fdignore_path: Optional[str] = None,
pathspec: Optional[PathSpec] = None,
) -> Iterator[TDirEntry]:
try:
entries = os.scandir(path)
except FileNotFoundError:
logger.info('Directory not found "%s"', path)
return
except PermissionError:
logger.info('No permissions to read directory "%s"', path)
return
if pathspec is None:
if fdignore_path is not None:
with open(fdignore_path, 'rt') as f:
pathspec = PathSpec.from_lines('gitwildmatch', f)
else:
logger.info('Loading default fdignore')
fdignore_bytes = pkgutil.get_data('human_activities.etc',
'human-activities.fdignore')
if fdignore_bytes:
pathspec = PathSpec.from_lines(
'gitwildmatch',
fdignore_bytes.decode().splitlines())
else:
logger.error('Failed to load default fdignoe')
pathspec = PathSpec.from_lines('gitwildmatch', [])
for entry in entries:
if entry.is_symlink():
continue
if is_hidden(entry) or pathspec.match_file(entry.path):
continue
if entry.is_file():
yield entry
elif entry.is_dir():
yield from find_files_python(entry.path,
fdignore_path,
pathspec=pathspec)
def get_ebignore_list():
location = get_ebignore_location()
if not os.path.isfile(location):
return None
with codecs.open(location, 'r', encoding='utf-8') as f:
spec = PathSpec.from_lines('gitwildmatch', f)
ignore_list = {f for f in spec.match_tree(get_project_root())}
ignore_list.add('.ebignore')
return ignore_list
def __init__(
self,
linked_dirs=True,
linked_files=True,
match=None,
):
self.linked_dirs = linked_dirs
self.linked_files = linked_files
self._match_patterns = tuple('*') if match is None else tuple(match)
if self._match_patterns != tuple('*'):
self._path_spec = PathSpec.from_lines(GitWildMatchPattern,
self.match_patterns)
else:
self._path_spec = None
def __init__(self, build_file_parser, project_tree, build_ignore_patterns=None):
"""Create a BuildFileAddressMapper.
:param build_file_parser: An instance of BuildFileParser
:param build_file_type: A subclass of BuildFile used to construct and cache BuildFile objects
"""
self._build_file_parser = build_file_parser
self._spec_path_to_address_map_map = {} # {spec_path: {address: addressable}} mapping
if isinstance(project_tree, ProjectTree):
self._project_tree = project_tree
else:
# If project_tree is BuildFile class actually.
# TODO(tabishev): Remove after transition period.
self._project_tree = project_tree._get_project_tree(self.root_dir)
self._build_ignore_patterns = PathSpec.from_lines(GitIgnorePattern, build_ignore_patterns or [])
def get_gitignore(root: Path, no_gitignore: bool = False) -> PathSpec:
"""Return a PathSpec matching gitignore content, if present.
:param root: root path to search for `.gitignore`.
:param no_gitignore: `config.no_gitignore` value (default=False).
:returns: PathSpec matching gitignore content, if present.
"""
lines: List[str] = []
if not no_gitignore:
path = os.path.join(root, GITIGNORE)
if os.path.isfile(path):
if os.access(path, os.R_OK):
with tokenize.open(path) as ignore_file:
lines = ignore_file.readlines()
return PathSpec.from_lines(GitWildMatchPattern, lines)
def find_matches(
path: str,
include_patterns: Optional[Union[str, Iterable[str]]] = None,
output_absolute_paths=False,
) -> Iterable:
_include_patterns = resolve_include_patterns(include_patterns)
path_spec = PathSpec.from_lines(patterns.GitWildMatchPattern,
_include_patterns)
matches = path_spec.match_tree(path)
if output_absolute_paths:
matches = (os.path.join(path, m) for m in matches)
return matches
def patch_files_walk(repo_top, path, ignored):
"""Yields string for every valid patch file in [path] whose file name does
not match the wildmatch patterns in [ignored], treated relative to
[repo_top]. If another `thcrap_ignore.txt` is found along the directory
hierarchy, its contents are added to a copy of [ignored], which is then
used for this directory and its subdirectories."""
local_ignore = thcrap_ignore_get(path)
if len(local_ignore) >= 1:
ignored = set(ignored).union(local_ignore)
spec = PathSpec.from_lines('gitwildmatch', ignored)
for i in os.scandir(path):
if spec.match_file(os.path.relpath(i.path, repo_top)) == False:
if i.is_dir():
yield from patch_files_walk(repo_top, i.path, ignored)
else:
yield i.path
def configure_project(self, targets, debug_port):
jvm_targets = [t for t in targets if t.has_label("jvm") or t.has_label("java") or isinstance(t, Resources)]
if self.intransitive:
jvm_targets = set(self.context.target_roots).intersection(jvm_targets)
build_ignore_patterns = self.context.options.for_global_scope().ignore_patterns or []
build_ignore_patterns.extend(
BuildFile._spec_excludes_to_gitignore_syntax(
os.path.realpath(get_buildroot()), self.context.options.for_global_scope().spec_excludes
)
)
project = Project(
self.project_name,
self.python,
self.skip_java,
self.skip_scala,
self.use_source_root,
get_buildroot(),
debug_port,
self.context,
jvm_targets,
not self.intransitive,
self.TargetUtil(self.context),
None,
PathSpec.from_lines(GitIgnorePattern, build_ignore_patterns),
)
if self.python:
python_source_paths = self.get_options().python_source_paths
python_test_paths = self.get_options().python_test_paths
python_lib_paths = self.get_options().python_lib_paths
project.configure_python(python_source_paths, python_test_paths, python_lib_paths)
extra_source_paths = self.get_options().extra_jvm_source_paths
extra_test_paths = self.get_options().extra_jvm_test_paths
all_targets = project.configure_jvm(extra_source_paths, extra_test_paths)
return all_targets, project
def ignore(*args):
return PathSpec.from_lines('gitwildmatch', args)
def _create_ignore_spec(self, build_ignore_patterns): return PathSpec.from_lines(GitIgnorePattern, build_ignore_patterns or [])
