def __init__(self, configFile, git, script):
self.configFile = configFile
self.configuration = Configuration().loadJsonFile(configFile)
if not "license" in self.configuration:
self.configuration["license"] = None
# Compile the patterns for future use
for p in self.configuration["patterns"]:
if "commentEnd" not in p:
p["commentEnd"] = ""
if "prolog" not in p:
p["prolog"] = []
p["include"] = PathSpec(map(GitWildMatchPattern, p["include"]))
if "exclude" in p:
p["exclude"] = PathSpec(map(GitWildMatchPattern, p["exclude"]))
self.git = git
self.python = sys.executable
self.script = script
# Change to the git repository directory
Out, Err, Code = self.git("rev-parse", "--show-toplevel")
Result = Out.splitlines()
self.repoDir = Result[0]
os.chdir(self.repoDir)
self.header = Header(self.git, self.configuration["copyright"],
self.configuration["license"])
return
def _add_spec_excludes_to_build_ignore_patterns(build_root, build_ignore_patterns=None, spec_excludes=None):
if not build_ignore_patterns:
build_ignore_patterns = PathSpec.from_lines(GitIgnorePattern, [])
patterns = list(build_ignore_patterns.patterns)
patterns.extend(PathSpec.from_lines(
GitIgnorePattern,
BuildFile._spec_excludes_to_gitignore_syntax(build_root, spec_excludes)).patterns)
return PathSpec(patterns)
def _load_ignore(at_path, parent_spec, ignores):
ignore_file = at_path / ".gitignore"
if not ignore_file.exists():
return parent_spec
lines = ignore_file.read_text().split(os.linesep)
spec = PathSpec.from_lines("gitwildmatch", lines)
spec = PathSpec(parent_spec.patterns + spec.patterns)
ignores[at_path] = spec
return spec
def filter_files(self, files):
if self.include:
spec = PathSpec(map(GitWildMatchPattern, self.include))
files = list(spec.match_files(files))
if self.exclude:
spec = PathSpec(map(GitWildMatchPattern, self.exclude))
exclude = list(spec.match_files(files))
files = list(filter(lambda f: f not in exclude, files))
return files
def reload_reloadignore(self):
try:
# have to specify full path here, otherwise file is not found
with open(self.launch_params.working_directory + '/.reloadignore', 'r') as fh:
self.spec = PathSpec.from_lines('gitwildmatch', fh)
except IOError as e:
if e.errno == 2:
# may happen if file is deleted and inotifyevent is triggered for that
# print("'.reloadignore' not found. Using default spec.")
self.spec = PathSpec.from_lines('gitwildmatch', DEFAULT_RELOADIGNORE.splitlines())
else:
raise
def extract_bower_zipfile(self, zip_file, dest, expected_version=None):
bower_json = None
root = None
deps_installed = []
for info in zip_file.infolist():
if PurePath(info.filename).name == "bower.json":
with zip_file.open(info) as f:
bower_json = json.load(f)
root = str(PurePath(info.filename).parent)
break
version = bower_json["version"]
if expected_version is not None:
expected_version = Bower.clean_semver(expected_version)
if not semver.match(version, expected_version):
click.secho("error: versions do not match ({} =/= {})".format(
version, expected_version))
raise InvalidPackageError
if "dependencies" in bower_json:
for package, version in bower_json["dependencies"].items():
url = Bower.get_package_url(package)
deps_installed.extend(self.get_bower_package(
url, dest=dest, version=version))
ignore_patterns = [GitIgnorePattern(ig) for ig in bower_json["ignore"]]
path_spec = PathSpec(ignore_patterns)
namelist = [path for path in zip_file.namelist()
if PurePath(path).parts[0] == root]
ignored = list(path_spec.match_files(namelist))
for path in namelist:
dest_path = PurePath(
bower_json["name"],
*PurePath(path).parts[1:])
if path in ignored:
continue
for path in ignored:
for parent in PurePath(path):
if parent in ignored:
continue
if path.endswith("/"):
if list(path_spec.match_files([str(dest_path)])):
ignored.append(PurePath(path))
elif not (dest / dest_path).is_dir():
(dest / dest_path).mkdir(parents=True)
else:
target_path = dest / dest_path.parent / dest_path.name
source = zip_file.open(path)
target = target_path.open("wb")
with source, target:
shutil.copyfileobj(source, target)
deps_installed.append((bower_json["name"], bower_json["version"]))
return deps_installed
def extract_bower_zipfile(self, zip_file, dest, expected_version=None):
bower_json = None
root = None
deps_installed = []
for info in zip_file.infolist():
if PurePath(info.filename).name == "bower.json":
with zip_file.open(info) as f:
bower_json = json.load(f)
root = str(PurePath(info.filename).parent)
break
version = bower_json["version"]
if expected_version is not None:
expected_version = Bower.clean_semver(expected_version)
if not semver.match(version, expected_version):
click.secho("error: versions do not match ({} =/= {})".format(
version, expected_version))
raise InvalidPackageError
if "dependencies" in bower_json:
for package, version in bower_json["dependencies"].items():
url = Bower.get_package_url(package)
deps_installed.extend(
self.get_bower_package(url, dest=dest, version=version))
ignore_patterns = [GitIgnorePattern(ig) for ig in bower_json["ignore"]]
path_spec = PathSpec(ignore_patterns)
namelist = [
path for path in zip_file.namelist()
if PurePath(path).parts[0] == root
]
ignored = list(path_spec.match_files(namelist))
for path in namelist:
dest_path = PurePath(bower_json["name"], *PurePath(path).parts[1:])
if path in ignored:
continue
for path in ignored:
for parent in PurePath(path):
if parent in ignored:
continue
if path.endswith("/"):
if list(path_spec.match_files([str(dest_path)])):
ignored.append(PurePath(path))
elif not (dest / dest_path).is_dir():
(dest / dest_path).mkdir(parents=True)
else:
target_path = dest / dest_path.parent / dest_path.name
source = zip_file.open(path)
target = target_path.open("wb")
with source, target:
shutil.copyfileobj(source, target)
deps_installed.append((bower_json["name"], bower_json["version"]))
return deps_installed
def reload_ignore_config():
global spec
with reload_lock:
try:
with open('.reloadignore', 'r') as fh:
spec = PathSpec.from_lines('gitwildmatch', fh)
except IOError as e:
if e.errno == 2:
logger.info("'.reloadignore' not found. Using default spec.")
spec = PathSpec.from_lines('gitwildmatch', DEFAULT_RELOADIGNORE.splitlines())
else:
raise
def _add_spec_excludes_to_build_ignore_patterns(build_root,
build_ignore_patterns=None,
spec_excludes=None):
if not build_ignore_patterns:
build_ignore_patterns = PathSpec.from_lines(GitIgnorePattern, [])
patterns = list(build_ignore_patterns.patterns)
patterns.extend(
PathSpec.from_lines(
GitIgnorePattern,
BuildFile._spec_excludes_to_gitignore_syntax(
build_root, spec_excludes)).patterns)
return PathSpec(patterns)
class DirectoryWatcher:
"""Iterator that detects and yields file changes by polling the filesystem."""
path: FileSystemPath
interval: float = 0.6
ignore: PathSpec = field(init=False)
ignore_file: Optional[FileSystemPath] = extra_field(default=None)
ignore_patterns: Sequence[str] = extra_field(default=())
files: Dict[str, float] = extra_field(init=False, default_factory=dict)
def __post_init__(self):
self.ignore_patterns = list(self.ignore_patterns)
if self.ignore_file:
ignore_file = Path(self.ignore_file)
if ignore_file.parts == (ignore_file.name,):
for directory in Path(self.path, ignore_file).parents:
if (path := (directory / ignore_file)).is_file():
ignore_file = path
break
if ignore_file.is_file():
self.ignore_patterns += [
pattern
for line in ignore_file.read_text().splitlines()
if not line.startswith("#") and (pattern := line.strip())
]
self.ignore = PathSpec.from_lines("gitwildmatch", self.ignore_patterns)
def iter_files(
paths: Iterable[Path],
include: Pattern[str],
exclude: Pattern[str],
gitignore: PathSpec,
) -> Generator[Path, None, None]:
"""
Iterate through all files matching given parameters.
Highly influenced by Black (https://github.com/psf/black).
"""
for child in paths:
normalized = child.relative_to(ROOT_PATH).as_posix()
if gitignore.match_file(normalized):
continue
normalized = f"/{normalized}"
if child.is_dir():
normalized += "/"
exclude_match = exclude.search(normalized)
if exclude_match is not None and exclude_match.group(0):
continue
if child.is_dir():
yield from iter_files(child.iterdir(), include, exclude, gitignore)
elif child.is_file():
if include.search(normalized) is not None:
yield child
def walk_not_git_ignored(path, keep, extra_ignore):
spec = PathSpec.from_lines("gitwildmatch", [".git"] + extra_ignore)
ignores = {}
# detect git folder, collect ignores up to root
at = path
while True:
git_exist = (at / ".git").exists()
if git_exist:
ignores[at.parent] = spec
# go down back and load all ignores
for part in (".", ) + path.relative_to(at).parts:
at = at / part
spec = _load_ignore(at, spec, ignores)
break
if at == at.parent:
ignores[path] = spec
break
at = at.parent
# now walk from root, collect new ignores and evaluate
for root, dirs, files in os.walk(str(path)):
root_path = Path(root).relative_to(path) # current path
current_path = path / root
parent_spec = ignores.get(current_path) or next(
ignores[p] for p in current_path.parents if p in ignores)
spec = _load_ignore(Path(root), parent_spec, ignores)
for file_name in files:
result = root_path / file_name
if (file_name != ".gitignore" and keep(result)
and not spec.match_file(str(result))):
yield path / result
for cur_dir in list(dirs):
if spec.match_file(str(root_path / cur_dir)):
dirs.remove(cur_dir)
def __init__(self,
parser,
build_patterns=None,
build_ignore_patterns=None,
exclude_target_regexps=None,
subproject_roots=None):
"""Create an AddressMapper.
Both the set of files that define a mappable BUILD files and the parser used to parse those
files can be customized. See the `pants.engine.parsers` module for example parsers.
:param parser: The BUILD file parser to use.
:type parser: An instance of :class:`pants.engine.parser.Parser`.
:param tuple build_patterns: A tuple of fnmatch-compatible patterns for identifying BUILD files
used to resolve addresses.
:param list build_ignore_patterns: A list of path ignore patterns used when searching for BUILD files.
:param list exclude_target_regexps: A list of regular expressions for excluding targets.
"""
self.parser = parser
self.build_patterns = build_patterns or (b'BUILD', b'BUILD.*')
self.build_ignore_patterns = PathSpec.from_lines(
GitWildMatchPattern, build_ignore_patterns or [])
self._exclude_target_regexps = exclude_target_regexps or []
self.exclude_patterns = [
re.compile(pattern) for pattern in self._exclude_target_regexps
]
self.subproject_roots = subproject_roots or []
def get_snakefiles_in_dir(
path: Path, include: Pattern[str], exclude: Pattern[str], gitignore: PathSpec
) -> Iterator[Path]:
"""Generate all files under `path` whose paths are not excluded by the
`exclude` regex, but are included by the `include` regex.
Adapted from
https://github.com/psf/black/blob/ce14fa8b497bae2b50ec48b3bd7022573a59cdb1/black.py#L3519-L3573
"""
for child in path.iterdir():
# First ignore files matching .gitignore
if gitignore.match_file(child.as_posix()):
logging.debug(f"Ignoring: {child} matches .gitignore file content")
continue
# Then ignore with `exclude` option.
normalized_path = str(child.resolve().as_posix())
exclude_match = exclude.search(normalized_path)
if exclude_match and exclude_match.group(0):
logging.debug(f"Excluded: {child} matched the --exclude regular expression")
continue
if child.is_dir():
yield from get_snakefiles_in_dir(child, include, exclude, gitignore)
elif child.is_file():
include_match = include.search(child.name)
if include_match:
logging.debug(
f"Included: {child} matched the --include regular expression"
)
yield child
else:
logging.debug(
f"Ignoring: {child} did not match the --include regular expression"
)
def __init__(self,
symbol_table_cls,
parser_cls,
build_pattern=None,
build_ignore_patterns=None,
exclude_target_regexps=None):
"""Create an AddressMapper.
Both the set of files that define a mappable BUILD files and the parser used to parse those
files can be customized. See the `pants.engine.parsers` module for example parsers.
:param symbol_table_cls: The symbol table cls to expose a symbol table dict.
:type symbol_table_cls: A :class:`pants.engine.parser.SymbolTable`.
:param parser_cls: The BUILD file parser cls to use.
:type parser_cls: A :class:`pants.engine.parser.Parser`.
:param string build_pattern: A fnmatch-compatible pattern for identifying BUILD files used
to resolve addresses; by default looks for `BUILD*` files.
:param list build_ignore_patterns: A list of path ignore patterns used when searching for BUILD files.
:param list exclude_target_regexps: A list of regular expressions for excluding targets.
"""
self.symbol_table_cls = symbol_table_cls
self.parser_cls = parser_cls
self.build_pattern = build_pattern or 'BUILD*'
self.build_ignore_patterns = PathSpec.from_lines(
GitIgnorePattern, build_ignore_patterns or [])
self._exclude_target_regexps = exclude_target_regexps or []
self.exclude_patterns = [
re.compile(pattern) for pattern in self._exclude_target_regexps
]
def configure_project(self, targets, debug_port):
jvm_targets = [t for t in targets if t.has_label('jvm') or t.has_label('java') or
isinstance(t, Resources)]
if self.intransitive:
jvm_targets = set(self.context.target_roots).intersection(jvm_targets)
build_ignore_patterns = self.context.options.for_global_scope().build_ignore or []
project = Project(self.project_name,
self.python,
self.skip_java,
self.skip_scala,
self.use_source_root,
get_buildroot(),
debug_port,
self.context,
jvm_targets,
not self.intransitive,
self.TargetUtil(self.context),
PathSpec.from_lines(GitIgnorePattern, build_ignore_patterns))
if self.python:
python_source_paths = self.get_options().python_source_paths
python_test_paths = self.get_options().python_test_paths
python_lib_paths = self.get_options().python_lib_paths
project.configure_python(python_source_paths, python_test_paths, python_lib_paths)
extra_source_paths = self.get_options().extra_jvm_source_paths
extra_test_paths = self.get_options().extra_jvm_test_paths
all_targets = project.configure_jvm(extra_source_paths, extra_test_paths)
return all_targets, project
def __init__(self,
symbol_table_cls,
parser_cls,
build_patterns=None,
build_ignore_patterns=None,
exclude_target_regexps=None,
subproject_roots=None):
"""Create an AddressMapper.
Both the set of files that define a mappable BUILD files and the parser used to parse those
files can be customized. See the `pants.engine.parsers` module for example parsers.
:param symbol_table_cls: The symbol table cls to expose a symbol table dict.
:type symbol_table_cls: A :class:`pants.engine.parser.SymbolTable`.
:param parser_cls: The BUILD file parser cls to use.
:type parser_cls: A :class:`pants.engine.parser.Parser`.
:param tuple build_patterns: A tuple of fnmatch-compatible patterns for identifying BUILD files
used to resolve addresses.
:param list build_ignore_patterns: A list of path ignore patterns used when searching for BUILD files.
:param list exclude_target_regexps: A list of regular expressions for excluding targets.
"""
self.symbol_table_cls = symbol_table_cls
self.parser_cls = parser_cls
self.build_patterns = build_patterns or (b'BUILD', b'BUILD.*')
self.build_ignore_patterns = PathSpec.from_lines(GitWildMatchPattern, build_ignore_patterns or [])
self._exclude_target_regexps = exclude_target_regexps or []
self.exclude_patterns = [re.compile(pattern) for pattern in self._exclude_target_regexps]
self.subproject_roots = subproject_roots or []
def configure_project(self, targets, debug_port):
jvm_targets = [
t for t in targets
if isinstance(t, (JvmTarget, JarLibrary, Resources))
]
if self.intransitive:
jvm_targets = set(
self.context.target_roots).intersection(jvm_targets)
build_ignore_patterns = self.context.options.for_global_scope(
).build_ignore or []
project = Project(
self.project_name, self.python,
self.skip_java, self.skip_scala, self.use_source_root,
get_buildroot(), debug_port, self.context, jvm_targets,
not self.intransitive, self.TargetUtil(self.context),
PathSpec.from_lines(GitWildMatchPattern, build_ignore_patterns))
if self.python:
python_source_paths = self.get_options().python_source_paths
python_test_paths = self.get_options().python_test_paths
python_lib_paths = self.get_options().python_lib_paths
project.configure_python(python_source_paths, python_test_paths,
python_lib_paths)
extra_source_paths = self.get_options().extra_jvm_source_paths
extra_test_paths = self.get_options().extra_jvm_test_paths
all_targets = project.configure_jvm(extra_source_paths,
extra_test_paths)
return all_targets, project
def get_pathspec(directory):
ignore_file = get_gitignore_file(directory)
if not ignore_file:
return _null_spec
with open(ignore_file) as f:
spec = PathSpec.from_lines('gitwildmatch', f.readlines())
return spec
def gen_python_files(
paths: Iterable[Path],
root: Path,
gitignore: PathSpec,
) -> Iterator[Path]:
"""Generate all files under `paths`.
Files listed in .gitignore are not considered.
"""
for child in paths:
normalized_path = normalize_path(child, root)
if normalized_path is None:
continue
# First ignore files matching .gitignore
if gitignore.match_file(normalized_path):
continue
if child.is_dir():
yield from gen_python_files(
child.iterdir(),
root,
gitignore,
)
elif child.is_file() and str(child).endswith(".py"):
yield child
def get_files_in_dir(path: Path, root: Path, gitignore: PathSpec) -> List[Path]:
assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}"
for child in path.iterdir():
# First ignore files matching .gitignore
if gitignore.match_file(child.as_posix()):
continue
# Then ignore with `exclude` option.
try:
normalized_path = "/" + child.resolve().relative_to(root).as_posix()
except OSError:
continue
except ValueError:
if child.is_symlink():
continue
raise
if child.is_dir():
normalized_path += "/"
if child.is_dir():
yield from get_files_in_dir(child, root, gitignore)
elif child.is_file():
yield child
def file_indexer(queue, path, ignore_patterns, max_files):
from pathspec import PathSpec
def _list_files(dirname):
files = []
try:
basenames = os.listdir(dirname)
except Exception:
pass
else:
for basename in basenames:
path = os.path.join(dirname, basename)
if ignore_spec.match_file(path):
continue
if os.path.isdir(path):
files += _list_files(path)
else:
files.append(path)
if len(files) > max_files:
raise ValueError(u'Too many files')
return files
ignore_spec = PathSpec.from_lines('gitwildmatch', ignore_patterns)
try:
files = _list_files(path)
except ValueError:
files = None
queue.put(files)
def configure_project(self, targets, debug_port):
jvm_targets = [t for t in targets if t.has_label('jvm') or t.has_label('java') or
isinstance(t, Resources)]
if self.intransitive:
jvm_targets = set(self.context.target_roots).intersection(jvm_targets)
build_ignore_patterns = self.context.options.for_global_scope().ignore_patterns or []
build_ignore_patterns.extend(BuildFile._spec_excludes_to_gitignore_syntax(
os.path.realpath(get_buildroot()), self.context.options.for_global_scope().spec_excludes))
project = Project(self.project_name,
self.python,
self.skip_java,
self.skip_scala,
self.use_source_root,
get_buildroot(),
debug_port,
self.context,
jvm_targets,
not self.intransitive,
self.TargetUtil(self.context),
None,
PathSpec.from_lines(GitIgnorePattern, build_ignore_patterns))
if self.python:
python_source_paths = self.get_options().python_source_paths
python_test_paths = self.get_options().python_test_paths
python_lib_paths = self.get_options().python_lib_paths
project.configure_python(python_source_paths, python_test_paths, python_lib_paths)
extra_source_paths = self.get_options().extra_jvm_source_paths
extra_test_paths = self.get_options().extra_jvm_test_paths
all_targets = project.configure_jvm(extra_source_paths, extra_test_paths)
return all_targets, project
def post_order_lexicographic(top: str, ignore_pathspec: pathspec.PathSpec = None):
"""
iterates a file system in the order necessary to generate composite tree hashes, bypassing ignored paths.
:param top: the directory being iterated
:param ignore_pathspec: the pathspec of ignore patterns to match file exclusions against
:return: yields results in folder chunks, in the order necessary for composite directory hashes
"""
# create a sorted list of our immediate children
names = os.listdir(top)
names.sort()
# list of tuples. each tuple contains the child name and whether the child is a directory.
children = []
for name in names:
file_path = os.path.join(top, name)
if ignore_pathspec and ignore_pathspec.match_file(file_path):
if os.path.basename(os.path.normpath(file_path)) != ascmhl_folder_name:
logger.verbose(f'ignoring filepath {file_path}')
continue
path = join(top, name)
children.append((name, isdir(path)))
# if directory, yield children recursively in post order until exhausted.
for name, is_dir in children:
if is_dir:
path = join(top, name)
if not os.path.islink(path):
for x in post_order_lexicographic(path, ignore_pathspec):
yield x
# now that all children have been traversed, yield the top (current) directory and all of it's sorted children.
yield top, children
def __init__(self, ignore_file_path, tree):
assert os.path.isabs(ignore_file_path)
self.ignore_file_path = ignore_file_path
self.dirname = os.path.normpath(os.path.dirname(ignore_file_path))
with tree.open(ignore_file_path, encoding="utf-8") as fobj:
self.ignore_spec = PathSpec.from_lines(GitWildMatchPattern, fobj)
def get_gitignore(root):
"""Return a PathSpec matching gitignore content if present."""
gitignore = root / ".gitignore"
lines = []
if gitignore.is_file():
with gitignore.open(encoding="utf-8") as gf:
lines = gf.readlines()
return PathSpec.from_lines("gitwildmatch", lines)
def get_gitignore(root: Path) -> PathSpec:
""" Return a PathSpec matching gitignore content if present."""
gitignore = root / ".gitignore"
lines: List[str] = []
if gitignore.is_file():
with gitignore.open() as gf:
lines = gf.readlines()
return PathSpec.from_lines("gitwildmatch", lines)
def mock_dvcignore(dvcignore_path, patterns):
mock_ignore_file_handler = Mock()
with patch.object(
mock_ignore_file_handler,
"read_patterns",
return_value=PathSpec.from_lines(GitWildMatchPattern, patterns),
):
ignore_file = DvcIgnoreFromFile(dvcignore_path,
mock_ignore_file_handler)
return ignore_file
def _get_match_filter(dir_abspath, ignore, **kwargs):
"""Helper to construct a function for filtering of paths.
"""
ignore = [] if ignore is None else list(ignore)
ignore = _parse_ignorefile(dir_abspath) + ignore
match_spec = _get_match_spec(ignore=ignore, **kwargs)
path_spec = PathSpec.from_lines(GitWildMatchPattern, match_spec)
return path_spec.match_files
def gen_template_files_in_dir(
path: Path,
root: Path,
include: Pattern[str],
exclude: Pattern[str],
report: "Report",
gitignore: PathSpec,
) -> Iterator[Path]:
"""Generate all files under `path` whose paths are not excluded by the
`exclude` regex, but are included by the `include` regex.
Symbolic links pointing outside of the `root` directory are ignored.
`report` is where output about exclusions goes.
"""
assert (root.is_absolute()
), f"INTERNAL ERROR: `root` must be absolute but is {root}"
for child in path.iterdir():
# First ignore files matching .gitignore
if gitignore.match_file(child.as_posix()):
report.path_ignored(child, "matches the .gitignore file content")
continue
# Then ignore with `exclude` option.
try:
normalized_path = "/" + child.resolve().relative_to(
root).as_posix()
except OSError as e:
report.path_ignored(child, f"cannot be read because {e}")
continue
except ValueError:
if child.is_symlink():
report.path_ignored(
child, f"is a symbolic link that points outside {root}")
continue
raise
if child.is_dir():
normalized_path += "/"
exclude_match = exclude.search(normalized_path)
if exclude_match and exclude_match.group(0):
report.path_ignored(child,
"matches the --exclude regular expression")
continue
if child.is_dir():
yield from gen_template_files_in_dir(child, root, include, exclude,
report, gitignore)
elif child.is_file():
include_match = include.search(normalized_path)
if include_match:
yield child
def gitignore(self):
"""Load a repo's .gitignore file for path matching usage."""
path = pjoin(self.options.target_repo.location, '.gitignore')
patterns = []
try:
with open(path) as f:
patterns = f.readlines()
except FileNotFoundError:
pass
except IOError as e:
logger.warning(f'failed reading {path!r}: {e}')
return PathSpec.from_lines('gitwildmatch', patterns)
def get_backup_filepaths(self, dpath):
"""Return filtered list of file paths."""
fname = os.path.join(self.BASE_DIR, dpath, 'backup.paths')
if not os.path.exists(fname):
return
backup_spec = open(fname).read()
spec = PathSpec.from_lines(GitWildMatchPattern,
backup_spec.splitlines())
return [
os.path.join(dpath, p)
for p in spec.match_tree(os.path.join(self.BASE_DIR, dpath))
]
def gitignore(self):
"""Load a repo's .gitignore and .git/info/exclude files for path matching."""
patterns = []
for path in ('.gitignore', '.git/info/exclude'):
try:
with open(pjoin(self.options.target_repo.location, path)) as f:
patterns.extend(f)
except FileNotFoundError:
pass
except IOError as e:
logger.warning(f'failed reading {path!r}: {e}')
return PathSpec.from_lines('gitwildmatch', patterns)
def _apply_exclude_patterns(names, exclude_filter):
"""Exclude matched patterns from passed names."""
included = set(names)
# Assume old way for easier testing
if hasattr(exclude_filter, '__iter__'):
exclude_filter = PathSpec.from_lines('gitwildmatch', exclude_filter)
for excluded in exclude_filter.match_files(names):
included.discard(excluded)
return sorted(included)
def get_gitignore(root: Path) -> PathSpec:
"""Return a PathSpec matching gitignore content if present."""
gitignore = root / ".gitignore"
lines: List[str] = []
if gitignore.is_file():
with gitignore.open(encoding="utf-8") as gf:
lines = gf.readlines()
try:
return PathSpec.from_lines("gitwildmatch", lines)
except GitWildMatchPatternError as e:
err(f"Could not parse {gitignore}: {e}")
raise
def __init__(self, build_file_parser, project_tree, build_ignore_patterns=None, exclude_target_regexps=None):
"""Create a BuildFileAddressMapper.
:param build_file_parser: An instance of BuildFileParser
:param build_file_type: A subclass of BuildFile used to construct and cache BuildFile objects
"""
self._build_file_parser = build_file_parser
self._spec_path_to_address_map_map = {} # {spec_path: {address: addressable}} mapping
self._project_tree = project_tree
self._build_ignore_patterns = PathSpec.from_lines(GitIgnorePattern, build_ignore_patterns or [])
self._exclude_target_regexps = exclude_target_regexps or []
self._exclude_patterns = [re.compile(pattern) for pattern in self._exclude_target_regexps]
def get_ebignore_list():
location = get_ebignore_location()
if not os.path.isfile(location):
return None
with codecs.open(location, 'r', encoding='utf-8') as f:
spec = PathSpec.from_lines('gitwildmatch', f)
ignore_list = [f for f in spec.match_tree(get_project_root())]
ignore_list.append('.ebignore')
return ignore_list
def __init__(self, build_file_parser, project_tree, build_ignore_patterns=None):
"""Create a BuildFileAddressMapper.
:param build_file_parser: An instance of BuildFileParser
:param build_file_type: A subclass of BuildFile used to construct and cache BuildFile objects
"""
self._build_file_parser = build_file_parser
self._spec_path_to_address_map_map = {} # {spec_path: {address: addressable}} mapping
if isinstance(project_tree, ProjectTree):
self._project_tree = project_tree
else:
# If project_tree is BuildFile class actually.
# TODO(tabishev): Remove after transition period.
self._project_tree = project_tree._get_project_tree(self.root_dir)
self._build_ignore_patterns = PathSpec.from_lines(GitIgnorePattern, build_ignore_patterns or [])
def patch_files_walk(repo_top, path, ignored):
"""Yields string for every valid patch file in [path] whose file name does
not match the wildmatch patterns in [ignored], treated relative to
[repo_top]. If another `thcrap_ignore.txt` is found along the directory
hierarchy, its contents are added to a copy of [ignored], which is then
used for this directory and its subdirectories."""
local_ignore = thcrap_ignore_get(path)
if len(local_ignore) >= 1:
ignored = set(ignored).union(local_ignore)
spec = PathSpec.from_lines('gitwildmatch', ignored)
for i in os.scandir(path):
if spec.match_file(os.path.relpath(i.path, repo_top)) == False:
if i.is_dir():
yield from patch_files_walk(repo_top, i.path, ignored)
else:
yield i.path
def configure_project(self, targets, debug_port):
jvm_targets = [t for t in targets if t.has_label("jvm") or t.has_label("java") or isinstance(t, Resources)]
if self.intransitive:
jvm_targets = set(self.context.target_roots).intersection(jvm_targets)
build_ignore_patterns = self.context.options.for_global_scope().ignore_patterns or []
build_ignore_patterns.extend(
BuildFile._spec_excludes_to_gitignore_syntax(
os.path.realpath(get_buildroot()), self.context.options.for_global_scope().spec_excludes
)
)
project = Project(
self.project_name,
self.python,
self.skip_java,
self.skip_scala,
self.use_source_root,
get_buildroot(),
debug_port,
self.context,
jvm_targets,
not self.intransitive,
self.TargetUtil(self.context),
None,
PathSpec.from_lines(GitIgnorePattern, build_ignore_patterns),
)
if self.python:
python_source_paths = self.get_options().python_source_paths
python_test_paths = self.get_options().python_test_paths
python_lib_paths = self.get_options().python_lib_paths
project.configure_python(python_source_paths, python_test_paths, python_lib_paths)
extra_source_paths = self.get_options().extra_jvm_source_paths
extra_test_paths = self.get_options().extra_jvm_test_paths
all_targets = project.configure_jvm(extra_source_paths, extra_test_paths)
return all_targets, project
def _create_ignore_spec(self, build_ignore_patterns): return PathSpec.from_lines(GitIgnorePattern, build_ignore_patterns or [])
def ignore(*args):
return PathSpec.from_lines('gitwildmatch', args)