def regenerate(self, bazel_targets: Iterable[str], cwd: str = ".") -> None:
targets = bazel_utils.expand_bazel_target_dirs(
self.workspace_dir,
[t for t in bazel_targets if not t.startswith("@")],
require_build_file=False,
cwd=cwd,
)
for target in targets:
assert target.startswith(
"//"), "Target must be absolute: " + target
pkg, _, _ = target.partition(":")
pkg_path = bazel_utils.normalize_relative_target_to_os_path(
pkg[2:])
target_dir = os.path.join(self.workspace_dir, pkg_path)
if target_dir in self.visited_dirs:
continue
self.visited_dirs.add(target_dir)
if not os.path.exists(os.path.join(target_dir, "BUILD.in")):
continue
if target_dir in self.generated_files:
continue
if self.dry_run:
continue
out = os.path.join(target_dir, "BUILD.gen_empty")
open(out, "w").close()
self.generated_files[target_dir].append(out)
def maybe_traverse_non_bzl(self, expanded_target):
if self.skip_deps_generation:
return
pkg, _, name = expanded_target.partition(":")
if not name:
name = os.path.basename(pkg)
expanded_target = pkg + ":" + name
if expanded_target in self.visited_non_bzl_targets:
return
self.visited_non_bzl_targets.add(expanded_target)
# Note that expanded_target is guaranteed to be an absolute target.
pkg_path = bazel_utils.normalize_relative_target_to_os_path(pkg[2:])
_, parsed = self.parsed_cache.get_build(
os.path.join(self.workspace_dir, pkg_path))
if parsed is None:
return
try:
rule = parsed.get_rule(name)
except KeyError:
return
self.regenerate(
build_parser.maybe_expand_attribute(rule.attr_map.get("deps", [])),
cwd=os.path.join(self.workspace_dir, pkg_path),
)
def _collect_local_targets(self, pkg, parsed):
path_prefix = pkg[2:]
pkg_path = bazel_utils.normalize_relative_target_to_os_path(
path_prefix)
# NOTE: We purposely ignore dbx_py_binary targets, even through it's
# technically allowed to use dbx_py_binary as deps.
for lib in parsed.get_rules_by_types(PY_LIBRARY_RULE_TYPES):
name = lib.attr_map["name"]
target = pkg + ":" + name
srcs = build_parser.maybe_expand_attribute(
lib.attr_map.get("srcs", []))
for src in srcs:
assert src.endswith(
".py"), "Invalid python src %s in %s" % (src, pkg)
file_path = os.path.join(pkg_path, src)
# Clip the file_path relative to the python_path if applicable.
if self.python_path:
if file_path == self.python_path:
file_path = file_path[len(self.python_path):]
elif file_path.startswith(self.python_path + "/"):
file_path = file_path[len(self.python_path) + 1:]
module_path = PythonPathMapping.convert_from_file_path_to_module(
file_path)
if module_path in self.local_module_targets:
other, other_size = self.local_module_targets[module_path]
other_pkg, _, other_name = other.partition(":")
if not other_name:
other_name = os.path.basename(other_pkg)
# Use the target with the more specific pkg path, or the
# target with the least srcs
overwrite = False
if len(other_pkg) < len(pkg):
overwrite = True
elif other_pkg == pkg:
if other_size > len(srcs):
overwrite = True
elif (other_size == len(srcs)
and os.path.basename(other_pkg) == other_name):
overwrite = True # use the most specific target name
if overwrite:
self.local_module_targets[module_path] = (target,
len(srcs))
print(("WARNING: Module %s specified in multiple targets: "
"%s vs %s (autogen_deps may pick the incorrect "
"target)") % (module_path, other, target))
continue
self.local_module_targets[module_path] = (target, len(srcs))
def regenerate(self, bazel_targets: Iterable[str], cwd: str = ".") -> None:
targets = bazel_utils.expand_bazel_target_dirs(
self.workspace_dir,
[t for t in bazel_targets if not t.startswith("@")],
require_build_file=False,
cwd=cwd,
)
for target in targets:
assert target.startswith(
"//"), "Target must be absolute: " + target
target_dir = bazel_utils.normalize_relative_target_to_os_path(
target[2:])
if target_dir in self.visited_dirs:
continue
self.visited_dirs.add(target_dir)
build_bzl = os.path.join(self.workspace_dir, target_dir,
BUILD_INPUT)
if not os.path.isfile(build_bzl):
continue
parsed = build_parser.parse_file(build_bzl)
pip_rules = parsed.get_rules_by_types(PIP_GEN_RULE_TYPES)
if not pip_rules:
if self.verbose:
print("No pip targets found in %s/%s" %
(target_dir, BUILD_INPUT))
continue
if not self.skip_deps_generation:
for rule in pip_rules:
self.regenerate(
build_parser.maybe_expand_attribute(
rule.attr_map.get("deps", [])),
cwd=os.path.join(self.workspace_dir, target_dir),
)
if self.verbose:
head = "(dry run) " if self.dry_run else ""
print(
head + "Processing pip targets in %s: %s" %
(target_dir, [rule.attr_map["name"]
for rule in pip_rules]))
if self.dry_run:
continue
self.process_pip_rules(target_dir, pip_rules)
def compute_self_modules(self, pkg, srcs):
target_dir = pkg[2:]
trim_prefix = 0
if self.python_path and target_dir.startswith(self.python_path):
trim_prefix = len(self.python_path) + 1
pkg_path = bazel_utils.normalize_relative_target_to_os_path(target_dir)
self_modules = set()
for src in srcs:
file_path = os.path.join(pkg_path, src)
module_path = PythonPathMapping.convert_from_file_path_to_module(file_path)
self_modules.add(module_path[trim_prefix:])
return self_modules
def regenerate(self, bazel_targets: Iterable[str], cwd: str = ".") -> None:
targets = bazel_utils.expand_bazel_targets(
self.workspace_dir,
[t for t in bazel_targets if not t.startswith("@")],
require_build_file=False,
cwd=cwd,
)
for target in targets:
assert target.startswith("//"), "Target must be absolute: " + target
pkg, _, _ = target.partition(":")
target_dir = bazel_utils.normalize_relative_target_to_os_path(pkg[2:])
_, parsed = self.parsed_cache.get_bzl(
os.path.join(self.workspace_dir, target_dir)
)
if not parsed:
self.maybe_traverse_non_bzl(target)
continue
if pkg in self.visited_bzl_dirs:
continue
self.visited_bzl_dirs.add(pkg)
py_rules = parsed.get_rules_by_types(PY_RULE_TYPES)
if not py_rules:
if self.cfg.verbose:
print("No py targets found in %s:%s" % (pkg, BUILD_INPUT))
continue
if self.cfg.verbose:
head = "(dry run) " if self.cfg.dry_run else ""
print(
head
+ "Processing py targets in %s: %s"
% (pkg, [rule.attr_map["name"] for rule in py_rules])
)
if self.cfg.dry_run:
continue
self.generate_build_file(pkg, py_rules)
def compute_deps(
self,
python_path,
pkg,
rule_type,
name,
srcs,
stub_srcs,
main,
pip_main,
validate,
is_py3_compatible,
):
srcs = (srcs or []) + (stub_srcs or [])
if main:
srcs = srcs + [main]
mapping = self.python_path_mappings.get(python_path)
self_modules = mapping.compute_self_modules(pkg, srcs)
target_dir = bazel_utils.normalize_relative_target_to_os_path(pkg[2:])
all_deps = set() # type: ignore[var-annotated]
all_unknown_imports = set() # type: ignore[var-annotated]
all_unknown_froms = set() # type: ignore[var-annotated]
for src in set(srcs):
src = os.path.join(target_dir, src)
module_path = PythonPathMapping.convert_from_file_path_to_module(
src)
filename, parsed = mapping.find_closest_bzl_or_build(module_path)
if not filename:
raise bazel_utils.BazelError(
"Cannot locate %s:%s's source (or its closest BUILD / "
"BUILD.in file): %s/%s" % (pkg, name, target_dir, src))
pkg_path = os.path.dirname(filename).replace(
self.workspace_dir, "/")
src_pkg = bazel_utils.normalize_os_path_to_target(pkg_path)
if src_pkg != pkg:
print(("WARNING: Skipping %s from %s:%s deps computation "
"since it belongs to %s") % (src, pkg, name, src_pkg))
continue
import_set, from_set = parse_imports(
self.workspace_dir,
src,
py3_compatible=is_py3_compatible or src.endswith(".pyi"),
)
import_deps, unknown_imports = mapping.find_import_targets(
src_pkg, self_modules, import_set)
all_deps.update(import_deps)
all_unknown_imports.update(unknown_imports)
if validate:
assert not unknown_imports, (
"Unable to locate modules %s (imported by %s) in any "
"library target (NOTE: bin and test targets are "
"ignored)") % (unknown_imports, src)
from_deps, unknown_froms = mapping.find_from_targets(
src_pkg, self_modules, from_set)
all_deps.update(from_deps)
all_unknown_froms.update(unknown_froms)
if validate:
assert not unknown_froms, (
"Unable to locate modules %s (imported by %s) in any "
"library target (NOTE: bin and test targets are "
"ignored)") % (unknown_froms, src)
import_deps, unknown_imports = mapping.find_import_targets(
pkg, self_modules, [])
all_deps.update(import_deps)
all_unknown_imports.update(unknown_imports)
if pip_main:
all_deps.add(pip_main)
all_deps.discard("%s:%s" % (pkg, name))
if name == os.path.basename(target_dir):
all_deps.discard("%s" % pkg)
return sort_deps(pkg, all_deps), all_unknown_imports, all_unknown_froms
