def regenerate(self, bazel_targets: Iterable[str], cwd: str = ".") -> None: targets = bazel_utils.expand_bazel_target_dirs( self.workspace_dir, [t for t in bazel_targets if not t.startswith("@")], require_build_file=False, cwd=cwd, ) for target in targets: assert target.startswith( "//"), "Target must be absolute: " + target pkg, _, _ = target.partition(":") pkg_path = bazel_utils.normalize_relative_target_to_os_path( pkg[2:]) target_dir = os.path.join(self.workspace_dir, pkg_path) if target_dir in self.visited_dirs: continue self.visited_dirs.add(target_dir) if not os.path.exists(os.path.join(target_dir, "BUILD.in")): continue if target_dir in self.generated_files: continue if self.dry_run: continue out = os.path.join(target_dir, "BUILD.gen_empty") open(out, "w").close() self.generated_files[target_dir].append(out)
def maybe_traverse_non_bzl(self, expanded_target): if self.skip_deps_generation: return pkg, _, name = expanded_target.partition(":") if not name: name = os.path.basename(pkg) expanded_target = pkg + ":" + name if expanded_target in self.visited_non_bzl_targets: return self.visited_non_bzl_targets.add(expanded_target) # Note that expanded_target is guaranteed to be an absolute target. pkg_path = bazel_utils.normalize_relative_target_to_os_path(pkg[2:]) _, parsed = self.parsed_cache.get_build( os.path.join(self.workspace_dir, pkg_path)) if parsed is None: return try: rule = parsed.get_rule(name) except KeyError: return self.regenerate( build_parser.maybe_expand_attribute(rule.attr_map.get("deps", [])), cwd=os.path.join(self.workspace_dir, pkg_path), )
def _collect_local_targets(self, pkg, parsed): path_prefix = pkg[2:] pkg_path = bazel_utils.normalize_relative_target_to_os_path( path_prefix) # NOTE: We purposely ignore dbx_py_binary targets, even through it's # technically allowed to use dbx_py_binary as deps. for lib in parsed.get_rules_by_types(PY_LIBRARY_RULE_TYPES): name = lib.attr_map["name"] target = pkg + ":" + name srcs = build_parser.maybe_expand_attribute( lib.attr_map.get("srcs", [])) for src in srcs: assert src.endswith( ".py"), "Invalid python src %s in %s" % (src, pkg) file_path = os.path.join(pkg_path, src) # Clip the file_path relative to the python_path if applicable. if self.python_path: if file_path == self.python_path: file_path = file_path[len(self.python_path):] elif file_path.startswith(self.python_path + "/"): file_path = file_path[len(self.python_path) + 1:] module_path = PythonPathMapping.convert_from_file_path_to_module( file_path) if module_path in self.local_module_targets: other, other_size = self.local_module_targets[module_path] other_pkg, _, other_name = other.partition(":") if not other_name: other_name = os.path.basename(other_pkg) # Use the target with the more specific pkg path, or the # target with the least srcs overwrite = False if len(other_pkg) < len(pkg): overwrite = True elif other_pkg == pkg: if other_size > len(srcs): overwrite = True elif (other_size == len(srcs) and os.path.basename(other_pkg) == other_name): overwrite = True # use the most specific target name if overwrite: self.local_module_targets[module_path] = (target, len(srcs)) print(("WARNING: Module %s specified in multiple targets: " "%s vs %s (autogen_deps may pick the incorrect " "target)") % (module_path, other, target)) continue self.local_module_targets[module_path] = (target, len(srcs))
def regenerate(self, bazel_targets: Iterable[str], cwd: str = ".") -> None: targets = bazel_utils.expand_bazel_target_dirs( self.workspace_dir, [t for t in bazel_targets if not t.startswith("@")], require_build_file=False, cwd=cwd, ) for target in targets: assert target.startswith( "//"), "Target must be absolute: " + target target_dir = bazel_utils.normalize_relative_target_to_os_path( target[2:]) if target_dir in self.visited_dirs: continue self.visited_dirs.add(target_dir) build_bzl = os.path.join(self.workspace_dir, target_dir, BUILD_INPUT) if not os.path.isfile(build_bzl): continue parsed = build_parser.parse_file(build_bzl) pip_rules = parsed.get_rules_by_types(PIP_GEN_RULE_TYPES) if not pip_rules: if self.verbose: print("No pip targets found in %s/%s" % (target_dir, BUILD_INPUT)) continue if not self.skip_deps_generation: for rule in pip_rules: self.regenerate( build_parser.maybe_expand_attribute( rule.attr_map.get("deps", [])), cwd=os.path.join(self.workspace_dir, target_dir), ) if self.verbose: head = "(dry run) " if self.dry_run else "" print( head + "Processing pip targets in %s: %s" % (target_dir, [rule.attr_map["name"] for rule in pip_rules])) if self.dry_run: continue self.process_pip_rules(target_dir, pip_rules)
def compute_self_modules(self, pkg, srcs): target_dir = pkg[2:] trim_prefix = 0 if self.python_path and target_dir.startswith(self.python_path): trim_prefix = len(self.python_path) + 1 pkg_path = bazel_utils.normalize_relative_target_to_os_path(target_dir) self_modules = set() for src in srcs: file_path = os.path.join(pkg_path, src) module_path = PythonPathMapping.convert_from_file_path_to_module(file_path) self_modules.add(module_path[trim_prefix:]) return self_modules
def regenerate(self, bazel_targets: Iterable[str], cwd: str = ".") -> None: targets = bazel_utils.expand_bazel_targets( self.workspace_dir, [t for t in bazel_targets if not t.startswith("@")], require_build_file=False, cwd=cwd, ) for target in targets: assert target.startswith("//"), "Target must be absolute: " + target pkg, _, _ = target.partition(":") target_dir = bazel_utils.normalize_relative_target_to_os_path(pkg[2:]) _, parsed = self.parsed_cache.get_bzl( os.path.join(self.workspace_dir, target_dir) ) if not parsed: self.maybe_traverse_non_bzl(target) continue if pkg in self.visited_bzl_dirs: continue self.visited_bzl_dirs.add(pkg) py_rules = parsed.get_rules_by_types(PY_RULE_TYPES) if not py_rules: if self.cfg.verbose: print("No py targets found in %s:%s" % (pkg, BUILD_INPUT)) continue if self.cfg.verbose: head = "(dry run) " if self.cfg.dry_run else "" print( head + "Processing py targets in %s: %s" % (pkg, [rule.attr_map["name"] for rule in py_rules]) ) if self.cfg.dry_run: continue self.generate_build_file(pkg, py_rules)
def compute_deps( self, python_path, pkg, rule_type, name, srcs, stub_srcs, main, pip_main, validate, is_py3_compatible, ): srcs = (srcs or []) + (stub_srcs or []) if main: srcs = srcs + [main] mapping = self.python_path_mappings.get(python_path) self_modules = mapping.compute_self_modules(pkg, srcs) target_dir = bazel_utils.normalize_relative_target_to_os_path(pkg[2:]) all_deps = set() # type: ignore[var-annotated] all_unknown_imports = set() # type: ignore[var-annotated] all_unknown_froms = set() # type: ignore[var-annotated] for src in set(srcs): src = os.path.join(target_dir, src) module_path = PythonPathMapping.convert_from_file_path_to_module( src) filename, parsed = mapping.find_closest_bzl_or_build(module_path) if not filename: raise bazel_utils.BazelError( "Cannot locate %s:%s's source (or its closest BUILD / " "BUILD.in file): %s/%s" % (pkg, name, target_dir, src)) pkg_path = os.path.dirname(filename).replace( self.workspace_dir, "/") src_pkg = bazel_utils.normalize_os_path_to_target(pkg_path) if src_pkg != pkg: print(("WARNING: Skipping %s from %s:%s deps computation " "since it belongs to %s") % (src, pkg, name, src_pkg)) continue import_set, from_set = parse_imports( self.workspace_dir, src, py3_compatible=is_py3_compatible or src.endswith(".pyi"), ) import_deps, unknown_imports = mapping.find_import_targets( src_pkg, self_modules, import_set) all_deps.update(import_deps) all_unknown_imports.update(unknown_imports) if validate: assert not unknown_imports, ( "Unable to locate modules %s (imported by %s) in any " "library target (NOTE: bin and test targets are " "ignored)") % (unknown_imports, src) from_deps, unknown_froms = mapping.find_from_targets( src_pkg, self_modules, from_set) all_deps.update(from_deps) all_unknown_froms.update(unknown_froms) if validate: assert not unknown_froms, ( "Unable to locate modules %s (imported by %s) in any " "library target (NOTE: bin and test targets are " "ignored)") % (unknown_froms, src) import_deps, unknown_imports = mapping.find_import_targets( pkg, self_modules, []) all_deps.update(import_deps) all_unknown_imports.update(unknown_imports) if pip_main: all_deps.add(pip_main) all_deps.discard("%s:%s" % (pkg, name)) if name == os.path.basename(target_dir): all_deps.discard("%s" % pkg) return sort_deps(pkg, all_deps), all_unknown_imports, all_unknown_froms