def all_roots(self): """Return all known source roots. Returns a generator over (source root, list of langs, category) triples. Note: Requires a directory walk to match actual directories against patterns. However we don't descend into source roots, once found, so this should be fast in practice. Note: Does not follow symlinks. """ project_tree = get_project_tree(self._options) fixed_roots = set() for root, langs, category in self._trie.fixed(): if project_tree.exists(root): yield self._source_root_factory.create(root, langs, category) fixed_roots.add(root) for relpath, dirnames, _ in project_tree.walk('', topdown=True): match = self._trie.find(relpath) if match: if not any( fixed_root.startswith(relpath) for fixed_root in fixed_roots): yield match # Found a source root not a prefix of any fixed roots. del dirnames[:] # Don't continue to walk into it.
def _init_graph(self, use_engine, pants_ignore_patterns, build_ignore_patterns, exclude_target_regexps, target_specs, graph_helper=None): """Determines the BuildGraph, AddressMapper and spec_roots for a given run. :param bool use_engine: Whether or not to use the v2 engine to construct the BuildGraph. :param list pants_ignore_patterns: The pants ignore patterns from '--pants-ignore'. :param list build_ignore_patterns: The build ignore patterns from '--build-ignore', applied during BUILD file searching. :param list exclude_target_regexps: Regular expressions for targets to be excluded. :param list target_specs: The original target specs. :param LegacyGraphHelper graph_helper: A LegacyGraphHelper to use for graph construction, if available. This would usually come from the daemon. :returns: A tuple of (BuildGraph, AddressMapper, spec_roots). """ # N.B. Use of the daemon implies use of the v2 engine. if graph_helper or use_engine: # The daemon may provide a `graph_helper`. If that's present, use it for graph construction. graph_helper = graph_helper or EngineInitializer.setup_legacy_graph( pants_ignore_patterns, build_ignore_patterns=build_ignore_patterns, exclude_target_regexps=exclude_target_regexps) target_roots = TargetRoots.create(options=self._options, build_root=self._root_dir, change_calculator=graph_helper.change_calculator) return graph_helper.create_build_graph(target_roots, self._root_dir) else: address_mapper = BuildFileAddressMapper( self._build_file_parser, get_project_tree(self._global_options), build_ignore_patterns, exclude_target_regexps) return MutableBuildGraph(address_mapper), address_mapper, target_specs
def _select_buildgraph_and_address_mapper(self, use_engine, path_ignore_patterns, build_ignore_patterns, daemon_buildgraph=None): """Selects a BuildGraph and AddressMapper to use then constructs them and returns them. :param bool use_engine: Whether or not to use the v2 engine to construct the BuildGraph. :param list path_ignore_patterns: The path ignore patterns from `--pants-ignore`. :param LegacyBuildGraph daemon_buildgraph: A cached graph to reuse, if available. :returns a tuple of the graph and the address mapper. """ if daemon_buildgraph is not None: # NB: The daemon may provide a buildgraph. In that case, we ignore the use_engine option, # since using the engine is implied by using the daemon. However, it is possible for the # daemon to pass a non-engine backed build graph instance, so we fall back to that. if isinstance(daemon_buildgraph, LegacyBuildGraph): return daemon_buildgraph, LegacyAddressMapper(daemon_buildgraph, self._root_dir) else: return daemon_buildgraph, daemon_buildgraph._address_mapper elif use_engine: root_specs = EngineInitializer.parse_commandline_to_spec_roots(options=self._options, build_root=self._root_dir) graph_helper = EngineInitializer.setup_legacy_graph(path_ignore_patterns) graph = graph_helper.create_graph(root_specs) return graph, LegacyAddressMapper(graph, self._root_dir) else: address_mapper = BuildFileAddressMapper(self._build_file_parser, get_project_tree(self._global_options), build_ignore_patterns, exclude_target_regexps=self._global_options.exclude_target_regexp) return MutableBuildGraph(address_mapper), address_mapper
def _init_graph(self, use_engine, pants_ignore_patterns, build_ignore_patterns, exclude_target_regexps, target_specs, workdir, graph_helper=None, subproject_build_roots=None): """Determine the BuildGraph, AddressMapper and spec_roots for a given run. :param bool use_engine: Whether or not to use the v2 engine to construct the BuildGraph. :param list pants_ignore_patterns: The pants ignore patterns from '--pants-ignore'. :param list build_ignore_patterns: The build ignore patterns from '--build-ignore', applied during BUILD file searching. :param str workdir: The pants workdir. :param list exclude_target_regexps: Regular expressions for targets to be excluded. :param list target_specs: The original target specs. :param LegacyGraphHelper graph_helper: A LegacyGraphHelper to use for graph construction, if available. This would usually come from the daemon. :returns: A tuple of (BuildGraph, AddressMapper, opt Scheduler, spec_roots). """ # N.B. Use of the daemon implies use of the v2 engine. if graph_helper or use_engine: # The daemon may provide a `graph_helper`. If that's present, use it for graph construction. if not graph_helper: native = Native.create(self._global_options) native.set_panic_handler() graph_helper = EngineInitializer.setup_legacy_graph( pants_ignore_patterns, workdir, self._global_options.build_file_imports, native=native, build_file_aliases=self._build_config.registered_aliases(), build_ignore_patterns=build_ignore_patterns, exclude_target_regexps=exclude_target_regexps, subproject_roots=subproject_build_roots, include_trace_on_error=self._options.for_global_scope( ).print_exception_stacktrace) target_roots = TargetRoots.create( options=self._options, build_root=self._root_dir, change_calculator=graph_helper.change_calculator) graph, address_mapper = graph_helper.create_build_graph( target_roots, self._root_dir) return graph, address_mapper, graph_helper.scheduler, target_roots.as_specs( ) else: spec_roots = TargetRoots.parse_specs(target_specs, self._root_dir) address_mapper = BuildFileAddressMapper( self._build_file_parser, get_project_tree(self._global_options), build_ignore_patterns, exclude_target_regexps, subproject_build_roots) return MutableBuildGraph( address_mapper), address_mapper, None, spec_roots
def configure_target(target): if target not in analyzed_targets: analyzed_targets.add(target) self.has_scala = not self.skip_scala and (self.has_scala or is_scala(target)) # Hack for java_sources and Eclipse/IntelliJ: add java_sources to project if isinstance(target, ScalaLibrary): for java_source in target.java_sources: configure_target(java_source) # Resources are already in the target set if target.has_resources: resources_by_basedir = defaultdict(set) for resources in target.resources: analyzed_targets.add(resources) resources_by_basedir[resources.target_base].update(relative_sources(resources)) for basedir, resources in resources_by_basedir.items(): self.resource_extensions.update(Project.extract_resource_extensions(resources)) configure_source_sets(basedir, resources, is_test=target.is_test, resources_only=True) if target.has_sources(): test = target.is_test self.has_tests = self.has_tests or test base = target.target_base configure_source_sets(base, relative_sources(target), is_test=test, resources_only=isinstance(target, Resources)) # TODO(Garrett Malmquist): This is dead code, and should be redone/reintegrated. # Other BUILD files may specify sources in the same directory as this target. Those BUILD # files might be in parent directories (globs('a/b/*.java')) or even children directories if # this target globs children as well. Gather all these candidate BUILD files to test for # sources they own that live in the directories this targets sources live in. target_dirset = find_source_basedirs(target) if not isinstance(target.address, BuildFileAddress): return [] # Siblings only make sense for BUILD files. candidates = OrderedSet() dir_relpath = target.address.spec_path project_tree = get_project_tree(self.context.options.for_global_scope()) for descendant in BuildFile.scan_build_files(project_tree, dir_relpath, build_ignore_patterns=self.build_ignore_patterns): candidates.update(self.target_util.get_all_addresses(descendant)) if not self._is_root_relpath(dir_relpath): ancestors = self._collect_ancestor_build_files(project_tree, os.path.dirname(dir_relpath), self.build_ignore_patterns) for ancestor in ancestors: candidates.update(self.target_util.get_all_addresses(ancestor)) def is_sibling(target): return source_target(target) and target_dirset.intersection(find_source_basedirs(target)) return filter(is_sibling, [self.target_util.get(a) for a in candidates if a != target.address])
def all_roots(self): """Return all known source roots. Returns a generator over (source root, list of langs, category) triples. Note: Requires a directory walk to match actual directories against patterns. However we don't descend into source roots, once found, so this should be fast in practice. Note: Does not follow symlinks. """ project_tree = get_project_tree(self._options) fixed_roots = set() for root, langs, category in self._trie.fixed(): if project_tree.exists(root): yield self._source_root_factory.create(root, langs, category) fixed_roots.add(root) for relpath, dirnames, _ in project_tree.walk('', topdown=True): match = self._trie.find(relpath) if match: if not any(fixed_root.startswith(relpath) for fixed_root in fixed_roots): yield match # Found a source root not a prefix of any fixed roots. del dirnames[:] # Don't continue to walk into it.
def configure_target(target): if target not in analyzed_targets: analyzed_targets.add(target) self.has_scala = not self.skip_scala and (self.has_scala or is_scala(target)) # Hack for java_sources and Eclipse/IntelliJ: add java_sources to project if isinstance(target, ScalaLibrary): for java_source in target.java_sources: configure_target(java_source) # Resources are already in the target set if target.has_resources: resources_by_basedir = defaultdict(set) for resources in target.resources: analyzed_targets.add(resources) resources_by_basedir[resources.target_base].update( relative_sources(resources)) for basedir, resources in resources_by_basedir.items(): self.resource_extensions.update( Project.extract_resource_extensions(resources)) configure_source_sets(basedir, resources, is_test=_is_test(target), resources_only=True) if target.has_sources(): test = _is_test(target) self.has_tests = self.has_tests or test base = target.target_base configure_source_sets(base, relative_sources(target), is_test=test, resources_only=isinstance( target, Resources)) # TODO(Garrett Malmquist): This is dead code, and should be redone/reintegrated. # Other BUILD files may specify sources in the same directory as this target. Those BUILD # files might be in parent directories (globs('a/b/*.java')) or even children directories if # this target globs children as well. Gather all these candidate BUILD files to test for # sources they own that live in the directories this targets sources live in. target_dirset = find_source_basedirs(target) if not isinstance(target.address, BuildFileAddress): return [] # Siblings only make sense for BUILD files. candidates = OrderedSet() dir_relpath = target.address.spec_path project_tree = get_project_tree( self.context.options.for_global_scope()) for descendant in BuildFile.scan_build_files( project_tree, dir_relpath, build_ignore_patterns=self.build_ignore_patterns): candidates.update( self.target_util.get_all_addresses(descendant)) if not self._is_root_relpath(dir_relpath): ancestors = self._collect_ancestor_build_files( project_tree, os.path.dirname(dir_relpath), self.build_ignore_patterns) for ancestor in ancestors: candidates.update( self.target_util.get_all_addresses(ancestor)) def is_sibling(target): return source_target( target) and target_dirset.intersection( find_source_basedirs(target)) return filter(is_sibling, [ self.target_util.get(a) for a in candidates if a != target.address ])