def all_roots(self):
"""Return all known source roots.
Returns a generator over (source root, list of langs, category) triples.
Note: Requires a directory walk to match actual directories against patterns.
However we don't descend into source roots, once found, so this should be fast in practice.
Note: Does not follow symlinks.
"""
project_tree = get_project_tree(self._options)
fixed_roots = set()
for root, langs, category in self._trie.fixed():
if project_tree.exists(root):
yield self._source_root_factory.create(root, langs, category)
fixed_roots.add(root)
for relpath, dirnames, _ in project_tree.walk('', topdown=True):
match = self._trie.find(relpath)
if match:
if not any(
fixed_root.startswith(relpath)
for fixed_root in fixed_roots):
yield match # Found a source root not a prefix of any fixed roots.
del dirnames[:] # Don't continue to walk into it.
def _init_graph(self, use_engine, pants_ignore_patterns, build_ignore_patterns,
exclude_target_regexps, target_specs, graph_helper=None):
"""Determines the BuildGraph, AddressMapper and spec_roots for a given run.
:param bool use_engine: Whether or not to use the v2 engine to construct the BuildGraph.
:param list pants_ignore_patterns: The pants ignore patterns from '--pants-ignore'.
:param list build_ignore_patterns: The build ignore patterns from '--build-ignore',
applied during BUILD file searching.
:param list exclude_target_regexps: Regular expressions for targets to be excluded.
:param list target_specs: The original target specs.
:param LegacyGraphHelper graph_helper: A LegacyGraphHelper to use for graph construction,
if available. This would usually come from the daemon.
:returns: A tuple of (BuildGraph, AddressMapper, spec_roots).
"""
# N.B. Use of the daemon implies use of the v2 engine.
if graph_helper or use_engine:
# The daemon may provide a `graph_helper`. If that's present, use it for graph construction.
graph_helper = graph_helper or EngineInitializer.setup_legacy_graph(
pants_ignore_patterns,
build_ignore_patterns=build_ignore_patterns,
exclude_target_regexps=exclude_target_regexps)
target_roots = TargetRoots.create(options=self._options,
build_root=self._root_dir,
change_calculator=graph_helper.change_calculator)
return graph_helper.create_build_graph(target_roots, self._root_dir)
else:
address_mapper = BuildFileAddressMapper(
self._build_file_parser,
get_project_tree(self._global_options),
build_ignore_patterns,
exclude_target_regexps)
return MutableBuildGraph(address_mapper), address_mapper, target_specs
def _select_buildgraph_and_address_mapper(self, use_engine, path_ignore_patterns, build_ignore_patterns, daemon_buildgraph=None):
"""Selects a BuildGraph and AddressMapper to use then constructs them and returns them.
:param bool use_engine: Whether or not to use the v2 engine to construct the BuildGraph.
:param list path_ignore_patterns: The path ignore patterns from `--pants-ignore`.
:param LegacyBuildGraph daemon_buildgraph: A cached graph to reuse, if available.
:returns a tuple of the graph and the address mapper.
"""
if daemon_buildgraph is not None:
# NB: The daemon may provide a buildgraph. In that case, we ignore the use_engine option,
# since using the engine is implied by using the daemon. However, it is possible for the
# daemon to pass a non-engine backed build graph instance, so we fall back to that.
if isinstance(daemon_buildgraph, LegacyBuildGraph):
return daemon_buildgraph, LegacyAddressMapper(daemon_buildgraph, self._root_dir)
else:
return daemon_buildgraph, daemon_buildgraph._address_mapper
elif use_engine:
root_specs = EngineInitializer.parse_commandline_to_spec_roots(options=self._options,
build_root=self._root_dir)
graph_helper = EngineInitializer.setup_legacy_graph(path_ignore_patterns)
graph = graph_helper.create_graph(root_specs)
return graph, LegacyAddressMapper(graph, self._root_dir)
else:
address_mapper = BuildFileAddressMapper(self._build_file_parser,
get_project_tree(self._global_options),
build_ignore_patterns,
exclude_target_regexps=self._global_options.exclude_target_regexp)
return MutableBuildGraph(address_mapper), address_mapper
def _init_graph(self,
use_engine,
pants_ignore_patterns,
build_ignore_patterns,
exclude_target_regexps,
target_specs,
workdir,
graph_helper=None,
subproject_build_roots=None):
"""Determine the BuildGraph, AddressMapper and spec_roots for a given run.
:param bool use_engine: Whether or not to use the v2 engine to construct the BuildGraph.
:param list pants_ignore_patterns: The pants ignore patterns from '--pants-ignore'.
:param list build_ignore_patterns: The build ignore patterns from '--build-ignore',
applied during BUILD file searching.
:param str workdir: The pants workdir.
:param list exclude_target_regexps: Regular expressions for targets to be excluded.
:param list target_specs: The original target specs.
:param LegacyGraphHelper graph_helper: A LegacyGraphHelper to use for graph construction,
if available. This would usually come from the daemon.
:returns: A tuple of (BuildGraph, AddressMapper, opt Scheduler, spec_roots).
"""
# N.B. Use of the daemon implies use of the v2 engine.
if graph_helper or use_engine:
# The daemon may provide a `graph_helper`. If that's present, use it for graph construction.
if not graph_helper:
native = Native.create(self._global_options)
native.set_panic_handler()
graph_helper = EngineInitializer.setup_legacy_graph(
pants_ignore_patterns,
workdir,
self._global_options.build_file_imports,
native=native,
build_file_aliases=self._build_config.registered_aliases(),
build_ignore_patterns=build_ignore_patterns,
exclude_target_regexps=exclude_target_regexps,
subproject_roots=subproject_build_roots,
include_trace_on_error=self._options.for_global_scope(
).print_exception_stacktrace)
target_roots = TargetRoots.create(
options=self._options,
build_root=self._root_dir,
change_calculator=graph_helper.change_calculator)
graph, address_mapper = graph_helper.create_build_graph(
target_roots, self._root_dir)
return graph, address_mapper, graph_helper.scheduler, target_roots.as_specs(
)
else:
spec_roots = TargetRoots.parse_specs(target_specs, self._root_dir)
address_mapper = BuildFileAddressMapper(
self._build_file_parser,
get_project_tree(self._global_options), build_ignore_patterns,
exclude_target_regexps, subproject_build_roots)
return MutableBuildGraph(
address_mapper), address_mapper, None, spec_roots
def configure_target(target):
if target not in analyzed_targets:
analyzed_targets.add(target)
self.has_scala = not self.skip_scala and (self.has_scala or is_scala(target))
# Hack for java_sources and Eclipse/IntelliJ: add java_sources to project
if isinstance(target, ScalaLibrary):
for java_source in target.java_sources:
configure_target(java_source)
# Resources are already in the target set
if target.has_resources:
resources_by_basedir = defaultdict(set)
for resources in target.resources:
analyzed_targets.add(resources)
resources_by_basedir[resources.target_base].update(relative_sources(resources))
for basedir, resources in resources_by_basedir.items():
self.resource_extensions.update(Project.extract_resource_extensions(resources))
configure_source_sets(basedir, resources, is_test=target.is_test,
resources_only=True)
if target.has_sources():
test = target.is_test
self.has_tests = self.has_tests or test
base = target.target_base
configure_source_sets(base, relative_sources(target), is_test=test,
resources_only=isinstance(target, Resources))
# TODO(Garrett Malmquist): This is dead code, and should be redone/reintegrated.
# Other BUILD files may specify sources in the same directory as this target. Those BUILD
# files might be in parent directories (globs('a/b/*.java')) or even children directories if
# this target globs children as well. Gather all these candidate BUILD files to test for
# sources they own that live in the directories this targets sources live in.
target_dirset = find_source_basedirs(target)
if not isinstance(target.address, BuildFileAddress):
return [] # Siblings only make sense for BUILD files.
candidates = OrderedSet()
dir_relpath = target.address.spec_path
project_tree = get_project_tree(self.context.options.for_global_scope())
for descendant in BuildFile.scan_build_files(project_tree, dir_relpath,
build_ignore_patterns=self.build_ignore_patterns):
candidates.update(self.target_util.get_all_addresses(descendant))
if not self._is_root_relpath(dir_relpath):
ancestors = self._collect_ancestor_build_files(project_tree, os.path.dirname(dir_relpath),
self.build_ignore_patterns)
for ancestor in ancestors:
candidates.update(self.target_util.get_all_addresses(ancestor))
def is_sibling(target):
return source_target(target) and target_dirset.intersection(find_source_basedirs(target))
return filter(is_sibling, [self.target_util.get(a) for a in candidates if a != target.address])
def all_roots(self):
"""Return all known source roots.
Returns a generator over (source root, list of langs, category) triples.
Note: Requires a directory walk to match actual directories against patterns.
However we don't descend into source roots, once found, so this should be fast in practice.
Note: Does not follow symlinks.
"""
project_tree = get_project_tree(self._options)
fixed_roots = set()
for root, langs, category in self._trie.fixed():
if project_tree.exists(root):
yield self._source_root_factory.create(root, langs, category)
fixed_roots.add(root)
for relpath, dirnames, _ in project_tree.walk('', topdown=True):
match = self._trie.find(relpath)
if match:
if not any(fixed_root.startswith(relpath) for fixed_root in fixed_roots):
yield match # Found a source root not a prefix of any fixed roots.
del dirnames[:] # Don't continue to walk into it.
def configure_target(target):
if target not in analyzed_targets:
analyzed_targets.add(target)
self.has_scala = not self.skip_scala and (self.has_scala
or is_scala(target))
# Hack for java_sources and Eclipse/IntelliJ: add java_sources to project
if isinstance(target, ScalaLibrary):
for java_source in target.java_sources:
configure_target(java_source)
# Resources are already in the target set
if target.has_resources:
resources_by_basedir = defaultdict(set)
for resources in target.resources:
analyzed_targets.add(resources)
resources_by_basedir[resources.target_base].update(
relative_sources(resources))
for basedir, resources in resources_by_basedir.items():
self.resource_extensions.update(
Project.extract_resource_extensions(resources))
configure_source_sets(basedir,
resources,
is_test=_is_test(target),
resources_only=True)
if target.has_sources():
test = _is_test(target)
self.has_tests = self.has_tests or test
base = target.target_base
configure_source_sets(base,
relative_sources(target),
is_test=test,
resources_only=isinstance(
target, Resources))
# TODO(Garrett Malmquist): This is dead code, and should be redone/reintegrated.
# Other BUILD files may specify sources in the same directory as this target. Those BUILD
# files might be in parent directories (globs('a/b/*.java')) or even children directories if
# this target globs children as well. Gather all these candidate BUILD files to test for
# sources they own that live in the directories this targets sources live in.
target_dirset = find_source_basedirs(target)
if not isinstance(target.address, BuildFileAddress):
return [] # Siblings only make sense for BUILD files.
candidates = OrderedSet()
dir_relpath = target.address.spec_path
project_tree = get_project_tree(
self.context.options.for_global_scope())
for descendant in BuildFile.scan_build_files(
project_tree,
dir_relpath,
build_ignore_patterns=self.build_ignore_patterns):
candidates.update(
self.target_util.get_all_addresses(descendant))
if not self._is_root_relpath(dir_relpath):
ancestors = self._collect_ancestor_build_files(
project_tree, os.path.dirname(dir_relpath),
self.build_ignore_patterns)
for ancestor in ancestors:
candidates.update(
self.target_util.get_all_addresses(ancestor))
def is_sibling(target):
return source_target(
target) and target_dirset.intersection(
find_source_basedirs(target))
return filter(is_sibling, [
self.target_util.get(a)
for a in candidates if a != target.address
])