def test_full_graph_for_planner_example(self):
address_mapper = AddressMapper(
JsonParser(TARGET_TABLE),
prelude_glob_patterns=(),
build_patterns="*.BUILD.json",
)
rules = create_graph_rules(address_mapper) + create_fs_rules()
fullgraph_str = self.create_full_graph(rules)
print("---diagnostic------")
print(fullgraph_str)
print("/---diagnostic------")
in_root_rules = False
in_all_rules = False
all_rules = []
root_rule_lines = []
for line in fullgraph_str.splitlines():
if line.startswith(" // root subject types:"):
pass
elif line.startswith(" // root entries"):
in_root_rules = True
elif line.startswith(" // internal entries"):
in_all_rules = True
elif in_all_rules:
all_rules.append(line)
elif in_root_rules:
root_rule_lines.append(line)
else:
pass
self.assertTrue(6 < len(all_rules))
self.assertTrue(12 < len(root_rule_lines)) # 2 lines per entry
def resolve_addresses_with_origins_from_address_specs(
address_specs: AddressSpecs,
address_family: AddressFamily,
*,
tags: Optional[Iterable[str]] = None,
exclude_patterns: Optional[Iterable[str]] = None
) -> AddressesWithOrigins:
address_mapper = AddressMapper(
Parser(target_type_aliases=[], object_aliases=BuildFileAliases()),
tags=tags,
exclude_target_regexps=exclude_patterns,
)
snapshot = Snapshot(Digest("xx", 2), ("root/BUILD", ), ())
addresses_with_origins = run_rule(
addresses_with_origins_from_address_specs,
rule_args=[address_mapper, address_specs],
mock_gets=[
MockGet(product_type=Snapshot,
subject_type=PathGlobs,
mock=lambda _: snapshot),
MockGet(
product_type=AddressFamily,
subject_type=Dir,
mock=lambda _: address_family,
),
],
)
return cast(AddressesWithOrigins, addresses_with_origins)
def test_empty(self) -> None:
"""Test that parsing an empty BUILD file results in an empty AddressFamily."""
address_mapper = AddressMapper(
parser=JsonParser(TEST_TABLE),
prelude_glob_patterns=(),
build_file_imports_behavior=BuildFileImportsBehavior.error,
)
af = run_rule(
parse_address_family,
rule_args=[
address_mapper,
BuildFilePreludeSymbols(FrozenDict()),
Dir("/dev/null")
],
mock_gets=[
MockGet(
product_type=Snapshot,
subject_type=PathGlobs,
mock=lambda _: Snapshot(Digest("abc", 10),
("/dev/null/BUILD", ), ()),
),
MockGet(
product_type=FilesContent,
subject_type=Digest,
mock=lambda _: FilesContent(
[FileContent(path="/dev/null/BUILD", content=b"")]),
),
],
)
self.assertEqual(len(af.objects_by_name), 0)
def setUp(self) -> None:
# Set up a scheduler that supports address mapping.
address_mapper = AddressMapper(
parser=JsonParser(TARGET_TABLE),
prelude_glob_patterns=(),
build_file_imports_behavior=BuildFileImportsBehavior.error,
build_patterns=("*.BUILD.json", ),
)
# We add the `unhydrated_structs` rule because it is otherwise not used in the core engine.
rules = [unhydrated_structs
] + create_fs_rules() + create_graph_rules(address_mapper)
project_tree = self.mk_fs_tree(
os.path.join(os.path.dirname(__file__), "examples/mapper_test"))
self.build_root = project_tree.build_root
self.scheduler = self.mk_scheduler(rules=rules,
project_tree=project_tree)
self.a_b = Address.parse("a/b")
self.a_b_target = Target(
address=self.a_b,
dependencies=["//a/d/e"],
configurations=["//a/d/e", Struct(embedded="yes")],
type_alias="target",
)
def create(self, build_patterns=None, parser=None) -> SchedulerSession:
address_mapper = AddressMapper(
parser=parser, prelude_glob_patterns=(), build_patterns=build_patterns
)
@rule
def symbol_table_singleton() -> SymbolTable:
return TEST_TABLE
rules = create_fs_rules() + create_graph_rules(address_mapper) + [symbol_table_singleton]
project_tree = self.mk_fs_tree(os.path.join(os.path.dirname(__file__), "examples"))
return cast(SchedulerSession, self.mk_scheduler(rules=rules, project_tree=project_tree))
async def addresses_with_origins_from_address_specs(
address_mapper: AddressMapper,
address_specs: AddressSpecs) -> AddressesWithOrigins:
"""Given an AddressMapper and list of AddressSpecs, return matching AddressesWithOrigins.
:raises: :class:`ResolveError` if there were no matching AddressFamilies or no targets
were matched.
"""
# Snapshot all BUILD files covered by the AddressSpecs, then group by directory.
snapshot = await Get(
Snapshot,
PathGlobs,
address_specs.to_path_globs(
build_patterns=address_mapper.build_patterns,
build_ignore_patterns=address_mapper.build_ignore_patterns,
),
)
dirnames = {os.path.dirname(f) for f in snapshot.files}
address_families = await MultiGet(
Get(AddressFamily, Dir(d)) for d in dirnames)
address_family_by_directory = {af.namespace: af for af in address_families}
matched_addresses: OrderedSet[Address] = OrderedSet()
addr_to_origin: Dict[Address, AddressSpec] = {}
for address_spec in address_specs:
# These may raise ResolveError, depending on the type of spec.
addr_families_for_spec = address_spec.matching_address_families(
address_family_by_directory)
addr_target_pairs_for_spec = address_spec.matching_addresses(
addr_families_for_spec)
if isinstance(address_spec,
SingleAddress) and not addr_target_pairs_for_spec:
addr_family = assert_single_element(addr_families_for_spec)
raise _did_you_mean_exception(addr_family, address_spec.name)
for addr, _ in addr_target_pairs_for_spec:
# A target might be covered by multiple specs, so we take the most specific one.
addr_to_origin[addr] = AddressSpecs.more_specific(
addr_to_origin.get(addr), address_spec)
matched_addresses.update(
addr for (addr, tgt) in addr_target_pairs_for_spec
if (address_specs.filter_by_global_options is False
or address_mapper.matches_filter_options(addr, tgt)))
return AddressesWithOrigins(
AddressWithOrigin(address=addr, origin=addr_to_origin[addr])
for addr in matched_addresses)
def test_parse_address_family_empty() -> None:
"""Test that parsing an empty BUILD file results in an empty AddressFamily."""
address_mapper = AddressMapper(parser=Parser(
target_type_aliases=[], object_aliases=BuildFileAliases()))
af = run_rule(
parse_address_family,
rule_args=[
address_mapper,
BuildFilePreludeSymbols(FrozenDict()),
Dir("/dev/null")
],
mock_gets=[
MockGet(
product_type=DigestContents,
subject_type=PathGlobs,
mock=lambda _: DigestContents(
[FileContent(path="/dev/null/BUILD", content=b"")]),
),
],
)
assert len(af.name_to_target_adaptors) == 0
def test_empty(self) -> None:
"""Test that parsing an empty BUILD file results in an empty AddressFamily."""
address_mapper = AddressMapper(JsonParser(TEST_TABLE))
af = run_rule(
parse_address_family,
rule_args=[address_mapper, Dir("/dev/null")],
mock_gets=[
MockGet(
product_type=Snapshot,
subject_type=PathGlobs,
mock=lambda _: Snapshot(Digest("abc", 10),
("/dev/null/BUILD", ), ()),
),
MockGet(
product_type=FilesContent,
subject_type=Digest,
mock=lambda _: FilesContent(
[FileContent(path="/dev/null/BUILD", content=b"")]),
),
],
)
self.assertEqual(len(af.objects_by_name), 0)
def test_match_filter_options() -> None:
def make_target(target_name: str, **kwargs) -> TargetAdaptor:
parsed_address = Address("", target_name=target_name)
return TargetAdaptor(
type_alias="", name=parsed_address.target_name, address=parsed_address, **kwargs
)
untagged_target = make_target(target_name="//:untagged")
b_tagged_target = make_target(target_name="//:b-tagged", tags=["b"])
a_and_b_tagged_target = make_target(target_name="//:a-and-b-tagged", tags=["a", "b"])
none_tagged_target = make_target(target_name="//:none-tagged-target", tags=None)
parser = Parser(target_type_aliases=[], object_aliases=BuildFileAliases())
mapper = AddressMapper(parser, tags=["-a", "+b"])
def matches(tgt: TargetAdaptor) -> bool:
return mapper.matches_filter_options(tgt.kwargs["address"], tgt)
assert matches(untagged_target) is False
assert matches(b_tagged_target) is True
assert matches(a_and_b_tagged_target) is False
assert matches(none_tagged_target) is False
def _address_mapper(self) -> AddressMapper:
return AddressMapper(JsonParser(TEST_TABLE))
def setup_legacy_graph_extended(
pants_ignore_patterns: List[str],
use_gitignore: bool,
local_store_dir: str,
local_execution_root_dir: str,
named_caches_dir: str,
build_file_prelude_globs: Tuple[str, ...],
options_bootstrapper: OptionsBootstrapper,
build_configuration: BuildConfiguration,
execution_options: ExecutionOptions,
build_root: Optional[str] = None,
native: Optional[Native] = None,
glob_match_error_behavior:
GlobMatchErrorBehavior = GlobMatchErrorBehavior.warn,
build_ignore_patterns=None,
exclude_target_regexps=None,
subproject_roots=None,
include_trace_on_error: bool = True,
) -> LegacyGraphScheduler:
"""Construct and return the components necessary for LegacyBuildGraph construction.
:param local_store_dir: The directory to use for storing the engine's LMDB store in.
:param local_execution_root_dir: The directory to use for local execution sandboxes.
:param named_caches_dir: The base directory for named cache storage.
:param build_file_prelude_globs: Globs to match files to be prepended to all BUILD files.
:param build_root: A path to be used as the build root. If None, then default is used.
:param native: An instance of the native-engine subsystem.
:param options_bootstrapper: A `OptionsBootstrapper` object containing bootstrap options.
:param build_configuration: The `BuildConfiguration` object to get build file aliases from.
:param glob_match_error_behavior: How to behave if a glob specified for a target's sources or
bundles does not expand to anything.
:param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD
files, usually taken from the '--build-ignore' global option.
:param list exclude_target_regexps: A list of regular expressions for excluding targets.
:param list subproject_roots: Paths that correspond with embedded build roots
under the current build root.
:param include_trace_on_error: If True, when an error occurs, the error message will include
the graph trace.
:param execution_options: Option values for (remote) process execution.
"""
build_root = build_root or get_buildroot()
build_configuration = build_configuration or BuildConfigInitializer.get(
options_bootstrapper)
bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope(
)
build_file_aliases = build_configuration.registered_aliases()
rules = build_configuration.rules()
registered_target_types = RegisteredTargetTypes.create(
build_configuration.target_types())
symbol_table = _legacy_symbol_table(build_file_aliases,
registered_target_types)
execution_options = execution_options or DEFAULT_EXECUTION_OPTIONS
# Register "literal" subjects required for these rules.
parser = LegacyPythonCallbacksParser(symbol_table, build_file_aliases)
address_mapper = AddressMapper(
parser=parser,
prelude_glob_patterns=build_file_prelude_globs,
build_ignore_patterns=build_ignore_patterns,
exclude_target_regexps=exclude_target_regexps,
subproject_roots=subproject_roots,
)
@rule
def glob_match_error_behavior_singleton() -> GlobMatchErrorBehavior:
return glob_match_error_behavior
@rule
def build_configuration_singleton() -> BuildConfiguration:
return build_configuration
@rule
def symbol_table_singleton() -> SymbolTable:
return symbol_table
@rule
def registered_target_types_singleton() -> RegisteredTargetTypes:
return registered_target_types
@rule
def union_membership_singleton() -> UnionMembership:
return UnionMembership(build_configuration.union_rules())
@rule
def build_root_singleton() -> BuildRoot:
return cast(BuildRoot, BuildRoot.instance)
# Create a Scheduler containing graph and filesystem rules, with no installed goals. The
# LegacyBuildGraph will explicitly request the products it needs.
rules = (
RootRule(Console),
glob_match_error_behavior_singleton,
build_configuration_singleton,
symbol_table_singleton,
registered_target_types_singleton,
union_membership_singleton,
build_root_singleton,
*interactive_runner.rules(),
*graph.rules(),
*options_parsing.rules(),
*process.rules(),
*target.rules(),
*create_legacy_graph_tasks(),
*create_fs_rules(),
*create_platform_rules(),
*create_graph_rules(address_mapper),
*structs_rules(),
*changed_rules(),
*binary_tool_rules(),
*binary_util_rules(),
*rules,
)
goal_map = EngineInitializer._make_goal_map_from_rules(rules)
union_rules = build_configuration.union_rules()
scheduler = Scheduler(
native=native,
ignore_patterns=pants_ignore_patterns,
use_gitignore=use_gitignore,
build_root=build_root,
local_store_dir=local_store_dir,
local_execution_root_dir=local_execution_root_dir,
named_caches_dir=named_caches_dir,
rules=rules,
union_rules=union_rules,
execution_options=execution_options,
include_trace_on_error=include_trace_on_error,
visualize_to_dir=bootstrap_options.native_engine_visualize_to,
)
return LegacyGraphScheduler(scheduler, build_file_aliases, goal_map)
def _address_mapper(self) -> AddressMapper:
return AddressMapper(JsonParser(TEST_TABLE), prelude_glob_patterns=())
def _address_mapper(self) -> AddressMapper:
return AddressMapper(
JsonParser(TEST_TABLE),
prelude_glob_patterns=(),
build_file_imports_behavior=BuildFileImportsBehavior.error,
)
def setup_legacy_graph_extended(
options_bootstrapper: OptionsBootstrapper,
build_configuration: BuildConfiguration,
execution_options: ExecutionOptions,
*,
pants_ignore_patterns: List[str],
use_gitignore: bool,
local_store_dir: str,
local_execution_root_dir: str,
named_caches_dir: str,
build_root: Optional[str] = None,
native: Optional[Native] = None,
glob_match_error_behavior:
GlobMatchErrorBehavior = GlobMatchErrorBehavior.warn,
build_patterns: Optional[Iterable[str]] = None,
build_file_prelude_globs: Optional[Iterable[str]] = None,
build_ignore_patterns: Optional[Iterable[str]] = None,
tags: Optional[Iterable[str]] = None,
exclude_target_regexps: Optional[Iterable[str]] = None,
subproject_roots: Optional[Iterable[str]] = None,
include_trace_on_error: bool = True,
) -> LegacyGraphScheduler:
"""Construct and return the components necessary for LegacyBuildGraph construction."""
build_root = build_root or get_buildroot()
build_configuration = build_configuration or BuildConfigInitializer.get(
options_bootstrapper)
bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope(
)
execution_options = execution_options or DEFAULT_EXECUTION_OPTIONS
build_file_aliases = build_configuration.registered_aliases
rules = build_configuration.rules
union_membership = UnionMembership.from_rules(
build_configuration.union_rules)
registered_target_types = RegisteredTargetTypes.create(
build_configuration.target_types)
parser = Parser(target_type_aliases=registered_target_types.aliases,
object_aliases=build_file_aliases)
address_mapper = AddressMapper(
parser=parser,
prelude_glob_patterns=build_file_prelude_globs,
build_patterns=build_patterns,
build_ignore_patterns=build_ignore_patterns,
tags=tags,
exclude_target_regexps=exclude_target_regexps,
subproject_roots=subproject_roots,
)
@rule
def address_mapper_singleton() -> AddressMapper:
return address_mapper
@rule
def glob_match_error_behavior_singleton() -> GlobMatchErrorBehavior:
return glob_match_error_behavior
@rule
def build_configuration_singleton() -> BuildConfiguration:
return build_configuration
@rule
def registered_target_types_singleton() -> RegisteredTargetTypes:
return registered_target_types
@rule
def union_membership_singleton() -> UnionMembership:
return union_membership
@rule
def build_root_singleton() -> BuildRoot:
return cast(BuildRoot, BuildRoot.instance)
# Create a Scheduler containing graph and filesystem rules, with no installed goals.
rules = FrozenOrderedSet((
*collect_rules(locals()),
RootRule(Console),
*build_files.rules(),
*fs.rules(),
*graph.rules(),
*uuid.rules(),
*options_parsing.rules(),
*process.rules(),
*create_platform_rules(),
*changed_rules(),
*rules,
))
goal_map = EngineInitializer._make_goal_map_from_rules(rules)
def ensure_absolute_path(v: str) -> str:
return Path(v).resolve().as_posix()
scheduler = Scheduler(
native=native,
ignore_patterns=pants_ignore_patterns,
use_gitignore=use_gitignore,
build_root=build_root,
local_store_dir=ensure_absolute_path(local_store_dir),
local_execution_root_dir=ensure_absolute_path(
local_execution_root_dir),
named_caches_dir=ensure_absolute_path(named_caches_dir),
rules=rules,
union_membership=union_membership,
execution_options=execution_options,
include_trace_on_error=include_trace_on_error,
visualize_to_dir=bootstrap_options.native_engine_visualize_to,
)
return LegacyGraphScheduler(scheduler, build_file_aliases, goal_map)
