def parse_specs(
cls,
raw_specs: Iterable[str],
build_root: Optional[str] = None,
exclude_patterns: Optional[Iterable[str]] = None,
tags: Optional[Iterable[str]] = None,
) -> Specs:
"""Parse raw string specs into a Specs object."""
build_root = build_root or get_buildroot()
spec_parser = CmdLineSpecParser(build_root)
address_specs: OrderedSet[AddressSpec] = OrderedSet()
filesystem_specs: OrderedSet[FilesystemSpec] = OrderedSet()
for spec_str in raw_specs:
parsed_spec = spec_parser.parse_spec(spec_str)
if isinstance(parsed_spec, AddressSpec):
address_specs.add(parsed_spec)
else:
filesystem_specs.add(parsed_spec)
address_specs_collection = AddressSpecs(
dependencies=address_specs,
exclude_patterns=exclude_patterns if exclude_patterns else tuple(),
tags=tags,
)
filesystem_specs_collection = FilesystemSpecs(filesystem_specs)
return Specs(
address_specs=address_specs_collection,
filesystem_specs=filesystem_specs_collection,
)
def _init_engine(self, local_store_dir: Optional[str] = None) -> None:
if self._scheduler is not None:
return
options_bootstrapper = OptionsBootstrapper.create(
args=["--pants-config-files=[]"])
local_store_dir = (local_store_dir
or options_bootstrapper.bootstrap_options.
for_global_scope().local_store_dir)
# NB: This uses the long form of initialization because it needs to directly specify
# `cls.alias_groups` rather than having them be provided by bootstrap options.
graph_session = EngineInitializer.setup_legacy_graph_extended(
pants_ignore_patterns=[],
local_store_dir=local_store_dir,
build_file_imports_behavior=BuildFileImportsBehavior.error,
native=init_native(),
options_bootstrapper=options_bootstrapper,
build_root=self.build_root,
build_configuration=self.build_config(),
build_ignore_patterns=None,
).new_session(zipkin_trace_v2=False, build_id="buildid_for_test")
self._scheduler = graph_session.scheduler_session
self._build_graph, self._address_mapper = graph_session.create_build_graph(
Specs(address_specs=AddressSpecs([]),
filesystem_specs=FilesystemSpecs([])),
self._build_root(),
)
async def addresses_from_filesystem_specs(
filesystem_specs: FilesystemSpecs,
owners_not_found_behavior: OwnersNotFoundBehavior) -> Addresses:
"""Find the owner(s) for each FilesystemSpec."""
paths_per_include = await MultiGet(
Get(
Paths,
PathGlobs,
filesystem_specs.path_globs_for_spec(
spec,
owners_not_found_behavior.to_glob_match_error_behavior()),
) for spec in filesystem_specs.file_includes)
owners_per_include = await MultiGet(
Get(Owners, OwnersRequest(sources=paths.files))
for paths in paths_per_include)
addresses: set[Address] = set()
for spec, owners in zip(filesystem_specs.file_includes,
owners_per_include):
if (owners_not_found_behavior != OwnersNotFoundBehavior.ignore
and isinstance(spec, FileLiteralSpec) and not owners):
_log_or_raise_unmatched_owners(
[PurePath(str(spec))],
owners_not_found_behavior,
ignore_option="--owners-not-found-behavior=ignore",
)
addresses.update(owners)
return Addresses(sorted(addresses))
async def addresses_with_origins_from_filesystem_specs(
filesystem_specs: FilesystemSpecs,
global_options: GlobalOptions,
) -> AddressesWithOrigins:
"""Find the owner(s) for each FilesystemSpec while preserving the original FilesystemSpec those
owners come from.
"""
pathglobs_per_include = (filesystem_specs.path_globs_for_spec(spec)
for spec in filesystem_specs.includes)
snapshot_per_include = await MultiGet(Get[Snapshot](PathGlobs, pg)
for pg in pathglobs_per_include)
owners_per_include = await MultiGet(
Get[Owners](OwnersRequest(sources=snapshot.files))
for snapshot in snapshot_per_include)
result: List[AddressWithOrigin] = []
for spec, owners in zip(filesystem_specs.includes, owners_per_include):
if (global_options.owners_not_found_behavior !=
OwnersNotFoundBehavior.ignore
and isinstance(spec, FilesystemLiteralSpec)
and not owners.addresses):
file_path = PurePath(spec.to_spec_string())
msg = (
f"No owning targets could be found for the file `{file_path}`.\n\nPlease check "
f"that there is a BUILD file in `{file_path.parent}` with a target whose `sources` field "
f"includes `{file_path}`. See https://www.pantsbuild.org/build_files.html."
)
if global_options.owners_not_found_behavior == OwnersNotFoundBehavior.warn:
logger.warning(msg)
else:
raise ResolveError(msg)
result.extend(
AddressWithOrigin(address=bfa, origin=spec)
for bfa in owners.addresses)
return AddressesWithOrigins(result)
async def addresses_from_filesystem_specs(
filesystem_specs: FilesystemSpecs,
global_options: GlobalOptions) -> Addresses:
"""Find the owner(s) for each FilesystemSpec.
Every returned address will be a generated subtarget, meaning that each address will have
exactly one file in its `sources` field.
"""
owners_not_found_behavior = global_options.options.owners_not_found_behavior
paths_per_include = await MultiGet(
Get(
Paths,
PathGlobs,
filesystem_specs.path_globs_for_spec(
spec,
owners_not_found_behavior.to_glob_match_error_behavior()),
) for spec in filesystem_specs.includes)
owners_per_include = await MultiGet(
Get(Owners, OwnersRequest(sources=paths.files))
for paths in paths_per_include)
addresses: Set[Address] = set()
for spec, owners in zip(filesystem_specs.includes, owners_per_include):
if (owners_not_found_behavior != OwnersNotFoundBehavior.ignore
and isinstance(spec, FilesystemLiteralSpec) and not owners):
_log_or_raise_unmatched_owners(
[PurePath(str(spec))],
global_options.options.owners_not_found_behavior,
ignore_option="--owners-not-found-behavior=ignore",
)
addresses.update(owners)
return Addresses(sorted(addresses))
async def addresses_with_origins_from_filesystem_specs(
filesystem_specs: FilesystemSpecs, global_options: GlobalOptions,
) -> AddressesWithOrigins:
"""Find the owner(s) for each FilesystemSpec while preserving the original FilesystemSpec those
owners come from.
This will merge FilesystemSpecs that come from the same owning target into a single
FilesystemMergedSpec.
"""
pathglobs_per_include = (
filesystem_specs.path_globs_for_spec(
spec, global_options.options.owners_not_found_behavior.to_glob_match_error_behavior(),
)
for spec in filesystem_specs.includes
)
snapshot_per_include = await MultiGet(
Get[Snapshot](PathGlobs, pg) for pg in pathglobs_per_include
)
owners_per_include = await MultiGet(
Get[Owners](OwnersRequest(sources=snapshot.files)) for snapshot in snapshot_per_include
)
addresses_to_specs: DefaultDict[
Address, List[Union[FilesystemLiteralSpec, FilesystemResolvedGlobSpec]]
] = defaultdict(list)
for spec, snapshot, owners in zip(
filesystem_specs.includes, snapshot_per_include, owners_per_include
):
if (
global_options.options.owners_not_found_behavior != OwnersNotFoundBehavior.ignore
and isinstance(spec, FilesystemLiteralSpec)
and not owners.addresses
):
file_path = PurePath(spec.to_spec_string())
msg = (
f"No owning targets could be found for the file `{file_path}`.\n\nPlease check "
f"that there is a BUILD file in `{file_path.parent}` with a target whose `sources` field "
f"includes `{file_path}`. See https://pants.readme.io/docs/targets for more "
"information on target definitions.\n"
"If you would like to ignore un-owned files, please pass `--owners-not-found-behavior=ignore`."
)
if global_options.options.owners_not_found_behavior == OwnersNotFoundBehavior.warn:
logger.warning(msg)
else:
raise ResolveError(msg)
# We preserve what literal files any globs resolved to. This allows downstream goals to be
# more precise in which files they operate on.
origin: Union[FilesystemLiteralSpec, FilesystemResolvedGlobSpec] = (
spec
if isinstance(spec, FilesystemLiteralSpec)
else FilesystemResolvedGlobSpec(glob=spec.glob, files=snapshot.files)
)
for address in owners.addresses:
addresses_to_specs[address].append(origin)
return AddressesWithOrigins(
AddressWithOrigin(
address, specs[0] if len(specs) == 1 else FilesystemMergedSpec.create(specs)
)
for address, specs in addresses_to_specs.items()
)
def create(
cls,
options_bootstrapper: OptionsBootstrapper,
options: Options,
session: SchedulerSession,
build_root: Optional[str] = None,
) -> Specs:
specs = cls.parse_specs(raw_specs=options.specs, build_root=build_root)
changed_options = ChangedOptions.from_options(
options.for_scope("changed"))
logger.debug("specs are: %s", specs)
logger.debug("changed_options are: %s", changed_options)
if specs.provided and changed_options.provided:
changed_name = "--changed-since" if changed_options.since else "--changed-diffspec"
if specs.filesystem_specs and specs.address_specs:
specs_description = "target and file arguments"
elif specs.filesystem_specs:
specs_description = "file arguments"
else:
specs_description = "target arguments"
raise InvalidSpecConstraint(
f"You used `{changed_name}` at the same time as using {specs_description}. Please "
"use only one.")
if not changed_options.provided:
return specs
scm = get_scm()
if not scm:
raise InvalidSpecConstraint(
"The `--changed-*` options are not available without a recognized SCM (usually "
"Git).")
changed_request = ChangedRequest(
sources=tuple(changed_options.changed_files(scm=scm)),
dependees=changed_options.dependees,
)
(changed_addresses, ) = session.product_request(
ChangedAddresses, [Params(changed_request, options_bootstrapper)])
logger.debug("changed addresses: %s", changed_addresses)
address_specs = []
filesystem_specs = []
for address in cast(ChangedAddresses, changed_addresses):
if not address.is_base_target:
# TODO: Should adjust Specs parsing to support parsing the disambiguated file
# Address, which would bypass-rediscovering owners.
filesystem_specs.append(FilesystemLiteralSpec(
address.filename))
else:
address_specs.append(
SingleAddress(address.spec_path, address.target_name))
return Specs(
AddressSpecs(address_specs, filter_by_global_options=True),
FilesystemSpecs(filesystem_specs),
)
def __init__(
self,
root_dir: str,
options_bootstrapper: OptionsBootstrapper,
options: Options,
build_config: BuildConfiguration,
run_tracker: RunTracker,
reporting: Reporting,
graph_session: LegacyGraphSession,
specs: Specs,
exiter=sys.exit,
) -> None:
"""
:param root_dir: The root directory of the pants workspace (aka the "build root").
:param options: The global, pre-initialized Options instance.
:param build_config: A pre-initialized BuildConfiguration instance.
:param run_tracker: The global, pre-initialized/running RunTracker instance.
:param reporting: The global, pre-initialized Reporting instance.
:param graph_session: The graph session for this run.
:param specs: The specs for this run, i.e. either the address or filesystem specs.
:param func exiter: A function that accepts an exit code value and exits. (for tests, Optional)
"""
self._root_dir = root_dir
self._options_bootstrapper = options_bootstrapper
self._options = options
self._build_config = build_config
self._run_tracker = run_tracker
self._reporting = reporting
self._graph_session = graph_session
self._specs = specs
self._exiter = exiter
self._global_options = options.for_global_scope()
self._fail_fast = self._global_options.fail_fast
self._explain = self._global_options.explain
self._kill_nailguns = self._global_options.kill_nailguns
# V1 tasks do not understand FilesystemSpecs, so we eagerly convert them into AddressSpecs.
if self._specs.filesystem_specs.dependencies:
(owned_addresses,
) = self._graph_session.scheduler_session.product_request(
Addresses, [
Params(self._specs.filesystem_specs,
self._options_bootstrapper)
])
updated_address_specs = AddressSpecs(
dependencies=tuple(
SingleAddress(a.spec_path, a.target_name)
for a in owned_addresses),
tags=self._specs.address_specs.matcher.tags,
exclude_patterns=self._specs.address_specs.matcher.
exclude_patterns,
)
self._specs = Specs(
address_specs=updated_address_specs,
filesystem_specs=FilesystemSpecs([]),
)
async def addresses_with_origins_from_filesystem_specs(
filesystem_specs: FilesystemSpecs,
global_options: GlobalOptions,
) -> AddressesWithOrigins:
"""Find the owner(s) for each FilesystemSpec while preserving the original FilesystemSpec those
owners come from.
Every returned address will be a generated subtarget, meaning that each address will have
exactly one file in its `sources` field.
"""
owners_not_found_behavior = global_options.options.owners_not_found_behavior
paths_per_include = await MultiGet(
Get(
Paths,
PathGlobs,
filesystem_specs.path_globs_for_spec(
spec, owners_not_found_behavior.to_glob_match_error_behavior()
),
)
for spec in filesystem_specs.includes
)
owners_per_include = await MultiGet(
Get(Owners, OwnersRequest(sources=paths.files)) for paths in paths_per_include
)
addresses_to_specs: Dict[Address, FilesystemSpec] = {}
for spec, owners in zip(filesystem_specs.includes, owners_per_include):
if (
owners_not_found_behavior != OwnersNotFoundBehavior.ignore
and isinstance(spec, FilesystemLiteralSpec)
and not owners
):
_log_or_raise_unmatched_owners(
[PurePath(str(spec))],
global_options.options.owners_not_found_behavior,
ignore_option="--owners-not-found-behavior=ignore",
)
for address in owners:
# A target might be covered by multiple specs, so we take the most specific one.
addresses_to_specs[address] = FilesystemSpecs.more_specific(
addresses_to_specs.get(address), spec
)
return AddressesWithOrigins(
AddressWithOrigin(address, spec) for address, spec in addresses_to_specs.items()
)
def calculate_specs(
options_bootstrapper: OptionsBootstrapper,
options: Options,
session: SchedulerSession,
*,
build_root: Optional[str] = None,
) -> Specs:
"""Determine the specs for a given Pants run."""
build_root = build_root or get_buildroot()
specs = SpecsParser(build_root).parse_specs(options.specs)
changed_options = ChangedOptions.from_options(options.for_scope("changed"))
logger.debug("specs are: %s", specs)
logger.debug("changed_options are: %s", changed_options)
if specs.provided and changed_options.provided:
changed_name = "--changed-since" if changed_options.since else "--changed-diffspec"
if specs.filesystem_specs and specs.address_specs:
specs_description = "target and file arguments"
elif specs.filesystem_specs:
specs_description = "file arguments"
else:
specs_description = "target arguments"
raise InvalidSpecConstraint(
f"You used `{changed_name}` at the same time as using {specs_description}. Please "
"use only one.")
if not changed_options.provided:
return specs
git = get_git()
if not git:
raise InvalidSpecConstraint(
"The `--changed-*` options are only available if Git is used for the repository."
)
changed_request = ChangedRequest(
sources=tuple(changed_options.changed_files(git)),
dependees=changed_options.dependees,
)
(changed_addresses, ) = session.product_request(
ChangedAddresses, [Params(changed_request, options_bootstrapper)])
logger.debug("changed addresses: %s", changed_addresses)
address_specs = []
for address in cast(ChangedAddresses, changed_addresses):
address_input = AddressInput.parse(address.spec)
address_specs.append(
AddressLiteralSpec(
path_component=address_input.path_component,
# NB: AddressInput.target_component may be None, but AddressLiteralSpec expects a
# string.
target_component=address_input.target_component
or address.target_name,
))
return Specs(AddressSpecs(address_specs, filter_by_global_options=True),
FilesystemSpecs([]))
async def sources_snapshots_from_filesystem_specs(
filesystem_specs: FilesystemSpecs, global_options: GlobalOptions,
) -> SourcesSnapshots:
"""Resolve the snapshot associated with the provided filesystem specs."""
snapshot = await Get[Snapshot](
PathGlobs,
filesystem_specs.to_path_globs(
global_options.options.owners_not_found_behavior.to_glob_match_error_behavior()
),
)
return SourcesSnapshots([SourcesSnapshot(snapshot)])
def create(
cls,
options: Options,
session: SchedulerSession,
build_root: Optional[str] = None,
exclude_patterns: Optional[Iterable[str]] = None,
tags: Optional[Iterable[str]] = None,
) -> Specs:
specs = cls.parse_specs(
raw_specs=options.specs,
build_root=build_root,
exclude_patterns=exclude_patterns,
tags=tags,
)
changed_options = ChangedOptions.from_options(
options.for_scope("changed"))
logger.debug("specs are: %s", specs)
logger.debug("changed_options are: %s", changed_options)
if changed_options.is_actionable(
) and specs.provided_specs.dependencies:
# We've been provided both a change request and specs.
raise InvalidSpecConstraint(
"Multiple target selection methods provided. Please use only one of "
"`--changed-*`, address specs, or filesystem specs.")
if changed_options.is_actionable():
scm = get_scm()
if not scm:
raise InvalidSpecConstraint(
"The `--changed-*` options are not available without a recognized SCM (usually Git)."
)
changed_request = ChangedRequest(
sources=tuple(changed_options.changed_files(scm=scm)),
include_dependees=changed_options.include_dependees,
)
(changed_addresses, ) = session.product_request(
ChangedAddresses, [changed_request])
logger.debug("changed addresses: %s", changed_addresses.addresses)
dependencies = tuple(
SingleAddress(a.spec_path, a.target_name)
for a in changed_addresses.addresses)
return Specs(
address_specs=AddressSpecs(
dependencies=dependencies,
exclude_patterns=exclude_patterns,
tags=tags,
),
filesystem_specs=FilesystemSpecs([]),
)
return specs
def test_filesystem_specs_literal_file(self) -> None:
self.create_files("demo", ["f1.txt", "f2.txt"])
self.add_to_build_file("demo", "target(sources=['*.txt'])")
spec = FilesystemLiteralSpec("demo/f1.txt")
result = self.request_single_product(
AddressesWithOrigins, Params(FilesystemSpecs([spec]), create_options_bootstrapper())
)
assert len(result) == 1
assert result[0] == AddressWithOrigin(
Address("demo", relative_file_path="f1.txt", target_name="demo"), origin=spec
)
def test_filesystem_specs_nonexistent_file(self) -> None:
specs = FilesystemSpecs([FilesystemLiteralSpec("demo/fake.txt")])
with pytest.raises(ExecutionError) as exc:
self.request_single_product(
AddressesWithOrigins, Params(specs, create_options_bootstrapper()),
)
assert 'Unmatched glob from file arguments: "demo/fake.txt"' in str(exc.value)
ignore_errors_result = self.request_single_product(
AddressesWithOrigins,
Params(specs, create_options_bootstrapper(args=["--owners-not-found-behavior=ignore"])),
)
assert not ignore_errors_result
def test_filesystem_specs_no_owner(self) -> None:
self.create_file("no_owners/f.txt")
# Error for literal specs.
with pytest.raises(ExecutionError) as exc:
self.request_single_product(
AddressesWithOrigins,
Params(
FilesystemSpecs([FilesystemLiteralSpec("no_owners/f.txt")]),
create_options_bootstrapper(),
),
)
assert "No owning targets could be found for the file `no_owners/f.txt`" in str(exc.value)
# Do not error for glob specs.
glob_file_result = self.request_single_product(
AddressesWithOrigins,
Params(
FilesystemSpecs([FilesystemGlobSpec("no_owners/*.txt")]),
create_options_bootstrapper(),
),
)
assert not glob_file_result
def parse_specs(self, specs: Iterable[str]) -> Specs:
address_specs: OrderedSet[AddressSpec] = OrderedSet()
filesystem_specs: OrderedSet[FilesystemSpec] = OrderedSet()
for spec_str in specs:
parsed_spec = self.parse_spec(spec_str)
if isinstance(parsed_spec, AddressSpec):
address_specs.add(parsed_spec)
else:
filesystem_specs.add(parsed_spec)
return Specs(
AddressSpecs(address_specs, filter_by_global_options=True),
FilesystemSpecs(filesystem_specs),
)
def resolve_address_specs(
self,
specs: Iterable[AddressSpec],
bootstrapper: Optional[OptionsBootstrapper] = None
) -> Set[AddressWithOrigin]:
result = self.request_single_product(
AddressesWithOrigins,
Params(
Specs(AddressSpecs(specs, filter_by_global_options=True),
FilesystemSpecs([])),
bootstrapper or create_options_bootstrapper(),
),
)
return set(result)
def assert_python_requirements(
rule_runner: RuleRunner,
build_file_entry: str,
requirements_txt: str,
*,
expected_file_dep: PythonRequirementsFile,
expected_targets: Iterable[PythonRequirementLibrary],
requirements_txt_relpath: str = "requirements.txt",
) -> None:
rule_runner.write_files({"BUILD": build_file_entry, requirements_txt_relpath: requirements_txt})
targets = rule_runner.request(
Targets,
[Specs(AddressSpecs([DescendantAddresses("")]), FilesystemSpecs([]))],
)
assert {expected_file_dep, *expected_targets} == set(targets)
def assert_poetry_requirements(
rule_runner: RuleRunner,
build_file_entry: str,
pyproject_toml: str,
*,
expected_file_dep: PythonRequirementsFile,
expected_targets: Iterable[PythonRequirementLibrary],
pyproject_toml_relpath: str = "pyproject.toml",
) -> None:
rule_runner.write_files({"BUILD": build_file_entry, pyproject_toml_relpath: pyproject_toml})
targets = rule_runner.request(
Targets,
[Specs(AddressSpecs([DescendantAddresses("")]), FilesystemSpecs([]))],
)
assert {expected_file_dep, *expected_targets} == set(targets)
def test_filesystem_specs_glob(self) -> None:
self.create_files("demo", ["f1.txt", "f2.txt"])
self.add_to_build_file("demo", "target(sources=['*.txt'])")
spec = FilesystemGlobSpec("demo/*.txt")
result = self.request_single_product(
AddressesWithOrigins, Params(FilesystemSpecs([spec]), create_options_bootstrapper()),
)
assert result == AddressesWithOrigins(
[
AddressWithOrigin(
Address("demo", relative_file_path="f1.txt", target_name="demo"), origin=spec
),
AddressWithOrigin(
Address("demo", relative_file_path="f2.txt", target_name="demo"), origin=spec
),
]
)
# If a glob and a literal spec both resolve to the same file, the literal spec should be
# used as it's more precise.
literal_spec = FilesystemLiteralSpec("demo/f1.txt")
result = self.request_single_product(
AddressesWithOrigins,
Params(FilesystemSpecs([spec, literal_spec]), create_options_bootstrapper()),
)
assert result == AddressesWithOrigins(
[
AddressWithOrigin(
Address("demo", relative_file_path="f1.txt", target_name="demo"),
origin=literal_spec,
),
AddressWithOrigin(
Address("demo", relative_file_path="f2.txt", target_name="demo"), origin=spec
),
]
)
def test_filesystem_specs_more_specific() -> None:
literal = FilesystemLiteralSpec("foo.txt")
glob = FilesystemGlobSpec("*.txt")
assert literal == FilesystemSpecs.more_specific(literal, None)
assert literal == FilesystemSpecs.more_specific(literal, glob)
assert literal == FilesystemSpecs.more_specific(None, literal)
assert literal == FilesystemSpecs.more_specific(glob, literal)
assert glob == FilesystemSpecs.more_specific(None, glob)
assert glob == FilesystemSpecs.more_specific(glob, None)
def assert_pipenv_requirements(
rule_runner: RuleRunner,
build_file_entry: str,
pipfile_lock: dict,
*,
expected_file_dep: PythonRequirementsFile,
expected_targets: Iterable[PythonRequirementLibrary],
pipfile_lock_relpath: str = "Pipfile.lock",
) -> None:
rule_runner.add_to_build_file("", f"{build_file_entry}\n")
rule_runner.create_file(pipfile_lock_relpath, dumps(pipfile_lock))
targets = rule_runner.request(
Targets,
[Specs(AddressSpecs([DescendantAddresses("")]), FilesystemSpecs([]))],
)
assert {expected_file_dep, *expected_targets} == set(targets)
def assert_python_requirements(
rule_runner: RuleRunner,
build_file_entry: str,
requirements_txt: str,
*,
expected_file_dep: PythonRequirementsFile,
expected_targets: Iterable[PythonRequirementLibrary],
requirements_txt_relpath: str = "requirements.txt",
) -> None:
rule_runner.add_to_build_file("", f"{build_file_entry}\n")
rule_runner.create_file(requirements_txt_relpath, requirements_txt)
targets = rule_runner.request_product(
Targets,
[
Specs(AddressSpecs([DescendantAddresses("")]), FilesystemSpecs([])),
create_options_bootstrapper(),
],
)
assert {expected_file_dep, *expected_targets} == set(targets)
def parse_specs(cls,
raw_specs: Iterable[str],
*,
build_root: Optional[str] = None) -> Specs:
"""Parse raw string specs into a Specs object."""
build_root = build_root or get_buildroot()
spec_parser = CmdLineSpecParser(build_root)
address_specs: OrderedSet[AddressSpec] = OrderedSet()
filesystem_specs: OrderedSet[FilesystemSpec] = OrderedSet()
for spec_str in raw_specs:
parsed_spec = spec_parser.parse_spec(spec_str)
if isinstance(parsed_spec, AddressSpec):
address_specs.add(parsed_spec)
else:
filesystem_specs.add(parsed_spec)
return Specs(
AddressSpecs(address_specs, filter_by_global_options=True),
FilesystemSpecs(filesystem_specs),
)
def test_specs_to_dirs() -> None:
assert specs_to_dirs(Specs(AddressSpecs([]),
FilesystemSpecs([]))) == ("", )
assert specs_to_dirs(
Specs(AddressSpecs([AddressLiteralSpec("src/python/foo")]),
FilesystemSpecs([]))) == ("src/python/foo", )
assert specs_to_dirs(
Specs(AddressSpecs([]),
FilesystemSpecs([DirLiteralSpec("src/python/foo")
]))) == ("src/python/foo", )
assert specs_to_dirs(
Specs(
AddressSpecs([
AddressLiteralSpec("src/python/foo"),
AddressLiteralSpec("src/python/bar")
]),
FilesystemSpecs([]),
)) == ("src/python/foo", "src/python/bar")
with pytest.raises(ValueError):
specs_to_dirs(
Specs(AddressSpecs([]),
FilesystemSpecs([FileLiteralSpec("src/python/foo.py")])))
with pytest.raises(ValueError):
specs_to_dirs(
Specs(AddressSpecs([AddressLiteralSpec("src/python/bar", "tgt")]),
FilesystemSpecs([])))
with pytest.raises(ValueError):
specs_to_dirs(
Specs(
AddressSpecs([
AddressLiteralSpec("src/python/bar",
target_component=None,
generated_component="gen")
]),
FilesystemSpecs([]),
))
def _init_engine(self, local_store_dir: Optional[str] = None) -> None:
if self._scheduler is not None:
return
options_bootstrapper = OptionsBootstrapper.create(
args=["--pants-config-files=[]", *self.additional_options]
)
global_options = options_bootstrapper.bootstrap_options.for_global_scope()
local_store_dir = local_store_dir or global_options.local_store_dir
local_execution_root_dir = global_options.local_execution_root_dir
named_caches_dir = global_options.named_caches_dir
# NB: This uses the long form of initialization because it needs to directly specify
# `cls.alias_groups` rather than having them be provided by bootstrap options.
graph_session = EngineInitializer.setup_legacy_graph_extended(
pants_ignore_patterns=[],
use_gitignore=False,
local_store_dir=local_store_dir,
local_execution_root_dir=local_execution_root_dir,
named_caches_dir=named_caches_dir,
build_file_prelude_globs=(),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
native=init_native(),
options_bootstrapper=options_bootstrapper,
build_root=self.build_root,
build_configuration=self.build_config(),
build_ignore_patterns=None,
execution_options=ExecutionOptions.from_bootstrap_options(global_options),
).new_session(
zipkin_trace_v2=False, build_id="buildid_for_test", should_report_workunits=True
)
self._scheduler = graph_session.scheduler_session
self._build_graph, self._address_mapper = graph_session.create_build_graph(
Specs(address_specs=AddressSpecs([]), filesystem_specs=FilesystemSpecs([])),
self._build_root(),
)
async def sources_snapshots_from_filesystem_specs(
filesystem_specs: FilesystemSpecs, ) -> SourcesSnapshots:
"""Resolve the snapshot associated with the provided filesystem specs."""
snapshot = await Get[Snapshot](PathGlobs, filesystem_specs.to_path_globs())
return SourcesSnapshots([SourcesSnapshot(snapshot)])
def test_tailor_rule(rule_runner: RuleRunner) -> None:
with mock_console(rule_runner.options_bootstrapper) as (console,
stdio_reader):
workspace = Workspace(rule_runner.scheduler)
union_membership = UnionMembership(
{PutativeTargetsRequest: [MockPutativeTargetsRequest]})
specs = Specs(address_specs=AddressSpecs(tuple()),
filesystem_specs=FilesystemSpecs(tuple()))
run_rule_with_mocks(
tailor.tailor,
rule_args=[
create_goal_subsystem(
TailorSubsystem,
build_file_name="BUILD",
build_file_header="",
build_file_indent=" ",
alias_mapping={"fortran_library": "my_fortran_lib"},
),
console,
workspace,
union_membership,
specs,
],
mock_gets=[
MockGet(
output_type=PutativeTargets,
input_type=PutativeTargetsRequest,
mock=lambda req: PutativeTargets([
PutativeTarget.
for_target_type(FortranTests, "src/fortran/foo",
"tests", ["bar1_test.f90"]),
PutativeTarget.for_target_type(
FortranLibrary, "src/fortran/baz", "baz",
["qux1.f90"]),
PutativeTarget.for_target_type(
FortranLibrary,
"src/fortran/conflict",
"conflict",
["conflict1.f90", "conflict2.f90"],
),
]),
),
MockGet(
output_type=UniquelyNamedPutativeTargets,
input_type=PutativeTargets,
mock=lambda pts: UniquelyNamedPutativeTargets(
PutativeTargets([
pt.rename("conflict0")
if pt.name == "conflict" else pt for pt in pts
])),
),
MockGet(
output_type=DisjointSourcePutativeTarget,
input_type=PutativeTarget,
# This test exists to test the console output, which isn't affected by
# whether the sources of a putative target were modified due to conflict,
# so we don't bother to inject such modifications. The BUILD file content
# generation, which is so affected, is tested separately above.
mock=lambda pt: DisjointSourcePutativeTarget(pt),
),
MockGet(
output_type=EditedBuildFiles,
input_type=EditBuildFilesRequest,
mock=lambda _: EditedBuildFiles(
# We test that the created digest contains what we expect above, and we
# don't need to test here that writing digests to the Workspace works.
# So the empty digest is sufficient.
digest=EMPTY_DIGEST,
created_paths=("src/fortran/baz/BUILD", ),
updated_paths=(
"src/fortran/foo/BUILD",
"src/fortran/conflict/BUILD",
),
),
),
],
union_membership=union_membership,
)
stdout_str = stdio_reader.get_stdout()
assert "Created src/fortran/baz/BUILD:\n - Added my_fortran_lib target baz" in stdout_str
assert "Updated src/fortran/foo/BUILD:\n - Added fortran_tests target tests" in stdout_str
assert (
"Updated src/fortran/conflict/BUILD:\n - Added my_fortran_lib target conflict0"
) in stdout_str
def create(
cls,
options: Options,
session: SchedulerSession,
build_root: Optional[str] = None,
exclude_patterns: Optional[Iterable[str]] = None,
tags: Optional[Iterable[str]] = None,
) -> Specs:
specs = cls.parse_specs(
raw_specs=options.specs,
build_root=build_root,
exclude_patterns=exclude_patterns,
tags=tags,
)
changed_options = ChangedOptions.from_options(
options.for_scope('changed'))
owned_files = options.for_global_scope().owner_of
logger.debug('specs are: %s', specs)
logger.debug('changed_options are: %s', changed_options)
logger.debug('owned_files are: %s', owned_files)
targets_specified = sum(1
for item in (changed_options.is_actionable(),
owned_files,
specs.provided_specs.dependencies)
if item)
if targets_specified > 1:
# We've been provided more than one of: a change request, an owner request, or specs.
raise InvalidSpecConstraint(
'Multiple target selection methods provided. Please use only one of '
'`--changed-*`, `--owner-of`, address specs, or filesystem specs.'
)
if changed_options.is_actionable():
scm = get_scm()
if not scm:
raise InvalidSpecConstraint(
'The `--changed-*` options are not available without a recognized SCM (usually git).'
)
changed_request = ChangedRequest(
sources=tuple(changed_options.changed_files(scm=scm)),
include_dependees=changed_options.include_dependees,
)
changed_addresses, = session.product_request(
ChangedAddresses, [changed_request])
logger.debug('changed addresses: %s', changed_addresses.addresses)
dependencies = tuple(
SingleAddress(a.spec_path, a.target_name)
for a in changed_addresses.addresses)
return Specs(
address_specs=AddressSpecs(
dependencies=dependencies,
exclude_patterns=exclude_patterns,
tags=tags,
),
filesystem_specs=FilesystemSpecs([]),
)
if owned_files:
owner_request = OwnersRequest(sources=tuple(owned_files))
owner_request.validate(
pants_bin_name=options.for_global_scope().pants_bin_name)
owners, = session.product_request(Owners, [owner_request])
logger.debug('owner addresses: %s', owners.addresses)
dependencies = tuple(
SingleAddress(a.spec_path, a.target_name)
for a in owners.addresses)
return Specs(address_specs=AddressSpecs(
dependencies=dependencies,
exclude_patterns=exclude_patterns,
tags=tags,
),
filesystem_specs=FilesystemSpecs([]))
return specs