def parse_spec(self, spec: str) -> Union[AddressSpec, FilesystemSpec]: """Parse the given spec into an `AddressSpec` or `FilesystemSpec` object. :raises: CmdLineSpecParser.BadSpecError if the address selector could not be parsed. """ if spec.endswith("::"): spec_path = spec[:-len("::")] return DescendantAddresses( directory=self._normalize_spec_path(spec_path)) if spec.endswith(":"): spec_path = spec[:-len(":")] return SiblingAddresses( directory=self._normalize_spec_path(spec_path)) if ":" in spec: spec_parts = spec.rsplit(":", 1) spec_path = self._normalize_spec_path(spec_parts[0]) return AddressLiteralSpec(path_component=spec_path, target_component=spec_parts[1]) if spec.startswith("!"): return FilesystemIgnoreSpec(spec[1:]) if "*" in spec: return FilesystemGlobSpec(spec) if PurePath(spec).suffix: return FilesystemLiteralSpec(self._normalize_spec_path(spec)) spec_path = self._normalize_spec_path(spec) if Path(self._root_dir, spec_path).is_file(): return FilesystemLiteralSpec(spec_path) # Else we apply address shorthand, i.e. `src/python/pants/util` -> # `src/python/pants/util:util` return AddressLiteralSpec(path_component=spec_path, target_component=PurePath(spec).name)
def parse_spec(self, spec: str) -> Union[AddressSpec, FilesystemSpec]: """Parse the given spec into an `AddressSpec` or `FilesystemSpec` object. :raises: CmdLineSpecParser.BadSpecError if the address selector could not be parsed. """ if spec.endswith("::"): spec_path = spec[:-len("::")] return DescendantAddresses( directory=self._normalize_spec_path(spec_path)) if spec.endswith(":"): spec_path = spec[:-len(":")] return SiblingAddresses( directory=self._normalize_spec_path(spec_path)) if ":" in spec: spec_parts = spec.rsplit(":", 1) spec_path = self._normalize_spec_path(spec_parts[0]) return SingleAddress(directory=spec_path, name=spec_parts[1]) if spec.startswith("!"): return FilesystemIgnoreSpec(spec[1:]) if "*" in spec: return FilesystemGlobSpec(spec) if PurePath(spec).suffix: return FilesystemLiteralSpec(self._normalize_spec_path(spec)) spec_path = self._normalize_spec_path(spec) if Path(self._root_dir, spec_path).is_file(): return FilesystemLiteralSpec(spec_path) # Else we apply address shorthand, i.e. `src/python/pants/util` -> `src/python/pants/util:util` # TODO: Figure out what this should look like if (once?) filesystem specs allow directories. # Should this shorthand be removed so that directories may be unambiguously resolved to a # FilesystemSpec? If so, do we still allow the shorthand in BUILD files? return SingleAddress(directory=spec_path, name=PurePath(spec).name)
def test_precise_file_args(self) -> None: target = self.make_target_with_origin( [self.good_source, self.bad_source], origin=FilesystemLiteralSpec(self.good_source.path)) result = self.run_flake8([target]) assert result.exit_code == 0 assert result.stdout.strip() == ""
def test_precise_file_args(self) -> None: self.create_python_test_target([self.good_source, self.bad_source]) file_arg = FilesystemLiteralSpec(PurePath(self.package, self.good_source.path).as_posix()) result = self.run_pytest(origin=file_arg) assert result.status == Status.SUCCESS assert "test_good.py ." in result.stdout assert "test_bad.py F" not in result.stdout
def test_filesystem_specs(self) -> None: # Literal file arg. sources_field1_all_sources = SOURCES1.source_file_absolute_paths sources_field1_slice = slice(0, 1) sources_field1 = self.mock_sources_field_with_origin( SOURCES1, origin=FilesystemLiteralSpec(sources_field1_all_sources[0])) # Glob file arg that matches the entire `sources`. sources_field2_all_sources = SOURCES2.source_file_absolute_paths sources_field2_slice = slice(0, len(sources_field2_all_sources)) sources_field2_origin = FilesystemResolvedGlobSpec( f"{SOURCES2.source_root}/*.py", files=tuple(sources_field2_all_sources)) sources_field2 = self.mock_sources_field_with_origin( SOURCES2, origin=sources_field2_origin) # Glob file arg that only matches a subset of the `sources` _and_ includes resolved # files not owned by the target. sources_field3_all_sources = SOURCES3.source_file_absolute_paths sources_field3_slice = slice(0, 1) sources_field3_origin = FilesystemResolvedGlobSpec( f"{SOURCES3.source_root}/*.java", files=tuple( PurePath(SOURCES3.source_root, name).as_posix() for name in [SOURCES3.source_files[0], "other_target.java", "j.tmp.java"]), ) sources_field3 = self.mock_sources_field_with_origin( SOURCES3, origin=sources_field3_origin) def assert_file_args_resolved( sources_field_with_origin: Tuple[SourcesField, OriginSpec], all_sources: List[str], expected_slice: slice, ) -> None: assert self.get_all_source_files([sources_field_with_origin ]) == all_sources assert (self.get_specified_source_files( [sources_field_with_origin]) == all_sources[expected_slice]) assert_file_args_resolved(sources_field1, sources_field1_all_sources, sources_field1_slice) assert_file_args_resolved(sources_field2, sources_field2_all_sources, sources_field2_slice) assert_file_args_resolved(sources_field3, sources_field3_all_sources, sources_field3_slice) combined_sources_fields = [ sources_field1, sources_field2, sources_field3 ] assert self.get_all_source_files(combined_sources_fields) == sorted([ *sources_field1_all_sources, *sources_field2_all_sources, *sources_field3_all_sources ]) assert self.get_specified_source_files( combined_sources_fields) == sorted([ *sources_field1_all_sources[sources_field1_slice], *sources_field2_all_sources[sources_field2_slice], *sources_field3_all_sources[sources_field3_slice], ])
def test_precise_file_args(self) -> None: target = self.make_target_with_origin( [self.good_source, self.bad_source], origin=FilesystemLiteralSpec(self.good_source.path) ) result = self.run_bandit([target]) assert result.exit_code == 0 assert "No issues identified." in result.stdout
def test_precise_file_args(self) -> None: target = self.make_target_with_origin( [self.good_source, self.bad_source], origin=FilesystemLiteralSpec(self.good_source.path)) result = self.run_pylint([target]) assert result.exit_code == 0 assert "Your code has been rated at 10.00/10" in result.stdout.strip()
def parse_spec(self, spec: str) -> AddressSpec | FilesystemSpec: """Parse the given spec into an `AddressSpec` or `FilesystemSpec` object. :raises: CmdLineSpecParser.BadSpecError if the address selector could not be parsed. """ if spec.endswith("::"): spec_path = spec[:-len("::")] return DescendantAddresses( directory=self._normalize_spec_path(spec_path)) if spec.endswith(":"): spec_path = spec[:-len(":")] return SiblingAddresses( directory=self._normalize_spec_path(spec_path)) if ":" in spec or "#" in spec: tgt_parts = spec.split(":", maxsplit=1) path_component = tgt_parts[0] if len(tgt_parts) == 1: target_component = None generated_parts = path_component.split("#", maxsplit=1) if len(generated_parts) == 1: generated_component = None else: path_component, generated_component = generated_parts else: generated_parts = tgt_parts[1].split("#", maxsplit=1) if len(generated_parts) == 1: target_component = generated_parts[0] generated_component = None else: target_component, generated_component = generated_parts return AddressLiteralSpec( path_component=self._normalize_spec_path(path_component), target_component=target_component, generated_component=generated_component, ) if spec.startswith("!"): return FilesystemIgnoreSpec(spec[1:]) if "*" in spec: return FilesystemGlobSpec(spec) if PurePath(spec).suffix: return FilesystemLiteralSpec(self._normalize_spec_path(spec)) spec_path = self._normalize_spec_path(spec) if Path(self._root_dir, spec_path).is_file(): return FilesystemLiteralSpec(spec_path) # Else we apply address shorthand, i.e. `src/python/pants/util` -> # `src/python/pants/util:util` return AddressLiteralSpec(spec_path, None, None)
def test_precise_file_args(self) -> None: target = self.make_target_with_origin( [self.good_source, self.bad_source], origin=FilesystemLiteralSpec(self.good_source.path)) lint_result, fmt_result = self.run_docformatter([target]) assert lint_result == LintResult.noop() assert fmt_result.digest == self.get_digest( [self.good_source, self.bad_source])
def create( cls, options_bootstrapper: OptionsBootstrapper, options: Options, session: SchedulerSession, build_root: Optional[str] = None, ) -> Specs: specs = cls.parse_specs(raw_specs=options.specs, build_root=build_root) changed_options = ChangedOptions.from_options( options.for_scope("changed")) logger.debug("specs are: %s", specs) logger.debug("changed_options are: %s", changed_options) if specs.provided and changed_options.provided: changed_name = "--changed-since" if changed_options.since else "--changed-diffspec" if specs.filesystem_specs and specs.address_specs: specs_description = "target and file arguments" elif specs.filesystem_specs: specs_description = "file arguments" else: specs_description = "target arguments" raise InvalidSpecConstraint( f"You used `{changed_name}` at the same time as using {specs_description}. Please " "use only one.") if not changed_options.provided: return specs scm = get_scm() if not scm: raise InvalidSpecConstraint( "The `--changed-*` options are not available without a recognized SCM (usually " "Git).") changed_request = ChangedRequest( sources=tuple(changed_options.changed_files(scm=scm)), dependees=changed_options.dependees, ) (changed_addresses, ) = session.product_request( ChangedAddresses, [Params(changed_request, options_bootstrapper)]) logger.debug("changed addresses: %s", changed_addresses) address_specs = [] filesystem_specs = [] for address in cast(ChangedAddresses, changed_addresses): if not address.is_base_target: # TODO: Should adjust Specs parsing to support parsing the disambiguated file # Address, which would bypass-rediscovering owners. filesystem_specs.append(FilesystemLiteralSpec( address.filename)) else: address_specs.append( SingleAddress(address.spec_path, address.target_name)) return Specs( AddressSpecs(address_specs, filter_by_global_options=True), FilesystemSpecs(filesystem_specs), )
def test_configuration() -> None: class UnrelatedField(StringField): alias = "unrelated_field" default = "default" value: str class UnrelatedTarget(Target): alias = "unrelated_target" core_fields = (UnrelatedField, ) class NoFieldsTarget(Target): alias = "no_fields_target" core_fields = () @dataclass(frozen=True) class FortranConfiguration(Configuration): required_fields = (FortranSources, ) sources: FortranSources unrelated_field: UnrelatedField @dataclass(frozen=True) class UnrelatedFieldConfiguration(ConfigurationWithOrigin): required_fields = () unrelated_field: UnrelatedField fortran_addr = Address.parse(":fortran") fortran_tgt = FortranTarget({}, address=fortran_addr) unrelated_addr = Address.parse(":unrelated") unrelated_tgt = UnrelatedTarget({UnrelatedField.alias: "configured"}, address=unrelated_addr) no_fields_addr = Address.parse(":no_fields") no_fields_tgt = NoFieldsTarget({}, address=no_fields_addr) assert FortranConfiguration.is_valid(fortran_tgt) is True assert FortranConfiguration.is_valid(unrelated_tgt) is False assert FortranConfiguration.is_valid(no_fields_tgt) is False # When no fields are required, every target is valid. for tgt in [fortran_tgt, unrelated_tgt, no_fields_tgt]: assert UnrelatedFieldConfiguration.is_valid(tgt) is True valid_fortran_config = FortranConfiguration.create(fortran_tgt) assert valid_fortran_config.address == fortran_addr assert valid_fortran_config.unrelated_field.value == UnrelatedField.default with pytest.raises(KeyError): FortranConfiguration.create(unrelated_tgt) origin = FilesystemLiteralSpec("f.txt") assert (UnrelatedFieldConfiguration.create( TargetWithOrigin(fortran_tgt, origin)).origin == origin) assert (UnrelatedFieldConfiguration.create( TargetWithOrigin(unrelated_tgt, origin)).unrelated_field.value == "configured") assert (UnrelatedFieldConfiguration.create( TargetWithOrigin( no_fields_tgt, origin)).unrelated_field.value == UnrelatedField.default)
def test_precise_file_args(self) -> None: target = self.make_target_with_origin( [self.good_source, self.bad_source], origin=FilesystemLiteralSpec(self.good_source.path) ) lint_result, fmt_result = self.run_black([target]) assert lint_result.exit_code == 0 assert "1 file would be left unchanged" in lint_result.stderr assert "1 file left unchanged" in fmt_result.stderr assert fmt_result.digest == self.get_digest([self.good_source, self.bad_source])
def test_precise_file_args(self) -> None: target = self.make_target_with_origin( [self.good_source, self.bad_source], origin=FilesystemLiteralSpec(self.good_source.path)) lint_result, fmt_result = self.run_isort([target]) assert lint_result.exit_code == 0 assert lint_result.stdout == "" assert fmt_result.stdout == "" assert fmt_result.digest == self.get_digest( [self.good_source, self.bad_source])
def test_filesystem_specs_literal_file(self) -> None: self.create_files("demo", ["f1.txt", "f2.txt"]) self.add_to_build_file("demo", "target(sources=['*.txt'])") spec = FilesystemLiteralSpec("demo/f1.txt") result = self.request_single_product( AddressesWithOrigins, Params(FilesystemSpecs([spec]), create_options_bootstrapper()) ) assert len(result) == 1 assert result[0] == AddressWithOrigin( Address("demo", relative_file_path="f1.txt", target_name="demo"), origin=spec )
def test_precise_file_args(self) -> None: target = self.make_target_with_origin( [self.good_source, self.bad_source], origin=FilesystemLiteralSpec(self.good_source.path)) lint_results, fmt_result = self.run_docformatter([target]) assert len(lint_results) == 1 assert lint_results[0].exit_code == 0 assert lint_results[0].stderr == "" assert fmt_result.output == self.get_digest( [self.good_source, self.bad_source]) assert fmt_result.did_change is False
def test_filesystem_specs_more_specific() -> None: literal = FilesystemLiteralSpec("foo.txt") glob = FilesystemGlobSpec("*.txt") assert literal == FilesystemSpecs.more_specific(literal, None) assert literal == FilesystemSpecs.more_specific(literal, glob) assert literal == FilesystemSpecs.more_specific(None, literal) assert literal == FilesystemSpecs.more_specific(glob, literal) assert glob == FilesystemSpecs.more_specific(None, glob) assert glob == FilesystemSpecs.more_specific(glob, None)
def test_filesystem_specs_nonexistent_file(self) -> None: specs = FilesystemSpecs([FilesystemLiteralSpec("demo/fake.txt")]) with pytest.raises(ExecutionError) as exc: self.request_single_product( AddressesWithOrigins, Params(specs, create_options_bootstrapper()), ) assert 'Unmatched glob from file arguments: "demo/fake.txt"' in str(exc.value) ignore_errors_result = self.request_single_product( AddressesWithOrigins, Params(specs, create_options_bootstrapper(args=["--owners-not-found-behavior=ignore"])), ) assert not ignore_errors_result
def test_filesystem_specs(self) -> None: # Literal file arg. target1_all_sources = self.SOURCES1.source_file_absolute_paths target1_slice = slice(0, 1) target1 = self.mock_target(self.SOURCES1, origin=FilesystemLiteralSpec( target1_all_sources[0])) # Glob file arg that matches the entire target's `sources`. target2_all_sources = self.SOURCES2.source_file_absolute_paths target2_slice = slice(0, len(target2_all_sources)) target2_origin = FilesystemResolvedGlobSpec( f"{self.SOURCES2.source_root}/*.py", files=tuple(target2_all_sources)) target2 = self.mock_target(self.SOURCES2, origin=target2_origin) # Glob file arg that only matches a subset of the target's `sources` _and_ includes resolved # files not owned by the target. target3_all_sources = self.SOURCES3.source_file_absolute_paths target3_slice = slice(0, 1) target3_origin = FilesystemResolvedGlobSpec( f"{self.SOURCES3.source_root}/*.java", files=tuple( PurePath(self.SOURCES3.source_root, name).as_posix() for name in [ self.SOURCES3.source_files[0], "other_target.java", "j.tmp.java" ]), ) target3 = self.mock_target(self.SOURCES3, origin=target3_origin) def assert_file_args_resolved(target: TargetAdaptorWithOrigin, all_sources: List[str], expected_slice: slice) -> None: assert self.get_all_source_files([target]) == all_sources assert self.get_specified_source_files( [target]) == all_sources[expected_slice] assert_file_args_resolved(target1, target1_all_sources, target1_slice) assert_file_args_resolved(target2, target2_all_sources, target2_slice) assert_file_args_resolved(target3, target3_all_sources, target3_slice) combined_targets = [target1, target2, target3] assert self.get_all_source_files(combined_targets) == sorted( [*target1_all_sources, *target2_all_sources, *target3_all_sources]) assert self.get_specified_source_files(combined_targets) == sorted([ *target1_all_sources[target1_slice], *target2_all_sources[target2_slice], *target3_all_sources[target3_slice], ])
def test_resolve_addresses(self) -> None: """This tests that we correctly handle resolving from both address and filesystem specs.""" self.create_file("fs_spec/f.txt") self.add_to_build_file("fs_spec", "target(sources=['f.txt'])") self.create_file("address_spec/f.txt") self.add_to_build_file("address_spec", "target(sources=['f.txt'])") no_interaction_specs = ["fs_spec/f.txt", "address_spec:address_spec"] # If a generated subtarget's original base target is included via an address spec, # we will still include the generated subtarget for consistency. When we expand Targets # into their base targets this redundancy is removed, but during Address expansion we # get literal matches. self.create_files("multiple_files", ["f1.txt", "f2.txt"]) self.add_to_build_file("multiple_files", "target(sources=['*.txt'])") multiple_files_specs = ["multiple_files/f2.txt", "multiple_files:multiple_files"] specs = SpecsCalculator.parse_specs([*no_interaction_specs, *multiple_files_specs]) result = self.request_single_product( AddressesWithOrigins, Params(specs, create_options_bootstrapper()) ) assert set(result) == { AddressWithOrigin( Address("fs_spec", relative_file_path="f.txt"), origin=FilesystemLiteralSpec("fs_spec/f.txt"), ), AddressWithOrigin( Address("address_spec"), origin=SingleAddress("address_spec", "address_spec"), ), AddressWithOrigin( Address("multiple_files"), origin=SingleAddress("multiple_files", "multiple_files"), ), AddressWithOrigin( Address("multiple_files", relative_file_path="f2.txt"), origin=FilesystemLiteralSpec(file="multiple_files/f2.txt"), ), }
def test_filesystem_specs_no_owner(self) -> None: self.create_file("no_owners/f.txt") # Error for literal specs. with pytest.raises(ExecutionError) as exc: self.request_single_product( AddressesWithOrigins, Params( FilesystemSpecs([FilesystemLiteralSpec("no_owners/f.txt")]), create_options_bootstrapper(), ), ) assert "No owning targets could be found for the file `no_owners/f.txt`" in str(exc.value) # Do not error for glob specs. glob_file_result = self.request_single_product( AddressesWithOrigins, Params( FilesystemSpecs([FilesystemGlobSpec("no_owners/*.txt")]), create_options_bootstrapper(), ), ) assert not glob_file_result
def test_filesystem_specs_glob(self) -> None: self.create_files("demo", ["f1.txt", "f2.txt"]) self.add_to_build_file("demo", "target(sources=['*.txt'])") spec = FilesystemGlobSpec("demo/*.txt") result = self.request_single_product( AddressesWithOrigins, Params(FilesystemSpecs([spec]), create_options_bootstrapper()), ) assert result == AddressesWithOrigins( [ AddressWithOrigin( Address("demo", relative_file_path="f1.txt", target_name="demo"), origin=spec ), AddressWithOrigin( Address("demo", relative_file_path="f2.txt", target_name="demo"), origin=spec ), ] ) # If a glob and a literal spec both resolve to the same file, the literal spec should be # used as it's more precise. literal_spec = FilesystemLiteralSpec("demo/f1.txt") result = self.request_single_product( AddressesWithOrigins, Params(FilesystemSpecs([spec, literal_spec]), create_options_bootstrapper()), ) assert result == AddressesWithOrigins( [ AddressWithOrigin( Address("demo", relative_file_path="f1.txt", target_name="demo"), origin=literal_spec, ), AddressWithOrigin( Address("demo", relative_file_path="f2.txt", target_name="demo"), origin=spec ), ] )
def test_specs_to_dirs() -> None: assert specs_to_dirs(Specs(AddressSpecs([]), FilesystemSpecs([]))) == ("", ) assert specs_to_dirs( Specs(AddressSpecs([AddressLiteralSpec("src/python/foo")]), FilesystemSpecs([]))) == ("src/python/foo", ) assert (specs_to_dirs( Specs( AddressSpecs([ AddressLiteralSpec("src/python/foo"), AddressLiteralSpec("src/python/bar") ]), FilesystemSpecs([]), )) == ("src/python/foo", "src/python/bar")) with pytest.raises(ValueError): specs_to_dirs( Specs( AddressSpecs([]), FilesystemSpecs([FilesystemLiteralSpec("src/python/foo.py")]))) with pytest.raises(ValueError): specs_to_dirs( Specs(AddressSpecs([AddressLiteralSpec("src/python/bar", "tgt")]), FilesystemSpecs([]))) with pytest.raises(ValueError): specs_to_dirs( Specs( AddressSpecs([ AddressLiteralSpec("src/python/bar", target_component=None, generated_component="gen") ]), FilesystemSpecs([]), ))
def literal(file: str) -> FilesystemLiteralSpec: return FilesystemLiteralSpec(file)
def test_find_valid_field_sets(self) -> None: origin = FilesystemLiteralSpec("f.txt") valid_tgt = FortranTarget({}, address=Address.parse(":valid")) valid_tgt_with_origin = TargetWithOrigin(valid_tgt, origin) invalid_tgt = self.InvalidTarget({}, address=Address.parse(":invalid")) invalid_tgt_with_origin = TargetWithOrigin(invalid_tgt, origin) def find_valid_field_sets( superclass: Type, targets_with_origins: Iterable[TargetWithOrigin], *, error_if_no_valid_targets: bool = False, expect_single_config: bool = False, ) -> TargetsToValidFieldSets: request = TargetsToValidFieldSetsRequest( superclass, goal_description="fake", error_if_no_valid_targets=error_if_no_valid_targets, expect_single_field_set=expect_single_config, ) return self.request_single_product( TargetsToValidFieldSets, Params(request, TargetsWithOrigins(targets_with_origins),), ) valid = find_valid_field_sets( self.FieldSetSuperclass, [valid_tgt_with_origin, invalid_tgt_with_origin] ) assert valid.targets == (valid_tgt,) assert valid.targets_with_origins == (valid_tgt_with_origin,) assert valid.field_sets == ( self.FieldSetSubclass1.create(valid_tgt), self.FieldSetSubclass2.create(valid_tgt), ) with pytest.raises(ExecutionError) as exc: find_valid_field_sets( self.FieldSetSuperclass, [valid_tgt_with_origin], expect_single_config=True ) assert AmbiguousImplementationsException.__name__ in str(exc.value) with pytest.raises(ExecutionError) as exc: find_valid_field_sets( self.FieldSetSuperclass, [ valid_tgt_with_origin, TargetWithOrigin(FortranTarget({}, address=Address.parse(":valid2")), origin), ], expect_single_config=True, ) assert TooManyTargetsException.__name__ in str(exc.value) no_valid_targets = find_valid_field_sets(self.FieldSetSuperclass, [invalid_tgt_with_origin]) assert no_valid_targets.targets == () assert no_valid_targets.targets_with_origins == () assert no_valid_targets.field_sets == () with pytest.raises(ExecutionError) as exc: find_valid_field_sets( self.FieldSetSuperclass, [invalid_tgt_with_origin], error_if_no_valid_targets=True ) assert NoValidTargetsException.__name__ in str(exc.value) valid_with_origin = find_valid_field_sets( self.FieldSetSuperclassWithOrigin, [valid_tgt_with_origin, invalid_tgt_with_origin] ) assert valid_with_origin.targets == (valid_tgt,) assert valid_with_origin.targets_with_origins == (valid_tgt_with_origin,) assert valid_with_origin.field_sets == ( self.FieldSetSubclassWithOrigin.create(valid_tgt_with_origin), )