def test_no_address_no_family(self) -> None:
spec = SingleAddress("a/c", "c")
# Does not exist.
with self.assertRaises(Exception):
self.resolve(spec)
build_file = os.path.join(self.build_root, "a/c", "c.BUILD.json")
with safe_open(build_file, "w") as fp:
fp.write('{"type_alias": "struct", "name": "c"}')
self.scheduler.invalidate_files(["a/c"])
# Success.
resolved = self.resolve(spec)
self.assertEqual(1, len(resolved))
self.assertEqual(
[Struct(address=Address.parse("a/c"), type_alias="struct")],
resolved)
def test_duplicated(self):
"""Test that matching the same Spec twice succeeds."""
address = SingleAddress('a', 'a')
address_mapper = AddressMapper(JsonParser(TestTable()))
snapshot = Snapshot(DirectoryDigest(text_type('xx'), 2),
(Path('a/BUILD', File('a/BUILD')), ))
address_family = AddressFamily(
'a', {'a': ('a/BUILD', 'this is an object!')})
bfas = run_rule(
addresses_from_address_families, address_mapper,
Specs([address, address]), {
(Snapshot, PathGlobs): lambda _: snapshot,
(AddressFamily, Dir): lambda _: address_family,
})
self.assertEqual(len(bfas.dependencies), 1)
self.assertEqual(bfas.dependencies[0].spec, 'a:a')
def test_no_address_no_family(self):
spec = SingleAddress('a/c', 'c')
# Does not exist.
self.assertEqual(0, len(self.resolve(spec)))
# Exists on disk, but not yet in memory.
directory = 'a/c'
build_file = os.path.join(self.build_root, directory, 'c.BUILD.json')
with safe_open(build_file, 'w') as fp:
fp.write('{"type_alias": "struct", "name": "c"}')
self.assertEqual(0, len(self.resolve(spec)))
# Success.
self.scheduler.invalidate_files([directory])
resolved = self.resolve(spec)
self.assertEqual(1, len(resolved))
self.assertEqual(Struct(name='c', type_alias='struct'),
resolved[0].struct)
def test_no_address_no_family(self):
spec = SingleAddress('a/c', None)
# Should fail: does not exist.
with self.assertRaises(ResolveError):
self.resolve(spec)
# Exists on disk, but not yet in memory.
build_file = os.path.join(self.build_root, 'a/c/c.BUILD.json')
with safe_open(build_file, 'w') as fp:
fp.write('{"type_alias": "struct", "name": "c"}')
with self.assertRaises(ResolveError):
self.resolve(spec)
# Success.
self.scheduler.product_graph.invalidate()
resolved = self.resolve(spec)
self.assertEqual(1, len(resolved))
self.assertEqual(Struct(name='c', type_alias='struct'), resolved[0].struct)
def single_target_run(
self, *, console: MockConsole, program_text: bytes, address_spec: str,
) -> Run:
workspace = Workspace(self.scheduler)
interactive_runner = InteractiveRunner(self.scheduler)
class TestBinaryConfiguration(BinaryConfiguration):
required_fields = ()
class TestBinaryTarget(Target):
alias = "binary"
core_fields = ()
address = Address.parse(address_spec)
target = TestBinaryTarget({}, address=address)
target_with_origin = TargetWithOrigin(
target, SingleAddress(address.spec_path, address.target_name)
)
config = TestBinaryConfiguration.create(target)
res = run_rule(
run,
rule_args=[
console,
workspace,
interactive_runner,
BuildRoot(),
create_goal_subsystem(RunOptions, args=[]),
create_subsystem(GlobalOptions, pants_workdir=self.pants_workdir),
],
mock_gets=[
MockGet(
product_type=TargetsToValidConfigurations,
subject_type=TargetsToValidConfigurationsRequest,
mock=lambda _: TargetsToValidConfigurations({target_with_origin: [config]}),
),
MockGet(
product_type=CreatedBinary,
subject_type=TestBinaryConfiguration,
mock=lambda _: self.create_mock_binary(program_text),
),
],
)
return cast(Run, res)
def make_target_with_origin(
self,
source_files: List[FileContent],
*,
interpreter_constraints: Optional[str] = None,
origin: Optional[OriginSpec] = None,
dependencies: Optional[List[Address]] = None,
) -> PythonTargetAdaptorWithOrigin:
input_snapshot = self.request_single_product(Snapshot, InputFilesContent(source_files))
adaptor_kwargs = dict(
sources=EagerFilesetWithSpec(self.source_root, {"globs": []}, snapshot=input_snapshot),
address=Address.parse(f"{self.source_root}:target"),
dependencies=dependencies or [],
)
if interpreter_constraints:
adaptor_kwargs["compatibility"] = interpreter_constraints
if origin is None:
origin = SingleAddress(directory=self.source_root, name="target")
return PythonTargetAdaptorWithOrigin(PythonTargetAdaptor(**adaptor_kwargs), origin)
def test_single_non_test_target(self):
bfaddr = BuildFileAddress(None, 'bin', 'some/dir')
target_adaptor = PythonBinaryAdaptor(type_alias='python_binary')
with self.captured_logging(logging.INFO):
# Note that this is not the same error message the end user will see, as we're resolving
# union Get requests in run_rule, not the real engine. But this test still asserts that
# we error when we expect to error.
with self.assertRaisesRegex(AssertionError, r'Rule requested: .* which cannot be satisfied.'):
run_rule(
coordinator_of_tests,
HydratedTarget(bfaddr.to_address(), target_adaptor, ()),
UnionMembership(union_rules={TestTarget: [PythonTestsAdaptor]}),
AddressProvenanceMap(bfaddr_to_spec={
bfaddr: SingleAddress(directory='some/dir', name='bin')
}),
{
(TestResult, TestTarget):
lambda _: TestResult(status=Status.SUCCESS, stdout='foo', stderr=''),
})
def parse_spec(self, spec):
"""Parse the given spec into a `specs.Spec` object.
:param spec: a single spec string.
:return: a single specs.Specs object.
:raises: CmdLineSpecParser.BadSpecError if the address selector could not be parsed.
"""
if spec.endswith('::'):
spec_path = spec[:-len('::')]
return DescendantAddresses(self._normalize_spec_path(spec_path))
elif spec.endswith(':'):
spec_path = spec[:-len(':')]
return SiblingAddresses(self._normalize_spec_path(spec_path))
else:
spec_parts = spec.rsplit(':', 1)
return SingleAddress(
self._normalize_spec_path(spec_parts[0]),
spec_parts[1] if len(spec_parts) > 1 else None)
def test_no_address_no_family(self):
spec = SingleAddress('a/c', 'c')
# Does not exist.
with self.assertRaises(Exception):
self.resolve(spec)
build_file = os.path.join(self.build_root, 'a/c', 'c.BUILD.json')
with safe_open(build_file, 'w') as fp:
fp.write('{"type_alias": "struct", "name": "c"}')
# Exists on disk, but not yet in memory.
with self.assertRaises(Exception):
self.resolve(spec)
self.scheduler.invalidate_files(['a/c'])
# Success.
resolved = self.resolve(spec)
self.assertEqual(1, len(resolved))
self.assertEqual([Struct(name='c', type_alias='struct')], [r.struct for r in resolved])
def run_black_and_isort(
self,
source_files: List[FileContent],
*,
name: str,
extra_args: Optional[List[str]] = None) -> LanguageFmtResults:
for source_file in source_files:
self.create_file(source_file.path, source_file.content.decode())
target = PythonLibrary({}, address=Address.parse(f"test:{name}"))
origin = SingleAddress(directory="test", name=name)
targets = PythonFmtTargets(
TargetsWithOrigins([TargetWithOrigin(target, origin)]))
args = [
"--backend-packages2=['pants.backend.python.lint.black', 'pants.backend.python.lint.isort']",
*(extra_args or []),
]
results = self.request_single_product(
LanguageFmtResults,
Params(targets, create_options_bootstrapper(args=args)),
)
return results
def run_pytest(
self,
*,
passthrough_args: Optional[str] = None,
origin: Optional[OriginSpec] = None,
) -> TestResult:
args = [
"--backend-packages2=pants.backend.python",
# pin to lower versions so that we can run Python 2 tests
"--pytest-version=pytest>=4.6.6,<4.7",
"--pytest-pytest-plugins=['zipp==1.0.0']",
]
if passthrough_args:
args.append(f"--pytest-args='{passthrough_args}'")
options_bootstrapper = create_options_bootstrapper(args=args)
if origin is None:
origin = SingleAddress(directory=self.source_root, name="target")
# TODO: We must use the V1 target's `_sources_field.sources` field to set the TargetAdaptor's
# sources attribute. The adaptor will not auto-populate this field. However, it will
# auto-populate things like `dependencies` and this was not necessary before using
# PythonTestsAdaptorWithOrigin. Why is this necessary in test code?
v1_target = self.target(f"{self.source_root}:target")
adaptor = PythonTestsAdaptor(
address=v1_target.address.to_address(),
sources=v1_target._sources_field.sources,
)
params = Params(
PytestRunner(PythonTestsAdaptorWithOrigin(adaptor, origin)),
options_bootstrapper)
test_result = self.request_single_product(TestResult, params)
debug_request = self.request_single_product(TestDebugRequest, params)
debug_result = InteractiveRunner(
self.scheduler).run_local_interactive_process(debug_request.ipr)
if test_result.status == Status.SUCCESS:
assert debug_result.process_exit_code == 0
else:
assert debug_result.process_exit_code != 0
return test_result
def test_more_specific():
single_address = SingleAddress(directory="foo/bar", name="baz")
sibling_addresses = SiblingAddresses(directory="foo/bar")
ascendant_addresses = AscendantAddresses(directory="foo/bar")
descendant_addresses = DescendantAddresses(directory="foo/bar")
assert single_address == more_specific(single_address, None)
assert single_address == more_specific(single_address, sibling_addresses)
assert single_address == more_specific(single_address, ascendant_addresses)
assert single_address == more_specific(single_address,
descendant_addresses)
assert single_address == more_specific(None, single_address)
assert single_address == more_specific(sibling_addresses, single_address)
assert single_address == more_specific(ascendant_addresses, single_address)
assert single_address == more_specific(descendant_addresses,
single_address)
assert sibling_addresses == more_specific(sibling_addresses, None)
assert sibling_addresses == more_specific(sibling_addresses,
ascendant_addresses)
assert sibling_addresses == more_specific(sibling_addresses,
descendant_addresses)
assert sibling_addresses == more_specific(None, sibling_addresses)
assert sibling_addresses == more_specific(ascendant_addresses,
sibling_addresses)
assert sibling_addresses == more_specific(descendant_addresses,
sibling_addresses)
assert ascendant_addresses == more_specific(ascendant_addresses, None)
assert ascendant_addresses == more_specific(ascendant_addresses,
descendant_addresses)
assert ascendant_addresses == more_specific(None, ascendant_addresses)
assert ascendant_addresses == more_specific(descendant_addresses,
ascendant_addresses)
assert descendant_addresses == more_specific(descendant_addresses, None)
assert descendant_addresses == more_specific(None, descendant_addresses)
def test_address_specs(self) -> None:
sources_field1 = self.mock_sources_field_with_origin(
SOURCES1,
origin=SingleAddress(directory=SOURCES1.source_root, name="lib"))
sources_field2 = self.mock_sources_field_with_origin(
SOURCES2, origin=SiblingAddresses(SOURCES2.source_root))
sources_field3 = self.mock_sources_field_with_origin(
SOURCES3, origin=DescendantAddresses(SOURCES3.source_root))
sources_field4 = self.mock_sources_field_with_origin(
SOURCES1, origin=AscendantAddresses(SOURCES1.source_root))
def assert_all_source_files_resolved(sources_field_with_origin: Tuple[
SourcesField, OriginSpec], sources: TargetSources) -> None:
expected = sources.source_file_absolute_paths
assert self.get_all_source_files([sources_field_with_origin
]) == expected
assert self.get_specified_source_files([sources_field_with_origin
]) == expected
assert_all_source_files_resolved(sources_field1, SOURCES1)
assert_all_source_files_resolved(sources_field2, SOURCES2)
assert_all_source_files_resolved(sources_field3, SOURCES3)
assert_all_source_files_resolved(sources_field4, SOURCES1)
# NB: sources_field1 and sources_field3 refer to the same files. We should be able to
# handle this gracefully.
combined_sources_fields = [
sources_field1, sources_field2, sources_field3, sources_field4
]
combined_expected = sorted([
*SOURCES1.source_file_absolute_paths,
*SOURCES2.source_file_absolute_paths,
*SOURCES3.source_file_absolute_paths,
])
assert self.get_all_source_files(
combined_sources_fields) == combined_expected
assert self.get_specified_source_files(
combined_sources_fields) == combined_expected
def run_black_and_isort(
self,
source_files: List[FileContent],
*,
extra_args: Optional[List[str]] = None) -> LanguageFmtResults:
input_snapshot = self.request_single_product(
Snapshot, InputFilesContent(source_files))
adaptor = TargetAdaptor(
sources=EagerFilesetWithSpec("test", {"globs": []},
snapshot=input_snapshot),
address=Address.parse("test:target"),
)
origin = SingleAddress(directory="test", name="target")
formatters = PythonFormatters(
(TargetAdaptorWithOrigin(adaptor, origin), ))
args = [
"--backend-packages2=['pants.backend.python.lint.black', 'pants.backend.python.lint.isort']",
*(extra_args or []),
]
results = self.request_single_product(
LanguageFmtResults,
Params(formatters, create_options_bootstrapper(args=args)),
)
return results
def test_address_specs(self) -> None:
target1 = self.mock_target(self.SOURCES1,
origin=SingleAddress(
directory=self.SOURCES1.source_root,
name="lib"))
target2 = self.mock_target(self.SOURCES2,
origin=SiblingAddresses(
self.SOURCES2.source_root))
target3 = self.mock_target(self.SOURCES3,
origin=DescendantAddresses(
self.SOURCES3.source_root))
target4 = self.mock_target(self.SOURCES1,
origin=AscendantAddresses(
self.SOURCES1.source_root))
def assert_all_source_files_resolved(target: TargetAdaptorWithOrigin,
sources: TargetSources) -> None:
expected = sources.source_file_absolute_paths
assert self.get_all_source_files([target]) == expected
assert self.get_specified_source_files([target]) == expected
assert_all_source_files_resolved(target1, self.SOURCES1)
assert_all_source_files_resolved(target2, self.SOURCES2)
assert_all_source_files_resolved(target3, self.SOURCES3)
assert_all_source_files_resolved(target4, self.SOURCES1)
# NB: target1 and target4 refer to the same files. We should be able to handle this
# gracefully.
combined_targets = [target1, target2, target3, target4]
combined_expected = sorted([
*self.SOURCES1.source_file_absolute_paths,
*self.SOURCES2.source_file_absolute_paths,
*self.SOURCES3.source_file_absolute_paths,
])
assert self.get_all_source_files(combined_targets) == combined_expected
assert self.get_specified_source_files(
combined_targets) == combined_expected
def create(cls,
options,
session,
symbol_table,
build_root=None,
exclude_patterns=None,
tags=None):
"""
:param Options options: An `Options` instance to use.
:param session: The Scheduler session
:param symbol_table: The symbol table
:param string build_root: The build root.
"""
# Determine the literal target roots.
spec_roots = cls.parse_specs(target_specs=options.target_specs,
build_root=build_root,
exclude_patterns=exclude_patterns,
tags=tags)
# Determine `Changed` arguments directly from options to support pre-`Subsystem`
# initialization paths.
changed_options = options.for_scope('changed')
changed_request = ChangedRequest.from_options(changed_options)
# Determine the `--owner-of=` arguments provided from the global options
owned_files = options.for_global_scope().owner_of
logger.debug('spec_roots are: %s', spec_roots)
logger.debug('changed_request is: %s', changed_request)
logger.debug('owned_files are: %s', owned_files)
targets_specified = sum(1 for item in (changed_request.is_actionable(),
owned_files, spec_roots)
if item)
if targets_specified > 1:
# We've been provided more than one of: a change request, an owner request, or spec roots.
raise InvalidSpecConstraint(
'Multiple target selection methods provided. Please use only one of '
'--changed-*, --owner-of, or target specs')
if changed_request.is_actionable():
scm = get_scm()
if not scm:
raise InvalidSpecConstraint(
'The --changed-* options are not available without a recognized SCM (usually git).'
)
changed_files = cls.changed_files(
scm,
changes_since=changed_request.changes_since,
diffspec=changed_request.diffspec)
# We've been provided no spec roots (e.g. `./pants list`) AND a changed request. Compute
# alternate target roots.
request = OwnersRequest(sources=tuple(changed_files),
include_dependees=str(
changed_request.include_dependees))
changed_addresses, = session.product_request(
BuildFileAddresses, [request])
logger.debug('changed addresses: %s', changed_addresses)
dependencies = tuple(
SingleAddress(a.spec_path, a.target_name)
for a in changed_addresses)
return TargetRoots([
Specs(dependencies=dependencies,
exclude_patterns=exclude_patterns,
tags=tags)
])
if owned_files:
# We've been provided no spec roots (e.g. `./pants list`) AND a owner request. Compute
# alternate target roots.
request = OwnersRequest(sources=tuple(owned_files),
include_dependees=str('none'))
owner_addresses, = session.product_request(BuildFileAddresses,
[request])
logger.debug('owner addresses: %s', owner_addresses)
dependencies = tuple(
SingleAddress(a.spec_path, a.target_name)
for a in owner_addresses)
return TargetRoots([
Specs(dependencies=dependencies,
exclude_patterns=exclude_patterns,
tags=tags)
])
return TargetRoots(spec_roots)
def create(
cls,
options: Options,
session: SchedulerSession,
build_root: Optional[str] = None,
exclude_patterns: Optional[Iterable[str]] = None,
tags: Optional[Iterable[str]] = None,
) -> Specs:
specs = cls.parse_specs(
raw_specs=options.specs,
build_root=build_root,
exclude_patterns=exclude_patterns,
tags=tags,
)
changed_options = ChangedOptions.from_options(
options.for_scope('changed'))
owned_files = options.for_global_scope().owner_of
logger.debug('specs are: %s', specs)
logger.debug('changed_options are: %s', changed_options)
logger.debug('owned_files are: %s', owned_files)
targets_specified = sum(1
for item in (changed_options.is_actionable(),
owned_files,
specs.provided_specs.dependencies)
if item)
if targets_specified > 1:
# We've been provided more than one of: a change request, an owner request, or specs.
raise InvalidSpecConstraint(
'Multiple target selection methods provided. Please use only one of '
'`--changed-*`, `--owner-of`, address specs, or filesystem specs.'
)
if changed_options.is_actionable():
scm = get_scm()
if not scm:
raise InvalidSpecConstraint(
'The `--changed-*` options are not available without a recognized SCM (usually git).'
)
changed_request = ChangedRequest(
sources=tuple(changed_options.changed_files(scm=scm)),
include_dependees=changed_options.include_dependees,
)
changed_addresses, = session.product_request(
ChangedAddresses, [changed_request])
logger.debug('changed addresses: %s', changed_addresses.addresses)
dependencies = tuple(
SingleAddress(a.spec_path, a.target_name)
for a in changed_addresses.addresses)
return Specs(
address_specs=AddressSpecs(
dependencies=dependencies,
exclude_patterns=exclude_patterns,
tags=tags,
),
filesystem_specs=FilesystemSpecs([]),
)
if owned_files:
owner_request = OwnersRequest(sources=tuple(owned_files))
owner_request.validate(
pants_bin_name=options.for_global_scope().pants_bin_name)
owners, = session.product_request(Owners, [owner_request])
logger.debug('owner addresses: %s', owners.addresses)
dependencies = tuple(
SingleAddress(a.spec_path, a.target_name)
for a in owners.addresses)
return Specs(address_specs=AddressSpecs(
dependencies=dependencies,
exclude_patterns=exclude_patterns,
tags=tags,
),
filesystem_specs=FilesystemSpecs([]))
return specs
def test_scan_specs_bad_spec(self):
self.create_build_files()
mapper = self.address_mapper
with self.assertRaises(AddressMapper.BuildFileScanError) as cm:
mapper.scan_specs([SingleAddress('dir_a', 'd')])
self.assertIn('does not match any targets.', str(cm.exception))
def test_scan_specs(self):
self.create_build_files()
mapper = self.address_mapper
addresses = mapper.scan_specs([SingleAddress('dir_a', 'a'), SiblingAddresses('')])
self.assertEqual(addresses,
{Address('', 'a'), Address('', 'b'), Address('', 'c'), Address('dir_a', 'a')})
def test_scan_specs_bad_spec(self):
with temporary_dir() as build_root:
self.create_build_files(build_root)
mapper = self.create_address_mapper(build_root)
with self.assertRaises(AddressMapper.BuildFileScanError):
mapper.scan_specs([SingleAddress('dir_a', 'd')])
def single(directory, name=None):
return SingleAddress(directory, name)
def test_resolve(self) -> None:
resolved = self.resolve(SingleAddress("a/b", "b"))
self.assertEqual(1, len(resolved))
self.assertEqual(self.a_b, resolved[0].address)
def inject_addresses_closure(self, addresses):
addresses = set(addresses) - set(self._target_by_address.keys())
if not addresses:
return
for _ in self._inject_specs([SingleAddress(a.spec_path, a.target_name) for a in addresses]):
pass
def single(directory: str, name: Optional[str] = None) -> SingleAddress:
name = name if name is not None else os.path.basename(directory)
return SingleAddress(directory, name)
def single(directory, name=None):
name = name if name is not None else os.path.basename(directory)
return SingleAddress(directory, name)
def test_scan_address_specs_bad_spec(self) -> None:
self.create_build_files()
mapper = self.address_mapper
with self.assertRaises(AddressMapper.BuildFileScanError) as cm:
mapper.scan_address_specs([SingleAddress("dir_a", "d")])
self.assertIn("does not match any targets.", str(cm.exception))
def test_resolve(self):
resolved = self.resolve(SingleAddress('a/b', None))
self.assertEqual(1, len(resolved))
self.assertEqual(self.a_b, resolved[0].address)
def create(cls,
options,
session,
symbol_table,
build_root=None,
exclude_patterns=None,
tags=None):
"""
:param Options options: An `Options` instance to use.
:param session: The Scheduler session
:param symbol_table: The symbol table
:param string build_root: The build root.
"""
# Determine the literal target roots.
spec_roots = cls.parse_specs(target_specs=options.target_specs,
build_root=build_root,
exclude_patterns=exclude_patterns,
tags=tags)
# Determine `Changed` arguments directly from options to support pre-`Subsystem`
# initialization paths.
changed_options = options.for_scope('changed')
changed_request = ChangedRequest.from_options(changed_options)
# Determine the `--owner-of=` arguments provided from the global options
owned_files = options.for_global_scope().owner_of
logger.debug('spec_roots are: %s', spec_roots)
logger.debug('changed_request is: %s', changed_request)
logger.debug('owned_files are: %s', owned_files)
scm = get_scm()
change_calculator = ChangeCalculator(scheduler=session,
symbol_table=symbol_table,
scm=scm) if scm else None
owner_calculator = OwnerCalculator(
scheduler=session,
symbol_table=symbol_table) if owned_files else None
targets_specified = sum(1 for item in (changed_request.is_actionable(),
owned_files, spec_roots)
if item)
if targets_specified > 1:
# We've been provided a more than one of: a change request, an owner request, or spec roots.
raise InvalidSpecConstraint(
'Multiple target selection methods provided. Please use only one of '
'--changed-*, --owner-of, or target specs')
if change_calculator and changed_request.is_actionable():
# We've been provided no spec roots (e.g. `./pants list`) AND a changed request. Compute
# alternate target roots.
changed_addresses = change_calculator.changed_target_addresses(
changed_request)
logger.debug('changed addresses: %s', changed_addresses)
dependencies = tuple(
SingleAddress(a.spec_path, a.target_name)
for a in changed_addresses)
return TargetRoots([
Specs(dependencies=dependencies,
exclude_patterns=exclude_patterns,
tags=tags)
])
if owner_calculator and owned_files:
# We've been provided no spec roots (e.g. `./pants list`) AND a owner request. Compute
# alternate target roots.
owner_addresses = owner_calculator.owner_target_addresses(
owned_files)
logger.debug('owner addresses: %s', owner_addresses)
dependencies = tuple(
SingleAddress(a.spec_path, a.target_name)
for a in owner_addresses)
return TargetRoots([
Specs(dependencies=dependencies,
exclude_patterns=exclude_patterns,
tags=tags)
])
return TargetRoots(spec_roots)
