def test_no_address_no_family(self) -> None: spec = SingleAddress("a/c", "c") # Does not exist. with self.assertRaises(Exception): self.resolve(spec) build_file = os.path.join(self.build_root, "a/c", "c.BUILD.json") with safe_open(build_file, "w") as fp: fp.write('{"type_alias": "struct", "name": "c"}') self.scheduler.invalidate_files(["a/c"]) # Success. resolved = self.resolve(spec) self.assertEqual(1, len(resolved)) self.assertEqual( [Struct(address=Address.parse("a/c"), type_alias="struct")], resolved)
def test_duplicated(self): """Test that matching the same Spec twice succeeds.""" address = SingleAddress('a', 'a') address_mapper = AddressMapper(JsonParser(TestTable())) snapshot = Snapshot(DirectoryDigest(text_type('xx'), 2), (Path('a/BUILD', File('a/BUILD')), )) address_family = AddressFamily( 'a', {'a': ('a/BUILD', 'this is an object!')}) bfas = run_rule( addresses_from_address_families, address_mapper, Specs([address, address]), { (Snapshot, PathGlobs): lambda _: snapshot, (AddressFamily, Dir): lambda _: address_family, }) self.assertEqual(len(bfas.dependencies), 1) self.assertEqual(bfas.dependencies[0].spec, 'a:a')
def test_no_address_no_family(self): spec = SingleAddress('a/c', 'c') # Does not exist. self.assertEqual(0, len(self.resolve(spec))) # Exists on disk, but not yet in memory. directory = 'a/c' build_file = os.path.join(self.build_root, directory, 'c.BUILD.json') with safe_open(build_file, 'w') as fp: fp.write('{"type_alias": "struct", "name": "c"}') self.assertEqual(0, len(self.resolve(spec))) # Success. self.scheduler.invalidate_files([directory]) resolved = self.resolve(spec) self.assertEqual(1, len(resolved)) self.assertEqual(Struct(name='c', type_alias='struct'), resolved[0].struct)
def test_no_address_no_family(self): spec = SingleAddress('a/c', None) # Should fail: does not exist. with self.assertRaises(ResolveError): self.resolve(spec) # Exists on disk, but not yet in memory. build_file = os.path.join(self.build_root, 'a/c/c.BUILD.json') with safe_open(build_file, 'w') as fp: fp.write('{"type_alias": "struct", "name": "c"}') with self.assertRaises(ResolveError): self.resolve(spec) # Success. self.scheduler.product_graph.invalidate() resolved = self.resolve(spec) self.assertEqual(1, len(resolved)) self.assertEqual(Struct(name='c', type_alias='struct'), resolved[0].struct)
def single_target_run( self, *, console: MockConsole, program_text: bytes, address_spec: str, ) -> Run: workspace = Workspace(self.scheduler) interactive_runner = InteractiveRunner(self.scheduler) class TestBinaryConfiguration(BinaryConfiguration): required_fields = () class TestBinaryTarget(Target): alias = "binary" core_fields = () address = Address.parse(address_spec) target = TestBinaryTarget({}, address=address) target_with_origin = TargetWithOrigin( target, SingleAddress(address.spec_path, address.target_name) ) config = TestBinaryConfiguration.create(target) res = run_rule( run, rule_args=[ console, workspace, interactive_runner, BuildRoot(), create_goal_subsystem(RunOptions, args=[]), create_subsystem(GlobalOptions, pants_workdir=self.pants_workdir), ], mock_gets=[ MockGet( product_type=TargetsToValidConfigurations, subject_type=TargetsToValidConfigurationsRequest, mock=lambda _: TargetsToValidConfigurations({target_with_origin: [config]}), ), MockGet( product_type=CreatedBinary, subject_type=TestBinaryConfiguration, mock=lambda _: self.create_mock_binary(program_text), ), ], ) return cast(Run, res)
def make_target_with_origin( self, source_files: List[FileContent], *, interpreter_constraints: Optional[str] = None, origin: Optional[OriginSpec] = None, dependencies: Optional[List[Address]] = None, ) -> PythonTargetAdaptorWithOrigin: input_snapshot = self.request_single_product(Snapshot, InputFilesContent(source_files)) adaptor_kwargs = dict( sources=EagerFilesetWithSpec(self.source_root, {"globs": []}, snapshot=input_snapshot), address=Address.parse(f"{self.source_root}:target"), dependencies=dependencies or [], ) if interpreter_constraints: adaptor_kwargs["compatibility"] = interpreter_constraints if origin is None: origin = SingleAddress(directory=self.source_root, name="target") return PythonTargetAdaptorWithOrigin(PythonTargetAdaptor(**adaptor_kwargs), origin)
def test_single_non_test_target(self): bfaddr = BuildFileAddress(None, 'bin', 'some/dir') target_adaptor = PythonBinaryAdaptor(type_alias='python_binary') with self.captured_logging(logging.INFO): # Note that this is not the same error message the end user will see, as we're resolving # union Get requests in run_rule, not the real engine. But this test still asserts that # we error when we expect to error. with self.assertRaisesRegex(AssertionError, r'Rule requested: .* which cannot be satisfied.'): run_rule( coordinator_of_tests, HydratedTarget(bfaddr.to_address(), target_adaptor, ()), UnionMembership(union_rules={TestTarget: [PythonTestsAdaptor]}), AddressProvenanceMap(bfaddr_to_spec={ bfaddr: SingleAddress(directory='some/dir', name='bin') }), { (TestResult, TestTarget): lambda _: TestResult(status=Status.SUCCESS, stdout='foo', stderr=''), })
def parse_spec(self, spec): """Parse the given spec into a `specs.Spec` object. :param spec: a single spec string. :return: a single specs.Specs object. :raises: CmdLineSpecParser.BadSpecError if the address selector could not be parsed. """ if spec.endswith('::'): spec_path = spec[:-len('::')] return DescendantAddresses(self._normalize_spec_path(spec_path)) elif spec.endswith(':'): spec_path = spec[:-len(':')] return SiblingAddresses(self._normalize_spec_path(spec_path)) else: spec_parts = spec.rsplit(':', 1) return SingleAddress( self._normalize_spec_path(spec_parts[0]), spec_parts[1] if len(spec_parts) > 1 else None)
def test_no_address_no_family(self): spec = SingleAddress('a/c', 'c') # Does not exist. with self.assertRaises(Exception): self.resolve(spec) build_file = os.path.join(self.build_root, 'a/c', 'c.BUILD.json') with safe_open(build_file, 'w') as fp: fp.write('{"type_alias": "struct", "name": "c"}') # Exists on disk, but not yet in memory. with self.assertRaises(Exception): self.resolve(spec) self.scheduler.invalidate_files(['a/c']) # Success. resolved = self.resolve(spec) self.assertEqual(1, len(resolved)) self.assertEqual([Struct(name='c', type_alias='struct')], [r.struct for r in resolved])
def run_black_and_isort( self, source_files: List[FileContent], *, name: str, extra_args: Optional[List[str]] = None) -> LanguageFmtResults: for source_file in source_files: self.create_file(source_file.path, source_file.content.decode()) target = PythonLibrary({}, address=Address.parse(f"test:{name}")) origin = SingleAddress(directory="test", name=name) targets = PythonFmtTargets( TargetsWithOrigins([TargetWithOrigin(target, origin)])) args = [ "--backend-packages2=['pants.backend.python.lint.black', 'pants.backend.python.lint.isort']", *(extra_args or []), ] results = self.request_single_product( LanguageFmtResults, Params(targets, create_options_bootstrapper(args=args)), ) return results
def run_pytest( self, *, passthrough_args: Optional[str] = None, origin: Optional[OriginSpec] = None, ) -> TestResult: args = [ "--backend-packages2=pants.backend.python", # pin to lower versions so that we can run Python 2 tests "--pytest-version=pytest>=4.6.6,<4.7", "--pytest-pytest-plugins=['zipp==1.0.0']", ] if passthrough_args: args.append(f"--pytest-args='{passthrough_args}'") options_bootstrapper = create_options_bootstrapper(args=args) if origin is None: origin = SingleAddress(directory=self.source_root, name="target") # TODO: We must use the V1 target's `_sources_field.sources` field to set the TargetAdaptor's # sources attribute. The adaptor will not auto-populate this field. However, it will # auto-populate things like `dependencies` and this was not necessary before using # PythonTestsAdaptorWithOrigin. Why is this necessary in test code? v1_target = self.target(f"{self.source_root}:target") adaptor = PythonTestsAdaptor( address=v1_target.address.to_address(), sources=v1_target._sources_field.sources, ) params = Params( PytestRunner(PythonTestsAdaptorWithOrigin(adaptor, origin)), options_bootstrapper) test_result = self.request_single_product(TestResult, params) debug_request = self.request_single_product(TestDebugRequest, params) debug_result = InteractiveRunner( self.scheduler).run_local_interactive_process(debug_request.ipr) if test_result.status == Status.SUCCESS: assert debug_result.process_exit_code == 0 else: assert debug_result.process_exit_code != 0 return test_result
def test_more_specific(): single_address = SingleAddress(directory="foo/bar", name="baz") sibling_addresses = SiblingAddresses(directory="foo/bar") ascendant_addresses = AscendantAddresses(directory="foo/bar") descendant_addresses = DescendantAddresses(directory="foo/bar") assert single_address == more_specific(single_address, None) assert single_address == more_specific(single_address, sibling_addresses) assert single_address == more_specific(single_address, ascendant_addresses) assert single_address == more_specific(single_address, descendant_addresses) assert single_address == more_specific(None, single_address) assert single_address == more_specific(sibling_addresses, single_address) assert single_address == more_specific(ascendant_addresses, single_address) assert single_address == more_specific(descendant_addresses, single_address) assert sibling_addresses == more_specific(sibling_addresses, None) assert sibling_addresses == more_specific(sibling_addresses, ascendant_addresses) assert sibling_addresses == more_specific(sibling_addresses, descendant_addresses) assert sibling_addresses == more_specific(None, sibling_addresses) assert sibling_addresses == more_specific(ascendant_addresses, sibling_addresses) assert sibling_addresses == more_specific(descendant_addresses, sibling_addresses) assert ascendant_addresses == more_specific(ascendant_addresses, None) assert ascendant_addresses == more_specific(ascendant_addresses, descendant_addresses) assert ascendant_addresses == more_specific(None, ascendant_addresses) assert ascendant_addresses == more_specific(descendant_addresses, ascendant_addresses) assert descendant_addresses == more_specific(descendant_addresses, None) assert descendant_addresses == more_specific(None, descendant_addresses)
def test_address_specs(self) -> None: sources_field1 = self.mock_sources_field_with_origin( SOURCES1, origin=SingleAddress(directory=SOURCES1.source_root, name="lib")) sources_field2 = self.mock_sources_field_with_origin( SOURCES2, origin=SiblingAddresses(SOURCES2.source_root)) sources_field3 = self.mock_sources_field_with_origin( SOURCES3, origin=DescendantAddresses(SOURCES3.source_root)) sources_field4 = self.mock_sources_field_with_origin( SOURCES1, origin=AscendantAddresses(SOURCES1.source_root)) def assert_all_source_files_resolved(sources_field_with_origin: Tuple[ SourcesField, OriginSpec], sources: TargetSources) -> None: expected = sources.source_file_absolute_paths assert self.get_all_source_files([sources_field_with_origin ]) == expected assert self.get_specified_source_files([sources_field_with_origin ]) == expected assert_all_source_files_resolved(sources_field1, SOURCES1) assert_all_source_files_resolved(sources_field2, SOURCES2) assert_all_source_files_resolved(sources_field3, SOURCES3) assert_all_source_files_resolved(sources_field4, SOURCES1) # NB: sources_field1 and sources_field3 refer to the same files. We should be able to # handle this gracefully. combined_sources_fields = [ sources_field1, sources_field2, sources_field3, sources_field4 ] combined_expected = sorted([ *SOURCES1.source_file_absolute_paths, *SOURCES2.source_file_absolute_paths, *SOURCES3.source_file_absolute_paths, ]) assert self.get_all_source_files( combined_sources_fields) == combined_expected assert self.get_specified_source_files( combined_sources_fields) == combined_expected
def run_black_and_isort( self, source_files: List[FileContent], *, extra_args: Optional[List[str]] = None) -> LanguageFmtResults: input_snapshot = self.request_single_product( Snapshot, InputFilesContent(source_files)) adaptor = TargetAdaptor( sources=EagerFilesetWithSpec("test", {"globs": []}, snapshot=input_snapshot), address=Address.parse("test:target"), ) origin = SingleAddress(directory="test", name="target") formatters = PythonFormatters( (TargetAdaptorWithOrigin(adaptor, origin), )) args = [ "--backend-packages2=['pants.backend.python.lint.black', 'pants.backend.python.lint.isort']", *(extra_args or []), ] results = self.request_single_product( LanguageFmtResults, Params(formatters, create_options_bootstrapper(args=args)), ) return results
def test_address_specs(self) -> None: target1 = self.mock_target(self.SOURCES1, origin=SingleAddress( directory=self.SOURCES1.source_root, name="lib")) target2 = self.mock_target(self.SOURCES2, origin=SiblingAddresses( self.SOURCES2.source_root)) target3 = self.mock_target(self.SOURCES3, origin=DescendantAddresses( self.SOURCES3.source_root)) target4 = self.mock_target(self.SOURCES1, origin=AscendantAddresses( self.SOURCES1.source_root)) def assert_all_source_files_resolved(target: TargetAdaptorWithOrigin, sources: TargetSources) -> None: expected = sources.source_file_absolute_paths assert self.get_all_source_files([target]) == expected assert self.get_specified_source_files([target]) == expected assert_all_source_files_resolved(target1, self.SOURCES1) assert_all_source_files_resolved(target2, self.SOURCES2) assert_all_source_files_resolved(target3, self.SOURCES3) assert_all_source_files_resolved(target4, self.SOURCES1) # NB: target1 and target4 refer to the same files. We should be able to handle this # gracefully. combined_targets = [target1, target2, target3, target4] combined_expected = sorted([ *self.SOURCES1.source_file_absolute_paths, *self.SOURCES2.source_file_absolute_paths, *self.SOURCES3.source_file_absolute_paths, ]) assert self.get_all_source_files(combined_targets) == combined_expected assert self.get_specified_source_files( combined_targets) == combined_expected
def create(cls, options, session, symbol_table, build_root=None, exclude_patterns=None, tags=None): """ :param Options options: An `Options` instance to use. :param session: The Scheduler session :param symbol_table: The symbol table :param string build_root: The build root. """ # Determine the literal target roots. spec_roots = cls.parse_specs(target_specs=options.target_specs, build_root=build_root, exclude_patterns=exclude_patterns, tags=tags) # Determine `Changed` arguments directly from options to support pre-`Subsystem` # initialization paths. changed_options = options.for_scope('changed') changed_request = ChangedRequest.from_options(changed_options) # Determine the `--owner-of=` arguments provided from the global options owned_files = options.for_global_scope().owner_of logger.debug('spec_roots are: %s', spec_roots) logger.debug('changed_request is: %s', changed_request) logger.debug('owned_files are: %s', owned_files) targets_specified = sum(1 for item in (changed_request.is_actionable(), owned_files, spec_roots) if item) if targets_specified > 1: # We've been provided more than one of: a change request, an owner request, or spec roots. raise InvalidSpecConstraint( 'Multiple target selection methods provided. Please use only one of ' '--changed-*, --owner-of, or target specs') if changed_request.is_actionable(): scm = get_scm() if not scm: raise InvalidSpecConstraint( 'The --changed-* options are not available without a recognized SCM (usually git).' ) changed_files = cls.changed_files( scm, changes_since=changed_request.changes_since, diffspec=changed_request.diffspec) # We've been provided no spec roots (e.g. `./pants list`) AND a changed request. Compute # alternate target roots. request = OwnersRequest(sources=tuple(changed_files), include_dependees=str( changed_request.include_dependees)) changed_addresses, = session.product_request( BuildFileAddresses, [request]) logger.debug('changed addresses: %s', changed_addresses) dependencies = tuple( SingleAddress(a.spec_path, a.target_name) for a in changed_addresses) return TargetRoots([ Specs(dependencies=dependencies, exclude_patterns=exclude_patterns, tags=tags) ]) if owned_files: # We've been provided no spec roots (e.g. `./pants list`) AND a owner request. Compute # alternate target roots. request = OwnersRequest(sources=tuple(owned_files), include_dependees=str('none')) owner_addresses, = session.product_request(BuildFileAddresses, [request]) logger.debug('owner addresses: %s', owner_addresses) dependencies = tuple( SingleAddress(a.spec_path, a.target_name) for a in owner_addresses) return TargetRoots([ Specs(dependencies=dependencies, exclude_patterns=exclude_patterns, tags=tags) ]) return TargetRoots(spec_roots)
def create( cls, options: Options, session: SchedulerSession, build_root: Optional[str] = None, exclude_patterns: Optional[Iterable[str]] = None, tags: Optional[Iterable[str]] = None, ) -> Specs: specs = cls.parse_specs( raw_specs=options.specs, build_root=build_root, exclude_patterns=exclude_patterns, tags=tags, ) changed_options = ChangedOptions.from_options( options.for_scope('changed')) owned_files = options.for_global_scope().owner_of logger.debug('specs are: %s', specs) logger.debug('changed_options are: %s', changed_options) logger.debug('owned_files are: %s', owned_files) targets_specified = sum(1 for item in (changed_options.is_actionable(), owned_files, specs.provided_specs.dependencies) if item) if targets_specified > 1: # We've been provided more than one of: a change request, an owner request, or specs. raise InvalidSpecConstraint( 'Multiple target selection methods provided. Please use only one of ' '`--changed-*`, `--owner-of`, address specs, or filesystem specs.' ) if changed_options.is_actionable(): scm = get_scm() if not scm: raise InvalidSpecConstraint( 'The `--changed-*` options are not available without a recognized SCM (usually git).' ) changed_request = ChangedRequest( sources=tuple(changed_options.changed_files(scm=scm)), include_dependees=changed_options.include_dependees, ) changed_addresses, = session.product_request( ChangedAddresses, [changed_request]) logger.debug('changed addresses: %s', changed_addresses.addresses) dependencies = tuple( SingleAddress(a.spec_path, a.target_name) for a in changed_addresses.addresses) return Specs( address_specs=AddressSpecs( dependencies=dependencies, exclude_patterns=exclude_patterns, tags=tags, ), filesystem_specs=FilesystemSpecs([]), ) if owned_files: owner_request = OwnersRequest(sources=tuple(owned_files)) owner_request.validate( pants_bin_name=options.for_global_scope().pants_bin_name) owners, = session.product_request(Owners, [owner_request]) logger.debug('owner addresses: %s', owners.addresses) dependencies = tuple( SingleAddress(a.spec_path, a.target_name) for a in owners.addresses) return Specs(address_specs=AddressSpecs( dependencies=dependencies, exclude_patterns=exclude_patterns, tags=tags, ), filesystem_specs=FilesystemSpecs([])) return specs
def test_scan_specs_bad_spec(self): self.create_build_files() mapper = self.address_mapper with self.assertRaises(AddressMapper.BuildFileScanError) as cm: mapper.scan_specs([SingleAddress('dir_a', 'd')]) self.assertIn('does not match any targets.', str(cm.exception))
def test_scan_specs(self): self.create_build_files() mapper = self.address_mapper addresses = mapper.scan_specs([SingleAddress('dir_a', 'a'), SiblingAddresses('')]) self.assertEqual(addresses, {Address('', 'a'), Address('', 'b'), Address('', 'c'), Address('dir_a', 'a')})
def test_scan_specs_bad_spec(self): with temporary_dir() as build_root: self.create_build_files(build_root) mapper = self.create_address_mapper(build_root) with self.assertRaises(AddressMapper.BuildFileScanError): mapper.scan_specs([SingleAddress('dir_a', 'd')])
def single(directory, name=None): return SingleAddress(directory, name)
def test_resolve(self) -> None: resolved = self.resolve(SingleAddress("a/b", "b")) self.assertEqual(1, len(resolved)) self.assertEqual(self.a_b, resolved[0].address)
def inject_addresses_closure(self, addresses): addresses = set(addresses) - set(self._target_by_address.keys()) if not addresses: return for _ in self._inject_specs([SingleAddress(a.spec_path, a.target_name) for a in addresses]): pass
def single(directory: str, name: Optional[str] = None) -> SingleAddress: name = name if name is not None else os.path.basename(directory) return SingleAddress(directory, name)
def single(directory, name=None): name = name if name is not None else os.path.basename(directory) return SingleAddress(directory, name)
def test_scan_address_specs_bad_spec(self) -> None: self.create_build_files() mapper = self.address_mapper with self.assertRaises(AddressMapper.BuildFileScanError) as cm: mapper.scan_address_specs([SingleAddress("dir_a", "d")]) self.assertIn("does not match any targets.", str(cm.exception))
def test_resolve(self): resolved = self.resolve(SingleAddress('a/b', None)) self.assertEqual(1, len(resolved)) self.assertEqual(self.a_b, resolved[0].address)
def create(cls, options, session, symbol_table, build_root=None, exclude_patterns=None, tags=None): """ :param Options options: An `Options` instance to use. :param session: The Scheduler session :param symbol_table: The symbol table :param string build_root: The build root. """ # Determine the literal target roots. spec_roots = cls.parse_specs(target_specs=options.target_specs, build_root=build_root, exclude_patterns=exclude_patterns, tags=tags) # Determine `Changed` arguments directly from options to support pre-`Subsystem` # initialization paths. changed_options = options.for_scope('changed') changed_request = ChangedRequest.from_options(changed_options) # Determine the `--owner-of=` arguments provided from the global options owned_files = options.for_global_scope().owner_of logger.debug('spec_roots are: %s', spec_roots) logger.debug('changed_request is: %s', changed_request) logger.debug('owned_files are: %s', owned_files) scm = get_scm() change_calculator = ChangeCalculator(scheduler=session, symbol_table=symbol_table, scm=scm) if scm else None owner_calculator = OwnerCalculator( scheduler=session, symbol_table=symbol_table) if owned_files else None targets_specified = sum(1 for item in (changed_request.is_actionable(), owned_files, spec_roots) if item) if targets_specified > 1: # We've been provided a more than one of: a change request, an owner request, or spec roots. raise InvalidSpecConstraint( 'Multiple target selection methods provided. Please use only one of ' '--changed-*, --owner-of, or target specs') if change_calculator and changed_request.is_actionable(): # We've been provided no spec roots (e.g. `./pants list`) AND a changed request. Compute # alternate target roots. changed_addresses = change_calculator.changed_target_addresses( changed_request) logger.debug('changed addresses: %s', changed_addresses) dependencies = tuple( SingleAddress(a.spec_path, a.target_name) for a in changed_addresses) return TargetRoots([ Specs(dependencies=dependencies, exclude_patterns=exclude_patterns, tags=tags) ]) if owner_calculator and owned_files: # We've been provided no spec roots (e.g. `./pants list`) AND a owner request. Compute # alternate target roots. owner_addresses = owner_calculator.owner_target_addresses( owned_files) logger.debug('owner addresses: %s', owner_addresses) dependencies = tuple( SingleAddress(a.spec_path, a.target_name) for a in owner_addresses) return TargetRoots([ Specs(dependencies=dependencies, exclude_patterns=exclude_patterns, tags=tags) ]) return TargetRoots(spec_roots)