def classify_source_files( paths: Iterable[str]) -> dict[type[Target], set[str]]: """Returns a dict of target type -> files that belong to targets of that type.""" tests_filespec = Filespec( includes=list(PythonTestsGeneratingSourcesField.default)) test_utils_filespec = Filespec( includes=list(PythonTestUtilsGeneratingSourcesField.default)) path_to_file_name = {path: os.path.basename(path) for path in paths} test_file_names = set( matches_filespec(tests_filespec, paths=path_to_file_name.values())) test_util_file_names = set( matches_filespec(test_utils_filespec, paths=path_to_file_name.values())) test_files = { path for path, file_name in path_to_file_name.items() if file_name in test_file_names } test_util_files = { path for path, file_name in path_to_file_name.items() if file_name in test_util_file_names } library_files = set(paths) - test_files - test_util_files return { PythonTestsGeneratorTarget: test_files, PythonTestUtilsGeneratorTarget: test_util_files, PythonSourcesGeneratorTarget: library_files, }
def classify_source_files( paths: Iterable[str]) -> dict[type[Target], set[str]]: """Returns a dict of target type -> files that belong to targets of that type.""" scalatest_filespec = Filespec( includes=list(ScalatestTestsGeneratorSourcesField.default)) junit_filespec = Filespec( includes=list(ScalaJunitTestsGeneratorSourcesField.default)) scalatest_files = { path for path in paths if os.path.basename(path) in set( matches_filespec(scalatest_filespec, paths=[os.path.basename(path) for path in paths])) } junit_files = { path for path in paths if os.path.basename(path) in set( matches_filespec(junit_filespec, paths=[os.path.basename(path) for path in paths])) } sources_files = set(paths) - scalatest_files - junit_files return { ScalaJunitTestsGeneratorTarget: junit_files, ScalaSourcesGeneratorTarget: sources_files, ScalatestTestsGeneratorTarget: scalatest_files, }
def assert_rule_match(self, glob, expected_matches, negate=False): """Tests that in-memory glob matching matches lazy-filesystem traversing globs.""" if negate: assertMatch, match_state = self.assertFalse, 'erroneously matches' else: assertMatch, match_state = self.assertTrue, "doesn't match" # Confirm in-memory behaviour. for expected in expected_matches: assertMatch( matches_filespec(expected, {'globs': [glob]}), '{} {} path `{}`'.format(glob, match_state, expected), ) # And confirm that it matches on-disk behaviour. for expected in expected_matches: if expected.endswith('/'): self.create_dir(expected) else: self.create_file(expected) snapshot, = self.scheduler.product_request(Snapshot, [PathGlobs([glob])]) if negate: subset = set(expected_matches).intersection(set(snapshot.files)) self.assertEquals(subset, set(), '{} {} path(s) {}'.format(glob, match_state, subset)) else: self.assertEquals(sorted(expected_matches), sorted(snapshot.files))
async def find_fortran_targets( req: PutativeFortranTargetsRequest, all_owned_sources: AllOwnedSources) -> PutativeTargets: all_fortran_files = await Get(Paths, PathGlobs, req.path_globs("*.f90")) unowned_shell_files = set(all_fortran_files.files) - set(all_owned_sources) tests_filespec = Filespec(includes=list(FortranTestsSources.default)) test_filenames = set( matches_filespec( tests_filespec, paths=[os.path.basename(path) for path in unowned_shell_files])) test_files = { path for path in unowned_shell_files if os.path.basename(path) in test_filenames } sources_files = set(unowned_shell_files) - test_files classified_unowned_shell_files = { FortranTestsTarget: test_files, FortranLibraryTarget: sources_files, } pts = [] for tgt_type, paths in classified_unowned_shell_files.items(): for dirname, filenames in group_by_dir(paths).items(): name = "tests" if tgt_type == FortranTestsTarget else None pts.append( PutativeTarget.for_target_type( tgt_type, path=dirname, name=name, triggering_sources=sorted(filenames))) return PutativeTargets(pts)
def assert_rule_match( self, glob: str, expected_matches: Tuple[str, ...], negate: bool = False, ) -> None: """Tests that in-memory glob matching matches lazy-filesystem traversing globs.""" if negate: assertMatch, match_state = self.assertFalse, 'erroneously matches' else: assertMatch, match_state = self.assertTrue, "doesn't match" # Confirm in-memory behaviour. for expected in expected_matches: assertMatch( matches_filespec(expected, {'globs': [glob]}), f'{glob} {match_state} path `{expected}`', ) # And confirm that it matches on-disk behaviour. for expected in expected_matches: if expected.endswith('/'): self.create_dir(expected) else: self.create_file(expected) snapshot = self.request_single_product(Snapshot, PathGlobs([glob])) if negate: subset = set(expected_matches).intersection(set(snapshot.files)) self.assertEquals(subset, set(), f'{glob} {match_state} path(s) {subset}') else: self.assertEquals(sorted(expected_matches), sorted(snapshot.files))
def assert_rule_match(self, glob, expected_matches, negate=False): """Tests that in-memory glob matching matches lazy-filesystem traversing globs.""" if negate: assertMatch, match_state = self.assertFalse, 'erroneously matches' else: assertMatch, match_state = self.assertTrue, "doesn't match" # Confirm in-memory behaviour. for expected in expected_matches: assertMatch( matches_filespec(expected, {'globs': [glob]}), '{} {} path `{}`'.format(glob, match_state, expected), ) # And confirm that it matches on-disk behaviour. for expected in expected_matches: if expected.endswith('/'): self.create_dir(expected) else: self.create_file(expected) snapshot, = self.scheduler.product_request(Snapshot, [PathGlobs([glob])]) if negate: subset = set(expected_matches).intersection(set(snapshot.files)) self.assertEquals( subset, set(), '{} {} path(s) {}'.format(glob, match_state, subset)) else: self.assertEquals(sorted(expected_matches), sorted(snapshot.files))
async def find_owners(owners_request: OwnersRequest) -> Owners: # Determine which of the sources are live and which are deleted. sources_set_snapshot = await Get(Snapshot, PathGlobs(owners_request.sources)) live_files = FrozenOrderedSet(sources_set_snapshot.files) deleted_files = FrozenOrderedSet(s for s in owners_request.sources if s not in live_files) live_dirs = FrozenOrderedSet(os.path.dirname(s) for s in live_files) deleted_dirs = FrozenOrderedSet(os.path.dirname(s) for s in deleted_files) matching_addresses: OrderedSet[Address] = OrderedSet() unmatched_sources = set(owners_request.sources) for live in (True, False): # Walk up the buildroot looking for targets that would conceivably claim changed sources. # For live files, we use expanded Targets, which have file level precision but which are # only created for existing files. For deleted files we use UnexpandedTargets, which have # the original declared glob. candidate_targets: Iterable[Target] if live: if not live_dirs: continue sources_set = live_files candidate_specs = tuple( AscendantAddresses(directory=d) for d in live_dirs) candidate_targets = await Get(Targets, AddressSpecs(candidate_specs)) else: if not deleted_dirs: continue sources_set = deleted_files candidate_specs = tuple( AscendantAddresses(directory=d) for d in deleted_dirs) candidate_targets = await Get(UnexpandedTargets, AddressSpecs(candidate_specs)) build_file_addresses = await MultiGet( Get(BuildFileAddress, Address, tgt.address) for tgt in candidate_targets) for candidate_tgt, bfa in zip(candidate_targets, build_file_addresses): matching_files = set( matches_filespec(candidate_tgt.get(Sources).filespec, paths=sources_set)) if not matching_files and bfa.rel_path not in sources_set: continue unmatched_sources -= matching_files matching_addresses.add(candidate_tgt.address) if (unmatched_sources and owners_request.owners_not_found_behavior != OwnersNotFoundBehavior.ignore): _log_or_raise_unmatched_owners( [PurePath(path) for path in unmatched_sources], owners_request.owners_not_found_behavior) return Owners(matching_addresses)
def classify_source_files(paths: Iterable[str]) -> dict[type[Target], set[str]]: """Returns a dict of target type -> files that belong to targets of that type.""" tests_filespec = Filespec(includes=list(Shunit2TestsGeneratorSourcesField.default)) test_filenames = set( matches_filespec(tests_filespec, paths=[os.path.basename(path) for path in paths]) ) test_files = {path for path in paths if os.path.basename(path) in test_filenames} sources_files = set(paths) - test_files return {Shunit2TestsGeneratorTarget: test_files, ShellSourcesGeneratorTarget: sources_files}
def assert_rule_match(self, glob, expected_matches, negate=False): if negate: asserter, match_state = self.assertFalse, 'erroneously matches' else: asserter, match_state = self.assertTrue, "doesn't match" for expected in expected_matches: asserter( matches_filespec(expected, {'globs': [glob]}), '{} {} path `{}`'.format(glob, match_state, expected), )
def validate_build_file_name(self, build_file_patterns: tuple[str, ...]) -> None: """Check that the specified BUILD file name works with the repository's BUILD file patterns.""" filespec = Filespec(includes=list(build_file_patterns)) if not bool(matches_filespec(filespec, paths=[self.build_file_name])): raise ValueError( f"The option `[{self.options_scope}].build_file_name` is set to " f"`{self.build_file_name}`, which is not compatible with " f"`[GLOBAL].build_patterns`: {sorted(build_file_patterns)}. This means that " "generated BUILD files would be ignored.\n\n" "To fix, please update the options so that they are compatible." )
def classify_source_files( paths: Iterable[str]) -> dict[type[Target], set[str]]: """Returns a dict of target type -> files that belong to targets of that type.""" tests_filespec = Filespec(includes=list(PythonTestsSources.default)) test_filenames = set( matches_filespec(tests_filespec, paths=[os.path.basename(path) for path in paths])) test_files = { path for path in paths if os.path.basename(path) in test_filenames } library_files = set(paths) - test_files return {PythonTests: test_files, PythonLibrary: library_files}
def filter_by_ignores( self, putative_targets: Iterable[PutativeTarget], build_file_ignores: tuple[str, ...]) -> Iterator[PutativeTarget]: ignore_paths_filespec = Filespec( includes=[*self.ignore_paths, *build_file_ignores]) for ptgt in putative_targets: is_ignored_file = bool( matches_filespec( ignore_paths_filespec, paths=[os.path.join(ptgt.path, self.build_file_name)], )) if is_ignored_file: continue # Note that `tailor` can only generate explicit targets, so we don't need to # worry about generated address syntax (`#`) or file address syntax. address = f"{ptgt.path or '//'}:{ptgt.name}" if address in self.ignore_adding_targets: continue yield ptgt
def assert_rule_match(rule_runner: RuleRunner, glob: str, paths: Tuple[str, ...], *, should_match: bool) -> None: # Confirm in-memory behavior. matched_filespec = matches_filespec({"includes": [glob]}, paths=paths) if should_match: assert matched_filespec == paths else: assert not matched_filespec # Confirm on-disk behavior. for expected_match in paths: if expected_match.endswith("/"): rule_runner.create_dir(expected_match) else: rule_runner.create_file(expected_match) snapshot = rule_runner.request_product(Snapshot, [PathGlobs([glob])]) if should_match: assert sorted(paths) == sorted(snapshot.files) else: assert not snapshot.files
async def find_owners(owners_request: OwnersRequest) -> Owners: # Determine which of the sources are live and which are deleted. sources_paths = await Get(Paths, PathGlobs(owners_request.sources)) live_files = FrozenOrderedSet(sources_paths.files) deleted_files = FrozenOrderedSet(s for s in owners_request.sources if s not in live_files) live_dirs = FrozenOrderedSet(os.path.dirname(s) for s in live_files) deleted_dirs = FrozenOrderedSet(os.path.dirname(s) for s in deleted_files) def create_live_and_deleted_gets( *, filter_by_global_options: bool ) -> tuple[Get[FilteredTargets | Targets, RawSpecsWithoutFileOwners], Get[ UnexpandedTargets, RawSpecsWithoutFileOwners], ]: """Walk up the buildroot looking for targets that would conceivably claim changed sources. For live files, we use Targets, which causes generated targets to be used rather than their target generators. For deleted files we use UnexpandedTargets, which have the original declared `sources` globs from target generators. We ignore unrecognized files, which can happen e.g. when finding owners for deleted files. """ live_raw_specs = RawSpecsWithoutFileOwners( ancestor_globs=tuple( AncestorGlobSpec(directory=d) for d in live_dirs), filter_by_global_options=filter_by_global_options, description_of_origin="<owners rule - unused>", unmatched_glob_behavior=GlobMatchErrorBehavior.ignore, ) live_get: Get[FilteredTargets | Targets, RawSpecsWithoutFileOwners] = ( Get(FilteredTargets, RawSpecsWithoutFileOwners, live_raw_specs) if filter_by_global_options else Get( Targets, RawSpecsWithoutFileOwners, live_raw_specs)) deleted_get = Get( UnexpandedTargets, RawSpecsWithoutFileOwners( ancestor_globs=tuple( AncestorGlobSpec(directory=d) for d in deleted_dirs), filter_by_global_options=filter_by_global_options, description_of_origin="<owners rule - unused>", unmatched_glob_behavior=GlobMatchErrorBehavior.ignore, ), ) return live_get, deleted_get live_get, deleted_get = create_live_and_deleted_gets( filter_by_global_options=owners_request.filter_by_global_options) live_candidate_tgts, deleted_candidate_tgts = await MultiGet( live_get, deleted_get) matching_addresses: OrderedSet[Address] = OrderedSet() unmatched_sources = set(owners_request.sources) for live in (True, False): candidate_tgts: Sequence[Target] if live: candidate_tgts = live_candidate_tgts sources_set = live_files else: candidate_tgts = deleted_candidate_tgts sources_set = deleted_files build_file_addresses = await MultiGet( Get( BuildFileAddress, BuildFileAddressRequest( tgt.address, description_of_origin="<owners rule - cannot trigger>"), ) for tgt in candidate_tgts) for candidate_tgt, bfa in zip(candidate_tgts, build_file_addresses): matching_files = set( matches_filespec(candidate_tgt.get(SourcesField).filespec, paths=sources_set)) # Also consider secondary ownership, meaning it's not a `SourcesField` field with # primary ownership, but the target still should match the file. We can't use # `tgt.get()` because this is a mixin, and there technically may be >1 field. secondary_owner_fields = tuple( field for field in candidate_tgt.field_values.values() if isinstance(field, SecondaryOwnerMixin)) for secondary_owner_field in secondary_owner_fields: matching_files.update( matches_filespec(secondary_owner_field.filespec, paths=sources_set)) if not matching_files and bfa.rel_path not in sources_set: continue unmatched_sources -= matching_files matching_addresses.add(candidate_tgt.address) if (unmatched_sources and owners_request.owners_not_found_behavior != OwnersNotFoundBehavior.ignore): _log_or_raise_unmatched_owners( [PurePath(path) for path in unmatched_sources], owners_request.owners_not_found_behavior) return Owners(matching_addresses)
def matches(self, path): if not self._populated: raise self.NotPopulatedError() return matches_filespec(path, self.filespec)
def matches(self, path): return self.sources.matches(path) or matches_filespec( path, self.filespec)
def matches(self, path): return matches_filespec(path, self.filespec)
def _match_source(self, source, fileset): return fileset.matches(source) or matches_filespec( source, fileset.filespec)
async def find_owners(owners_request: OwnersRequest) -> Owners: # Determine which of the sources are live and which are deleted. sources_paths = await Get(Paths, PathGlobs(owners_request.sources)) live_files = FrozenOrderedSet(sources_paths.files) deleted_files = FrozenOrderedSet(s for s in owners_request.sources if s not in live_files) live_dirs = FrozenOrderedSet(os.path.dirname(s) for s in live_files) deleted_dirs = FrozenOrderedSet(os.path.dirname(s) for s in deleted_files) # Walk up the buildroot looking for targets that would conceivably claim changed sources. # For live files, we use expanded Targets, which have file level precision but which are # only created for existing files. For deleted files we use UnexpandedTargets, which have # the original declared glob. live_candidate_specs = tuple( AscendantAddresses(directory=d) for d in live_dirs) deleted_candidate_specs = tuple( AscendantAddresses(directory=d) for d in deleted_dirs) live_candidate_tgts, deleted_candidate_tgts = await MultiGet( Get(Targets, AddressSpecs(live_candidate_specs)), Get(UnexpandedTargets, AddressSpecs(deleted_candidate_specs)), ) matching_addresses: OrderedSet[Address] = OrderedSet() unmatched_sources = set(owners_request.sources) for live in (True, False): candidate_tgts: Sequence[Target] if live: candidate_tgts = live_candidate_tgts sources_set = live_files else: candidate_tgts = deleted_candidate_tgts sources_set = deleted_files build_file_addresses = await MultiGet( Get(BuildFileAddress, Address, tgt.address) for tgt in candidate_tgts) for candidate_tgt, bfa in zip(candidate_tgts, build_file_addresses): matching_files = set( matches_filespec(candidate_tgt.get(Sources).filespec, paths=sources_set)) # Also consider secondary ownership, meaning it's not a `Sources` field with primary # ownership, but the target still should match the file. We can't use `tgt.get()` # because this is a mixin, and there technically may be >1 field. secondary_owner_fields = tuple( field # type: ignore[misc] for field in candidate_tgt.field_values.values() if isinstance(field, SecondaryOwnerMixin)) for secondary_owner_field in secondary_owner_fields: matching_files.update( matches_filespec(secondary_owner_field.filespec, paths=sources_set)) if not matching_files and bfa.rel_path not in sources_set: continue unmatched_sources -= matching_files matching_addresses.add(candidate_tgt.address) if (unmatched_sources and owners_request.owners_not_found_behavior != OwnersNotFoundBehavior.ignore): _log_or_raise_unmatched_owners( [PurePath(path) for path in unmatched_sources], owners_request.owners_not_found_behavior) return Owners(matching_addresses)
def _match_source(self, source, fileset): return fileset.matches(source) or matches_filespec(source, fileset.filespec)
def matches(self, path): return self.sources.matches(path) or matches_filespec(path, self.filespec)