def _register(cls, source_root_dir, mutable, *allowed_target_types):
"""Registers a source root.
:param string source_root_dir: The source root directory against which we resolve source paths,
relative to the build root.
:param list allowed_target_types: Optional list of target types. If specified, we enforce that
only targets of those types appear under this source root.
"""
# Temporary delegation to the new implementation, until this entire file goes away.
SourceRootConfig.global_instance().get_source_roots().add_source_root(
source_root_dir)
source_root_dir = SourceRoot._relative_to_buildroot(source_root_dir)
types = cls._TYPES_BY_ROOT.get(source_root_dir)
if types is None:
types = OrderedSet()
cls._TYPES_BY_ROOT[source_root_dir] = types
for allowed_target_type in allowed_target_types:
types.add(allowed_target_type)
roots = cls._ROOTS_BY_TYPE.get(allowed_target_type)
if roots is None:
roots = OrderedSet()
cls._ROOTS_BY_TYPE[allowed_target_type] = roots
roots.add(source_root_dir)
cls._SOURCE_ROOT_TREE.add_root(source_root_dir, allowed_target_types,
mutable)
def _register(cls, source_root_dir, mutable, *allowed_target_types):
"""Registers a source root.
:param string source_root_dir: The source root directory against which we resolve source paths,
relative to the build root.
:param list allowed_target_types: Optional list of target types. If specified, we enforce that
only targets of those types appear under this source root.
"""
# Temporary delegation to the new implementation, until this entire file goes away.
SourceRootConfig.global_instance().get_source_roots().add_source_root(source_root_dir)
source_root_dir = SourceRoot._relative_to_buildroot(source_root_dir)
types = cls._TYPES_BY_ROOT.get(source_root_dir)
if types is None:
types = OrderedSet()
cls._TYPES_BY_ROOT[source_root_dir] = types
for allowed_target_type in allowed_target_types:
types.add(allowed_target_type)
roots = cls._ROOTS_BY_TYPE.get(allowed_target_type)
if roots is None:
roots = OrderedSet()
cls._ROOTS_BY_TYPE[allowed_target_type] = roots
roots.add(source_root_dir)
cls._SOURCE_ROOT_TREE.add_root(source_root_dir, allowed_target_types, mutable)
def assert_sources(
self,
expected_files,
expected_packages,
expected_namespace_packages,
expected_package_data,
addrs,
):
srcs = self.request_single_product(
SetupPySources,
Params(
SetupPySourcesRequest(Targets(
[self.tgt(addr) for addr in addrs]),
py2=False),
SourceRootConfig.global_instance(),
),
)
chroot_snapshot = self.request_single_product(Snapshot,
Params(srcs.digest))
assert sorted(expected_files) == sorted(chroot_snapshot.files)
assert sorted(expected_packages) == sorted(srcs.packages)
assert sorted(expected_namespace_packages) == sorted(
srcs.namespace_packages)
assert expected_package_data == dict(srcs.package_data)
async def strip_source_root(
hydrated_target: HydratedTarget,
source_root_config: SourceRootConfig) -> SourceRootStrippedSources:
"""Relativize targets to their source root, e.g.
`src/python/pants/util/strutil.py` -> `pants/util/strutil.py ."""
target_adaptor = hydrated_target.adaptor
source_roots = source_root_config.get_source_roots()
# TODO: make TargetAdaptor return a 'sources' field with an empty snapshot instead of raising to
# simplify the hasattr() checks here!
if not hasattr(target_adaptor, 'sources'):
return SourceRootStrippedSources(snapshot=EMPTY_SNAPSHOT)
digest = target_adaptor.sources.snapshot.directory_digest
source_root = source_roots.find_by_path(target_adaptor.address.spec_path)
if source_root is None:
# If we found no source root, use the target's dir.
# Note that when --source-unmatched is 'create' (the default) we'll never return None,
# but will return the target's dir. This check allows this code to work even if
# --source-unmatched is 'fail'.
source_root_path = target_adaptor.address.spec_path
else:
source_root_path = source_root.path
# Loose `Files`, as opposed to `Resources` or `Target`s, have no (implied) package
# structure and so we do not remove their source root like we normally do, so that filesystem
# APIs may still access the files. See pex_build_util.py's `_create_source_dumper`.
if target_adaptor.type_alias == Files.alias():
source_root_path = ''
resulting_digest = await Get[Digest](DirectoryWithPrefixToStrip(
directory_digest=digest, prefix=source_root_path))
resulting_snapshot = await Get[Snapshot](Digest, resulting_digest)
return SourceRootStrippedSources(snapshot=resulting_snapshot)
async def strip_source_root(
hydrated_target: HydratedTarget,
source_root_config: SourceRootConfig) -> SourceRootStrippedSources:
"""Relativize targets to their source root, e.g.
`src/python/pants/util/strutil.py` -> `pants/util/strutil.py ."""
target_adaptor = hydrated_target.adaptor
source_roots = source_root_config.get_source_roots()
# TODO: make TargetAdaptor return a 'sources' field with an empty snapshot instead of raising to
# simplify the hasattr() checks here!
if not hasattr(target_adaptor, 'sources'):
return SourceRootStrippedSources(snapshot=EMPTY_SNAPSHOT)
digest = target_adaptor.sources.snapshot.directory_digest
source_root = source_roots.find_by_path(target_adaptor.address.spec_path)
# Loose `Files`, as opposed to `Resources` or `Target`s, have no (implied) package
# structure and so we do not remove their source root like we normally do, so that filesystem
# APIs may still access the files. See pex_build_util.py's `_create_source_dumper`.
if target_adaptor.type_alias == Files.alias():
source_root = None
resulting_digest = await Get(
Digest,
DirectoryWithPrefixToStrip(
directory_digest=digest,
prefix=source_root.path if source_root else ""))
resulting_snapshot = await Get(Snapshot, Digest, resulting_digest)
return SourceRootStrippedSources(snapshot=resulting_snapshot)
def __init__(self,
options,
run_tracker,
target_roots,
requested_goals=None,
target_base=None,
build_graph=None,
build_file_parser=None,
address_mapper=None,
console_outstream=None,
scm=None,
workspace=None,
invalidation_report=None):
self._options = options
self.build_graph = build_graph
self.build_file_parser = build_file_parser
self.address_mapper = address_mapper
self.run_tracker = run_tracker
self._log = self.Log(run_tracker)
self._target_base = target_base or Target
self._products = Products()
self._buildroot = get_buildroot()
self._source_roots = SourceRootConfig.global_instance(
).get_source_roots()
self._lock = OwnerPrintingInterProcessFileLock(
os.path.join(self._buildroot, '.pants.workdir.file_lock'))
self._java_sysprops = None # Computed lazily.
self.requested_goals = requested_goals or []
self._console_outstream = console_outstream or sys.stdout
self._scm = scm or get_scm()
self._workspace = workspace or (ScmWorkspace(self._scm)
if self._scm else None)
self._replace_targets(target_roots)
self._invalidation_report = invalidation_report
def all_roots(source_root_config: SourceRootConfig) -> AllSourceRoots:
source_roots = source_root_config.get_source_roots()
all_paths: Set[str] = set()
for path in source_roots.traverse():
if path.startswith("^/"):
all_paths.add(f"{path[2:]}/")
else:
all_paths.add(f"**/{path}/")
snapshot = yield Get(Snapshot, PathGlobs(include=tuple(all_paths)))
all_source_roots: Set[SourceRoot] = set()
# The globs above can match on subdirectories of the source roots.
# For instance, `src/*/` might match 'src/rust/' as well as
# 'src/rust/engine/process_execution/bazel_protos/src/gen'.
# So we use find_by_path to verify every candidate source root.
for directory in snapshot.dirs:
match: SourceRoot = source_roots.find_by_path(directory)
if match:
all_source_roots.add(match)
yield AllSourceRoots(all_source_roots)
def __init__(self, options, run_tracker, target_roots,
requested_goals=None, target_base=None, build_graph=None,
build_file_parser=None, build_configuration=None,
address_mapper=None, console_outstream=None, scm=None,
workspace=None, invalidation_report=None, scheduler=None):
self._options = options
# We register a callback that will cause build graph edits to invalidate our caches, and we hold
# a handle directly to the callback function to ensure that it is not GC'd until the context is.
self.build_graph = build_graph
self._clear_target_cache_handle = self._clear_target_cache
self._targets_cache = dict()
self.build_graph.add_invalidation_callback(self._clear_target_cache_handle)
self._build_file_parser = build_file_parser
self.build_configuration = build_configuration
self.address_mapper = address_mapper
self.run_tracker = run_tracker
self._log = run_tracker.logger
self._target_base = target_base or Target
self._products = Products()
self._buildroot = get_buildroot()
self._source_roots = SourceRootConfig.global_instance().get_source_roots()
self._lock = OwnerPrintingInterProcessFileLock(os.path.join(self._buildroot, '.pants.workdir.file_lock'))
self._java_sysprops = None # Computed lazily.
self.requested_goals = requested_goals or []
self._console_outstream = console_outstream or sys.stdout.buffer
self._scm = scm or get_scm()
self._workspace = workspace or (ScmWorkspace(self._scm) if self._scm else None)
self._replace_targets(target_roots)
self._invalidation_report = invalidation_report
self._scheduler = scheduler
def __init__(self, options, run_tracker, target_roots,
requested_goals=None, target_base=None, build_graph=None,
build_file_parser=None, address_mapper=None, console_outstream=None, scm=None,
workspace=None, spec_excludes=None, invalidation_report=None):
"""
:API: public
"""
deprecated_conditional(lambda: spec_excludes is not None,
'0.0.75',
'Use address_mapper#build_ignore_patterns instead.')
self._options = options
self.build_graph = build_graph
self.build_file_parser = build_file_parser
self.address_mapper = address_mapper
self.run_tracker = run_tracker
self._log = self.Log(run_tracker)
self._target_base = target_base or Target
self._products = Products()
self._buildroot = get_buildroot()
self._source_roots = SourceRootConfig.global_instance().get_source_roots()
self._lock = OwnerPrintingPIDLockFile(os.path.join(self._buildroot, '.pants.run'))
self._java_sysprops = None # Computed lazily.
self.requested_goals = requested_goals or []
self._console_outstream = console_outstream or sys.stdout
self._scm = scm or get_scm()
self._workspace = workspace or (ScmWorkspace(self._scm) if self._scm else None)
self._spec_excludes = spec_excludes
self._replace_targets(target_roots)
self._invalidation_report = invalidation_report
def create_python_awslambda(self, addr: str) -> Tuple[str, bytes]:
lambdex_setup = self.request_single_product(
LambdexSetup,
Params(
PythonSetup.global_instance(),
PythonNativeCode.global_instance(),
SubprocessEnvironment.global_instance(),
Lambdex.global_instance(),
))
target = self.request_single_product(HydratedTarget,
Address.parse(addr))
created_awslambda = self.request_single_product(
CreatedAWSLambda,
Params(
target.adaptor,
lambdex_setup,
SourceRootConfig.global_instance(),
PythonSetup.global_instance(),
PythonNativeCode.global_instance(),
SubprocessEnvironment.global_instance(),
))
files_content = list(
self.request_single_product(FilesContent,
Params(created_awslambda.digest)))
assert len(files_content) == 1
return created_awslambda.name, files_content[0].content
def __init__(
self,
options,
run_tracker,
target_roots,
requested_goals=None,
target_base=None,
build_graph=None,
build_file_parser=None,
address_mapper=None,
console_outstream=None,
scm=None,
workspace=None,
invalidation_report=None,
):
self._options = options
self.build_graph = build_graph
self.build_file_parser = build_file_parser
self.address_mapper = address_mapper
self.run_tracker = run_tracker
self._log = self.Log(run_tracker)
self._target_base = target_base or Target
self._products = Products()
self._buildroot = get_buildroot()
self._source_roots = SourceRootConfig.global_instance().get_source_roots()
self._lock = OwnerPrintingInterProcessFileLock(os.path.join(self._buildroot, ".pants.workdir.file_lock"))
self._java_sysprops = None # Computed lazily.
self.requested_goals = requested_goals or []
self._console_outstream = console_outstream or sys.stdout
self._scm = scm or get_scm()
self._workspace = workspace or (ScmWorkspace(self._scm) if self._scm else None)
self._replace_targets(target_roots)
self._invalidation_report = invalidation_report
def source_root(self):
""":returns: the source root for these sources, or None if they're not under a source root."""
# TODO: It's a shame that we have to access the singleton directly here, instead of getting
# the SourceRoots instance from context, as tasks do. In the new engine we could inject
# this into the target, rather than have it reach out for global singletons.
return SourceRootConfig.global_instance().get_source_roots(
).find_by_path(self.rel_path)
def test_all_roots_with_root_at_buildroot(self):
options = {
"pants_ignore": [],
"root_patterns": ["/"],
}
options.update(self.options[""]) # We need inherited values for pants_workdir etc.
self.context(
for_subsystems=[SourceRootConfig], options={SourceRootConfig.options_scope: options}
)
source_root_config = SourceRootConfig.global_instance()
# This function mocks out reading real directories off the file system
def provider_rule(path_globs: PathGlobs) -> Snapshot:
dirs = ("foo",) # A python package at the buildroot.
return Snapshot(Digest("abcdef", 10), (), dirs)
output = run_rule(
list_roots.all_roots,
rule_args=[source_root_config],
mock_gets=[
MockGet(product_type=Snapshot, subject_type=PathGlobs, mock=provider_rule),
MockGet(
product_type=OptionalSourceRoot,
subject_type=SourceRootRequest,
mock=lambda req: OptionalSourceRoot(SourceRoot(".")),
),
],
)
self.assertEqual({SourceRoot(".")}, set(output))
async def get_sources(request: SetupPySourcesRequest,
source_root_config: SourceRootConfig) -> SetupPySources:
targets = request.hydrated_targets
stripped_srcs_list = await MultiGet(
Get[SourceRootStrippedSources](HydratedTarget, target)
for target in targets)
# Create a chroot with all the sources, and any ancestor __init__.py files that might be needed
# for imports to work. Note that if a repo has multiple exported targets under a single ancestor
# package, then that package must be a namespace package, which in Python 3 means it must not
# have an __init__.py. We don't validate this here, because it would require inspecting *all*
# targets, whether or not they are in the target set for this run - basically the entire repo.
# So it's the repo owners' responsibility to ensure __init__.py hygiene.
stripped_srcs_digests = [
stripped_sources.snapshot.directory_digest
for stripped_sources in stripped_srcs_list
]
ancestor_init_pys = await Get[AncestorInitPyFiles](HydratedTargets,
targets)
sources_digest = await Get[Digest](DirectoriesToMerge(directories=tuple(
[*stripped_srcs_digests, *ancestor_init_pys.digests])))
init_pys_snapshot = await Get[Snapshot](SnapshotSubset(
sources_digest, PathGlobs(['**/__init__.py'])))
init_py_contents = await Get[FilesContent](
Digest, init_pys_snapshot.directory_digest)
packages, namespace_packages, package_data = find_packages(
source_roots=source_root_config.get_source_roots(),
tgts_and_stripped_srcs=list(zip(targets, stripped_srcs_list)),
init_py_contents=init_py_contents,
py2=request.py2)
return SetupPySources(digest=sources_digest,
packages=packages,
namespace_packages=namespace_packages,
package_data=package_data)
def source_root(self):
""":returns: the source root for these sources, or None if they're not under a source root."""
# TODO: It's a shame that we have to access the singleton directly here, instead of getting
# the SourceRoots instance from context, as tasks do. In the new engine we could inject
# this into the target, rather than have it reach out for global singletons.
return SourceRootConfig.global_instance().get_source_roots(
).find_by_path(self.rel_path)
def test_collapse_source_root(self):
self.context(for_subsystems=[SourceRootConfig], options={
SourceRootConfig.options_scope: {
'source_roots': {
'/src/java': [],
'/tests/java': [],
'/some/other': []
},
'unmatched': 'fail'
}
})
source_roots = SourceRootConfig.global_instance().get_source_roots()
source_set_list = []
self.assertEquals([], Project._collapse_by_source_root(source_roots, source_set_list))
source_sets = [
SourceSet('/repo-root', 'src/java', 'org/pantsbuild/app', False),
SourceSet('/repo-root', 'tests/java', 'org/pantsbuild/app', True),
SourceSet('/repo-root', 'some/other', 'path', False),
]
results = Project._collapse_by_source_root(source_roots, source_sets)
self.assertEquals(SourceSet('/repo-root', 'src/java', '', False), results[0])
self.assertFalse(results[0].is_test)
self.assertEquals(SourceSet('/repo-root', 'tests/java', '', True), results[1])
self.assertTrue(results[1].is_test)
# If there is no registered source root, the SourceSet should be returned unmodified
self.assertEquals(source_sets[2], results[2])
self.assertFalse(results[2].is_test)
def assert_chroot(self, expected_files, expected_setup_kwargs, addr):
chroot = self.request_single_product(
SetupPyChroot,
Params(SetupPyChrootRequest(ExportedTarget(self.tgt(addr))),
SourceRootConfig.global_instance()))
snapshot = self.request_single_product(Snapshot, Params(chroot.digest))
assert sorted(expected_files) == sorted(snapshot.files)
kwargs = json.loads(chroot.setup_keywords_json)
assert expected_setup_kwargs == kwargs
def assert_error(self, addr: str, exc_cls: Type[Exception]):
with pytest.raises(ExecutionError) as excinfo:
self.request_single_product(
SetupPyChroot,
Params(SetupPyChrootRequest(ExportedTarget(self.tgt(addr))),
SourceRootConfig.global_instance()))
ex = excinfo.value
assert len(ex.wrapped_exceptions) == 1
assert type(ex.wrapped_exceptions[0]) == exc_cls
def target_base(self):
""":returns: the source root path for this target."""
# TODO: It's a shame that we have to access the singleton directly here, instead of getting
# the SourceRoots instance from context, as tasks do. In the new engine we could inject
# this into the target, rather than have it reach out for global singletons.
source_root = SourceRootConfig.global_instance().get_source_roots().find(self)
if not source_root:
raise TargetDefinitionException(self, 'Not under any configured source root.')
return source_root.path
async def create_coverage_config(
coverage_config_request: CoverageConfigRequest, source_root_config: SourceRootConfig
) -> CoverageConfig:
sources = await Get[SourceFiles](
AllSourceFilesRequest(
(tgt.get(Sources) for tgt in coverage_config_request.targets), strip_source_roots=False,
)
)
init_injected = await Get[InitInjectedSnapshot](InjectInitRequest(sources.snapshot))
source_roots = source_root_config.get_source_roots()
# Generate a map from source root stripped source to its source root. eg:
# {'pants/testutil/subsystem/util.py': 'src/python'}. This is so that coverage reports
# referencing /chroot/path/pants/testutil/subsystem/util.py can be mapped back to the actual
# sources they reference when generating coverage reports.
def stripped_file_with_source_root(file_name: str) -> Tuple[str, str]:
source_root_object = source_roots.find_by_path(file_name)
source_root = source_root_object.path if source_root_object is not None else ""
stripped_path = file_name[len(source_root) + 1 :]
return stripped_path, source_root
stripped_files_to_source_roots = dict(
stripped_file_with_source_root(filename)
for filename in sorted(init_injected.snapshot.files)
)
default_config = dedent(
"""
[run]
branch = True
timid = False
relative_files = True
"""
)
config_parser = configparser.ConfigParser()
config_parser.read_file(StringIO(default_config))
config_parser.set("run", "plugins", COVERAGE_PLUGIN_MODULE_NAME)
config_parser.add_section(COVERAGE_PLUGIN_MODULE_NAME)
config_parser.set(
COVERAGE_PLUGIN_MODULE_NAME,
"source_to_target_base",
json.dumps(stripped_files_to_source_roots),
)
config_parser.set(
COVERAGE_PLUGIN_MODULE_NAME, "test_time", json.dumps(coverage_config_request.is_test_time)
)
config_io_stream = StringIO()
config_parser.write(config_io_stream)
digest = await Get[Digest](
InputFilesContent(
[FileContent(".coveragerc", content=config_io_stream.getvalue().encode())]
)
)
return CoverageConfig(digest)
def target_base(self):
""":returns: the source root path for this target."""
# TODO: It's a shame that we have to access the singleton directly here, instead of getting
# the SourceRoots instance from context, as tasks do. In the new engine we could inject
# this into the target, rather than have it reach out for global singletons.
source_root = SourceRootConfig.global_instance().get_source_roots(
).find(self)
if not source_root:
raise TargetDefinitionException(
self, 'Not under any configured source root.')
return source_root.path
def assert_ancestor_init_py(self, expected_init_pys: Iterable[str],
addrs: Iterable[str]) -> None:
ancestor_init_py_files = self.request_single_product(
AncestorInitPyFiles,
Params(HydratedTargets([self.tgt(addr) for addr in addrs]),
SourceRootConfig.global_instance()))
snapshots = [
self.request_single_product(Snapshot, Params(digest))
for digest in ancestor_init_py_files.digests
]
init_py_files_found = set(
[file for snapshot in snapshots for file in snapshot.files])
# NB: Doesn't include the root __init__.py or the missing src/python/foo/bar/__init__.py.
assert sorted(expected_init_pys) == sorted(init_py_files_found)
def test_all_roots(self):
self.create_dir("contrib/go/examples/3rdparty/go")
self.create_dir("contrib/go/examples/src/go/src")
self.create_dir("src/java")
self.create_dir("src/python")
self.create_dir("src/kotlin")
self.create_dir("src/example/java")
self.create_dir("src/example/python")
self.create_dir("my/project/src/java")
self.create_dir("fixed/root/jvm")
self.create_dir("not/a/srcroot/java")
options = {
"pants_ignore": [],
"source_root_patterns": ["src/*", "src/example/*"],
"source_roots": {
# Fixed roots should trump patterns which would detect contrib/go/examples/src/go here.
"contrib/go/examples/src/go/src": ["go"],
# Dir does not exist, should not be listed as a root.
"java": ["java"],
},
}
options.update(self.options[""]
) # We need inherited values for pants_workdir etc.
self.context(for_subsystems=[SourceRootConfig],
options={SourceRootConfig.options_scope: options})
source_roots = SourceRootConfig.global_instance().get_source_roots()
# Ensure that we see any manually added roots.
source_roots.add_source_root("fixed/root/jvm", ("java", "scala"), TEST)
source_roots.all_roots()
self.assertEqual(
{
SourceRoot("contrib/go/examples/3rdparty/go",
("go", ), THIRDPARTY),
SourceRoot("contrib/go/examples/src/go/src", ("go", ), SOURCE),
SourceRoot("src/java", ("java", ), SOURCE),
SourceRoot("src/python", ("python", ), SOURCE),
SourceRoot("src/kotlin", ("kotlin", ), SOURCE),
SourceRoot("src/example/java", ("java", ), SOURCE),
SourceRoot("src/example/python", ("python", ), SOURCE),
SourceRoot("my/project/src/java", ("java", ), SOURCE),
SourceRoot("fixed/root/jvm", ("java", "scala"), TEST),
},
set(source_roots.all_roots()),
)
async def strip_source_roots_from_snapshot(
request: StripSnapshotRequest,
source_root_config: SourceRootConfig,
) -> SourceRootStrippedSources:
"""Removes source roots from a snapshot, e.g. `src/python/pants/util/strutil.py` ->
`pants/util/strutil.py`."""
if not request.snapshot.files:
return SourceRootStrippedSources(request.snapshot)
source_roots_object = source_root_config.get_source_roots()
def determine_source_root(path: str) -> str:
return cast(str, source_roots_object.strict_find_by_path(path).path)
if request.representative_path is not None:
source_root = determine_source_root(request.representative_path)
if source_root == ".":
return SourceRootStrippedSources(request.snapshot)
resulting_snapshot = await Get[Snapshot](RemovePrefix(
request.snapshot.digest, source_root))
return SourceRootStrippedSources(resulting_snapshot)
files_grouped_by_source_root = {
source_root: tuple(files)
for source_root, files in itertools.groupby(request.snapshot.files,
key=determine_source_root)
}
if len(files_grouped_by_source_root) == 1:
source_root = next(iter(files_grouped_by_source_root.keys()))
if source_root == ".":
return SourceRootStrippedSources(request.snapshot)
resulting_snapshot = await Get[Snapshot](RemovePrefix(
request.snapshot.digest, source_root))
return SourceRootStrippedSources(resulting_snapshot)
snapshot_subsets = await MultiGet(
Get[Snapshot](SnapshotSubset(request.snapshot.digest, PathGlobs(
files))) for files in files_grouped_by_source_root.values())
resulting_digests = await MultiGet(
Get[Digest](RemovePrefix(snapshot.digest, source_root))
for snapshot, source_root in zip(snapshot_subsets,
files_grouped_by_source_root.keys()))
resulting_snapshot = await Get[Snapshot](MergeDigests(resulting_digests))
return SourceRootStrippedSources(resulting_snapshot)
def test_all_roots(self):
self.create_dir('contrib/go/examples/3rdparty/go')
self.create_dir('contrib/go/examples/src/go/src')
self.create_dir('src/java')
self.create_dir('src/python')
self.create_dir('src/kotlin')
self.create_dir('src/example/java')
self.create_dir('src/example/python')
self.create_dir('my/project/src/java')
self.create_dir('fixed/root/jvm')
self.create_dir('not/a/srcroot/java')
options = {
'pants_ignore': [],
'source_root_patterns': ['src/*', 'src/example/*'],
'source_roots': {
# Fixed roots should trump patterns which would detect contrib/go/examples/src/go here.
'contrib/go/examples/src/go/src': ['go'],
# Dir does not exist, should not be listed as a root.
'java': ['java']
}
}
options.update(self.options['']
) # We need inherited values for pants_workdir etc.
self.context(for_subsystems=[SourceRootConfig],
options={SourceRootConfig.options_scope: options})
source_roots = SourceRootConfig.global_instance().get_source_roots()
# Ensure that we see any manually added roots.
source_roots.add_source_root('fixed/root/jvm', ('java', 'scala'), TEST)
source_roots.all_roots()
self.assertEqual(
{
SourceRoot('contrib/go/examples/3rdparty/go',
('go', ), THIRDPARTY),
SourceRoot('contrib/go/examples/src/go/src', ('go', ), SOURCE),
SourceRoot('src/java', ('java', ), SOURCE),
SourceRoot('src/python', ('python', ), SOURCE),
SourceRoot('src/kotlin', ('kotlin', ), SOURCE),
SourceRoot('src/example/java', ('java', ), SOURCE),
SourceRoot('src/example/python', ('python', ), SOURCE),
SourceRoot('my/project/src/java', ('java', ), SOURCE),
SourceRoot('fixed/root/jvm', ('java', 'scala'), TEST)
}, set(source_roots.all_roots()))
async def strip_source_roots_from_snapshot(
request: StripSnapshotRequest,
source_root_config: SourceRootConfig,
) -> SourceRootStrippedSources:
"""Removes source roots from a snapshot, e.g. `src/python/pants/util/strutil.py` ->
`pants/util/strutil.py`."""
source_roots_object = source_root_config.get_source_roots()
def determine_source_root(path: str) -> str:
source_root = source_roots_object.safe_find_by_path(path)
if source_root is not None:
return cast(str, source_root.path)
if source_root_config.options.unmatched == "fail":
raise NoSourceRootError(
f"Could not find a source root for `{path}`.")
# Otherwise, create a source root by using the parent directory.
return PurePath(path).parent.as_posix()
if request.representative_path is not None:
resulting_digest = await Get[Digest](DirectoryWithPrefixToStrip(
directory_digest=request.snapshot.directory_digest,
prefix=determine_source_root(request.representative_path),
))
resulting_snapshot = await Get[Snapshot](Digest, resulting_digest)
return SourceRootStrippedSources(snapshot=resulting_snapshot)
files_grouped_by_source_root = {
source_root: tuple(files)
for source_root, files in itertools.groupby(request.snapshot.files,
key=determine_source_root)
}
snapshot_subsets = await MultiGet(Get[Snapshot](SnapshotSubset(
directory_digest=request.snapshot.directory_digest,
globs=PathGlobs(files),
)) for files in files_grouped_by_source_root.values())
resulting_digests = await MultiGet(
Get[Digest](DirectoryWithPrefixToStrip(
directory_digest=snapshot.directory_digest, prefix=source_root))
for snapshot, source_root in zip(snapshot_subsets,
files_grouped_by_source_root.keys()))
merged_result = await Get[Digest](DirectoriesToMerge(resulting_digests))
resulting_snapshot = await Get[Snapshot](Digest, merged_result)
return SourceRootStrippedSources(resulting_snapshot)
def assert_stripped_source_file(self,
*,
original_path: str,
expected_path: str,
target_type_alias=None):
init_subsystem(SourceRootConfig)
adaptor = Mock()
adaptor.sources = Mock()
source_files = {original_path: "print('random python')"}
adaptor.sources.snapshot = self.make_snapshot(source_files)
adaptor.address = Mock()
adaptor.address.spec_path = original_path
if target_type_alias:
adaptor.type_alias = target_type_alias
target = HydratedTarget('some/target/address', adaptor, tuple())
stripped_sources = self.request_single_product(
SourceRootStrippedSources,
Params(target, SourceRootConfig.global_instance()))
self.assertEqual(stripped_sources.snapshot.files, (expected_path, ))
def __init__(self,
options,
run_tracker,
target_roots,
requested_goals=None,
target_base=None,
build_graph=None,
build_file_parser=None,
address_mapper=None,
console_outstream=None,
scm=None,
workspace=None,
spec_excludes=None,
invalidation_report=None):
"""
:API: public
"""
deprecated_conditional(
lambda: spec_excludes is not None, '0.0.75',
'Use address_mapper#build_ignore_patterns instead.')
self._options = options
self.build_graph = build_graph
self.build_file_parser = build_file_parser
self.address_mapper = address_mapper
self.run_tracker = run_tracker
self._log = self.Log(run_tracker)
self._target_base = target_base or Target
self._products = Products()
self._buildroot = get_buildroot()
self._source_roots = SourceRootConfig.global_instance(
).get_source_roots()
self._lock = OwnerPrintingPIDLockFile(
os.path.join(self._buildroot, '.pants.run'))
self._java_sysprops = None # Computed lazily.
self.requested_goals = requested_goals or []
self._console_outstream = console_outstream or sys.stdout
self._scm = scm or get_scm()
self._workspace = workspace or (ScmWorkspace(self._scm)
if self._scm else None)
self._spec_excludes = spec_excludes
self._replace_targets(target_roots)
self._invalidation_report = invalidation_report
async def get_ancestor_init_py(
targets: Targets, source_root_config: SourceRootConfig
) -> AncestorInitPyFiles:
"""Find any ancestor __init__.py files for the given targets.
Includes sibling __init__.py files. Returns the files stripped of their source roots.
"""
source_roots = source_root_config.get_source_roots()
sources = await Get[SourceFiles](
AllSourceFilesRequest(tgt[PythonSources] for tgt in targets if tgt.has_field(PythonSources))
)
# Find the ancestors of all dirs containing .py files, including those dirs themselves.
source_dir_ancestors: Set[Tuple[str, str]] = set() # Items are (src_root, path incl. src_root).
for fp in sources.snapshot.files:
source_dir_ancestor = os.path.dirname(fp)
source_root = source_root_or_raise(source_roots, fp)
# Do not allow the repository root to leak (i.e., '.' should not be a package in setup.py).
while source_dir_ancestor != source_root:
source_dir_ancestors.add((source_root, source_dir_ancestor))
source_dir_ancestor = os.path.dirname(source_dir_ancestor)
source_dir_ancestors_list = list(source_dir_ancestors) # To force a consistent order.
# Note that we must MultiGet single globs instead of a a single Get for all the globs, because
# we match each result to its originating glob (see use of zip below).
ancestor_init_py_snapshots = await MultiGet[Snapshot](
Get[Snapshot](PathGlobs, PathGlobs([os.path.join(source_dir_ancestor[1], "__init__.py")]))
for source_dir_ancestor in source_dir_ancestors_list
)
source_root_stripped_ancestor_init_pys = await MultiGet[Digest](
Get[Digest](
DirectoryWithPrefixToStrip(
directory_digest=snapshot.directory_digest, prefix=source_dir_ancestor[0]
)
)
for snapshot, source_dir_ancestor in zip(
ancestor_init_py_snapshots, source_dir_ancestors_list
)
)
return AncestorInitPyFiles(source_root_stripped_ancestor_init_pys)
def test_all_roots(self):
self.create_dir('contrib/go/examples/3rdparty/go')
self.create_dir('contrib/go/examples/src/go/src')
self.create_dir('src/java')
self.create_dir('src/python')
self.create_dir('src/example/java')
self.create_dir('src/example/python')
self.create_dir('my/project/src/java')
self.create_dir('fixed/root/jvm')
self.create_dir('not/a/srcroot/java')
options = {
'build_file_rev': None,
'pants_ignore': [],
'source_root_patterns': ['src/*', 'src/example/*'],
'source_roots': {
# Fixed roots should trump patterns which would detect contrib/go/examples/src/go here.
'contrib/go/examples/src/go/src': ['go'],
# Dir does not exist, should not be listed as a root.
'java': ['java']}
}
options.update(self.options['']) # We need inherited values for pants_workdir etc.
self.context(for_subsystems=[SourceRootConfig], options={
SourceRootConfig.options_scope: options
})
source_roots = SourceRootConfig.global_instance().get_source_roots()
# Ensure that we see any manually added roots.
source_roots.add_source_root('fixed/root/jvm', ('java', 'scala'), TEST)
source_roots.all_roots()
self.assertEquals({SourceRoot('contrib/go/examples/3rdparty/go', ('go',), THIRDPARTY),
SourceRoot('contrib/go/examples/src/go/src', ('go',), SOURCE),
SourceRoot('src/java', ('java',), SOURCE),
SourceRoot('src/python', ('python',), SOURCE),
SourceRoot('src/example/java', ('java',), SOURCE),
SourceRoot('src/example/python', ('python',), SOURCE),
SourceRoot('my/project/src/java', ('java',), SOURCE),
SourceRoot('fixed/root/jvm', ('java','scala'), TEST)},
set(source_roots.all_roots()))
async def construct_coverage_config(
source_root_config: SourceRootConfig,
coverage_config_request: CoveragercRequest) -> Coveragerc:
sources = await Get[SourceFiles](AllSourceFilesRequest(
(ht.adaptor for ht in coverage_config_request.hydrated_targets),
strip_source_roots=False,
))
init_injected = await Get[InitInjectedSnapshot](InjectInitRequest(
sources.snapshot))
source_roots = source_root_config.get_source_roots()
# Generate a map from source root stripped source to its source root. eg:
# {'pants/testutil/subsystem/util.py': 'src/python'}
# This is so coverage reports referencing /chroot/path/pants/testutil/subsystem/util.py can be mapped
# back to the actual sources they reference when generating coverage reports.
def source_root_stripped_source_and_source_root(
file_name: str) -> Tuple[str, str]:
source_root = source_roots.find_by_path(file_name)
source_root_path = source_root.path if source_root is not None else ""
source_root_stripped_path = file_name[len(source_root_path) + 1:]
return (source_root_stripped_path, source_root_path)
source_to_target_base = dict(
source_root_stripped_source_and_source_root(filename)
for filename in sorted(init_injected.snapshot.files))
config_parser = configparser.ConfigParser()
config_parser.read_file(StringIO(DEFAULT_COVERAGE_CONFIG))
ensure_section(config_parser, "run")
config_parser.set("run", "plugins", COVERAGE_PLUGIN_MODULE_NAME)
config_parser.add_section(COVERAGE_PLUGIN_MODULE_NAME)
config_parser.set(COVERAGE_PLUGIN_MODULE_NAME, "source_to_target_base",
json.dumps(source_to_target_base))
config_parser.set(COVERAGE_PLUGIN_MODULE_NAME, "test_time",
json.dumps(coverage_config_request.test_time))
config = StringIO()
config_parser.write(config)
coveragerc_digest = await Get[Digest](InputFilesContent,
get_coveragerc_input(
config.getvalue()))
return Coveragerc(coveragerc_digest)
async def all_roots(source_root_config: SourceRootConfig) -> AllSourceRoots:
source_root_pattern_matcher = source_root_config.get_pattern_matcher()
# Create globs corresponding to all source root patterns.
all_paths: Set[str] = set()
for path in source_root_pattern_matcher.get_patterns():
if path == "/":
all_paths.add("**")
elif path.startswith("/"):
all_paths.add(f"{path[1:]}/")
else:
all_paths.add(f"**/{path}/")
# Match the patterns against actual files, to find the roots that actually exist.
snapshot = await Get[Snapshot](PathGlobs(globs=sorted(all_paths)))
responses = await MultiGet(
Get[OptionalSourceRoot](SourceRootRequest(PurePath(d))) for d in snapshot.dirs
)
all_source_roots = {
response.source_root for response in responses if response.source_root is not None
}
return AllSourceRoots(all_source_roots)
async def list_backends(
backend_options: BackendsOptions,
source_roots_config: SourceRootConfig,
global_options: GlobalOptions,
console: Console,
) -> Backends:
source_roots = source_roots_config.get_source_roots()
discovered_register_pys = await Get[Snapshot](PathGlobs(
["**/*/register.py"]))
register_pys_content = await Get[FilesContent](
Digest, discovered_register_pys.digest)
backend_infos = tuple(
BackendInfo.create(fc, source_roots, global_options)
for fc in register_pys_content)
v1_backends = []
v2_backends = []
for backend in backend_infos:
if backend.is_v1:
v1_backends.append(backend)
if backend.is_v2:
v2_backends.append(backend)
with backend_options.line_oriented(console) as print_stdout:
if global_options.options.v1:
print_stdout(
format_section(v1_backends,
console,
version_number=1,
option_name="backend_packages"))
if global_options.options.v2:
print_stdout(
format_section(v2_backends,
console,
version_number=2,
option_name="backend_packages2"))
return Backends(exit_code=0)
def test_all_roots(self):
dirs = (
"contrib/go/examples/src/go/src",
"src/java",
"src/python",
"src/python/subdir/src/python", # We allow source roots under source roots.
"src/kotlin",
"my/project/src/java",
"src/example/java",
"src/example/python",
"fixed/root/jvm",
)
options = {
"pants_ignore": [],
"root_patterns": [
"src/*",
"src/example/*",
"contrib/go/examples/src/go/src",
# Dir does not exist, should not be listed as a root.
"java",
"fixed/root/jvm",
],
}
options.update(self.options[""]) # We need inherited values for pants_workdir etc.
self.context(
for_subsystems=[SourceRootConfig], options={SourceRootConfig.options_scope: options}
)
source_root_config = SourceRootConfig.global_instance()
# This function mocks out reading real directories off the file system.
def provider_rule(path_globs: PathGlobs) -> Snapshot:
return Snapshot(Digest("abcdef", 10), (), dirs)
def source_root_mock_rule(req: SourceRootRequest) -> OptionalSourceRoot:
for d in dirs:
if str(req.path).startswith(d):
return OptionalSourceRoot(SourceRoot(str(req.path)))
return OptionalSourceRoot(None)
output = run_rule(
list_roots.all_roots,
rule_args=[source_root_config],
mock_gets=[
MockGet(product_type=Snapshot, subject_type=PathGlobs, mock=provider_rule),
MockGet(
product_type=OptionalSourceRoot,
subject_type=SourceRootRequest,
mock=source_root_mock_rule,
),
],
)
self.assertEqual(
{
SourceRoot("contrib/go/examples/src/go/src"),
SourceRoot("src/java"),
SourceRoot("src/python"),
SourceRoot("src/python/subdir/src/python"),
SourceRoot("src/kotlin"),
SourceRoot("src/example/java"),
SourceRoot("src/example/python"),
SourceRoot("my/project/src/java"),
SourceRoot("fixed/root/jvm"),
},
set(output),
)
def test_all_roots(self):
SOURCE = SourceRootCategories.SOURCE
TEST = SourceRootCategories.TEST
THIRDPARTY = SourceRootCategories.THIRDPARTY
options = {
'pants_ignore': [],
'source_root_patterns': ['src/*', 'src/example/*'],
'source_roots': {
# Fixed roots should trump patterns which would detect contrib/go/examples/src/go here.
'contrib/go/examples/src/go/src': ['go'],
# Dir does not exist, should not be listed as a root.
'java': ['java']
}
}
options.update(self.options['']
) # We need inherited values for pants_workdir etc.
self.context(for_subsystems=[SourceRootConfig],
options={SourceRootConfig.options_scope: options})
source_root_config = SourceRootConfig.global_instance()
source_roots = source_root_config.get_source_roots()
# Ensure that we see any manually added roots.
source_roots.add_source_root('fixed/root/jvm', ('java', 'scala'), TEST)
# This function mocks out reading real directories off the file system
def provider_rule(path_globs: PathGlobs) -> Snapshot:
dirs = (
'contrib/go/examples/3rdparty/go',
'contrib/go/examples/src/go/src',
'src/java',
'src/python',
'src/kotlin',
'my/project/src/java',
'src/example/java',
'src/example/python',
'fixed/root/jvm',
# subdirectories of source roots should not show up in final output
'src/kotlin/additional/directories/that/might/get/matched/src/foo',
)
return Snapshot(Digest('abcdef', 10), (), dirs)
output = run_rule(list_roots.all_roots, source_root_config,
{(Snapshot, PathGlobs): provider_rule})
self.assertEqual(
{
SourceRoot('contrib/go/examples/3rdparty/go',
('go', ), THIRDPARTY),
SourceRoot('contrib/go/examples/src/go/src', ('go', ), SOURCE),
SourceRoot('src/java', ('java', ), SOURCE),
SourceRoot('src/python', ('python', ), SOURCE),
SourceRoot('src/kotlin', ('kotlin', ), SOURCE),
SourceRoot('src/example/java', ('java', ), SOURCE),
SourceRoot('src/example/python', ('python', ), SOURCE),
SourceRoot('my/project/src/java', ('java', ), SOURCE),
SourceRoot('fixed/root/jvm', ('java', 'scala'), TEST)
}, set(output))
def test_all_roots(self):
SOURCE = SourceRootCategories.SOURCE
TEST = SourceRootCategories.TEST
THIRDPARTY = SourceRootCategories.THIRDPARTY
options = {
"pants_ignore": [],
"source_root_patterns": ["src/*", "src/example/*"],
"source_roots": {
# Fixed roots should trump patterns which would detect contrib/go/examples/src/go here.
"contrib/go/examples/src/go/src": ["go"],
# Dir does not exist, should not be listed as a root.
"java": ["java"],
},
}
options.update(self.options[""]
) # We need inherited values for pants_workdir etc.
self.context(for_subsystems=[SourceRootConfig],
options={SourceRootConfig.options_scope: options})
source_root_config = SourceRootConfig.global_instance()
source_roots = source_root_config.get_source_roots()
# Ensure that we see any manually added roots.
source_roots.add_source_root("fixed/root/jvm", ("java", "scala"), TEST)
# This function mocks out reading real directories off the file system
def provider_rule(path_globs: PathGlobs) -> Snapshot:
dirs = (
"contrib/go/examples/3rdparty/go",
"contrib/go/examples/src/go/src",
"src/java",
"src/python",
"src/kotlin",
"my/project/src/java",
"src/example/java",
"src/example/python",
"fixed/root/jvm",
# subdirectories of source roots should not show up in final output
"src/kotlin/additional/directories/that/might/get/matched/src/foo",
)
return Snapshot(Digest("abcdef", 10), (), dirs)
output = run_rule(
list_roots.all_roots,
rule_args=[source_root_config],
mock_gets=[
MockGet(product_type=Snapshot,
subject_type=PathGlobs,
mock=provider_rule)
],
)
self.assertEqual(
{
SourceRoot("contrib/go/examples/3rdparty/go",
("go", ), THIRDPARTY),
SourceRoot("contrib/go/examples/src/go/src", ("go", ), SOURCE),
SourceRoot("src/java", ("java", ), SOURCE),
SourceRoot("src/python", ("python", ), SOURCE),
SourceRoot("src/kotlin", ("kotlin", ), SOURCE),
SourceRoot("src/example/java", ("java", ), SOURCE),
SourceRoot("src/example/python", ("python", ), SOURCE),
SourceRoot("my/project/src/java", ("java", ), SOURCE),
SourceRoot("fixed/root/jvm", ("java", "scala"), TEST),
},
set(output),
)