async def map_protobuf_to_python_modules(
protobuf_targets: AllProtobufTargets,
python_setup: PythonSetup,
_: PythonProtobufMappingMarker,
) -> FirstPartyPythonMappingImpl:
stripped_file_per_target = await MultiGet(
Get(StrippedFileName,
StrippedFileNameRequest(tgt[ProtobufSourceField].file_path))
for tgt in protobuf_targets)
resolves_to_modules_to_providers: DefaultDict[ResolveName, DefaultDict[
str, list[ModuleProvider]]] = defaultdict(lambda: defaultdict(list))
for tgt, stripped_file in zip(protobuf_targets, stripped_file_per_target):
resolve = tgt[PythonResolveField].normalized_value(python_setup)
# NB: We don't consider the MyPy plugin, which generates `_pb2.pyi`. The stubs end up
# sharing the same module as the implementation `_pb2.py`. Because both generated files
# come from the same original Protobuf target, we're covered.
module = proto_path_to_py_module(stripped_file.value, suffix="_pb2")
resolves_to_modules_to_providers[resolve][module].append(
ModuleProvider(tgt.address, ModuleProviderType.IMPL))
if tgt.get(ProtobufGrpcToggleField).value:
module = proto_path_to_py_module(stripped_file.value,
suffix="_pb2_grpc")
resolves_to_modules_to_providers[resolve][module].append(
ModuleProvider(tgt.address, ModuleProviderType.IMPL))
return FirstPartyPythonMappingImpl.create(resolves_to_modules_to_providers)
async def map_to_python_modules(
vcs_version_targets: AllVCSVersionTargets,
python_setup: PythonSetup,
_: PythonVCSVersionMappingMarker,
) -> FirstPartyPythonMappingImpl:
suffix = ".py"
targets = [
tgt
for tgt in vcs_version_targets
if cast(str, tgt[VersionGenerateToField].value).endswith(suffix)
]
stripped_files = await MultiGet(
Get(StrippedFileName, StrippedFileNameRequest(cast(str, tgt[VersionGenerateToField].value)))
for tgt in targets
)
resolves_to_modules_to_providers: DefaultDict[
ResolveName, DefaultDict[str, list[ModuleProvider]]
] = defaultdict(lambda: defaultdict(list))
for tgt, stripped_file in zip(targets, stripped_files):
resolve = tgt[PythonResolveField].normalized_value(python_setup)
module = stripped_file.value[: -len(suffix)].replace("/", ".")
resolves_to_modules_to_providers[resolve][module].append(
ModuleProvider(tgt.address, ModuleProviderType.IMPL)
)
return FirstPartyPythonMappingImpl.create(resolves_to_modules_to_providers)
def test_issue_15111(rule_runner: RuleRunner) -> None:
"""Ensure we can handle when a single address implement multiple modules.
This is currently only possible with third-party targets.
"""
rule_runner.write_files({
"BUILD":
"python_requirement(name='req', requirements=['docopt', 'types-docopt'])"
})
rule_runner.set_options(["--python-enable-resolves"])
result = rule_runner.request(ThirdPartyPythonModuleMapping, [])
assert result == ThirdPartyPythonModuleMapping({
"python-default":
FrozenDict({
"docopt": (
ModuleProvider(Address("", target_name="req"),
ModuleProviderType.IMPL),
ModuleProvider(Address("", target_name="req"),
ModuleProviderType.TYPE_STUB),
),
})
})
async def map_thrift_to_python_modules(
thrift_targets: AllThriftTargets,
python_setup: PythonSetup,
_: PythonThriftMappingMarker,
) -> FirstPartyPythonMappingImpl:
parsed_files = await MultiGet(
Get(ParsedThrift, ParsedThriftRequest(tgt[ThriftSourceField]))
for tgt in thrift_targets)
resolves_to_modules_to_providers: DefaultDict[ResolveName, DefaultDict[
str, list[ModuleProvider]]] = defaultdict(lambda: defaultdict(list))
for tgt, parsed in zip(thrift_targets, parsed_files):
resolve = tgt[PythonResolveField].normalized_value(python_setup)
provider = ModuleProvider(tgt.address, ModuleProviderType.IMPL)
m1, m2 = thrift_path_to_py_modules(
source_path=tgt[ThriftSourceField].file_path,
namespace=parsed.namespaces.get("py"))
resolves_to_modules_to_providers[resolve][m1].append(provider)
resolves_to_modules_to_providers[resolve][m2].append(provider)
return FirstPartyPythonMappingImpl.create(resolves_to_modules_to_providers)
def test_first_party_modules_mapping() -> None:
root_provider = ModuleProvider(Address("", relative_file_path="root.py"),
ModuleProviderType.IMPL)
util_provider = ModuleProvider(
Address("src/python/util", relative_file_path="strutil.py"),
ModuleProviderType.IMPL)
util_stubs_provider = ModuleProvider(
Address("src/python/util", relative_file_path="strutil.pyi"),
ModuleProviderType.TYPE_STUB)
test_provider = ModuleProvider(
Address("tests/python/project_test", relative_file_path="test.py"),
ModuleProviderType.IMPL)
mapping = FirstPartyPythonModuleMapping(
FrozenDict({
"default":
FrozenDict({
"root": (root_provider, ),
"util.strutil": (util_provider, util_stubs_provider),
"project_test.test": (test_provider, ),
"ambiguous": (root_provider, util_provider),
"util.ambiguous": (util_provider, test_provider),
"two_resolves": (root_provider, ),
}),
"another":
FrozenDict({"two_resolves": (test_provider, )}),
}))
def assert_addresses(mod: str,
expected: tuple[ModuleProvider, ...],
*,
resolve: str | None = None) -> None:
assert mapping.providers_for_module(mod, resolve=resolve) == expected
assert_addresses("root", (root_provider, ))
assert_addresses("root.func", (root_provider, ))
assert_addresses("root.submodule.func", ())
assert_addresses("util.strutil", (util_provider, util_stubs_provider))
assert_addresses("util.strutil.ensure_text",
(util_provider, util_stubs_provider))
assert_addresses("util", ())
assert_addresses("project_test.test", (test_provider, ))
assert_addresses("project_test.test.TestDemo", (test_provider, ))
assert_addresses("project_test", ())
assert_addresses("project.test", ())
assert_addresses("ambiguous", (root_provider, util_provider))
assert_addresses("ambiguous.func", (root_provider, util_provider))
assert_addresses("ambiguous.submodule.func", ())
assert_addresses("util.ambiguous", (util_provider, test_provider))
assert_addresses("util.ambiguous.Foo", (util_provider, test_provider))
assert_addresses("util.ambiguous.Foo.method", ())
assert_addresses("two_resolves", (root_provider, test_provider),
resolve=None)
assert_addresses("two_resolves.foo", (root_provider, test_provider),
resolve=None)
assert_addresses("two_resolves.foo.bar", (), resolve=None)
assert_addresses("two_resolves", (root_provider, ), resolve="default")
assert_addresses("two_resolves", (test_provider, ), resolve="another")
def test_map_third_party_modules_to_addresses(rule_runner: RuleRunner) -> None:
def req(
tgt_name: str,
req_str: str,
*,
modules: list[str] | None = None,
stub_modules: list[str] | None = None,
resolve: str = "default",
) -> str:
return dedent(f"""\
python_requirement(name='{tgt_name}', requirements=['{req_str}'],
modules={modules or []},
type_stub_modules={stub_modules or []},
resolve={repr(resolve)})
""")
build_file = "\n\n".join([
req("req1", "req1==1.2"),
req("un_normalized", "Un-Normalized-Project>3"),
req("file_dist", "file_dist@ file:///path/to/dist.whl"),
req("vcs_dist", "vcs_dist@ git+https://github.com/vcs/dist.git"),
req("modules", "foo==1", modules=["mapped_module"]),
# We extract the module from type stub dependencies.
req("typed-dep1", "typed-dep1-types"),
req("typed-dep2", "types-typed-dep2"),
req("typed-dep3", "typed-dep3-stubs"),
req("typed-dep4", "stubs-typed-dep4"),
req("typed-dep5", "typed-dep5-foo", stub_modules=["typed_dep5"]),
# A 3rd-party dependency can have both a type stub and implementation.
req("multiple_owners1", "multiple_owners==1"),
req("multiple_owners2", "multiple_owners==2", resolve="another"),
req("multiple_owners_types",
"types-multiple_owners==1",
resolve="another"),
# Only assume it's a type stubs dep if we are certain it's not an implementation.
req("looks_like_stubs",
"looks-like-stubs-types",
modules=["looks_like_stubs"]),
])
rule_runner.write_files({"BUILD": build_file})
rule_runner.set_options([
"--python-resolves={'default': '', 'another': ''}",
"--python-enable-resolves"
])
result = rule_runner.request(ThirdPartyPythonModuleMapping, [])
assert result == ThirdPartyPythonModuleMapping({
"another":
FrozenDict({
"multiple_owners": (
ModuleProvider(Address("", target_name="multiple_owners2"),
ModuleProviderType.IMPL),
ModuleProvider(
Address("", target_name="multiple_owners_types"),
ModuleProviderType.TYPE_STUB,
),
),
}),
"default":
FrozenDict({
"file_dist": (ModuleProvider(Address("", target_name="file_dist"),
ModuleProviderType.IMPL), ),
"looks_like_stubs":
(ModuleProvider(Address("", target_name="looks_like_stubs"),
ModuleProviderType.IMPL), ),
"mapped_module":
(ModuleProvider(Address("", target_name="modules"),
ModuleProviderType.IMPL), ),
"multiple_owners":
(ModuleProvider(Address("", target_name="multiple_owners1"),
ModuleProviderType.IMPL), ),
"req1": (ModuleProvider(Address("", target_name="req1"),
ModuleProviderType.IMPL), ),
"typed_dep1":
(ModuleProvider(Address("", target_name="typed-dep1"),
ModuleProviderType.TYPE_STUB), ),
"typed_dep2":
(ModuleProvider(Address("", target_name="typed-dep2"),
ModuleProviderType.TYPE_STUB), ),
"typed_dep3":
(ModuleProvider(Address("", target_name="typed-dep3"),
ModuleProviderType.TYPE_STUB), ),
"typed_dep4":
(ModuleProvider(Address("", target_name="typed-dep4"),
ModuleProviderType.TYPE_STUB), ),
"typed_dep5":
(ModuleProvider(Address("", target_name="typed-dep5"),
ModuleProviderType.TYPE_STUB), ),
"un_normalized_project":
(ModuleProvider(Address("", target_name="un_normalized"),
ModuleProviderType.IMPL), ),
"vcs_dist": (ModuleProvider(Address("", target_name="vcs_dist"),
ModuleProviderType.IMPL), ),
}),
})
def test_map_first_party_modules_to_addresses(rule_runner: RuleRunner) -> None:
rule_runner.set_options([
"--source-root-patterns=['src/python', 'tests/python', 'build-support']",
"--python-enable-resolves",
"--python-resolves={'python-default': '', 'another-resolve': ''}",
])
rule_runner.write_files({
"src/python/project/util/dirutil.py":
"",
"src/python/project/util/tarutil.py":
"",
"src/python/project/util/BUILD":
"python_sources(resolve='another-resolve')",
# A module with multiple owners, including type stubs.
"src/python/multiple_owners.py":
"",
"src/python/multiple_owners.pyi":
"",
"src/python/BUILD":
"python_sources()",
"build-support/multiple_owners.py":
"",
"build-support/BUILD":
"python_sources()",
# A package module.
"tests/python/project_test/demo_test/__init__.py":
"",
"tests/python/project_test/demo_test/BUILD":
"python_sources()",
# Check that plugin mappings work. Note that we duplicate one of the files with a normal
# python_source.
"src/python/protos/f1.proto":
"",
"src/python/protos/f2.proto":
"",
"src/python/protos/f2_pb2.py":
"",
"src/python/protos/BUILD":
dedent("""\
protobuf_sources(name='protos')
python_source(name='py', source="f2_pb2.py")
"""),
})
result = rule_runner.request(FirstPartyPythonModuleMapping, [])
assert result == FirstPartyPythonModuleMapping(
FrozenDict({
"another-resolve":
FrozenDict({
"project.util.dirutil": (ModuleProvider(
Address("src/python/project/util",
relative_file_path="dirutil.py"),
ModuleProviderType.IMPL,
), ),
"project.util.tarutil": (ModuleProvider(
Address("src/python/project/util",
relative_file_path="tarutil.py"),
ModuleProviderType.IMPL,
), ),
}),
"python-default":
FrozenDict({
"multiple_owners": (
ModuleProvider(
Address("build-support",
relative_file_path="multiple_owners.py"),
ModuleProviderType.IMPL,
),
ModuleProvider(
Address("src/python",
relative_file_path="multiple_owners.py"),
ModuleProviderType.IMPL,
),
ModuleProvider(
Address("src/python",
relative_file_path="multiple_owners.pyi"),
ModuleProviderType.TYPE_STUB,
),
),
"project_test.demo_test": (ModuleProvider(
Address(
"tests/python/project_test/demo_test",
relative_file_path="__init__.py",
),
ModuleProviderType.IMPL,
), ),
"protos.f1_pb2": (ModuleProvider(
Address(
"src/python/protos",
relative_file_path="f1.proto",
target_name="protos",
),
ModuleProviderType.IMPL,
), ),
"protos.f2_pb2": (
ModuleProvider(
Address("src/python/protos", target_name="py"),
ModuleProviderType.IMPL,
),
ModuleProvider(
Address(
"src/python/protos",
relative_file_path="f2.proto",
target_name="protos",
),
ModuleProviderType.IMPL,
),
),
}),
}))
def test_third_party_modules_mapping() -> None:
colors_provider = ModuleProvider(Address("", target_name="ansicolors"),
ModuleProviderType.IMPL)
colors_stubs_provider = ModuleProvider(
Address("", target_name="types-ansicolors"),
ModuleProviderType.TYPE_STUB)
pants_provider = ModuleProvider(Address("", target_name="pantsbuild"),
ModuleProviderType.IMPL)
pants_testutil_provider = ModuleProvider(
Address("", target_name="pantsbuild.testutil"),
ModuleProviderType.IMPL)
submodule_provider = ModuleProvider(Address("", target_name="submodule"),
ModuleProviderType.IMPL)
mapping = ThirdPartyPythonModuleMapping({
"default-resolve":
FrozenDict({
"colors": (colors_provider, colors_stubs_provider),
"pants": (pants_provider, ),
"req.submodule": (submodule_provider, ),
"pants.testutil": (pants_testutil_provider, ),
"two_resolves": (colors_provider, ),
}),
"another-resolve":
FrozenDict({"two_resolves": (pants_provider, )}),
})
def assert_addresses(mod: str,
expected: tuple[ModuleProvider, ...],
*,
resolve: str | None = None) -> None:
assert mapping.providers_for_module(mod, resolve) == expected
assert_addresses("colors", (colors_provider, colors_stubs_provider))
assert_addresses("colors.red", (colors_provider, colors_stubs_provider))
assert_addresses("pants", (pants_provider, ))
assert_addresses("pants.task", (pants_provider, ))
assert_addresses("pants.task.task", (pants_provider, ))
assert_addresses("pants.task.task.Task", (pants_provider, ))
assert_addresses("pants.testutil", (pants_testutil_provider, ))
assert_addresses("pants.testutil.foo", (pants_testutil_provider, ))
assert_addresses("req.submodule", (submodule_provider, ))
assert_addresses("req.submodule.foo", (submodule_provider, ))
assert_addresses("req.another", ())
assert_addresses("req", ())
assert_addresses("unknown", ())
assert_addresses("unknown.pants", ())
assert_addresses("two_resolves", (colors_provider, pants_provider),
resolve=None)
assert_addresses("two_resolves.foo", (colors_provider, pants_provider),
resolve=None)
assert_addresses("two_resolves.foo.bar", (colors_provider, pants_provider),
resolve=None)
assert_addresses("two_resolves", (colors_provider, ),
resolve="default-resolve")
assert_addresses("two_resolves", (pants_provider, ),
resolve="another-resolve")
def providers(addresses: list[Address]) -> tuple[ModuleProvider, ...]:
return tuple(
ModuleProvider(addr, ModuleProviderType.IMPL)
for addr in addresses)