def test_third_party_modules_mapping() -> None:
colors_addr = Address("", target_name="ansicolors")
pants_addr = Address("", target_name="pantsbuild")
mapping = ThirdPartyPythonModuleMapping({"colors": colors_addr, "pants": pants_addr})
assert mapping.address_for_module("colors") == colors_addr
assert mapping.address_for_module("colors.red") == colors_addr
assert mapping.address_for_module("pants") == pants_addr
assert mapping.address_for_module("pants.task") == pants_addr
assert mapping.address_for_module("pants.task.task") == pants_addr
assert mapping.address_for_module("pants.task.task.Task") == pants_addr
def test_third_party_modules_mapping() -> None:
colors_addr = Address("", target_name="ansicolors")
colors_stubs_addr = Address("", target_name="types-ansicolors")
pants_addr = Address("", target_name="pantsbuild")
pants_testutil_addr = Address("", target_name="pantsbuild.testutil")
submodule_addr = Address("", target_name="submodule")
mapping = ThirdPartyPythonModuleMapping(
mapping=FrozenDict(
{
"colors": (colors_addr, colors_stubs_addr),
"pants": (pants_addr,),
"req.submodule": (submodule_addr,),
"pants.testutil": (pants_testutil_addr,),
}
),
ambiguous_modules=FrozenDict({"ambiguous": (colors_addr, pants_addr)}),
)
def assert_addresses(
mod: str, expected: tuple[tuple[Address, ...], tuple[Address, ...]]
) -> None:
assert mapping.addresses_for_module(mod) == expected
unknown = ((), ())
colors = ((colors_addr, colors_stubs_addr), ())
assert_addresses("colors", colors)
assert_addresses("colors.red", colors)
pants = ((pants_addr,), ())
assert_addresses("pants", pants)
assert_addresses("pants.task", pants)
assert_addresses("pants.task.task", pants)
assert_addresses("pants.task.task.Task", pants)
testutil = ((pants_testutil_addr,), ())
assert_addresses("pants.testutil", testutil)
assert_addresses("pants.testutil.foo", testutil)
submodule = ((submodule_addr,), ())
assert_addresses("req.submodule", submodule)
assert_addresses("req.submodule.foo", submodule)
assert_addresses("req.another", unknown)
assert_addresses("req", unknown)
assert_addresses("unknown", unknown)
assert_addresses("unknown.pants", unknown)
ambiguous = ((), (colors_addr, pants_addr))
assert_addresses("ambiguous", ambiguous)
assert_addresses("ambiguous.foo", ambiguous)
assert_addresses("ambiguous.foo.bar", ambiguous)
def test_map_third_party_modules_to_addresses(rule_runner: RuleRunner) -> None:
rule_runner.add_to_build_file(
"3rdparty/python",
dedent(
"""\
python_requirement_library(
name='ansicolors',
requirements=['ansicolors==1.21'],
module_mapping={'ansicolors': ['colors']},
)
python_requirement_library(
name='req1',
requirements=['req1', 'two_owners'],
)
python_requirement_library(
name='un_normalized',
requirements=['Un-Normalized-Project>3', 'two_owners'],
)
python_requirement_library(
name='direct_references',
requirements=[
'pip@ git+https://github.com/pypa/pip.git', 'local_dist@ file:///path/to/dist.whl',
],
)
"""
),
)
result = rule_runner.request(ThirdPartyPythonModuleMapping, [])
assert result == ThirdPartyPythonModuleMapping(
mapping=FrozenDict(
{
"colors": Address("3rdparty/python", target_name="ansicolors"),
"local_dist": Address("3rdparty/python", target_name="direct_references"),
"pip": Address("3rdparty/python", target_name="direct_references"),
"req1": Address("3rdparty/python", target_name="req1"),
"un_normalized_project": Address("3rdparty/python", target_name="un_normalized"),
}
),
ambiguous_modules=FrozenDict(
{
"two_owners": (
Address("3rdparty/python", target_name="req1"),
Address("3rdparty/python", target_name="un_normalized"),
),
}
),
)
def test_issue_15111(rule_runner: RuleRunner) -> None:
"""Ensure we can handle when a single address implement multiple modules.
This is currently only possible with third-party targets.
"""
rule_runner.write_files({
"BUILD":
"python_requirement(name='req', requirements=['docopt', 'types-docopt'])"
})
rule_runner.set_options(["--python-enable-resolves"])
result = rule_runner.request(ThirdPartyPythonModuleMapping, [])
assert result == ThirdPartyPythonModuleMapping({
"python-default":
FrozenDict({
"docopt": (
ModuleProvider(Address("", target_name="req"),
ModuleProviderType.IMPL),
ModuleProvider(Address("", target_name="req"),
ModuleProviderType.TYPE_STUB),
),
})
})
def test_map_third_party_modules_to_addresses(rule_runner: RuleRunner) -> None:
def req(
tgt_name: str,
req_str: str,
*,
modules: list[str] | None = None,
stub_modules: list[str] | None = None,
) -> str:
return (
f"python_requirement(name='{tgt_name}', requirements=['{req_str}'], "
f"modules={modules or []},"
f"type_stub_modules={stub_modules or []})")
build_file = "\n\n".join([
req("req1", "req1==1.2"),
req("un_normalized", "Un-Normalized-Project>3"),
req("file_dist", "file_dist@ file:///path/to/dist.whl"),
req("vcs_dist", "vcs_dist@ git+https://github.com/vcs/dist.git"),
req("modules", "foo==1", modules=["mapped_module"]),
# We extract the module from type stub dependencies.
req("typed-dep1", "typed-dep1-types"),
req("typed-dep2", "types-typed-dep2"),
req("typed-dep3", "typed-dep3-stubs"),
req("typed-dep4", "stubs-typed-dep4"),
req("typed-dep5", "typed-dep5-foo", stub_modules=["typed_dep5"]),
# A 3rd-party dependency can have both a type stub and implementation.
req("req2", "req2==1"),
req("req2_types", "types-req2==1"),
req("req3", "req3==1"),
req("req3_types", "req3-types==1"),
req("req4", "req4==1"),
req("req4_types", "req4-stubs==1", stub_modules=["req4"]),
# Ambiguous.
req("ambiguous_t1", "ambiguous==1.2"),
req("ambiguous_t2", "ambiguous==1.3"),
req("ambiguous_stubs_t1", "ambiguous-stubs-types==1.3"),
req("ambiguous_stubs_t2", "types-ambiguous-stubs==1.3"),
# If there's ambiguity within type stubs or within implementations, then there should
# be ambiguity with the other category too.
req("ambiguous_again_t1", "ambiguous-again==1.2"),
req("ambiguous_again_t2", "ambiguous-again==1.3"),
req("ambiguous_again_t3", "ambiguous-again-types==1.3"),
req("ambiguous_again_stubby_t1", "ambiguous-again-stubby-types==1.2"),
req("ambiguous_again_stubby_t2", "types-ambiguous-again-stubby==1.3"),
req("ambiguous_again_stubby_t3", "ambiguous-again-stubby==1.3"),
# Only assume it's a type stubs dep if we are certain it's not an implementation.
req("looks_like_stubs",
"looks-like-stubs-types",
modules=["looks_like_stubs"]),
])
rule_runner.write_files({"BUILD": build_file})
result = rule_runner.request(ThirdPartyPythonModuleMapping, [])
assert result == ThirdPartyPythonModuleMapping(
mapping=FrozenDict({
"file_dist": (Address("", target_name="file_dist"), ),
"looks_like_stubs": (Address("",
target_name="looks_like_stubs"), ),
"mapped_module": (Address("", target_name="modules"), ),
"req1": (Address("", target_name="req1"), ),
"req2": (Address("", target_name="req2"),
Address("", target_name="req2_types")),
"req3": (Address("", target_name="req3"),
Address("", target_name="req3_types")),
"req4": (Address("", target_name="req4"),
Address("", target_name="req4_types")),
"typed_dep1": (Address("", target_name="typed-dep1"), ),
"typed_dep2": (Address("", target_name="typed-dep2"), ),
"typed_dep3": (Address("", target_name="typed-dep3"), ),
"typed_dep4": (Address("", target_name="typed-dep4"), ),
"typed_dep5": (Address("", target_name="typed-dep5"), ),
"un_normalized_project": (Address("",
target_name="un_normalized"), ),
"vcs_dist": (Address("", target_name="vcs_dist"), ),
}),
ambiguous_modules=FrozenDict({
"ambiguous": (
Address("", target_name="ambiguous_t1"),
Address("", target_name="ambiguous_t2"),
),
"ambiguous_again": (
Address("", target_name="ambiguous_again_t1"),
Address("", target_name="ambiguous_again_t2"),
Address("", target_name="ambiguous_again_t3"),
),
"ambiguous_again_stubby": (
Address("", target_name="ambiguous_again_stubby_t1"),
Address("", target_name="ambiguous_again_stubby_t2"),
Address("", target_name="ambiguous_again_stubby_t3"),
),
"ambiguous_stubs": (
Address("", target_name="ambiguous_stubs_t1"),
Address("", target_name="ambiguous_stubs_t2"),
),
}),
)
def test_map_third_party_modules_to_addresses(rule_runner: RuleRunner) -> None:
def req(
tgt_name: str,
req_str: str,
*,
modules: list[str] | None = None,
stub_modules: list[str] | None = None,
resolve: str = "default",
) -> str:
return dedent(f"""\
python_requirement(name='{tgt_name}', requirements=['{req_str}'],
modules={modules or []},
type_stub_modules={stub_modules or []},
resolve={repr(resolve)})
""")
build_file = "\n\n".join([
req("req1", "req1==1.2"),
req("un_normalized", "Un-Normalized-Project>3"),
req("file_dist", "file_dist@ file:///path/to/dist.whl"),
req("vcs_dist", "vcs_dist@ git+https://github.com/vcs/dist.git"),
req("modules", "foo==1", modules=["mapped_module"]),
# We extract the module from type stub dependencies.
req("typed-dep1", "typed-dep1-types"),
req("typed-dep2", "types-typed-dep2"),
req("typed-dep3", "typed-dep3-stubs"),
req("typed-dep4", "stubs-typed-dep4"),
req("typed-dep5", "typed-dep5-foo", stub_modules=["typed_dep5"]),
# A 3rd-party dependency can have both a type stub and implementation.
req("multiple_owners1", "multiple_owners==1"),
req("multiple_owners2", "multiple_owners==2", resolve="another"),
req("multiple_owners_types",
"types-multiple_owners==1",
resolve="another"),
# Only assume it's a type stubs dep if we are certain it's not an implementation.
req("looks_like_stubs",
"looks-like-stubs-types",
modules=["looks_like_stubs"]),
])
rule_runner.write_files({"BUILD": build_file})
rule_runner.set_options([
"--python-resolves={'default': '', 'another': ''}",
"--python-enable-resolves"
])
result = rule_runner.request(ThirdPartyPythonModuleMapping, [])
assert result == ThirdPartyPythonModuleMapping({
"another":
FrozenDict({
"multiple_owners": (
ModuleProvider(Address("", target_name="multiple_owners2"),
ModuleProviderType.IMPL),
ModuleProvider(
Address("", target_name="multiple_owners_types"),
ModuleProviderType.TYPE_STUB,
),
),
}),
"default":
FrozenDict({
"file_dist": (ModuleProvider(Address("", target_name="file_dist"),
ModuleProviderType.IMPL), ),
"looks_like_stubs":
(ModuleProvider(Address("", target_name="looks_like_stubs"),
ModuleProviderType.IMPL), ),
"mapped_module":
(ModuleProvider(Address("", target_name="modules"),
ModuleProviderType.IMPL), ),
"multiple_owners":
(ModuleProvider(Address("", target_name="multiple_owners1"),
ModuleProviderType.IMPL), ),
"req1": (ModuleProvider(Address("", target_name="req1"),
ModuleProviderType.IMPL), ),
"typed_dep1":
(ModuleProvider(Address("", target_name="typed-dep1"),
ModuleProviderType.TYPE_STUB), ),
"typed_dep2":
(ModuleProvider(Address("", target_name="typed-dep2"),
ModuleProviderType.TYPE_STUB), ),
"typed_dep3":
(ModuleProvider(Address("", target_name="typed-dep3"),
ModuleProviderType.TYPE_STUB), ),
"typed_dep4":
(ModuleProvider(Address("", target_name="typed-dep4"),
ModuleProviderType.TYPE_STUB), ),
"typed_dep5":
(ModuleProvider(Address("", target_name="typed-dep5"),
ModuleProviderType.TYPE_STUB), ),
"un_normalized_project":
(ModuleProvider(Address("", target_name="un_normalized"),
ModuleProviderType.IMPL), ),
"vcs_dist": (ModuleProvider(Address("", target_name="vcs_dist"),
ModuleProviderType.IMPL), ),
}),
})
def test_third_party_modules_mapping() -> None:
colors_provider = ModuleProvider(Address("", target_name="ansicolors"),
ModuleProviderType.IMPL)
colors_stubs_provider = ModuleProvider(
Address("", target_name="types-ansicolors"),
ModuleProviderType.TYPE_STUB)
pants_provider = ModuleProvider(Address("", target_name="pantsbuild"),
ModuleProviderType.IMPL)
pants_testutil_provider = ModuleProvider(
Address("", target_name="pantsbuild.testutil"),
ModuleProviderType.IMPL)
submodule_provider = ModuleProvider(Address("", target_name="submodule"),
ModuleProviderType.IMPL)
mapping = ThirdPartyPythonModuleMapping({
"default-resolve":
FrozenDict({
"colors": (colors_provider, colors_stubs_provider),
"pants": (pants_provider, ),
"req.submodule": (submodule_provider, ),
"pants.testutil": (pants_testutil_provider, ),
"two_resolves": (colors_provider, ),
}),
"another-resolve":
FrozenDict({"two_resolves": (pants_provider, )}),
})
def assert_addresses(mod: str,
expected: tuple[ModuleProvider, ...],
*,
resolve: str | None = None) -> None:
assert mapping.providers_for_module(mod, resolve) == expected
assert_addresses("colors", (colors_provider, colors_stubs_provider))
assert_addresses("colors.red", (colors_provider, colors_stubs_provider))
assert_addresses("pants", (pants_provider, ))
assert_addresses("pants.task", (pants_provider, ))
assert_addresses("pants.task.task", (pants_provider, ))
assert_addresses("pants.task.task.Task", (pants_provider, ))
assert_addresses("pants.testutil", (pants_testutil_provider, ))
assert_addresses("pants.testutil.foo", (pants_testutil_provider, ))
assert_addresses("req.submodule", (submodule_provider, ))
assert_addresses("req.submodule.foo", (submodule_provider, ))
assert_addresses("req.another", ())
assert_addresses("req", ())
assert_addresses("unknown", ())
assert_addresses("unknown.pants", ())
assert_addresses("two_resolves", (colors_provider, pants_provider),
resolve=None)
assert_addresses("two_resolves.foo", (colors_provider, pants_provider),
resolve=None)
assert_addresses("two_resolves.foo.bar", (colors_provider, pants_provider),
resolve=None)
assert_addresses("two_resolves", (colors_provider, ),
resolve="default-resolve")
assert_addresses("two_resolves", (pants_provider, ),
resolve="another-resolve")
def test_third_party_modules_mapping() -> None:
colors_addr = Address("", target_name="ansicolors")
pants_addr = Address("", target_name="pantsbuild")
submodule_addr = Address("", target_name="submodule")
mapping = ThirdPartyPythonModuleMapping(
mapping=FrozenDict(
{"colors": colors_addr, "pants": pants_addr, "req.submodule": submodule_addr}
),
ambiguous_modules=FrozenDict({"ambiguous": (colors_addr, pants_addr)}),
)
assert mapping.address_for_module("colors") == (colors_addr, ())
assert mapping.address_for_module("colors.red") == (colors_addr, ())
assert mapping.address_for_module("pants") == (pants_addr, ())
assert mapping.address_for_module("pants.task") == (pants_addr, ())
assert mapping.address_for_module("pants.task.task") == (pants_addr, ())
assert mapping.address_for_module("pants.task.task.Task") == (pants_addr, ())
assert mapping.address_for_module("req.submodule") == (submodule_addr, ())
assert mapping.address_for_module("req.submodule.foo") == (submodule_addr, ())
assert mapping.address_for_module("req.another") == (None, ())
assert mapping.address_for_module("req") == (None, ())
assert mapping.address_for_module("unknown") == (None, ())
assert mapping.address_for_module("unknown.pants") == (None, ())
assert mapping.address_for_module("ambiguous") == (None, (colors_addr, pants_addr))
assert mapping.address_for_module("ambiguous.foo") == (None, (colors_addr, pants_addr))
assert mapping.address_for_module("ambiguous.foo.bar") == (None, (colors_addr, pants_addr))
def find_python_runtime_library_or_raise_error(
module_mapping: ThirdPartyPythonModuleMapping,
codegen_address: Address,
runtime_library_module: str,
*,
resolve: str,
resolves_enabled: bool,
recommended_requirement_name: str,
recommended_requirement_url: str,
disable_inference_option: str,
) -> Address:
addresses = [
module_provider.addr
for module_provider in module_mapping.providers_for_module(
runtime_library_module, resolve=resolve
)
if module_provider.typ == ModuleProviderType.IMPL
]
if len(addresses) == 1:
return addresses[0]
for_resolve_str = f" for the resolve '{resolve}'" if resolves_enabled else ""
if not addresses:
resolve_note = softwrap(
(
f"""
Note that because `[python].enable_resolves` is set, you must specifically have a
`python_requirement` target that uses the same resolve '{resolve}' as the target
{codegen_address}. Alternatively, update {codegen_address} to use a different
resolve.
"""
)
if resolves_enabled
else ""
)
raise MissingPythonCodegenRuntimeLibrary(
softwrap(
f"""
No `python_requirement` target was found with the module `{runtime_library_module}`
in your project{for_resolve_str}, so the Python code generated from the target
{codegen_address} will not work properly. See
{doc_url('python-third-party-dependencies')} for how to add a requirement, such as
adding to requirements.txt. Usually you will want to use the
`{recommended_requirement_name}` project at {recommended_requirement_url}.
{resolve_note}
To ignore this error, set `{disable_inference_option} = false` in `pants.toml`.
"""
)
)
alternative_solution = softwrap(
(
f"""
Alternatively, change the resolve field for {codegen_address} to use a different resolve
from `[python].resolves`.
"""
)
if resolves_enabled
else (
f"""
Alternatively, if you do want to have multiple conflicting versions of the
`{runtime_library_module}` requirement, set `{disable_inference_option} = false` in
`pants.toml`. Then manually add a dependency on the relevant `python_requirement` target
to each target that directly depends on this generated code (e.g. `python_source` targets).
"""
)
)
raise AmbiguousPythonCodegenRuntimeLibrary(
softwrap(
f"""
Multiple `python_requirement` targets were found with the module `{runtime_library_module}`
in your project{for_resolve_str}, so it is ambiguous which to use for the runtime library
for the Python code generated from the the target {codegen_address}:
{sorted(addr.spec for addr in addresses)}
To fix, remove one of these `python_requirement` targets{for_resolve_str} so that
there is no ambiguity and Pants can infer a dependency. {alternative_solution}
"""
)
)