def test_third_party_modules_mapping() -> None: colors_addr = Address("", target_name="ansicolors") pants_addr = Address("", target_name="pantsbuild") mapping = ThirdPartyPythonModuleMapping({"colors": colors_addr, "pants": pants_addr}) assert mapping.address_for_module("colors") == colors_addr assert mapping.address_for_module("colors.red") == colors_addr assert mapping.address_for_module("pants") == pants_addr assert mapping.address_for_module("pants.task") == pants_addr assert mapping.address_for_module("pants.task.task") == pants_addr assert mapping.address_for_module("pants.task.task.Task") == pants_addr
def test_third_party_modules_mapping() -> None: colors_addr = Address("", target_name="ansicolors") colors_stubs_addr = Address("", target_name="types-ansicolors") pants_addr = Address("", target_name="pantsbuild") pants_testutil_addr = Address("", target_name="pantsbuild.testutil") submodule_addr = Address("", target_name="submodule") mapping = ThirdPartyPythonModuleMapping( mapping=FrozenDict( { "colors": (colors_addr, colors_stubs_addr), "pants": (pants_addr,), "req.submodule": (submodule_addr,), "pants.testutil": (pants_testutil_addr,), } ), ambiguous_modules=FrozenDict({"ambiguous": (colors_addr, pants_addr)}), ) def assert_addresses( mod: str, expected: tuple[tuple[Address, ...], tuple[Address, ...]] ) -> None: assert mapping.addresses_for_module(mod) == expected unknown = ((), ()) colors = ((colors_addr, colors_stubs_addr), ()) assert_addresses("colors", colors) assert_addresses("colors.red", colors) pants = ((pants_addr,), ()) assert_addresses("pants", pants) assert_addresses("pants.task", pants) assert_addresses("pants.task.task", pants) assert_addresses("pants.task.task.Task", pants) testutil = ((pants_testutil_addr,), ()) assert_addresses("pants.testutil", testutil) assert_addresses("pants.testutil.foo", testutil) submodule = ((submodule_addr,), ()) assert_addresses("req.submodule", submodule) assert_addresses("req.submodule.foo", submodule) assert_addresses("req.another", unknown) assert_addresses("req", unknown) assert_addresses("unknown", unknown) assert_addresses("unknown.pants", unknown) ambiguous = ((), (colors_addr, pants_addr)) assert_addresses("ambiguous", ambiguous) assert_addresses("ambiguous.foo", ambiguous) assert_addresses("ambiguous.foo.bar", ambiguous)
def test_map_third_party_modules_to_addresses(rule_runner: RuleRunner) -> None: rule_runner.add_to_build_file( "3rdparty/python", dedent( """\ python_requirement_library( name='ansicolors', requirements=['ansicolors==1.21'], module_mapping={'ansicolors': ['colors']}, ) python_requirement_library( name='req1', requirements=['req1', 'two_owners'], ) python_requirement_library( name='un_normalized', requirements=['Un-Normalized-Project>3', 'two_owners'], ) python_requirement_library( name='direct_references', requirements=[ 'pip@ git+https://github.com/pypa/pip.git', 'local_dist@ file:///path/to/dist.whl', ], ) """ ), ) result = rule_runner.request(ThirdPartyPythonModuleMapping, []) assert result == ThirdPartyPythonModuleMapping( mapping=FrozenDict( { "colors": Address("3rdparty/python", target_name="ansicolors"), "local_dist": Address("3rdparty/python", target_name="direct_references"), "pip": Address("3rdparty/python", target_name="direct_references"), "req1": Address("3rdparty/python", target_name="req1"), "un_normalized_project": Address("3rdparty/python", target_name="un_normalized"), } ), ambiguous_modules=FrozenDict( { "two_owners": ( Address("3rdparty/python", target_name="req1"), Address("3rdparty/python", target_name="un_normalized"), ), } ), )
def test_issue_15111(rule_runner: RuleRunner) -> None: """Ensure we can handle when a single address implement multiple modules. This is currently only possible with third-party targets. """ rule_runner.write_files({ "BUILD": "python_requirement(name='req', requirements=['docopt', 'types-docopt'])" }) rule_runner.set_options(["--python-enable-resolves"]) result = rule_runner.request(ThirdPartyPythonModuleMapping, []) assert result == ThirdPartyPythonModuleMapping({ "python-default": FrozenDict({ "docopt": ( ModuleProvider(Address("", target_name="req"), ModuleProviderType.IMPL), ModuleProvider(Address("", target_name="req"), ModuleProviderType.TYPE_STUB), ), }) })
def test_map_third_party_modules_to_addresses(rule_runner: RuleRunner) -> None: def req( tgt_name: str, req_str: str, *, modules: list[str] | None = None, stub_modules: list[str] | None = None, ) -> str: return ( f"python_requirement(name='{tgt_name}', requirements=['{req_str}'], " f"modules={modules or []}," f"type_stub_modules={stub_modules or []})") build_file = "\n\n".join([ req("req1", "req1==1.2"), req("un_normalized", "Un-Normalized-Project>3"), req("file_dist", "file_dist@ file:///path/to/dist.whl"), req("vcs_dist", "vcs_dist@ git+https://github.com/vcs/dist.git"), req("modules", "foo==1", modules=["mapped_module"]), # We extract the module from type stub dependencies. req("typed-dep1", "typed-dep1-types"), req("typed-dep2", "types-typed-dep2"), req("typed-dep3", "typed-dep3-stubs"), req("typed-dep4", "stubs-typed-dep4"), req("typed-dep5", "typed-dep5-foo", stub_modules=["typed_dep5"]), # A 3rd-party dependency can have both a type stub and implementation. req("req2", "req2==1"), req("req2_types", "types-req2==1"), req("req3", "req3==1"), req("req3_types", "req3-types==1"), req("req4", "req4==1"), req("req4_types", "req4-stubs==1", stub_modules=["req4"]), # Ambiguous. req("ambiguous_t1", "ambiguous==1.2"), req("ambiguous_t2", "ambiguous==1.3"), req("ambiguous_stubs_t1", "ambiguous-stubs-types==1.3"), req("ambiguous_stubs_t2", "types-ambiguous-stubs==1.3"), # If there's ambiguity within type stubs or within implementations, then there should # be ambiguity with the other category too. req("ambiguous_again_t1", "ambiguous-again==1.2"), req("ambiguous_again_t2", "ambiguous-again==1.3"), req("ambiguous_again_t3", "ambiguous-again-types==1.3"), req("ambiguous_again_stubby_t1", "ambiguous-again-stubby-types==1.2"), req("ambiguous_again_stubby_t2", "types-ambiguous-again-stubby==1.3"), req("ambiguous_again_stubby_t3", "ambiguous-again-stubby==1.3"), # Only assume it's a type stubs dep if we are certain it's not an implementation. req("looks_like_stubs", "looks-like-stubs-types", modules=["looks_like_stubs"]), ]) rule_runner.write_files({"BUILD": build_file}) result = rule_runner.request(ThirdPartyPythonModuleMapping, []) assert result == ThirdPartyPythonModuleMapping( mapping=FrozenDict({ "file_dist": (Address("", target_name="file_dist"), ), "looks_like_stubs": (Address("", target_name="looks_like_stubs"), ), "mapped_module": (Address("", target_name="modules"), ), "req1": (Address("", target_name="req1"), ), "req2": (Address("", target_name="req2"), Address("", target_name="req2_types")), "req3": (Address("", target_name="req3"), Address("", target_name="req3_types")), "req4": (Address("", target_name="req4"), Address("", target_name="req4_types")), "typed_dep1": (Address("", target_name="typed-dep1"), ), "typed_dep2": (Address("", target_name="typed-dep2"), ), "typed_dep3": (Address("", target_name="typed-dep3"), ), "typed_dep4": (Address("", target_name="typed-dep4"), ), "typed_dep5": (Address("", target_name="typed-dep5"), ), "un_normalized_project": (Address("", target_name="un_normalized"), ), "vcs_dist": (Address("", target_name="vcs_dist"), ), }), ambiguous_modules=FrozenDict({ "ambiguous": ( Address("", target_name="ambiguous_t1"), Address("", target_name="ambiguous_t2"), ), "ambiguous_again": ( Address("", target_name="ambiguous_again_t1"), Address("", target_name="ambiguous_again_t2"), Address("", target_name="ambiguous_again_t3"), ), "ambiguous_again_stubby": ( Address("", target_name="ambiguous_again_stubby_t1"), Address("", target_name="ambiguous_again_stubby_t2"), Address("", target_name="ambiguous_again_stubby_t3"), ), "ambiguous_stubs": ( Address("", target_name="ambiguous_stubs_t1"), Address("", target_name="ambiguous_stubs_t2"), ), }), )
def test_map_third_party_modules_to_addresses(rule_runner: RuleRunner) -> None: def req( tgt_name: str, req_str: str, *, modules: list[str] | None = None, stub_modules: list[str] | None = None, resolve: str = "default", ) -> str: return dedent(f"""\ python_requirement(name='{tgt_name}', requirements=['{req_str}'], modules={modules or []}, type_stub_modules={stub_modules or []}, resolve={repr(resolve)}) """) build_file = "\n\n".join([ req("req1", "req1==1.2"), req("un_normalized", "Un-Normalized-Project>3"), req("file_dist", "file_dist@ file:///path/to/dist.whl"), req("vcs_dist", "vcs_dist@ git+https://github.com/vcs/dist.git"), req("modules", "foo==1", modules=["mapped_module"]), # We extract the module from type stub dependencies. req("typed-dep1", "typed-dep1-types"), req("typed-dep2", "types-typed-dep2"), req("typed-dep3", "typed-dep3-stubs"), req("typed-dep4", "stubs-typed-dep4"), req("typed-dep5", "typed-dep5-foo", stub_modules=["typed_dep5"]), # A 3rd-party dependency can have both a type stub and implementation. req("multiple_owners1", "multiple_owners==1"), req("multiple_owners2", "multiple_owners==2", resolve="another"), req("multiple_owners_types", "types-multiple_owners==1", resolve="another"), # Only assume it's a type stubs dep if we are certain it's not an implementation. req("looks_like_stubs", "looks-like-stubs-types", modules=["looks_like_stubs"]), ]) rule_runner.write_files({"BUILD": build_file}) rule_runner.set_options([ "--python-resolves={'default': '', 'another': ''}", "--python-enable-resolves" ]) result = rule_runner.request(ThirdPartyPythonModuleMapping, []) assert result == ThirdPartyPythonModuleMapping({ "another": FrozenDict({ "multiple_owners": ( ModuleProvider(Address("", target_name="multiple_owners2"), ModuleProviderType.IMPL), ModuleProvider( Address("", target_name="multiple_owners_types"), ModuleProviderType.TYPE_STUB, ), ), }), "default": FrozenDict({ "file_dist": (ModuleProvider(Address("", target_name="file_dist"), ModuleProviderType.IMPL), ), "looks_like_stubs": (ModuleProvider(Address("", target_name="looks_like_stubs"), ModuleProviderType.IMPL), ), "mapped_module": (ModuleProvider(Address("", target_name="modules"), ModuleProviderType.IMPL), ), "multiple_owners": (ModuleProvider(Address("", target_name="multiple_owners1"), ModuleProviderType.IMPL), ), "req1": (ModuleProvider(Address("", target_name="req1"), ModuleProviderType.IMPL), ), "typed_dep1": (ModuleProvider(Address("", target_name="typed-dep1"), ModuleProviderType.TYPE_STUB), ), "typed_dep2": (ModuleProvider(Address("", target_name="typed-dep2"), ModuleProviderType.TYPE_STUB), ), "typed_dep3": (ModuleProvider(Address("", target_name="typed-dep3"), ModuleProviderType.TYPE_STUB), ), "typed_dep4": (ModuleProvider(Address("", target_name="typed-dep4"), ModuleProviderType.TYPE_STUB), ), "typed_dep5": (ModuleProvider(Address("", target_name="typed-dep5"), ModuleProviderType.TYPE_STUB), ), "un_normalized_project": (ModuleProvider(Address("", target_name="un_normalized"), ModuleProviderType.IMPL), ), "vcs_dist": (ModuleProvider(Address("", target_name="vcs_dist"), ModuleProviderType.IMPL), ), }), })
def test_third_party_modules_mapping() -> None: colors_provider = ModuleProvider(Address("", target_name="ansicolors"), ModuleProviderType.IMPL) colors_stubs_provider = ModuleProvider( Address("", target_name="types-ansicolors"), ModuleProviderType.TYPE_STUB) pants_provider = ModuleProvider(Address("", target_name="pantsbuild"), ModuleProviderType.IMPL) pants_testutil_provider = ModuleProvider( Address("", target_name="pantsbuild.testutil"), ModuleProviderType.IMPL) submodule_provider = ModuleProvider(Address("", target_name="submodule"), ModuleProviderType.IMPL) mapping = ThirdPartyPythonModuleMapping({ "default-resolve": FrozenDict({ "colors": (colors_provider, colors_stubs_provider), "pants": (pants_provider, ), "req.submodule": (submodule_provider, ), "pants.testutil": (pants_testutil_provider, ), "two_resolves": (colors_provider, ), }), "another-resolve": FrozenDict({"two_resolves": (pants_provider, )}), }) def assert_addresses(mod: str, expected: tuple[ModuleProvider, ...], *, resolve: str | None = None) -> None: assert mapping.providers_for_module(mod, resolve) == expected assert_addresses("colors", (colors_provider, colors_stubs_provider)) assert_addresses("colors.red", (colors_provider, colors_stubs_provider)) assert_addresses("pants", (pants_provider, )) assert_addresses("pants.task", (pants_provider, )) assert_addresses("pants.task.task", (pants_provider, )) assert_addresses("pants.task.task.Task", (pants_provider, )) assert_addresses("pants.testutil", (pants_testutil_provider, )) assert_addresses("pants.testutil.foo", (pants_testutil_provider, )) assert_addresses("req.submodule", (submodule_provider, )) assert_addresses("req.submodule.foo", (submodule_provider, )) assert_addresses("req.another", ()) assert_addresses("req", ()) assert_addresses("unknown", ()) assert_addresses("unknown.pants", ()) assert_addresses("two_resolves", (colors_provider, pants_provider), resolve=None) assert_addresses("two_resolves.foo", (colors_provider, pants_provider), resolve=None) assert_addresses("two_resolves.foo.bar", (colors_provider, pants_provider), resolve=None) assert_addresses("two_resolves", (colors_provider, ), resolve="default-resolve") assert_addresses("two_resolves", (pants_provider, ), resolve="another-resolve")
def test_third_party_modules_mapping() -> None: colors_addr = Address("", target_name="ansicolors") pants_addr = Address("", target_name="pantsbuild") submodule_addr = Address("", target_name="submodule") mapping = ThirdPartyPythonModuleMapping( mapping=FrozenDict( {"colors": colors_addr, "pants": pants_addr, "req.submodule": submodule_addr} ), ambiguous_modules=FrozenDict({"ambiguous": (colors_addr, pants_addr)}), ) assert mapping.address_for_module("colors") == (colors_addr, ()) assert mapping.address_for_module("colors.red") == (colors_addr, ()) assert mapping.address_for_module("pants") == (pants_addr, ()) assert mapping.address_for_module("pants.task") == (pants_addr, ()) assert mapping.address_for_module("pants.task.task") == (pants_addr, ()) assert mapping.address_for_module("pants.task.task.Task") == (pants_addr, ()) assert mapping.address_for_module("req.submodule") == (submodule_addr, ()) assert mapping.address_for_module("req.submodule.foo") == (submodule_addr, ()) assert mapping.address_for_module("req.another") == (None, ()) assert mapping.address_for_module("req") == (None, ()) assert mapping.address_for_module("unknown") == (None, ()) assert mapping.address_for_module("unknown.pants") == (None, ()) assert mapping.address_for_module("ambiguous") == (None, (colors_addr, pants_addr)) assert mapping.address_for_module("ambiguous.foo") == (None, (colors_addr, pants_addr)) assert mapping.address_for_module("ambiguous.foo.bar") == (None, (colors_addr, pants_addr))
def find_python_runtime_library_or_raise_error( module_mapping: ThirdPartyPythonModuleMapping, codegen_address: Address, runtime_library_module: str, *, resolve: str, resolves_enabled: bool, recommended_requirement_name: str, recommended_requirement_url: str, disable_inference_option: str, ) -> Address: addresses = [ module_provider.addr for module_provider in module_mapping.providers_for_module( runtime_library_module, resolve=resolve ) if module_provider.typ == ModuleProviderType.IMPL ] if len(addresses) == 1: return addresses[0] for_resolve_str = f" for the resolve '{resolve}'" if resolves_enabled else "" if not addresses: resolve_note = softwrap( ( f""" Note that because `[python].enable_resolves` is set, you must specifically have a `python_requirement` target that uses the same resolve '{resolve}' as the target {codegen_address}. Alternatively, update {codegen_address} to use a different resolve. """ ) if resolves_enabled else "" ) raise MissingPythonCodegenRuntimeLibrary( softwrap( f""" No `python_requirement` target was found with the module `{runtime_library_module}` in your project{for_resolve_str}, so the Python code generated from the target {codegen_address} will not work properly. See {doc_url('python-third-party-dependencies')} for how to add a requirement, such as adding to requirements.txt. Usually you will want to use the `{recommended_requirement_name}` project at {recommended_requirement_url}. {resolve_note} To ignore this error, set `{disable_inference_option} = false` in `pants.toml`. """ ) ) alternative_solution = softwrap( ( f""" Alternatively, change the resolve field for {codegen_address} to use a different resolve from `[python].resolves`. """ ) if resolves_enabled else ( f""" Alternatively, if you do want to have multiple conflicting versions of the `{runtime_library_module}` requirement, set `{disable_inference_option} = false` in `pants.toml`. Then manually add a dependency on the relevant `python_requirement` target to each target that directly depends on this generated code (e.g. `python_source` targets). """ ) ) raise AmbiguousPythonCodegenRuntimeLibrary( softwrap( f""" Multiple `python_requirement` targets were found with the module `{runtime_library_module}` in your project{for_resolve_str}, so it is ambiguous which to use for the runtime library for the Python code generated from the the target {codegen_address}: {sorted(addr.spec for addr in addresses)} To fix, remove one of these `python_requirement` targets{for_resolve_str} so that there is no ambiguity and Pants can infer a dependency. {alternative_solution} """ ) )