def rules():
return (
*collect_rules(),
*import_rules(),
UnionRule(InjectDependenciesRequest,
InjectPythonCloudFunctionHandlerDependency),
)
def rules():
return (
*collect_rules(),
*import_rules(),
UnionRule(InjectDependenciesRequest, InjectPexBinaryEntryPointDependency),
UnionRule(InjectDependenciesRequest, InjectPythonDistributionDependencies),
)
def rules():
return (
*collect_rules(),
*import_rules(),
UnionRule(TargetFilesGeneratorSettingsRequest, PythonFilesGeneratorSettingsRequest),
UnionRule(GenerateTargetsRequest, GenerateTargetsFromPexBinaries),
UnionRule(InjectDependenciesRequest, InjectPexBinaryEntryPointDependency),
UnionRule(InjectDependenciesRequest, InjectPythonDistributionDependencies),
UnionRule(ValidateDependenciesRequest, PythonValidateDependenciesRequest),
)
def rules():
return (
*collect_rules(),
*import_rules(),
UnionRule(GenerateTargetsRequest, GenerateTargetsFromPythonTests),
UnionRule(GenerateTargetsRequest, GenerateTargetsFromPythonSources),
UnionRule(GenerateTargetsRequest, GenerateTargetsFromPythonTestUtils),
UnionRule(InjectDependenciesRequest, InjectPexBinaryEntryPointDependency),
UnionRule(InjectDependenciesRequest, InjectPythonDistributionDependencies),
)
def test_pex_binary_targets() -> None:
rule_runner = RuleRunner(
rules=[
*target_types_rules.rules(),
*import_rules(),
*python_sources.rules(),
QueryRule(_TargetParametrizations, [_TargetParametrizationsRequest]),
],
target_types=[PexBinariesGeneratorTarget],
)
rule_runner.write_files(
{
"src/py/BUILD": dedent(
"""\
pex_binaries(
name="pexes",
entry_points=[
"f1.py",
"f2:foo",
"subdir.f.py",
"subdir.f:main",
],
overrides={
'f2:foo': {'tags': ['overridden']},
'subdir.f.py': {'tags': ['overridden']},
}
)
"""
),
}
)
def gen_pex_binary_tgt(entry_point: str, tags: list[str] | None = None) -> PexBinary:
return PexBinary(
{PexEntryPointField.alias: entry_point, Tags.alias: tags},
Address("src/py", target_name="pexes", generated_name=entry_point.replace(":", "-")),
residence_dir="src/py",
)
result = rule_runner.request(
_TargetParametrizations,
[
_TargetParametrizationsRequest(
Address("src/py", target_name="pexes"), description_of_origin="tests"
)
],
).parametrizations.values()
assert set(result) == {
gen_pex_binary_tgt("f1.py"),
gen_pex_binary_tgt("f2:foo", tags=["overridden"]),
gen_pex_binary_tgt("subdir.f.py", tags=["overridden"]),
gen_pex_binary_tgt("subdir.f:main"),
}
def imports_rule_runner() -> RuleRunner:
return RuleRunner(
rules=[
*import_rules(),
*target_types_rules.rules(),
*core_target_types_rules(),
*python_requirements.rules(),
QueryRule(InferredDependencies, [InferPythonImportDependencies]),
],
target_types=[
PythonSourceTarget,
PythonSourcesGeneratorTarget,
PythonRequirementTarget,
PythonRequirementsTargetGenerator,
],
objects={"parametrize": Parametrize},
)
def test_inject_pex_binary_entry_point_dependency() -> None:
rule_runner = RuleRunner(
rules=[
inject_pex_binary_entry_point_dependency,
resolve_pex_entry_point,
*import_rules(),
QueryRule(InjectedDependencies,
[InjectPexBinaryEntryPointDependency]),
],
target_types=[PexBinary, PythonRequirementLibrary, PythonLibrary],
)
rule_runner.add_to_build_file(
"",
dedent("""\
python_requirement_library(
name='ansicolors',
requirements=['ansicolors'],
module_mapping={'ansicolors': ['colors']},
)
"""),
)
rule_runner.create_files("project", ["app.py", "self.py"])
rule_runner.add_to_build_file(
"project",
dedent("""\
python_library(sources=['app.py'])
pex_binary(name='first_party', entry_point='project.app')
pex_binary(name='first_party_func', entry_point='project.app:func')
pex_binary(name='first_party_shorthand', entry_point='app.py:func')
pex_binary(name='first_party_shorthand_func', entry_point='app.py:func')
pex_binary(name='third_party', entry_point='colors')
pex_binary(name='third_party_func', entry_point='colors:func')
pex_binary(name='unrecognized', entry_point='who_knows.module')
"""),
)
def assert_injected(address: Address, *,
expected: Optional[Address]) -> None:
tgt = rule_runner.get_target(address)
injected = rule_runner.request(
InjectedDependencies,
[InjectPexBinaryEntryPointDependency(tgt[PexBinaryDependencies])],
)
assert injected == InjectedDependencies([expected] if expected else [])
assert_injected(
Address("project", target_name="first_party"),
expected=Address("project", relative_file_path="app.py"),
)
assert_injected(
Address("project", target_name="first_party_func"),
expected=Address("project", relative_file_path="app.py"),
)
assert_injected(
Address("project", target_name="first_party_shorthand"),
expected=Address("project", relative_file_path="app.py"),
)
assert_injected(
Address("project", target_name="first_party_shorthand_func"),
expected=Address("project", relative_file_path="app.py"),
)
assert_injected(
Address("project", target_name="third_party"),
expected=Address("", target_name="ansicolors"),
)
assert_injected(
Address("project", target_name="third_party_func"),
expected=Address("", target_name="ansicolors"),
)
assert_injected(Address("project", target_name="unrecognized"),
expected=None)
# Test that we can turn off the injection.
rule_runner.set_options(["--no-python-infer-entry-points"])
assert_injected(Address("project", target_name="first_party"),
expected=None)
def test_inject_python_distribution_dependencies() -> None:
rule_runner = RuleRunner(
rules=[
*target_types_rules.rules(),
*import_rules(),
*python_sources.rules(),
QueryRule(InjectedDependencies, [InjectPythonDistributionDependencies]),
],
target_types=[
PythonDistribution,
PythonRequirementTarget,
PythonSourcesGeneratorTarget,
PexBinary,
],
objects={"setup_py": PythonArtifact},
)
rule_runner.write_files(
{
"BUILD": dedent(
"""\
python_requirement(
name='ansicolors',
requirements=['ansicolors'],
modules=['colors'],
)
"""
),
"project/app.py": "",
"project/BUILD": dedent(
"""\
pex_binary(name="my_binary", entry_point="who_knows.module:main")
python_sources(name="my_library", sources=["app.py"])
python_distribution(
name="dist-a",
provides=setup_py(
name='my-dist-a'
),
entry_points={
"console_scripts":{
"my_cmd": ":my_binary",
},
},
)
python_distribution(
name="dist-b",
provides=setup_py(
name="my-dist-b"
),
entry_points={
"console_scripts":{
"b_cmd": "project.app:main",
"cmd_2": "//project:my_binary",
}
},
)
python_distribution(
name="third_dep",
provides=setup_py(name="my-third"),
entry_points={
"color-plugins":{
"my-ansi-colors": "colors",
}
}
)
python_distribution(
name="third_dep2",
provides=setup_py(
name="my-third",
entry_points={
"console_scripts":{
"my-cmd": ":my_binary",
"main": "project.app:main",
},
"color-plugins":{
"my-ansi-colors": "colors",
}
}
)
)
"""
),
"who_knows/module.py": "",
"who_knows/BUILD": dedent(
"""\
python_sources(name="random_lib", sources=["module.py"])
"""
),
}
)
def assert_injected(address: Address, expected: list[Address]) -> None:
tgt = rule_runner.get_target(address)
injected = rule_runner.request(
InjectedDependencies,
[InjectPythonDistributionDependencies(tgt[PythonDistributionDependenciesField])],
)
assert injected == InjectedDependencies(expected)
assert_injected(
Address("project", target_name="dist-a"),
[Address("project", target_name="my_binary")],
)
assert_injected(
Address("project", target_name="dist-b"),
[
Address("project", target_name="my_binary"),
Address("project", relative_file_path="app.py", target_name="my_library"),
],
)
assert_injected(
Address("project", target_name="third_dep"),
[
Address("", target_name="ansicolors"),
],
)
assert_injected(
Address("project", target_name="third_dep2"),
[
Address("", target_name="ansicolors"),
Address("project", target_name="my_binary"),
Address("project", relative_file_path="app.py", target_name="my_library"),
],
)
def test_infer_python_imports() -> None:
rule_runner = RuleRunner(
rules=[
*import_rules(),
QueryRule(InferredDependencies, [InferPythonImportDependencies])
],
target_types=[PythonLibrary, PythonRequirementLibrary],
)
rule_runner.add_to_build_file(
"3rdparty/python",
dedent("""\
python_requirement_library(
name='Django',
requirements=['Django==1.21'],
)
"""),
)
# If there's a `.py` and `.pyi` file for the same module, we should infer a dependency on both.
rule_runner.create_file("src/python/str_import/subdir/f.py")
rule_runner.create_file("src/python/str_import/subdir/f.pyi")
rule_runner.add_to_build_file("src/python/str_import/subdir",
"python_library()")
rule_runner.create_file("src/python/util/dep.py")
rule_runner.add_to_build_file("src/python/util", "python_library()")
rule_runner.create_file(
"src/python/app.py",
dedent("""\
import django
import unrecognized.module
from util.dep import Demo
from util import dep
"""),
)
rule_runner.create_file(
"src/python/f2.py",
dedent("""\
import typing
# Import from another file in the same target.
from app import main
# Dynamic string import.
importlib.import_module('str_import.subdir.f')
"""),
)
rule_runner.add_to_build_file("src/python", "python_library()")
def run_dep_inference(
address: Address,
*,
enable_string_imports: bool = False) -> InferredDependencies:
args = [
"--backend-packages=pants.backend.python",
"--source-root-patterns=src/python"
]
if enable_string_imports:
args.append("--python-infer-string-imports")
rule_runner.set_options(args,
env_inherit={"PATH", "PYENV_ROOT", "HOME"})
target = rule_runner.get_target(address)
return rule_runner.request(
InferredDependencies,
[InferPythonImportDependencies(target[PythonSources])])
build_address = Address("src/python")
assert run_dep_inference(build_address) == InferredDependencies(
[
Address("3rdparty/python", target_name="Django"),
Address("src/python", relative_file_path="app.py"),
Address("src/python/util", relative_file_path="dep.py"),
],
sibling_dependencies_inferrable=True,
)
file_address = Address("src/python", relative_file_path="f2.py")
assert run_dep_inference(file_address) == InferredDependencies(
[Address("src/python", relative_file_path="app.py")],
sibling_dependencies_inferrable=True,
)
assert run_dep_inference(
file_address, enable_string_imports=True) == InferredDependencies(
[
Address("src/python", relative_file_path="app.py"),
Address("src/python/str_import/subdir",
relative_file_path="f.py"),
Address("src/python/str_import/subdir",
relative_file_path="f.pyi"),
],
sibling_dependencies_inferrable=True,
)
def test_inject_pex_binary_entry_point_dependency(caplog) -> None:
rule_runner = RuleRunner(
rules=[
inject_pex_binary_entry_point_dependency,
resolve_pex_entry_point,
*import_rules(),
QueryRule(InjectedDependencies,
[InjectPexBinaryEntryPointDependency]),
],
target_types=[PexBinary, PythonRequirementLibrary, PythonLibrary],
)
rule_runner.add_to_build_file(
"",
dedent("""\
python_requirement_library(
name='ansicolors',
requirements=['ansicolors'],
module_mapping={'ansicolors': ['colors']},
)
"""),
)
rule_runner.create_file("project/app.py")
rule_runner.add_to_build_file(
"project",
dedent("""\
python_library(sources=['app.py'])
pex_binary(name='first_party', entry_point='project.app')
pex_binary(name='first_party_func', entry_point='project.app:func')
pex_binary(name='first_party_shorthand', entry_point='app.py')
pex_binary(name='first_party_shorthand_func', entry_point='app.py:func')
pex_binary(name='third_party', entry_point='colors')
pex_binary(name='third_party_func', entry_point='colors:func')
pex_binary(name='unrecognized', entry_point='who_knows.module')
"""),
)
def assert_injected(address: Address, *,
expected: Optional[Address]) -> None:
tgt = rule_runner.get_target(address)
injected = rule_runner.request(
InjectedDependencies,
[InjectPexBinaryEntryPointDependency(tgt[PexBinaryDependencies])],
)
assert injected == InjectedDependencies([expected] if expected else [])
assert_injected(
Address("project", target_name="first_party"),
expected=Address("project", relative_file_path="app.py"),
)
assert_injected(
Address("project", target_name="first_party_func"),
expected=Address("project", relative_file_path="app.py"),
)
assert_injected(
Address("project", target_name="first_party_shorthand"),
expected=Address("project", relative_file_path="app.py"),
)
assert_injected(
Address("project", target_name="first_party_shorthand_func"),
expected=Address("project", relative_file_path="app.py"),
)
assert_injected(
Address("project", target_name="third_party"),
expected=Address("", target_name="ansicolors"),
)
assert_injected(
Address("project", target_name="third_party_func"),
expected=Address("", target_name="ansicolors"),
)
assert_injected(Address("project", target_name="unrecognized"),
expected=None)
# Test that we can turn off the injection.
rule_runner.set_options(["--no-python-infer-entry-points"])
assert_injected(Address("project", target_name="first_party"),
expected=None)
rule_runner.set_options([])
# Warn if there's ambiguity, meaning we cannot infer.
caplog.clear()
rule_runner.create_file("project/ambiguous.py")
rule_runner.add_to_build_file(
"project",
dedent("""\
python_library(name="dep1", sources=["ambiguous.py"])
python_library(name="dep2", sources=["ambiguous.py"])
pex_binary(name="ambiguous", entry_point="ambiguous.py")
"""),
)
assert_injected(
Address("project", target_name="ambiguous"),
expected=None,
)
assert len(caplog.records) == 1
assert (
"project:ambiguous has the field `entry_point='ambiguous.py'`, which maps to the Python "
"module `project.ambiguous`") in caplog.text
assert "['project/ambiguous.py:dep1', 'project/ambiguous.py:dep2']" in caplog.text
# Test that ignores can disambiguate an otherwise ambiguous handler. Ensure we don't log a
# warning about ambiguity.
caplog.clear()
rule_runner.add_to_build_file(
"project",
dedent("""\
pex_binary(
name="disambiguated",
entry_point="ambiguous.py",
dependencies=["!./ambiguous.py:dep2"],
)
"""),
)
assert_injected(
Address("project", target_name="disambiguated"),
expected=Address("project",
target_name="dep1",
relative_file_path="ambiguous.py"),
)
assert not caplog.records
def test_generate_source_and_test_targets() -> None:
rule_runner = RuleRunner(
rules=[
*target_types_rules.rules(),
*import_rules(),
*python_sources.rules(),
QueryRule(GeneratedTargets, [GenerateTargetsFromPythonTests]),
QueryRule(GeneratedTargets, [GenerateTargetsFromPythonSources]),
QueryRule(GeneratedTargets, [GenerateTargetsFromPythonTestUtils]),
],
target_types=[
PythonTestsGeneratorTarget,
PythonSourcesGeneratorTarget,
PythonTestUtilsGeneratorTarget,
],
)
rule_runner.write_files({
"src/py/BUILD":
dedent("""\
python_sources(
name='lib',
sources=['**/*.py', '!**/*_test.py', '!**/conftest.py'],
overrides={'f1.py': {'tags': ['overridden']}},
)
python_tests(
name='tests',
sources=['**/*_test.py'],
overrides={'f1_test.py': {'tags': ['overridden']}},
)
python_test_utils(
name='test_utils',
sources=['**/conftest.py'],
overrides={'conftest.py': {'tags': ['overridden']}},
)
"""),
"src/py/f1.py":
"",
"src/py/f1_test.py":
"",
"src/py/conftest.py":
"",
"src/py/f2.py":
"",
"src/py/f2_test.py":
"",
"src/py/subdir/f.py":
"",
"src/py/subdir/f_test.py":
"",
"src/py/subdir/conftest.py":
"",
})
sources_generator = rule_runner.get_target(
Address("src/py", target_name="lib"))
tests_generator = rule_runner.get_target(
Address("src/py", target_name="tests"))
test_utils_generator = rule_runner.get_target(
Address("src/py", target_name="test_utils"))
def gen_source_tgt(rel_fp: str,
tags: list[str] | None = None,
*,
tgt_name: str) -> PythonSourceTarget:
return PythonSourceTarget(
{
SingleSourceField.alias: rel_fp,
Tags.alias: tags
},
Address("src/py", target_name=tgt_name, relative_file_path=rel_fp),
residence_dir=os.path.dirname(os.path.join("src/py", rel_fp)),
)
def gen_test_tgt(rel_fp: str,
tags: list[str] | None = None) -> PythonTestTarget:
return PythonTestTarget(
{
SingleSourceField.alias: rel_fp,
Tags.alias: tags
},
Address("src/py", target_name="tests", relative_file_path=rel_fp),
residence_dir=os.path.dirname(os.path.join("src/py", rel_fp)),
)
sources_generated = rule_runner.request(
GeneratedTargets,
[GenerateTargetsFromPythonSources(sources_generator)])
tests_generated = rule_runner.request(
GeneratedTargets, [GenerateTargetsFromPythonTests(tests_generator)])
test_utils_generated = rule_runner.request(
GeneratedTargets,
[GenerateTargetsFromPythonTestUtils(test_utils_generator)])
assert sources_generated == GeneratedTargets(
sources_generator,
{
gen_source_tgt("f1.py", tags=["overridden"], tgt_name="lib"),
gen_source_tgt("f2.py", tgt_name="lib"),
gen_source_tgt("subdir/f.py", tgt_name="lib"),
},
)
assert tests_generated == GeneratedTargets(
tests_generator,
{
gen_test_tgt("f1_test.py", tags=["overridden"]),
gen_test_tgt("f2_test.py"),
gen_test_tgt("subdir/f_test.py"),
},
)
assert test_utils_generated == GeneratedTargets(
test_utils_generator,
{
gen_source_tgt(
"conftest.py", tags=["overridden"], tgt_name="test_utils"),
gen_source_tgt("subdir/conftest.py", tgt_name="test_utils"),
},
)
def test_infer_python_assets(caplog) -> None:
rule_runner = RuleRunner(
rules=[
*import_rules(),
*target_types_rules.rules(),
*core_target_types_rules(),
QueryRule(InferredDependencies, [InferPythonImportDependencies]),
],
target_types=[
PythonSourcesGeneratorTarget,
PythonRequirementTarget,
ResourcesGeneratorTarget,
FilesGeneratorTarget,
],
)
rule_runner.write_files(
{
"src/python/data/BUILD": "resources(name='jsonfiles', sources=['*.json'])",
"src/python/data/db.json": "",
"src/python/data/db2.json": "",
"src/python/data/flavors.txt": "",
"configs/prod.txt": "",
"src/python/app.py": dedent(
"""\
pkgutil.get_data(__name__, "data/db.json")
pkgutil.get_data(__name__, "data/db2.json")
open("configs/prod.txt")
"""
),
"src/python/f.py": dedent(
"""\
idk_kinda_looks_resourcey = "data/db.json"
CustomResourceType("data/flavors.txt")
"""
),
"src/python/BUILD": dedent(
"""\
python_sources()
# Also test assets declared from parent dir
resources(
name="txtfiles",
sources=["data/*.txt"],
)
"""
),
"configs/BUILD": dedent(
"""\
files(
name="configs",
sources=["prod.txt"],
)
"""
),
}
)
def run_dep_inference(address: Address) -> InferredDependencies:
args = [
"--source-root-patterns=src/python",
"--python-infer-assets",
]
rule_runner.set_options(args, env_inherit={"PATH", "PYENV_ROOT", "HOME"})
target = rule_runner.get_target(address)
return rule_runner.request(
InferredDependencies, [InferPythonImportDependencies(target[PythonSourceField])]
)
assert run_dep_inference(
Address("src/python", relative_file_path="app.py")
) == InferredDependencies(
[
Address("src/python/data", target_name="jsonfiles", relative_file_path="db.json"),
Address("src/python/data", target_name="jsonfiles", relative_file_path="db2.json"),
Address("configs", target_name="configs", relative_file_path="prod.txt"),
],
)
assert run_dep_inference(
Address("src/python", relative_file_path="f.py")
) == InferredDependencies(
[
Address("src/python/data", target_name="jsonfiles", relative_file_path="db.json"),
Address("src/python", target_name="txtfiles", relative_file_path="data/flavors.txt"),
],
)
# Test handling of ambiguous assets. We should warn on the ambiguous dependency, but not warn
# on the disambiguated one and should infer a dep.
caplog.clear()
rule_runner.write_files(
{
"src/python/data/BUILD": dedent(
"""\
resources(name='jsonfiles', sources=['*.json'])
resources(name='also_jsonfiles', sources=['*.json'])
resources(name='txtfiles', sources=['*.txt'])
"""
),
"src/python/data/ambiguous.json": "",
"src/python/data/disambiguated_with_bang.json": "",
"src/python/app.py": dedent(
"""\
pkgutil.get_data(__name__, "data/ambiguous.json")
pkgutil.get_data(__name__, "data/disambiguated_with_bang.json")
"""
),
"src/python/BUILD": dedent(
"""\
python_sources(
name="main",
dependencies=['!./data/disambiguated_with_bang.json:also_jsonfiles'],
)
"""
),
# Both a resource relative to the module and file with conspicuously similar paths
"src/python/data/both_file_and_resource.txt": "",
"data/both_file_and_resource.txt": "",
"data/BUILD": "files(name='txtfiles', sources=['*.txt'])",
"src/python/assets_bag.py": "ImAPathType('data/both_file_and_resource.txt')",
}
)
assert run_dep_inference(
Address("src/python", target_name="main", relative_file_path="app.py")
) == InferredDependencies(
[
Address(
"src/python/data",
target_name="jsonfiles",
relative_file_path="disambiguated_with_bang.json",
),
],
)
assert len(caplog.records) == 1
assert "The target src/python/app.py:main uses `data/ambiguous.json`" in caplog.text
assert (
"['src/python/data/ambiguous.json:also_jsonfiles', 'src/python/data/ambiguous.json:jsonfiles']"
in caplog.text
)
assert "disambiguated_with_bang.py" not in caplog.text
caplog.clear()
assert run_dep_inference(
Address("src/python", target_name="main", relative_file_path="assets_bag.py")
) == InferredDependencies([])
assert len(caplog.records) == 1
assert (
"The target src/python/assets_bag.py:main uses `data/both_file_and_resource.txt`"
in caplog.text
)
assert (
"['data/both_file_and_resource.txt:txtfiles', 'src/python/data/both_file_and_resource.txt:txtfiles']"
in caplog.text
)
def test_generate_source_and_test_targets() -> None:
rule_runner = RuleRunner(
rules=[
*target_types_rules.rules(),
*import_rules(),
*python_sources.rules(),
QueryRule(_TargetParametrizations, [Address]),
],
target_types=[
PythonTestsGeneratorTarget,
PythonSourcesGeneratorTarget,
PythonTestUtilsGeneratorTarget,
PexBinariesGeneratorTarget,
],
)
rule_runner.write_files({
"src/py/BUILD":
dedent("""\
python_sources(
name='lib',
sources=['**/*.py', '!**/*_test.py', '!**/conftest.py'],
overrides={'f1.py': {'tags': ['overridden']}},
)
python_tests(
name='tests',
sources=['**/*_test.py'],
overrides={'f1_test.py': {'tags': ['overridden']}},
)
python_test_utils(
name='test_utils',
sources=['**/conftest.py'],
overrides={'conftest.py': {'tags': ['overridden']}},
)
pex_binaries(
name="pexes",
entry_points=[
"f1.py",
"f2:foo",
"subdir.f.py",
"subdir.f:main",
],
overrides={
'f2:foo': {'tags': ['overridden']},
'subdir.f.py': {'tags': ['overridden']},
}
)
"""),
"src/py/f1.py":
"",
"src/py/f1_test.py":
"",
"src/py/conftest.py":
"",
"src/py/f2.py":
"",
"src/py/f2_test.py":
"",
"src/py/subdir/f.py":
"",
"src/py/subdir/f_test.py":
"",
"src/py/subdir/conftest.py":
"",
})
def gen_source_tgt(rel_fp: str,
tags: list[str] | None = None,
*,
tgt_name: str) -> PythonSourceTarget:
return PythonSourceTarget(
{
SingleSourceField.alias: rel_fp,
Tags.alias: tags
},
Address("src/py", target_name=tgt_name, relative_file_path=rel_fp),
residence_dir=os.path.dirname(os.path.join("src/py", rel_fp)),
)
def gen_test_tgt(rel_fp: str,
tags: list[str] | None = None) -> PythonTestTarget:
return PythonTestTarget(
{
SingleSourceField.alias: rel_fp,
Tags.alias: tags
},
Address("src/py", target_name="tests", relative_file_path=rel_fp),
residence_dir=os.path.dirname(os.path.join("src/py", rel_fp)),
)
def gen_pex_binary_tgt(entry_point: str,
tags: list[str] | None = None) -> PexBinary:
return PexBinary(
{
PexEntryPointField.alias: entry_point,
Tags.alias: tags
},
Address("src/py",
target_name="pexes",
generated_name=entry_point.replace(":", "-")),
residence_dir="src/py",
)
sources_generated = rule_runner.request(
_TargetParametrizations,
[Address("src/py", target_name="lib")]).parametrizations
tests_generated = rule_runner.request(
_TargetParametrizations,
[Address("src/py", target_name="tests")]).parametrizations
test_utils_generated = rule_runner.request(
_TargetParametrizations,
[Address("src/py", target_name="test_utils")]).parametrizations
pex_binaries_generated = rule_runner.request(
_TargetParametrizations,
[Address("src/py", target_name="pexes")]).parametrizations
assert set(sources_generated.values()) == {
gen_source_tgt("f1.py", tags=["overridden"], tgt_name="lib"),
gen_source_tgt("f2.py", tgt_name="lib"),
gen_source_tgt("subdir/f.py", tgt_name="lib"),
}
assert set(tests_generated.values()) == {
gen_test_tgt("f1_test.py", tags=["overridden"]),
gen_test_tgt("f2_test.py"),
gen_test_tgt("subdir/f_test.py"),
}
assert set(test_utils_generated.values()) == {
gen_source_tgt("conftest.py",
tags=["overridden"],
tgt_name="test_utils"),
gen_source_tgt("subdir/conftest.py", tgt_name="test_utils"),
}
assert set(pex_binaries_generated.values()) == {
gen_pex_binary_tgt("f1.py"),
gen_pex_binary_tgt("f2:foo", tags=["overridden"]),
gen_pex_binary_tgt("subdir.f.py", tags=["overridden"]),
gen_pex_binary_tgt("subdir.f:main"),
}
def test_infer_python_strict(caplog) -> None:
rule_runner = RuleRunner(
rules=[
*import_rules(),
*target_types_rules.rules(),
QueryRule(InferredDependencies, [InferPythonImportDependencies]),
],
target_types=[
PythonSourcesGeneratorTarget,
PythonRequirementTarget,
PythonRequirementsFile,
],
context_aware_object_factories={
"python_requirements": PythonRequirementsCAOF
},
)
rule_runner.create_file(
"src/python/cheesey.py",
"import venezuelan_beaver_cheese",
)
rule_runner.add_to_build_file("src/python", "python_sources()")
def run_dep_inference(
address: Address,
unowned_dependency_behavior: str,
) -> InferredDependencies:
rule_runner.set_options(
[
"--backend-packages=pants.backend.python",
f"--python-infer-unowned-dependency-behavior={unowned_dependency_behavior}",
"--source-root-patterns=src/python",
],
env_inherit={"PATH", "PYENV_ROOT", "HOME"},
)
target = rule_runner.get_target(address)
return rule_runner.request(
InferredDependencies,
[InferPythonImportDependencies(target[PythonSourceField])],
)
# First test with "warning"
run_dep_inference(Address("src/python", relative_file_path="cheesey.py"),
"warning")
assert len(caplog.records) == 1
assert "The following imports in src/python/cheesey.py have no owners:" in caplog.text
assert " * venezuelan_beaver_cheese" in caplog.text
# Now test with "error"
caplog.clear()
with pytest.raises(ExecutionError) as exc_info:
run_dep_inference(
Address("src/python", relative_file_path="cheesey.py"), "error")
assert isinstance(exc_info.value.wrapped_exceptions[0],
UnownedDependencyError)
assert len(caplog.records
) == 2 # one for the error being raised and one for our message
assert "The following imports in src/python/cheesey.py have no owners:" in caplog.text
assert " * venezuelan_beaver_cheese" in caplog.text
caplog.clear()
# All modes should be fine if the module is explictly declared as a requirement
rule_runner.add_to_build_file(
"src/python",
dedent("""\
python_requirement(
name="venezuelan_beaver_cheese",
modules=["venezuelan_beaver_cheese"],
requirements=["venezuelan_beaver_cheese==1.0.0"],
)
python_sources(dependencies=[":venezuelan_beaver_cheese"])
"""),
overwrite=True,
)
for mode in UnownedDependencyUsage:
run_dep_inference(
Address("src/python", relative_file_path="cheesey.py"), mode.value)
assert not caplog.records
rule_runner.add_to_build_file("src/python",
"python_sources()",
overwrite=True) # Cleanup
# All modes should be fine if the module is implictly found via requirements.txt
rule_runner.create_file("src/python/requirements.txt",
"venezuelan_beaver_cheese==1.0.0")
rule_runner.add_to_build_file(
"src/python",
dedent("""\
python_requirements()
python_sources()
"""),
overwrite=True,
)
for mode in UnownedDependencyUsage:
run_dep_inference(
Address("src/python", relative_file_path="cheesey.py"), mode.value)
assert not caplog.records
# All modes should be fine if the module is owned by a first party
rule_runner.create_file("src/python/venezuelan_beaver_cheese.py")
rule_runner.add_to_build_file("src/python",
"python_sources()",
overwrite=True)
for mode in UnownedDependencyUsage:
run_dep_inference(
Address("src/python", relative_file_path="cheesey.py"), mode.value)
assert not caplog.records
def test_inject_pex_binary_entry_point_dependency(caplog) -> None:
rule_runner = RuleRunner(
rules=[
*target_types_rules.rules(),
*import_rules(),
QueryRule(InjectedDependencies, [InjectPexBinaryEntryPointDependency]),
],
target_types=[PexBinary, PythonRequirementTarget, PythonSourcesGeneratorTarget],
)
rule_runner.write_files(
{
"BUILD": dedent(
"""\
python_requirement(
name='ansicolors',
requirements=['ansicolors'],
modules=['colors'],
)
"""
),
"project/app.py": "",
"project/ambiguous.py": "",
"project/ambiguous_in_another_root.py": "",
"project/BUILD": dedent(
"""\
python_sources(sources=['app.py'])
pex_binary(name='first_party', entry_point='project.app')
pex_binary(name='first_party_func', entry_point='project.app:func')
pex_binary(name='first_party_shorthand', entry_point='app.py')
pex_binary(name='first_party_shorthand_func', entry_point='app.py:func')
pex_binary(name='third_party', entry_point='colors')
pex_binary(name='third_party_func', entry_point='colors:func')
pex_binary(name='unrecognized', entry_point='who_knows.module')
python_sources(name="dep1", sources=["ambiguous.py"])
python_sources(name="dep2", sources=["ambiguous.py"])
pex_binary(name="ambiguous", entry_point="ambiguous.py")
pex_binary(
name="disambiguated",
entry_point="ambiguous.py",
dependencies=["!./ambiguous.py:dep2"],
)
python_sources(
name="ambiguous_in_another_root", sources=["ambiguous_in_another_root.py"]
)
pex_binary(
name="another_root__file_used", entry_point="ambiguous_in_another_root.py"
)
pex_binary(
name="another_root__module_used",
entry_point="project.ambiguous_in_another_root",
)
"""
),
"src/py/project/ambiguous_in_another_root.py": "",
"src/py/project/BUILD.py": "python_sources()",
}
)
def assert_injected(address: Address, *, expected: Address | None) -> None:
tgt = rule_runner.get_target(address)
injected = rule_runner.request(
InjectedDependencies,
[InjectPexBinaryEntryPointDependency(tgt[PexBinaryDependenciesField])],
)
assert injected == InjectedDependencies([expected] if expected else [])
assert_injected(
Address("project", target_name="first_party"),
expected=Address("project", relative_file_path="app.py"),
)
assert_injected(
Address("project", target_name="first_party_func"),
expected=Address("project", relative_file_path="app.py"),
)
assert_injected(
Address("project", target_name="first_party_shorthand"),
expected=Address("project", relative_file_path="app.py"),
)
assert_injected(
Address("project", target_name="first_party_shorthand_func"),
expected=Address("project", relative_file_path="app.py"),
)
assert_injected(
Address("project", target_name="third_party"),
expected=Address("", target_name="ansicolors"),
)
assert_injected(
Address("project", target_name="third_party_func"),
expected=Address("", target_name="ansicolors"),
)
assert_injected(Address("project", target_name="unrecognized"), expected=None)
# Warn if there's ambiguity, meaning we cannot infer.
caplog.clear()
assert_injected(Address("project", target_name="ambiguous"), expected=None)
assert len(caplog.records) == 1
assert (
softwrap(
"""
project:ambiguous has the field `entry_point='ambiguous.py'`, which maps to the Python
module `project.ambiguous`
"""
)
in caplog.text
)
assert "['project/ambiguous.py:dep1', 'project/ambiguous.py:dep2']" in caplog.text
# Test that ignores can disambiguate an otherwise ambiguous entry point. Ensure we don't log a
# warning about ambiguity.
caplog.clear()
assert_injected(
Address("project", target_name="disambiguated"),
expected=Address("project", target_name="dep1", relative_file_path="ambiguous.py"),
)
assert not caplog.records
# Test that using a file path results in ignoring all targets which are not an ancestor. We can
# do this because we know the file name must be in the current directory or subdir of the
# `pex_binary`.
assert_injected(
Address("project", target_name="another_root__file_used"),
expected=Address(
"project",
target_name="ambiguous_in_another_root",
relative_file_path="ambiguous_in_another_root.py",
),
)
caplog.clear()
assert_injected(Address("project", target_name="another_root__module_used"), expected=None)
assert len(caplog.records) == 1
assert (
softwrap(
"""
['project/ambiguous_in_another_root.py:ambiguous_in_another_root',
'src/py/project/ambiguous_in_another_root.py']
"""
)
in caplog.text
)
# Test that we can turn off the injection.
rule_runner.set_options(["--no-python-infer-entry-points"])
assert_injected(Address("project", target_name="first_party"), expected=None)
def test_infer_python_imports(caplog) -> None:
rule_runner = RuleRunner(
rules=[
*import_rules(),
*target_types_rules.rules(),
QueryRule(InferredDependencies, [InferPythonImportDependencies]),
],
target_types=[PythonSourcesGeneratorTarget, PythonRequirementTarget],
)
rule_runner.add_to_build_file(
"3rdparty/python",
dedent("""\
python_requirement(
name='Django',
requirements=['Django==1.21'],
)
"""),
)
# If there's a `.py` and `.pyi` file for the same module, we should infer a dependency on both.
rule_runner.create_file("src/python/str_import/subdir/f.py")
rule_runner.create_file("src/python/str_import/subdir/f.pyi")
rule_runner.add_to_build_file("src/python/str_import/subdir",
"python_sources()")
rule_runner.create_file("src/python/util/dep.py")
rule_runner.add_to_build_file("src/python/util", "python_sources()")
rule_runner.create_file(
"src/python/app.py",
dedent("""\
import django
import unrecognized.module
from util.dep import Demo
from util import dep
"""),
)
rule_runner.create_file(
"src/python/f2.py",
dedent("""\
import typing
# Import from another file in the same target.
from app import main
# Dynamic string import.
importlib.import_module('str_import.subdir.f')
"""),
)
rule_runner.add_to_build_file("src/python", "python_sources()")
def run_dep_inference(
address: Address,
*,
enable_string_imports: bool = False) -> InferredDependencies:
args = [
"--backend-packages=pants.backend.python",
"--source-root-patterns=src/python"
]
if enable_string_imports:
args.append("--python-infer-string-imports")
rule_runner.set_options(args,
env_inherit={"PATH", "PYENV_ROOT", "HOME"})
target = rule_runner.get_target(address)
return rule_runner.request(
InferredDependencies,
[InferPythonImportDependencies(target[PythonSourceField])])
assert run_dep_inference(Address(
"src/python", relative_file_path="app.py")) == InferredDependencies([
Address("3rdparty/python", target_name="Django"),
Address("src/python/util", relative_file_path="dep.py"),
], )
addr = Address("src/python", relative_file_path="f2.py")
assert run_dep_inference(addr) == InferredDependencies(
[Address("src/python", relative_file_path="app.py")])
assert run_dep_inference(
addr, enable_string_imports=True) == InferredDependencies([
Address("src/python", relative_file_path="app.py"),
Address("src/python/str_import/subdir", relative_file_path="f.py"),
Address("src/python/str_import/subdir",
relative_file_path="f.pyi"),
], )
# Test handling of ambiguous imports. We should warn on the ambiguous dependency, but not warn
# on the disambiguated one and should infer a dep.
caplog.clear()
rule_runner.create_files("src/python/ambiguous",
["dep.py", "disambiguated_via_ignores.py"])
rule_runner.create_file(
"src/python/ambiguous/main.py",
"import ambiguous.dep\nimport ambiguous.disambiguated_via_ignores\n",
)
rule_runner.add_to_build_file(
"src/python/ambiguous",
dedent("""\
python_sources(name='dep1', sources=['dep.py', 'disambiguated_via_ignores.py'])
python_sources(name='dep2', sources=['dep.py', 'disambiguated_via_ignores.py'])
python_sources(
name='main',
sources=['main.py'],
dependencies=['!./disambiguated_via_ignores.py:dep2'],
)
"""),
)
assert run_dep_inference(
Address("src/python/ambiguous",
target_name="main",
relative_file_path="main.py")) == InferredDependencies([
Address(
"src/python/ambiguous",
target_name="dep1",
relative_file_path="disambiguated_via_ignores.py",
)
], )
assert len(caplog.records) == 1
assert "The target src/python/ambiguous/main.py:main imports `ambiguous.dep`" in caplog.text
assert "['src/python/ambiguous/dep.py:dep1', 'src/python/ambiguous/dep.py:dep2']" in caplog.text
assert "disambiguated_via_ignores.py" not in caplog.text
def test_infer_python_strict(caplog) -> None:
rule_runner = RuleRunner(
rules=[
*import_rules(),
*target_types_rules.rules(),
*core_target_types_rules(),
*python_requirements.rules(),
QueryRule(InferredDependencies, [InferPythonImportDependencies]),
],
target_types=[
PythonSourcesGeneratorTarget,
PythonRequirementTarget,
PythonRequirementsTargetGenerator,
],
)
rule_runner.write_files({
"src/python/cheesey.py":
dedent("""\
import venezuelan_beaver_cheese
"japanese.sage.derby"
"""),
"src/python/BUILD":
"python_sources()",
})
def run_dep_inference(
address: Address,
unowned_dependency_behavior: str,
) -> InferredDependencies:
rule_runner.set_options(
[
f"--python-infer-unowned-dependency-behavior={unowned_dependency_behavior}",
"--python-infer-string-imports",
"--source-root-patterns=src/python",
],
env_inherit={"PATH", "PYENV_ROOT", "HOME"},
)
target = rule_runner.get_target(address)
return rule_runner.request(
InferredDependencies,
[InferPythonImportDependencies(target[PythonSourceField])],
)
# First test with "warning"
run_dep_inference(Address("src/python", relative_file_path="cheesey.py"),
"warning")
assert len(caplog.records) == 1
assert "The following imports in src/python/cheesey.py have no owners:" in caplog.text
assert " * venezuelan_beaver_cheese (src/python/cheesey.py:1)" in caplog.text
assert "japanese.sage.derby" not in caplog.text
# Now test with "error"
caplog.clear()
with pytest.raises(ExecutionError) as exc_info:
run_dep_inference(
Address("src/python", relative_file_path="cheesey.py"), "error")
assert isinstance(exc_info.value.wrapped_exceptions[0],
UnownedDependencyError)
assert len(caplog.records
) == 2 # one for the error being raised and one for our message
assert "The following imports in src/python/cheesey.py have no owners:" in caplog.text
assert " * venezuelan_beaver_cheese (src/python/cheesey.py:1)" in caplog.text
assert "japanese.sage.derby" not in caplog.text
caplog.clear()
# All modes should be fine if the module is explicitly declared as a requirement
rule_runner.write_files({
"src/python/BUILD":
dedent("""\
python_requirement(
name="venezuelan_beaver_cheese",
modules=["venezuelan_beaver_cheese"],
requirements=["venezuelan_beaver_cheese==1.0.0"],
)
python_sources(dependencies=[":venezuelan_beaver_cheese"])
"""),
})
for mode in UnownedDependencyUsage:
run_dep_inference(
Address("src/python", relative_file_path="cheesey.py"), mode.value)
assert not caplog.records
# All modes should be fine if the module is implictly found via requirements.txt
rule_runner.write_files({
"src/python/requirements.txt":
"venezuelan_beaver_cheese==1.0.0",
"src/python/BUILD":
dedent("""\
python_requirements(name='reqs')
python_sources()
"""),
})
for mode in UnownedDependencyUsage:
run_dep_inference(
Address("src/python", relative_file_path="cheesey.py"), mode.value)
assert not caplog.records
# All modes should be fine if the module is owned by a first party
rule_runner.write_files({
"src/python/venezuelan_beaver_cheese.py": "",
"src/python/BUILD": "python_sources()",
})
for mode in UnownedDependencyUsage:
run_dep_inference(
Address("src/python", relative_file_path="cheesey.py"), mode.value)
assert not caplog.records
