def create_rule_runner() -> RuleRunner:
rule_runner = RuleRunner(
rules=[
*core_target_types_rules(),
*dependencies.rules(),
*docker_binary.rules(),
*docker_build_args.rules(),
*docker_build_context.rules(),
*docker_build_env.rules(),
*dockerfile.rules(),
*dockerfile_parser.rules(),
*package_image.rules(),
*package_pex_binary.rules(),
*pex_from_targets.rules(),
*shell_target_types_rules(),
*target_types_rules.rules(),
package.find_all_packageable_targets,
QueryRule(BuiltPackage, [PexBinaryFieldSet]),
QueryRule(DockerBuildContext, (DockerBuildContextRequest, )),
],
target_types=[
DockerImageTarget,
FilesGeneratorTarget,
PexBinary,
ShellSourcesGeneratorTarget,
ShellSourceTarget,
],
)
return rule_runner
def rule_runner() -> RuleRunner:
rule_runner = RuleRunner(
rules=[
*awslambda_python_rules(),
*awslambda_python_subsystem_rules(),
*core_target_types_rules(),
*package_pex_binary.rules(),
*python_target_types_rules(),
*target_rules(),
QueryRule(BuiltPackage, (PythonAwsLambdaFieldSet, )),
],
target_types=[
FileTarget,
FilesGeneratorTarget,
PexBinary,
PythonAWSLambda,
PythonRequirementTarget,
PythonRequirementTarget,
PythonSourcesGeneratorTarget,
RelocatedFiles,
ResourcesGeneratorTarget,
],
)
rule_runner.set_options([], env_inherit={"PATH", "PYENV_ROOT", "HOME"})
return rule_runner
def rule_runner():
rule_runner = RuleRunner(
rules=[
*war.rules(),
*jvm_tool.rules(),
*classpath.rules(),
*javac_rules(),
*jdk_rules.rules(),
*java_dep_inf_rules(),
*target_types_rules(),
*core_target_types_rules(),
*util_rules(),
*archive.rules(),
QueryRule(BuiltPackage, (PackageWarFileFieldSet,)),
],
target_types=[
JvmArtifactTarget,
JvmWarTarget,
FileTarget,
FilesGeneratorTarget,
RelocatedFiles,
],
)
rule_runner.set_options([], env_inherit=PYTHON_BOOTSTRAP_ENV)
return rule_runner
def test_infer_python_conftests() -> None:
rule_runner = RuleRunner(
rules=[
*ancestor_files.rules(),
*target_types_rules.rules(),
*core_target_types_rules(),
infer_python_conftest_dependencies,
SubsystemRule(PythonInferSubsystem),
QueryRule(InferredDependencies, (InferConftestDependencies,)),
],
target_types=[PythonTestsGeneratorTarget, PythonTestUtilsGeneratorTarget],
objects={"parametrize": Parametrize},
)
rule_runner.set_options(
[
"--source-root-patterns=src/python",
"--python-resolves={'a': '', 'b': ''}",
"--python-default-resolve=a",
"--python-enable-resolves",
],
env_inherit={"PATH", "PYENV_ROOT", "HOME"},
)
rule_runner.write_files(
{
"src/python/root/conftest.py": "",
"src/python/root/BUILD": "python_test_utils(resolve=parametrize('a', 'b'))",
"src/python/root/mid/conftest.py": "",
"src/python/root/mid/BUILD": "python_test_utils()",
"src/python/root/mid/leaf/conftest.py": "",
"src/python/root/mid/leaf/this_is_a_test.py": "",
"src/python/root/mid/leaf/BUILD": "python_test_utils()\npython_tests(name='tests')",
}
)
def run_dep_inference(address: Address) -> InferredDependencies:
target = rule_runner.get_target(address)
return rule_runner.request(
InferredDependencies,
[InferConftestDependencies(target[PythonSourceField])],
)
assert run_dep_inference(
Address(
"src/python/root/mid/leaf", target_name="tests", relative_file_path="this_is_a_test.py"
)
) == InferredDependencies(
[
Address(
"src/python/root", relative_file_path="conftest.py", parameters={"resolve": "a"}
),
Address("src/python/root/mid", relative_file_path="conftest.py"),
Address("src/python/root/mid/leaf", relative_file_path="conftest.py"),
],
)
def rule_runner() -> RuleRunner:
return RuleRunner(
rules=[
*package_pex_binary.rules(),
*pex_from_targets.rules(),
*target_types_rules.rules(),
*core_target_types_rules(),
QueryRule(BuiltPackage, [PexBinaryFieldSet]),
],
target_types=[PexBinary, Files, RelocatedFiles, Resources],
)
def test_infer_python_inits() -> None:
rule_runner = RuleRunner(
rules=[
*ancestor_files.rules(),
*target_types_rules.rules(),
*core_target_types_rules(),
infer_python_init_dependencies,
SubsystemRule(PythonInferSubsystem),
QueryRule(InferredDependencies, (InferInitDependencies, )),
],
target_types=[PythonSourcesGeneratorTarget],
)
rule_runner.set_options(
["--python-infer-inits", "--source-root-patterns=src/python"],
env_inherit={"PATH", "PYENV_ROOT", "HOME"},
)
rule_runner.write_files({
"src/python/root/__init__.py": "",
"src/python/root/BUILD": "python_sources()",
"src/python/root/mid/__init__.py": "",
"src/python/root/mid/BUILD": "python_sources()",
"src/python/root/mid/leaf/__init__.py": "",
"src/python/root/mid/leaf/f.py": "",
"src/python/root/mid/leaf/BUILD": "python_sources()",
"src/python/type_stub/__init__.pyi": "",
"src/python/type_stub/foo.pyi": "",
"src/python/type_stub/BUILD": "python_sources()",
})
def run_dep_inference(address: Address) -> InferredDependencies:
target = rule_runner.get_target(address)
return rule_runner.request(
InferredDependencies,
[InferInitDependencies(target[PythonSourceField])],
)
assert run_dep_inference(
Address(
"src/python/root/mid/leaf",
relative_file_path="f.py")) == InferredDependencies([
Address("src/python/root", relative_file_path="__init__.py"),
Address("src/python/root/mid",
relative_file_path="__init__.py"),
Address("src/python/root/mid/leaf",
relative_file_path="__init__.py"),
], )
assert run_dep_inference(
Address("src/python/type_stub",
relative_file_path="foo.pyi")) == InferredDependencies([
Address("src/python/type_stub",
relative_file_path="__init__.pyi")
])
def rule_runner() -> RuleRunner:
return RuleRunner(
rules=[
*awslambda_python_rules(),
*target_rules(),
*core_target_types_rules(),
QueryRule(BuiltPackage, (PythonAwsLambdaFieldSet, )),
],
target_types=[
PythonAWSLambda, PythonLibrary, Files, RelocatedFiles, Resources
],
)
def rule_runner() -> RuleRunner:
return RuleRunner(
rules=[
*context_rules(),
*core_target_types_rules(),
*package_pex_binary.rules(),
*pex_from_targets.rules(),
*target_types_rules.rules(),
QueryRule(BuiltPackage, [PexBinaryFieldSet]),
QueryRule(DockerBuildContext, (DockerBuildContextRequest, )),
],
target_types=[DockerImage, Files, PexBinary],
)
def rule_runner() -> RuleRunner:
return RuleRunner(
rules=[
*python_google_cloud_function_rules(),
*python_google_cloud_function_subsystem_rules(),
*target_rules(),
*python_target_types_rules(),
*core_target_types_rules(),
QueryRule(BuiltPackage, (PythonGoogleCloudFunctionFieldSet, )),
],
target_types=[
PythonGoogleCloudFunction, PythonLibrary, Files, RelocatedFiles,
Resources
],
)
def imports_rule_runner() -> RuleRunner:
return RuleRunner(
rules=[
*import_rules(),
*target_types_rules.rules(),
*core_target_types_rules(),
*python_requirements.rules(),
QueryRule(InferredDependencies, [InferPythonImportDependencies]),
],
target_types=[
PythonSourceTarget,
PythonSourcesGeneratorTarget,
PythonRequirementTarget,
PythonRequirementsTargetGenerator,
],
objects={"parametrize": Parametrize},
)
def rule_runner() -> RuleRunner:
return RuleRunner(
rules=[
*awslambda_python_rules(),
*awslambda_python_subsystem_rules(),
*target_rules(),
*python_target_types_rules(),
*core_target_types_rules(),
QueryRule(BuiltPackage, (PythonAwsLambdaFieldSet, )),
],
target_types=[
PythonAWSLambda,
PythonSourcesGeneratorTarget,
FilesGeneratorTarget,
RelocatedFiles,
ResourcesGeneratorTarget,
],
)
def rule_runner() -> RuleRunner:
rule_runner = RuleRunner(
rules=[
*core_target_types_rules(),
*coursier_fetch_rules(),
*lockfile.rules(),
*resources.rules(),
*classpath.rules(),
*util_rules(),
*testutil.rules(),
QueryRule(RenderedClasspath, (Addresses, )),
],
target_types=[
ResourcesGeneratorTarget,
ResourceTarget,
],
)
rule_runner.set_options(args=[], env_inherit=PYTHON_BOOTSTRAP_ENV)
return rule_runner
def rule_runner() -> RuleRunner:
rule_runner = RuleRunner(
rules=[
*package_pex_binary.rules(),
*pex_from_targets.rules(),
*target_types_rules.rules(),
*core_target_types_rules(),
QueryRule(BuiltPackage, [PexBinaryFieldSet]),
],
target_types=[
FileTarget,
FilesGeneratorTarget,
PexBinary,
PythonRequirementTarget,
PythonSourcesGeneratorTarget,
RelocatedFiles,
ResourcesGeneratorTarget,
],
)
rule_runner.set_options([], env_inherit={"PATH", "PYENV_ROOT", "HOME"})
return rule_runner
def chroot_rule_runner() -> RuleRunner:
return create_setup_py_rule_runner(
rules=[
*core_target_types_rules(),
determine_explicitly_provided_setup_kwargs,
generate_chroot,
generate_setup_py,
determine_finalized_setup_kwargs,
get_sources,
get_requirements,
get_owned_dependencies,
get_exporting_owner,
*python_sources.rules(),
*target_types_rules.rules(),
setup_kwargs_plugin,
SubsystemRule(SetupPyGeneration),
UnionRule(SetupKwargsRequest, PluginSetupKwargsRequest),
QueryRule(DistBuildChroot, (DistBuildChrootRequest,)),
QueryRule(DistBuildSources, (DistBuildChrootRequest,)),
QueryRule(FinalizedSetupKwargs, (GenerateSetupPyRequest,)),
]
)
def test_infer_python_imports(caplog) -> None:
rule_runner = RuleRunner(
rules=[
*import_rules(),
*target_types_rules.rules(),
*core_target_types_rules(),
QueryRule(InferredDependencies, [InferPythonImportDependencies]),
],
target_types=[PythonSourcesGeneratorTarget, PythonRequirementTarget],
)
rule_runner.write_files(
{
"3rdparty/python/BUILD": dedent(
"""\
python_requirement(
name='Django',
requirements=['Django==1.21'],
)
"""
),
# If there's a `.py` and `.pyi` file for the same module, we should infer a dependency on both.
"src/python/str_import/subdir/f.py": "",
"src/python/str_import/subdir/f.pyi": "",
"src/python/str_import/subdir/BUILD": "python_sources()",
"src/python/util/dep.py": "",
"src/python/util/BUILD": "python_sources()",
"src/python/app.py": dedent(
"""\
import django
import unrecognized.module
from util.dep import Demo
from util import dep
"""
),
"src/python/f2.py": dedent(
"""\
import typing
# Import from another file in the same target.
from app import main
# Dynamic string import.
importlib.import_module('str_import.subdir.f')
"""
),
"src/python/BUILD": "python_sources()",
}
)
def run_dep_inference(
address: Address, *, enable_string_imports: bool = False
) -> InferredDependencies:
args = ["--source-root-patterns=src/python"]
if enable_string_imports:
args.append("--python-infer-string-imports")
rule_runner.set_options(args, env_inherit={"PATH", "PYENV_ROOT", "HOME"})
target = rule_runner.get_target(address)
return rule_runner.request(
InferredDependencies, [InferPythonImportDependencies(target[PythonSourceField])]
)
assert run_dep_inference(
Address("src/python", relative_file_path="app.py")
) == InferredDependencies(
[
Address("3rdparty/python", target_name="Django"),
Address("src/python/util", relative_file_path="dep.py"),
],
)
addr = Address("src/python", relative_file_path="f2.py")
assert run_dep_inference(addr) == InferredDependencies(
[Address("src/python", relative_file_path="app.py")]
)
assert run_dep_inference(addr, enable_string_imports=True) == InferredDependencies(
[
Address("src/python", relative_file_path="app.py"),
Address("src/python/str_import/subdir", relative_file_path="f.py"),
Address("src/python/str_import/subdir", relative_file_path="f.pyi"),
],
)
# Test handling of ambiguous imports. We should warn on the ambiguous dependency, but not warn
# on the disambiguated one and should infer a dep.
caplog.clear()
rule_runner.write_files(
{
"src/python/ambiguous/dep.py": "",
"src/python/ambiguous/disambiguated_via_ignores.py": "",
"src/python/ambiguous/main.py": (
"import ambiguous.dep\nimport ambiguous.disambiguated_via_ignores\n"
),
"src/python/ambiguous/BUILD": dedent(
"""\
python_sources(name='dep1', sources=['dep.py', 'disambiguated_via_ignores.py'])
python_sources(name='dep2', sources=['dep.py', 'disambiguated_via_ignores.py'])
python_sources(
name='main',
sources=['main.py'],
dependencies=['!./disambiguated_via_ignores.py:dep2'],
)
"""
),
}
)
assert run_dep_inference(
Address("src/python/ambiguous", target_name="main", relative_file_path="main.py")
) == InferredDependencies(
[
Address(
"src/python/ambiguous",
target_name="dep1",
relative_file_path="disambiguated_via_ignores.py",
)
],
)
assert len(caplog.records) == 1
assert "The target src/python/ambiguous/main.py:main imports `ambiguous.dep`" in caplog.text
assert "['src/python/ambiguous/dep.py:dep1', 'src/python/ambiguous/dep.py:dep2']" in caplog.text
assert "disambiguated_via_ignores.py" not in caplog.text
def test_infer_python_inits(behavior: InitFilesInference) -> None:
rule_runner = RuleRunner(
rules=[
*ancestor_files.rules(),
*target_types_rules.rules(),
*core_target_types_rules(),
infer_python_init_dependencies,
SubsystemRule(PythonInferSubsystem),
QueryRule(InferredDependencies, (InferInitDependencies,)),
],
target_types=[PythonSourcesGeneratorTarget],
objects={"parametrize": Parametrize},
)
rule_runner.set_options(
[
f"--python-infer-init-files={behavior.value}",
"--python-resolves={'a': '', 'b': ''}",
"--python-default-resolve=a",
"--python-enable-resolves",
],
env_inherit=PYTHON_BOOTSTRAP_ENV,
)
rule_runner.write_files(
{
"src/python/root/__init__.py": "content",
"src/python/root/BUILD": "python_sources(resolve=parametrize('a', 'b'))",
"src/python/root/mid/__init__.py": "",
"src/python/root/mid/BUILD": "python_sources()",
"src/python/root/mid/leaf/__init__.py": "content",
"src/python/root/mid/leaf/f.py": "",
"src/python/root/mid/leaf/BUILD": "python_sources()",
"src/python/type_stub/__init__.pyi": "content",
"src/python/type_stub/foo.pyi": "",
"src/python/type_stub/BUILD": "python_sources()",
}
)
def check(address: Address, expected: list[Address]) -> None:
target = rule_runner.get_target(address)
result = rule_runner.request(
InferredDependencies,
[InferInitDependencies(target[PythonSourceField])],
)
if behavior == InitFilesInference.never:
assert not result
else:
assert result == InferredDependencies(expected)
check(
Address("src/python/root/mid/leaf", relative_file_path="f.py"),
[
Address(
"src/python/root", relative_file_path="__init__.py", parameters={"resolve": "a"}
),
*(
[]
if behavior is InitFilesInference.content_only
else [Address("src/python/root/mid", relative_file_path="__init__.py")]
),
Address("src/python/root/mid/leaf", relative_file_path="__init__.py"),
],
)
check(
Address("src/python/type_stub", relative_file_path="foo.pyi"),
[Address("src/python/type_stub", relative_file_path="__init__.pyi")],
)
def test_infer_python_assets(caplog) -> None:
rule_runner = RuleRunner(
rules=[
*import_rules(),
*target_types_rules.rules(),
*core_target_types_rules(),
QueryRule(InferredDependencies, [InferPythonImportDependencies]),
],
target_types=[
PythonSourcesGeneratorTarget,
PythonRequirementTarget,
ResourcesGeneratorTarget,
FilesGeneratorTarget,
],
)
rule_runner.write_files(
{
"src/python/data/BUILD": "resources(name='jsonfiles', sources=['*.json'])",
"src/python/data/db.json": "",
"src/python/data/db2.json": "",
"src/python/data/flavors.txt": "",
"configs/prod.txt": "",
"src/python/app.py": dedent(
"""\
pkgutil.get_data(__name__, "data/db.json")
pkgutil.get_data(__name__, "data/db2.json")
open("configs/prod.txt")
"""
),
"src/python/f.py": dedent(
"""\
idk_kinda_looks_resourcey = "data/db.json"
CustomResourceType("data/flavors.txt")
"""
),
"src/python/BUILD": dedent(
"""\
python_sources()
# Also test assets declared from parent dir
resources(
name="txtfiles",
sources=["data/*.txt"],
)
"""
),
"configs/BUILD": dedent(
"""\
files(
name="configs",
sources=["prod.txt"],
)
"""
),
}
)
def run_dep_inference(address: Address) -> InferredDependencies:
args = [
"--source-root-patterns=src/python",
"--python-infer-assets",
]
rule_runner.set_options(args, env_inherit={"PATH", "PYENV_ROOT", "HOME"})
target = rule_runner.get_target(address)
return rule_runner.request(
InferredDependencies, [InferPythonImportDependencies(target[PythonSourceField])]
)
assert run_dep_inference(
Address("src/python", relative_file_path="app.py")
) == InferredDependencies(
[
Address("src/python/data", target_name="jsonfiles", relative_file_path="db.json"),
Address("src/python/data", target_name="jsonfiles", relative_file_path="db2.json"),
Address("configs", target_name="configs", relative_file_path="prod.txt"),
],
)
assert run_dep_inference(
Address("src/python", relative_file_path="f.py")
) == InferredDependencies(
[
Address("src/python/data", target_name="jsonfiles", relative_file_path="db.json"),
Address("src/python", target_name="txtfiles", relative_file_path="data/flavors.txt"),
],
)
# Test handling of ambiguous assets. We should warn on the ambiguous dependency, but not warn
# on the disambiguated one and should infer a dep.
caplog.clear()
rule_runner.write_files(
{
"src/python/data/BUILD": dedent(
"""\
resources(name='jsonfiles', sources=['*.json'])
resources(name='also_jsonfiles', sources=['*.json'])
resources(name='txtfiles', sources=['*.txt'])
"""
),
"src/python/data/ambiguous.json": "",
"src/python/data/disambiguated_with_bang.json": "",
"src/python/app.py": dedent(
"""\
pkgutil.get_data(__name__, "data/ambiguous.json")
pkgutil.get_data(__name__, "data/disambiguated_with_bang.json")
"""
),
"src/python/BUILD": dedent(
"""\
python_sources(
name="main",
dependencies=['!./data/disambiguated_with_bang.json:also_jsonfiles'],
)
"""
),
# Both a resource relative to the module and file with conspicuously similar paths
"src/python/data/both_file_and_resource.txt": "",
"data/both_file_and_resource.txt": "",
"data/BUILD": "files(name='txtfiles', sources=['*.txt'])",
"src/python/assets_bag.py": "ImAPathType('data/both_file_and_resource.txt')",
}
)
assert run_dep_inference(
Address("src/python", target_name="main", relative_file_path="app.py")
) == InferredDependencies(
[
Address(
"src/python/data",
target_name="jsonfiles",
relative_file_path="disambiguated_with_bang.json",
),
],
)
assert len(caplog.records) == 1
assert "The target src/python/app.py:main uses `data/ambiguous.json`" in caplog.text
assert (
"['src/python/data/ambiguous.json:also_jsonfiles', 'src/python/data/ambiguous.json:jsonfiles']"
in caplog.text
)
assert "disambiguated_with_bang.py" not in caplog.text
caplog.clear()
assert run_dep_inference(
Address("src/python", target_name="main", relative_file_path="assets_bag.py")
) == InferredDependencies([])
assert len(caplog.records) == 1
assert (
"The target src/python/assets_bag.py:main uses `data/both_file_and_resource.txt`"
in caplog.text
)
assert (
"['data/both_file_and_resource.txt:txtfiles', 'src/python/data/both_file_and_resource.txt:txtfiles']"
in caplog.text
)
def test_infer_python_strict(caplog) -> None:
rule_runner = RuleRunner(
rules=[
*import_rules(),
*target_types_rules.rules(),
*core_target_types_rules(),
*python_requirements.rules(),
QueryRule(InferredDependencies, [InferPythonImportDependencies]),
],
target_types=[
PythonSourcesGeneratorTarget,
PythonRequirementTarget,
PythonRequirementsTargetGenerator,
],
)
rule_runner.write_files({
"src/python/cheesey.py":
dedent("""\
import venezuelan_beaver_cheese
"japanese.sage.derby"
"""),
"src/python/BUILD":
"python_sources()",
})
def run_dep_inference(
address: Address,
unowned_dependency_behavior: str,
) -> InferredDependencies:
rule_runner.set_options(
[
f"--python-infer-unowned-dependency-behavior={unowned_dependency_behavior}",
"--python-infer-string-imports",
"--source-root-patterns=src/python",
],
env_inherit={"PATH", "PYENV_ROOT", "HOME"},
)
target = rule_runner.get_target(address)
return rule_runner.request(
InferredDependencies,
[InferPythonImportDependencies(target[PythonSourceField])],
)
# First test with "warning"
run_dep_inference(Address("src/python", relative_file_path="cheesey.py"),
"warning")
assert len(caplog.records) == 1
assert "The following imports in src/python/cheesey.py have no owners:" in caplog.text
assert " * venezuelan_beaver_cheese (src/python/cheesey.py:1)" in caplog.text
assert "japanese.sage.derby" not in caplog.text
# Now test with "error"
caplog.clear()
with pytest.raises(ExecutionError) as exc_info:
run_dep_inference(
Address("src/python", relative_file_path="cheesey.py"), "error")
assert isinstance(exc_info.value.wrapped_exceptions[0],
UnownedDependencyError)
assert len(caplog.records
) == 2 # one for the error being raised and one for our message
assert "The following imports in src/python/cheesey.py have no owners:" in caplog.text
assert " * venezuelan_beaver_cheese (src/python/cheesey.py:1)" in caplog.text
assert "japanese.sage.derby" not in caplog.text
caplog.clear()
# All modes should be fine if the module is explicitly declared as a requirement
rule_runner.write_files({
"src/python/BUILD":
dedent("""\
python_requirement(
name="venezuelan_beaver_cheese",
modules=["venezuelan_beaver_cheese"],
requirements=["venezuelan_beaver_cheese==1.0.0"],
)
python_sources(dependencies=[":venezuelan_beaver_cheese"])
"""),
})
for mode in UnownedDependencyUsage:
run_dep_inference(
Address("src/python", relative_file_path="cheesey.py"), mode.value)
assert not caplog.records
# All modes should be fine if the module is implictly found via requirements.txt
rule_runner.write_files({
"src/python/requirements.txt":
"venezuelan_beaver_cheese==1.0.0",
"src/python/BUILD":
dedent("""\
python_requirements(name='reqs')
python_sources()
"""),
})
for mode in UnownedDependencyUsage:
run_dep_inference(
Address("src/python", relative_file_path="cheesey.py"), mode.value)
assert not caplog.records
# All modes should be fine if the module is owned by a first party
rule_runner.write_files({
"src/python/venezuelan_beaver_cheese.py": "",
"src/python/BUILD": "python_sources()",
})
for mode in UnownedDependencyUsage:
run_dep_inference(
Address("src/python", relative_file_path="cheesey.py"), mode.value)
assert not caplog.records
