def test_no_include_trace_error_multiple_paths_raises_executionerror(self): rules = [nested_raise, QueryRule(A, (B, ))] scheduler = self.scheduler(rules, include_trace_on_error=False) with self.assertRaises(ExecutionError) as cm: list(scheduler.product_request(A, subjects=[B(), B()])) self.assert_equal_with_printing( dedent(""" 2 Exceptions encountered: Exception: An exception for B Exception: An exception for B """).lstrip(), str(cm.exception), )
def rule_runner() -> RuleRunner: return RuleRunner( rules=[ *awslambda_python_rules(), *awslambda_python_subsystem_rules(), *target_rules(), *python_target_types_rules(), *core_target_types_rules(), QueryRule(BuiltPackage, (PythonAwsLambdaFieldSet, )), ], target_types=[ PythonAWSLambda, PythonLibrary, Files, RelocatedFiles, Resources ], )
def test_generate_source_targets() -> None: rule_runner = RuleRunner( rules=[ *target_types.rules(), QueryRule(_TargetParametrizations, [_TargetParametrizationsRequest]), ], target_types=[AvroSourcesGeneratorTarget], ) rule_runner.write_files({ "src/avro/BUILD": dedent("""\ avro_sources( name='lib', sources=['**/*.avsc', '**/*.avpr'], overrides={'f1.avsc': {'tags': ['overridden']}}, ) """), "src/avro/f1.avsc": "", "src/avro/f2.avpr": "", "src/avro/subdir/f.avsc": "", }) def gen_tgt(rel_fp: str, tags: list[str] | None = None) -> AvroSourceTarget: return AvroSourceTarget( { SingleSourceField.alias: rel_fp, Tags.alias: tags }, Address("src/avro", target_name="lib", relative_file_path=rel_fp), residence_dir=os.path.dirname(os.path.join("src/avro", rel_fp)), ) generated = rule_runner.request( _TargetParametrizations, [ _TargetParametrizationsRequest(Address("src/avro", target_name="lib"), description_of_origin="tests") ], ).parametrizations assert set(generated.values()) == { gen_tgt("f1.avsc", tags=["overridden"]), gen_tgt("f2.avpr"), gen_tgt("subdir/f.avsc"), }
def rule_runner() -> RuleRunner: rule_runner = RuleRunner( rules=[ *rules(), *docker_binary.rules(), QueryRule(PublishProcesses, [PublishDockerImageRequest]), QueryRule(DockerBinary, []), ], target_types=[DockerImageTarget], ) rule_runner.set_options( [], env_inherit={"PATH", "PYENV_ROOT", "HOME"}, ) rule_runner.write_files({ "src/default/BUILD": """docker_image()""", "src/skip-test/BUILD": """docker_image(skip_push=True)""", "src/registries/BUILD": """docker_image(registries=["@inhouse1", "@inhouse2"])""", }) return rule_runner
def rule_runner() -> RuleRunner: return RuleRunner( rules=[ format_python_target, *black_rules(), *isort_rules(), *black_subsystem_rules(), *isort_subsystem_rules(), *source_files.rules(), *config_files.rules(), QueryRule(LanguageFmtResults, (PythonFmtTargets, )), ], target_types=[PythonLibrary], )
def test_build_file_address() -> None: rule_runner = RuleRunner(rules=[QueryRule(BuildFileAddress, (Address, ))], target_types=[MockTgt]) rule_runner.create_file("helloworld/BUILD.ext", "mock_tgt()") def assert_bfa_resolved(address: Address) -> None: expected_bfa = BuildFileAddress(rel_path="helloworld/BUILD.ext", address=address) bfa = rule_runner.request(BuildFileAddress, [address]) assert bfa == expected_bfa assert_bfa_resolved(Address("helloworld")) # File addresses should use their base target to find the BUILD file. assert_bfa_resolved(Address("helloworld", relative_file_path="f.txt"))
def rule_runner() -> RuleRunner: return RuleRunner( rules=[ *package_pex_binary.rules(), *pex_from_targets.rules(), *target_types_rules.rules(), *core_target_types_rules(), QueryRule(BuiltPackage, [PexBinaryFieldSet]), ], target_types=[ PexBinary, FilesGeneratorTarget, RelocatedFiles, ResourcesGeneratorTarget ], )
def rules(): return [ QueryRule(PexPEX, ()), QueryRule(Pex, (PexRequest, )), QueryRule(VenvPex, (PexRequest, )), QueryRule(Process, (PexProcess, )), QueryRule(Process, (VenvPexProcess, )), QueryRule(ProcessResult, (Process, )), ]
def rule_runner() -> RuleRunner: rule_runner = RuleRunner( preserve_tmpdirs=True, rules=[ *classpath.rules(), *config_files.rules(), *coursier_fetch_rules(), *coursier_setup_rules(), *external_tool_rules(), *java_util_rules(), *javac_rules(), *junit_rules(), *scala_target_types_rules(), *scalac_rules(), *source_files.rules(), *target_types_rules(), *util_rules(), *non_jvm_dependencies_rules(), QueryRule(CoarsenedTargets, (Addresses,)), QueryRule(TestResult, (JunitTestFieldSet,)), ], target_types=[ FileTarget, FilesGeneratorTarget, RelocatedFiles, JvmArtifactTarget, JavaSourcesGeneratorTarget, JunitTestsGeneratorTarget, ScalaJunitTestsGeneratorTarget, ], ) rule_runner.set_options( # Makes JUnit output predictable and parseable across versions (#12933): args=["--junit-args=['--disable-ansi-colors','--details=flat','--details-theme=ascii']"], env_inherit=PYTHON_BOOTSTRAP_ENV, ) return rule_runner
def rule_runner() -> RuleRunner: return RuleRunner( rules=[ QueryRule(Digest, [CreateDigest]), QueryRule(DigestContents, [PathGlobs]), QueryRule(DigestEntries, [Digest]), QueryRule(DigestEntries, [PathGlobs]), QueryRule(Snapshot, [CreateDigest]), QueryRule(Snapshot, [DigestSubset]), QueryRule(Snapshot, [PathGlobs]), ], isolated_local_store=True, )
def rule_runner() -> RuleRunner: rule_runner = RuleRunner( rules=[ *dockerfile_rules(), *parser_rules(), *pex_rules(), QueryRule(DockerfileInfo, (DockerfileInfoRequest, )), ], target_types=[DockerImageTarget, PexBinary], ) rule_runner.set_options( [], env_inherit={"PATH", "PYENV_ROOT", "HOME"}, ) return rule_runner
def rule_runner() -> RuleRunner: return RuleRunner( rules=[ *config_files.rules(), *coursier_fetch_rules(), *coursier_setup_rules(), *external_tool_rules(), *source_files.rules(), *javac_rules(), *util_rules(), *target_types_rules(), *coursier_rules(), *java_util_rules.rules(), QueryRule(CheckResults, (JavacCheckRequest,)), QueryRule(FallibleCompiledClassfiles, (CompileJavaSourceRequest,)), QueryRule(CompiledClassfiles, (CompileJavaSourceRequest,)), QueryRule(CoarsenedTargets, (Addresses,)), ], target_types=[JvmDependencyLockfile, JavaSourcesGeneratorTarget, JvmArtifact], # TODO(#12293): use a fixed JDK version. bootstrap_args=[ "--javac-jdk=system", ], )
def rule_runner() -> RuleRunner: rule_runner = RuleRunner( rules=[ *archive.rules(), *shell_command_rules(), *source_files.rules(), *core_target_type_rules(), QueryRule(GeneratedSources, [GenerateFilesFromShellCommandRequest]), QueryRule(Process, [ShellCommandProcessRequest]), QueryRule(RunRequest, [RunShellCommand]), QueryRule(SourceFiles, [SourceFilesRequest]), QueryRule(TransitiveTargets, [TransitiveTargetsRequest]), ], target_types=[ ShellCommandTarget, ShellCommandRunTarget, ShellSourcesGeneratorTarget, ArchiveTarget, FilesGeneratorTarget, ], ) rule_runner.set_options([], env_inherit={"PATH"}) return rule_runner
def rule_runner() -> RuleRunner: rule_runner = RuleRunner( rules=[ *config_files.rules(), *coursier_fetch_rules(), *coursier_setup_rules(), *dep_inference_rules(), *external_tool_rules(), *java_target_rules(), *java_util_rules(), *javac_rules(), *source_files.rules(), *util_rules(), QueryRule(Addresses, [DependenciesRequest]), QueryRule(ThirdPartyPackageToArtifactMapping, []), ], target_types=[ JavaSourcesGeneratorTarget, JunitTestsGeneratorTarget, JvmArtifact ], ) rule_runner.set_options( args=[NAMED_RESOLVE_OPTIONS, DEFAULT_RESOLVE_OPTION], env_inherit=PYTHON_BOOTSTRAP_ENV) return rule_runner
def rule_runner() -> RuleRunner: rule_runner = RuleRunner( rules=[ *parser_rules(), *pex_rules(), inject_docker_dependencies, QueryRule(InjectedDependencies, (InjectDockerDependencies, )), ], target_types=[DockerImageTarget, PexBinary], ) rule_runner.set_options( [], env_inherit={"PATH", "PYENV_ROOT", "HOME"}, ) return rule_runner
def rule_runner() -> RuleRunner: return RuleRunner( rules=[ *python_google_cloud_function_rules(), *python_google_cloud_function_subsystem_rules(), *target_rules(), *python_target_types_rules(), *core_target_types_rules(), QueryRule(BuiltPackage, (PythonGoogleCloudFunctionFieldSet, )), ], target_types=[ PythonGoogleCloudFunction, PythonLibrary, Files, RelocatedFiles, Resources ], )
def rule_runner() -> RuleRunner: rule_runner = RuleRunner( rules=[ *coursier_fetch_rules(), *jdk_rules.rules(), *scalac_check_rules(), *scalac_rules(), *source_files.rules(), *target_types_rules(), *testutil.rules(), *util_rules(), *scala_dep_inf_rules(), QueryRule(CheckResults, (ScalacCheckRequest, )), QueryRule(CoarsenedTargets, (Addresses, )), QueryRule(FallibleClasspathEntry, (CompileScalaSourceRequest, )), QueryRule(RenderedClasspath, (CompileScalaSourceRequest, )), QueryRule(ClasspathEntry, (CompileScalaSourceRequest, )), ], target_types=[ JvmArtifactTarget, ScalaSourcesGeneratorTarget, ScalacPluginTarget ], ) rule_runner.set_options(args=[], env_inherit=PYTHON_BOOTSTRAP_ENV) return rule_runner
def rule_runner() -> RuleRunner: return RuleRunner( preserve_tmpdirs=True, rules=[ *coursier_fetch_rules(), *coursier_setup_rules(), *external_tool_rules(), *java_parser_launcher_rules(), *java_parser_rules(), *javac_rules(), *source_files.rules(), *util_rules(), *java_util_rules.rules(), QueryRule(FallibleJavaSourceDependencyAnalysisResult, (SourceFiles, )), QueryRule(JavaSourceDependencyAnalysis, (SourceFiles, )), QueryRule(SourceFiles, (SourceFilesRequest, )), ], target_types=[JvmDependencyLockfile, JavaSourceTarget], # TODO(#12293): use a fixed JDK version. bootstrap_args=[ "--javac-jdk=system", ], )
def chroot_rule_runner() -> RuleRunner: return create_setup_py_rule_runner(rules=[ determine_setup_kwargs, generate_chroot, get_sources, get_requirements, get_owned_dependencies, get_exporting_owner, *python_sources.rules(), *target_types_rules.rules(), setup_kwargs_plugin, SubsystemRule(SetupPyGeneration), UnionRule(SetupKwargsRequest, PluginSetupKwargsRequest), QueryRule(SetupPyChroot, (SetupPyChrootRequest, )), ])
def test_build_file_address() -> None: rule_runner = RuleRunner( rules=[QueryRule(BuildFileAddress, (Address,))], target_types=[MockTgt] ) rule_runner.write_files({"helloworld/BUILD.ext": "mock_tgt()"}) def assert_bfa_resolved(address: Address) -> None: expected_bfa = BuildFileAddress(address, "helloworld/BUILD.ext") bfa = rule_runner.request(BuildFileAddress, [address]) assert bfa == expected_bfa assert_bfa_resolved(Address("helloworld")) # Generated targets should use their target generator's BUILD file. assert_bfa_resolved(Address("helloworld", generated_name="f.txt")) assert_bfa_resolved(Address("helloworld", relative_file_path="f.txt"))
def test_resolve_pex_binary_entry_point() -> None: rule_runner = RuleRunner( rules=[ resolve_pex_entry_point, QueryRule(ResolvedPexEntryPoint, [ResolvePexEntryPointRequest]), ] ) def assert_resolved( *, entry_point: str | None, expected: EntryPoint | None, is_file: bool ) -> None: addr = Address("src/python/project") rule_runner.write_files( { "src/python/project/app.py": "", "src/python/project/f2.py": "", } ) ep_field = PexEntryPointField(entry_point, addr) result = rule_runner.request(ResolvedPexEntryPoint, [ResolvePexEntryPointRequest(ep_field)]) assert result.val == expected assert result.file_name_used == is_file # Full module provided. assert_resolved( entry_point="custom.entry_point", expected=EntryPoint("custom.entry_point"), is_file=False ) assert_resolved( entry_point="custom.entry_point:func", expected=EntryPoint.parse("custom.entry_point:func"), is_file=False, ) # File names are expanded into the full module path. assert_resolved(entry_point="app.py", expected=EntryPoint(module="project.app"), is_file=True) assert_resolved( entry_point="app.py:func", expected=EntryPoint(module="project.app", function="func"), is_file=True, ) with pytest.raises(ExecutionError): assert_resolved( entry_point="doesnt_exist.py", expected=EntryPoint("doesnt matter"), is_file=True ) # Resolving >1 file is an error. with pytest.raises(ExecutionError): assert_resolved(entry_point="*.py", expected=EntryPoint("doesnt matter"), is_file=True)
def test_generate_source_targets() -> None: rule_runner = RuleRunner( rules=[ *target_types.rules(), QueryRule(_TargetParametrizations, [_TargetParametrizationsRequest]), ], target_types=[WsdlSourcesGeneratorTarget], ) source_root = "src/wsdl" rule_runner.write_files({ f"{source_root}/BUILD": dedent("""\ wsdl_sources( name="lib", sources=["**/*.wsdl"] ) """), f"{source_root}/f1.wsdl": "", f"{source_root}/sub/f2.wsdl": "", }) def gen_tgt(rel_fp: str, tags: list[str] | None = None) -> WsdlSourceTarget: return WsdlSourceTarget( { SingleSourceField.alias: rel_fp, Tags.alias: tags }, Address(source_root, target_name="lib", relative_file_path=rel_fp), residence_dir=os.path.dirname(os.path.join(source_root, rel_fp)), ) generated = rule_runner.request( _TargetParametrizations, [ _TargetParametrizationsRequest(Address(source_root, target_name="lib"), description_of_origin="tests") ], ).parametrizations assert set(generated.values()) == { gen_tgt("f1.wsdl"), gen_tgt("sub/f2.wsdl"), }
def test_generate_source_targets() -> None: rule_runner = RuleRunner( rules=[ *target_types.rules(), QueryRule(GeneratedTargets, [GenerateTargetsFromProtobufSources]), ], target_types=[ProtobufSourcesGeneratorTarget], ) rule_runner.write_files({ "src/proto/BUILD": dedent("""\ protobuf_sources( name='lib', sources=['**/*.proto'], overrides={'f1.proto': {'tags': ['overridden']}}, ) """), "src/proto/f1.proto": "", "src/proto/f2.proto": "", "src/proto/subdir/f.proto": "", }) generator = rule_runner.get_target(Address("src/proto", target_name="lib")) def gen_tgt(rel_fp: str, tags: list[str] | None = None) -> ProtobufSourceTarget: return ProtobufSourceTarget( { SingleSourceField.alias: rel_fp, Tags.alias: tags }, Address("src/proto", target_name="lib", relative_file_path=rel_fp), residence_dir=os.path.dirname(os.path.join("src/proto", rel_fp)), ) generated = rule_runner.request( GeneratedTargets, [GenerateTargetsFromProtobufSources(generator)]) assert generated == GeneratedTargets( generator, { gen_tgt("f1.proto", tags=["overridden"]), gen_tgt("f2.proto"), gen_tgt("subdir/f.proto"), }, )
def test_infer_python_conftests() -> None: rule_runner = RuleRunner( rules=[ *ancestor_files.rules(), *target_types_rules.rules(), infer_python_conftest_dependencies, SubsystemRule(PythonInferSubsystem), QueryRule(InferredDependencies, (InferConftestDependencies, )), ], target_types=[ PythonTestsGeneratorTarget, PythonTestUtilsGeneratorTarget ], ) rule_runner.set_options( ["--source-root-patterns=src/python"], env_inherit={"PATH", "PYENV_ROOT", "HOME"}, ) rule_runner.create_file("src/python/root/conftest.py") rule_runner.add_to_build_file("src/python/root", "python_test_utils()") rule_runner.create_file("src/python/root/mid/conftest.py") rule_runner.add_to_build_file("src/python/root/mid", "python_test_utils()") rule_runner.create_file("src/python/root/mid/leaf/conftest.py") rule_runner.create_file("src/python/root/mid/leaf/this_is_a_test.py") rule_runner.add_to_build_file( "src/python/root/mid/leaf", "python_test_utils()\npython_tests(name='tests')") def run_dep_inference(address: Address) -> InferredDependencies: target = rule_runner.get_target(address) return rule_runner.request( InferredDependencies, [InferConftestDependencies(target[PythonSourceField])], ) assert run_dep_inference( Address( "src/python/root/mid/leaf", target_name="tests", relative_file_path="this_is_a_test.py")) == InferredDependencies([ Address("src/python/root", relative_file_path="conftest.py"), Address("src/python/root/mid", relative_file_path="conftest.py"), Address("src/python/root/mid/leaf", relative_file_path="conftest.py"), ], )
def test_infer_python_inits() -> None: rule_runner = RuleRunner( rules=[ *ancestor_files.rules(), *target_types_rules.rules(), infer_python_init_dependencies, SubsystemRule(PythonInferSubsystem), QueryRule(InferredDependencies, (InferInitDependencies, )), ], target_types=[PythonSourcesGeneratorTarget], ) rule_runner.set_options( [ "--backend-packages=pants.backend.python", "--python-infer-inits", "--source-root-patterns=src/python", ], env_inherit={"PATH", "PYENV_ROOT", "HOME"}, ) rule_runner.create_file("src/python/root/__init__.py") rule_runner.add_to_build_file("src/python/root", "python_sources()") rule_runner.create_file("src/python/root/mid/__init__.py") rule_runner.add_to_build_file("src/python/root/mid", "python_sources()") rule_runner.create_file("src/python/root/mid/leaf/__init__.py") rule_runner.create_file("src/python/root/mid/leaf/f.py") rule_runner.add_to_build_file("src/python/root/mid/leaf", "python_sources()") def run_dep_inference(address: Address) -> InferredDependencies: target = rule_runner.get_target(address) return rule_runner.request( InferredDependencies, [InferInitDependencies(target[PythonSourceField])], ) assert run_dep_inference( Address( "src/python/root/mid/leaf", relative_file_path="f.py")) == InferredDependencies([ Address("src/python/root", relative_file_path="__init__.py"), Address("src/python/root/mid", relative_file_path="__init__.py"), Address("src/python/root/mid/leaf", relative_file_path="__init__.py"), ], )
def rule_runner() -> RuleRunner: return RuleRunner(rules=[ *pex_rules(), QueryRule(Pex, (PexRequest, )), QueryRule(VenvPex, (PexRequest, )), QueryRule(Process, (PexProcess, )), QueryRule(Process, (VenvPexProcess, )), QueryRule(ProcessResult, (Process, )), QueryRule(PexResolveInfo, (VenvPex, )), ])
def test_use_params() -> None: rule_runner = RuleRunner(rules=[consumes_a_and_b, QueryRule(str, [A, B])]) # Confirm that we can pass in Params in order to provide multiple inputs to an execution. a, b = A(), B() result_str = rule_runner.request(str, [a, b]) assert result_str == consumes_a_and_b(a, b) # And confirm that a superset of Params is also accepted. result_str = rule_runner.request(str, [a, b, b"bytes aren't used by any rules"]) assert result_str == consumes_a_and_b(a, b) # But not a subset. expected_msg = "No installed QueryRules can compute str given input Params(A), but" with pytest.raises(Exception, match=re.escape(expected_msg)): rule_runner.request(str, [a])
def test_counters(self) -> None: @dataclass(frozen=True) class TrueResult: pass @rule(desc="a_rule") async def a_rule() -> TrueResult: proc = Process( ["/bin/sh", "-c", "true"], description="always true", ) _ = await Get(ProcessResult, MultiPlatformProcess({None: proc})) return TrueResult() scheduler, tracker, handler = self._fixture_for_rules( [ a_rule, QueryRule(TrueResult, tuple()), *process_rules(), *platform_rules() ], max_workunit_verbosity=LogLevel.TRACE, ) with handler.session(): scheduler.record_test_observation(128) scheduler.product_request(TrueResult, subjects=[0]) histograms_info = scheduler.get_observation_histograms() finished = list( itertools.chain.from_iterable(tracker.finished_workunit_chunks)) workunits_with_counters = [ item for item in finished if "counters" in item ] assert workunits_with_counters[0]["counters"][ "local_execution_requests"] == 1 assert workunits_with_counters[1]["counters"][ "local_cache_requests"] == 1 assert workunits_with_counters[1]["counters"][ "local_cache_requests_uncached"] == 1 assert histograms_info["version"] == 0 assert "histograms" in histograms_info assert "test_observation" in histograms_info["histograms"] assert ( histograms_info["histograms"]["test_observation"] == b"\x1c\x84\x93\x14\x00\x00\x00\x1fx\x9c\x93i\x99,\xcc\xc0\xc0\xc0\xcc\x00\x010\x9a\x11J3\xd9\x7f\x800\xfe32\x01\x00E\x0c\x03\x81" )
def test_infer_python_conftests() -> None: rule_runner = RuleRunner( rules=[ *ancestor_files.rules(), infer_python_conftest_dependencies, SubsystemRule(PythonInferSubsystem), QueryRule(InferredDependencies, (InferConftestDependencies, )), ], target_types=[PythonTests], ) rule_runner.set_options( [ "--backend-packages=pants.backend.python", "--source-root-patterns=src/python" ], env_inherit={"PATH", "PYENV_ROOT", "HOME"}, ) rule_runner.create_file("src/python/root/conftest.py") rule_runner.add_to_build_file("src/python/root", "python_tests()") rule_runner.create_file("src/python/root/mid/conftest.py") rule_runner.add_to_build_file("src/python/root/mid", "python_tests()") rule_runner.create_file("src/python/root/mid/leaf/conftest.py") rule_runner.create_file("src/python/root/mid/leaf/this_is_a_test.py") rule_runner.add_to_build_file("src/python/root/mid/leaf", "python_tests()") def run_dep_inference(address: Address) -> InferredDependencies: target = rule_runner.get_target(address) return rule_runner.request( InferredDependencies, [InferConftestDependencies(target[PythonSources])], ) assert run_dep_inference( Address("src/python/root/mid/leaf")) == InferredDependencies( [ Address("src/python/root", relative_file_path="conftest.py", target_name="root"), Address("src/python/root/mid", relative_file_path="conftest.py", target_name="mid"), ], sibling_dependencies_inferrable=False, )
def imports_rule_runner() -> RuleRunner: return RuleRunner( rules=[ *import_rules(), *target_types_rules.rules(), *core_target_types_rules(), *python_requirements.rules(), QueryRule(InferredDependencies, [InferPythonImportDependencies]), ], target_types=[ PythonSourceTarget, PythonSourcesGeneratorTarget, PythonRequirementTarget, PythonRequirementsTargetGenerator, ], objects={"parametrize": Parametrize}, )