def __init__(
self,
*,
ignore_patterns: list[str],
use_gitignore: bool,
build_root: str,
local_execution_root_dir: str,
named_caches_dir: str,
ca_certs_path: str | None,
rules: Iterable[Rule],
union_membership: UnionMembership,
execution_options: ExecutionOptions,
local_store_options: LocalStoreOptions,
executor: PyExecutor,
include_trace_on_error: bool = True,
visualize_to_dir: str | None = None,
validate_reachability: bool = True,
watch_filesystem: bool = True,
) -> None:
"""
:param ignore_patterns: A list of gitignore-style file patterns for pants to ignore.
:param use_gitignore: If set, pay attention to .gitignore files.
:param build_root: The build root as a string.
:param local_execution_root_dir: The directory to use for local execution sandboxes.
:param named_caches_dir: The directory to use as the root for named mutable caches.
:param ca_certs_path: Path to pem file for custom CA, if needed.
:param rules: A set of Rules which is used to compute values in the graph.
:param union_membership: All the registered and normalized union rules.
:param execution_options: Execution options for (remote) processes.
:param local_store_options: Options for the engine's LMDB store(s).
:param include_trace_on_error: Include the trace through the graph upon encountering errors.
:param validate_reachability: True to assert that all rules in an otherwise successfully
constructed rule graph are reachable: if a graph cannot be successfully constructed, it
is always a fatal error.
:param watch_filesystem: False if filesystem watching should be disabled.
"""
self.include_trace_on_error = include_trace_on_error
self._visualize_to_dir = visualize_to_dir
self._visualize_run_count = 0
# Validate and register all provided and intrinsic tasks.
rule_index = RuleIndex.create(rules)
tasks = register_rules(rule_index, union_membership)
# Create the native Scheduler and Session.
types = PyTypes(
file_digest=FileDigest,
snapshot=Snapshot,
paths=Paths,
file_content=FileContent,
file_entry=FileEntry,
directory=Directory,
digest_contents=DigestContents,
digest_entries=DigestEntries,
path_globs=PathGlobs,
merge_digests=MergeDigests,
add_prefix=AddPrefix,
remove_prefix=RemovePrefix,
create_digest=CreateDigest,
digest_subset=DigestSubset,
download_file=DownloadFile,
platform=Platform,
multi_platform_process=MultiPlatformProcess,
process_result=FallibleProcessResult,
process_result_metadata=ProcessResultMetadata,
coroutine=CoroutineType,
session_values=SessionValues,
run_id=RunId,
interactive_process=InteractiveProcess,
interactive_process_result=InteractiveProcessResult,
engine_aware_parameter=EngineAwareParameter,
)
remoting_options = PyRemotingOptions(
execution_enable=execution_options.remote_execution,
store_address=execution_options.remote_store_address,
execution_address=execution_options.remote_execution_address,
execution_process_cache_namespace=execution_options.
process_execution_cache_namespace,
instance_name=execution_options.remote_instance_name,
root_ca_certs_path=execution_options.remote_ca_certs_path,
store_headers=tuple(
execution_options.remote_store_headers.items()),
store_chunk_bytes=execution_options.remote_store_chunk_bytes,
store_chunk_upload_timeout=execution_options.
remote_store_chunk_upload_timeout_seconds,
store_rpc_retries=execution_options.remote_store_rpc_retries,
store_rpc_concurrency=execution_options.
remote_store_rpc_concurrency,
store_batch_api_size_limit=execution_options.
remote_store_batch_api_size_limit,
cache_warnings_behavior=execution_options.remote_cache_warnings.
value,
cache_eager_fetch=execution_options.remote_cache_eager_fetch,
cache_rpc_concurrency=execution_options.
remote_cache_rpc_concurrency,
execution_extra_platform_properties=tuple(
tuple(pair.split("=", 1)) for pair in
execution_options.remote_execution_extra_platform_properties),
execution_headers=tuple(
execution_options.remote_execution_headers.items()),
execution_overall_deadline_secs=execution_options.
remote_execution_overall_deadline_secs,
execution_rpc_concurrency=execution_options.
remote_execution_rpc_concurrency,
)
py_local_store_options = PyLocalStoreOptions(
store_dir=local_store_options.store_dir,
process_cache_max_size_bytes=local_store_options.
processes_max_size_bytes,
files_max_size_bytes=local_store_options.files_max_size_bytes,
directories_max_size_bytes=local_store_options.
directories_max_size_bytes,
lease_time_millis=LOCAL_STORE_LEASE_TIME_SECS * 1000,
shard_count=local_store_options.shard_count,
)
exec_stategy_opts = PyExecutionStrategyOptions(
local_cache=execution_options.process_execution_local_cache,
remote_cache_read=execution_options.remote_cache_read,
remote_cache_write=execution_options.remote_cache_write,
local_cleanup=execution_options.process_execution_local_cleanup,
local_parallelism=execution_options.
process_execution_local_parallelism,
local_enable_nailgun=execution_options.
process_execution_local_enable_nailgun,
remote_parallelism=execution_options.
process_execution_remote_parallelism,
)
self._py_scheduler = native_engine.scheduler_create(
executor,
tasks,
types,
build_root,
local_execution_root_dir,
named_caches_dir,
ca_certs_path,
ignore_patterns,
use_gitignore,
watch_filesystem,
remoting_options,
py_local_store_options,
exec_stategy_opts,
)
# If configured, visualize the rule graph before asserting that it is valid.
if self._visualize_to_dir is not None:
rule_graph_name = "rule_graph.dot"
self.visualize_rule_graph_to_file(
os.path.join(self._visualize_to_dir, rule_graph_name))
if validate_reachability:
native_engine.validate_reachability(self.py_scheduler)
def __init__(
self,
*,
native,
ignore_patterns: List[str],
use_gitignore: bool,
build_root: str,
local_store_dir: str,
local_execution_root_dir: str,
named_caches_dir: str,
ca_certs_path: Optional[str],
rules: Iterable[Rule],
union_membership: UnionMembership,
execution_options: ExecutionOptions,
executor: PyExecutor,
include_trace_on_error: bool = True,
visualize_to_dir: Optional[str] = None,
validate_reachability: bool = True,
) -> None:
"""
:param native: An instance of engine.native.Native.
:param ignore_patterns: A list of gitignore-style file patterns for pants to ignore.
:param use_gitignore: If set, pay attention to .gitignore files.
:param build_root: The build root as a string.
:param local_store_dir: The directory to use for storing the engine's LMDB store in.
:param local_execution_root_dir: The directory to use for local execution sandboxes.
:param named_caches_dir: The directory to use as the root for named mutable caches.
:param ca_certs_path: Path to pem file for custom CA, if needed.
:param rules: A set of Rules which is used to compute values in the graph.
:param union_membership: All the registered and normalized union rules.
:param execution_options: Execution options for (remote) processes.
:param include_trace_on_error: Include the trace through the graph upon encountering errors.
:param validate_reachability: True to assert that all rules in an otherwise successfully
constructed rule graph are reachable: if a graph cannot be successfully constructed, it
is always a fatal error.
"""
self._native = native
self.include_trace_on_error = include_trace_on_error
self._visualize_to_dir = visualize_to_dir
# Validate and register all provided and intrinsic tasks.
rule_index = RuleIndex.create(rules)
# Create the native Scheduler and Session.
tasks = self._register_rules(rule_index, union_membership)
types = PyTypes(
file_digest=FileDigest,
snapshot=Snapshot,
paths=Paths,
file_content=FileContent,
digest_contents=DigestContents,
path_globs=PathGlobs,
merge_digests=MergeDigests,
add_prefix=AddPrefix,
remove_prefix=RemovePrefix,
create_digest=CreateDigest,
digest_subset=DigestSubset,
download_file=DownloadFile,
platform=Platform,
multi_platform_process=MultiPlatformProcess,
process_result=FallibleProcessResultWithPlatform,
coroutine=CoroutineType,
session_values=SessionValues,
interactive_process_result=InteractiveProcessResult,
engine_aware_parameter=EngineAwareParameter,
)
self._scheduler = native.new_scheduler(
tasks=tasks,
build_root=build_root,
local_store_dir=local_store_dir,
local_execution_root_dir=local_execution_root_dir,
named_caches_dir=named_caches_dir,
ca_certs_path=ca_certs_path,
ignore_patterns=ignore_patterns,
use_gitignore=use_gitignore,
executor=executor,
execution_options=execution_options,
types=types,
)
# If configured, visualize the rule graph before asserting that it is valid.
if self._visualize_to_dir is not None:
rule_graph_name = "rule_graph.dot"
self.visualize_rule_graph_to_file(os.path.join(self._visualize_to_dir, rule_graph_name))
if validate_reachability:
self._native.lib.validate_reachability(self._scheduler)