def new_scheduler(
self,
tasks,
build_root: str,
local_store_dir: str,
local_execution_root_dir: str,
named_caches_dir: str,
ca_certs_path: Optional[str],
ignore_patterns: List[str],
use_gitignore: bool,
executor: PyExecutor,
execution_options: ExecutionOptions,
types: PyTypes,
) -> PyScheduler:
"""Create and return a native Scheduler."""
remoting_options = PyRemotingOptions(
execution_enable=execution_options.remote_execution,
store_address=execution_options.remote_store_address,
execution_address=execution_options.remote_execution_address,
execution_process_cache_namespace=execution_options.
process_execution_cache_namespace,
instance_name=execution_options.remote_instance_name,
root_ca_certs_path=execution_options.remote_ca_certs_path,
store_headers=tuple(
execution_options.remote_store_headers.items()),
store_chunk_bytes=execution_options.remote_store_chunk_bytes,
store_chunk_upload_timeout=execution_options.
remote_store_chunk_upload_timeout_seconds,
store_rpc_retries=execution_options.remote_store_rpc_retries,
cache_eager_fetch=execution_options.remote_cache_eager_fetch,
execution_extra_platform_properties=tuple(
tuple(pair.split("=", 1)) for pair in
execution_options.remote_execution_extra_platform_properties),
execution_headers=tuple(
execution_options.remote_execution_headers.items()),
execution_overall_deadline_secs=execution_options.
remote_execution_overall_deadline_secs,
)
exec_stategy_opts = PyExecutionStrategyOptions(
local_parallelism=execution_options.
process_execution_local_parallelism,
remote_parallelism=execution_options.
process_execution_remote_parallelism,
cleanup_local_dirs=execution_options.
process_execution_cleanup_local_dirs,
use_local_cache=execution_options.
process_execution_use_local_cache,
remote_cache_read=execution_options.remote_cache_read,
remote_cache_write=execution_options.remote_cache_write,
)
return cast(
PyScheduler,
self.lib.scheduler_create(
executor,
tasks,
types,
# Project tree.
build_root,
local_store_dir,
local_execution_root_dir,
named_caches_dir,
ca_certs_path,
ignore_patterns,
use_gitignore,
remoting_options,
exec_stategy_opts,
),
)
def __init__(
self,
*,
ignore_patterns: list[str],
use_gitignore: bool,
build_root: str,
local_execution_root_dir: str,
named_caches_dir: str,
ca_certs_path: str | None,
rules: Iterable[Rule],
union_membership: UnionMembership,
execution_options: ExecutionOptions,
local_store_options: LocalStoreOptions,
executor: PyExecutor,
include_trace_on_error: bool = True,
visualize_to_dir: str | None = None,
validate_reachability: bool = True,
watch_filesystem: bool = True,
) -> None:
"""
:param ignore_patterns: A list of gitignore-style file patterns for pants to ignore.
:param use_gitignore: If set, pay attention to .gitignore files.
:param build_root: The build root as a string.
:param local_execution_root_dir: The directory to use for local execution sandboxes.
:param named_caches_dir: The directory to use as the root for named mutable caches.
:param ca_certs_path: Path to pem file for custom CA, if needed.
:param rules: A set of Rules which is used to compute values in the graph.
:param union_membership: All the registered and normalized union rules.
:param execution_options: Execution options for (remote) processes.
:param local_store_options: Options for the engine's LMDB store(s).
:param include_trace_on_error: Include the trace through the graph upon encountering errors.
:param validate_reachability: True to assert that all rules in an otherwise successfully
constructed rule graph are reachable: if a graph cannot be successfully constructed, it
is always a fatal error.
:param watch_filesystem: False if filesystem watching should be disabled.
"""
self.include_trace_on_error = include_trace_on_error
self._visualize_to_dir = visualize_to_dir
self._visualize_run_count = 0
# Validate and register all provided and intrinsic tasks.
rule_index = RuleIndex.create(rules)
tasks = register_rules(rule_index, union_membership)
# Create the native Scheduler and Session.
types = PyTypes(
file_digest=FileDigest,
snapshot=Snapshot,
paths=Paths,
file_content=FileContent,
file_entry=FileEntry,
directory=Directory,
digest_contents=DigestContents,
digest_entries=DigestEntries,
path_globs=PathGlobs,
merge_digests=MergeDigests,
add_prefix=AddPrefix,
remove_prefix=RemovePrefix,
create_digest=CreateDigest,
digest_subset=DigestSubset,
download_file=DownloadFile,
platform=Platform,
multi_platform_process=MultiPlatformProcess,
process_result=FallibleProcessResult,
process_result_metadata=ProcessResultMetadata,
coroutine=CoroutineType,
session_values=SessionValues,
run_id=RunId,
interactive_process=InteractiveProcess,
interactive_process_result=InteractiveProcessResult,
engine_aware_parameter=EngineAwareParameter,
)
remoting_options = PyRemotingOptions(
execution_enable=execution_options.remote_execution,
store_address=execution_options.remote_store_address,
execution_address=execution_options.remote_execution_address,
execution_process_cache_namespace=execution_options.
process_execution_cache_namespace,
instance_name=execution_options.remote_instance_name,
root_ca_certs_path=execution_options.remote_ca_certs_path,
store_headers=tuple(
execution_options.remote_store_headers.items()),
store_chunk_bytes=execution_options.remote_store_chunk_bytes,
store_chunk_upload_timeout=execution_options.
remote_store_chunk_upload_timeout_seconds,
store_rpc_retries=execution_options.remote_store_rpc_retries,
store_rpc_concurrency=execution_options.
remote_store_rpc_concurrency,
store_batch_api_size_limit=execution_options.
remote_store_batch_api_size_limit,
cache_warnings_behavior=execution_options.remote_cache_warnings.
value,
cache_eager_fetch=execution_options.remote_cache_eager_fetch,
cache_rpc_concurrency=execution_options.
remote_cache_rpc_concurrency,
execution_extra_platform_properties=tuple(
tuple(pair.split("=", 1)) for pair in
execution_options.remote_execution_extra_platform_properties),
execution_headers=tuple(
execution_options.remote_execution_headers.items()),
execution_overall_deadline_secs=execution_options.
remote_execution_overall_deadline_secs,
execution_rpc_concurrency=execution_options.
remote_execution_rpc_concurrency,
)
py_local_store_options = PyLocalStoreOptions(
store_dir=local_store_options.store_dir,
process_cache_max_size_bytes=local_store_options.
processes_max_size_bytes,
files_max_size_bytes=local_store_options.files_max_size_bytes,
directories_max_size_bytes=local_store_options.
directories_max_size_bytes,
lease_time_millis=LOCAL_STORE_LEASE_TIME_SECS * 1000,
shard_count=local_store_options.shard_count,
)
exec_stategy_opts = PyExecutionStrategyOptions(
local_cache=execution_options.process_execution_local_cache,
remote_cache_read=execution_options.remote_cache_read,
remote_cache_write=execution_options.remote_cache_write,
local_cleanup=execution_options.process_execution_local_cleanup,
local_parallelism=execution_options.
process_execution_local_parallelism,
local_enable_nailgun=execution_options.
process_execution_local_enable_nailgun,
remote_parallelism=execution_options.
process_execution_remote_parallelism,
)
self._py_scheduler = native_engine.scheduler_create(
executor,
tasks,
types,
build_root,
local_execution_root_dir,
named_caches_dir,
ca_certs_path,
ignore_patterns,
use_gitignore,
watch_filesystem,
remoting_options,
py_local_store_options,
exec_stategy_opts,
)
# If configured, visualize the rule graph before asserting that it is valid.
if self._visualize_to_dir is not None:
rule_graph_name = "rule_graph.dot"
self.visualize_rule_graph_to_file(
os.path.join(self._visualize_to_dir, rule_graph_name))
if validate_reachability:
native_engine.validate_reachability(self.py_scheduler)
def new_scheduler(
self,
tasks,
root_subject_types,
build_root: str,
local_store_dir: str,
local_execution_root_dir: str,
named_caches_dir: str,
ignore_patterns: List[str],
use_gitignore: bool,
execution_options,
types: PyTypes,
) -> PyScheduler:
"""Create and return a native Scheduler."""
remoting_options = PyRemotingOptions(
execution_enable=execution_options.remote_execution,
store_servers=execution_options.remote_store_server,
execution_server=execution_options.remote_execution_server,
execution_process_cache_namespace=execution_options.
remote_execution_process_cache_namespace,
instance_name=execution_options.remote_instance_name,
root_ca_certs_path=execution_options.remote_ca_certs_path,
oauth_bearer_token_path=execution_options.
remote_oauth_bearer_token_path,
store_thread_count=execution_options.remote_store_thread_count,
store_chunk_bytes=execution_options.remote_store_chunk_bytes,
store_chunk_upload_timeout=execution_options.
remote_store_connection_limit,
store_rpc_retries=execution_options.
remote_store_chunk_upload_timeout_seconds,
store_connection_limit=execution_options.remote_store_rpc_retries,
execution_extra_platform_properties=tuple(
tuple(pair.split("=", 1)) for pair in
execution_options.remote_execution_extra_platform_properties),
execution_headers=tuple(
(k, v)
for (k,
v) in execution_options.remote_execution_headers.items()),
execution_overall_deadline_secs=execution_options.
remote_execution_overall_deadline_secs,
)
exec_stategy_opts = PyExecutionStrategyOptions(
local_parallelism=execution_options.
process_execution_local_parallelism,
remote_parallelism=execution_options.
process_execution_remote_parallelism,
cleanup_local_dirs=execution_options.
process_execution_cleanup_local_dirs,
speculation_delay=execution_options.
process_execution_speculation_delay,
speculation_strategy=execution_options.
process_execution_speculation_strategy,
use_local_cache=execution_options.
process_execution_use_local_cache,
local_enable_nailgun=execution_options.
process_execution_local_enable_nailgun,
)
return self.lib.scheduler_create(
self._executor,
tasks,
types,
# Project tree.
build_root,
local_store_dir,
local_execution_root_dir,
named_caches_dir,
ignore_patterns,
use_gitignore,
root_subject_types,
remoting_options,
exec_stategy_opts,
)