def mk_scheduler(
self,
tmp_path: Path,
rules,
include_trace_on_error: bool = True,
max_workunit_verbosity: LogLevel = LogLevel.DEBUG,
) -> SchedulerSession:
"""Creates a SchedulerSession for a Scheduler with the given Rules installed."""
build_root = tmp_path / "build_root"
build_root.mkdir(parents=True, exist_ok=True)
local_execution_root_dir = os.path.realpath(safe_mkdtemp())
named_caches_dir = os.path.realpath(safe_mkdtemp())
scheduler = Scheduler(
ignore_patterns=[],
use_gitignore=False,
build_root=build_root.as_posix(),
local_execution_root_dir=local_execution_root_dir,
named_caches_dir=named_caches_dir,
ca_certs_path=None,
rules=rules,
union_membership=UnionMembership({}),
executor=self._executor,
execution_options=DEFAULT_EXECUTION_OPTIONS,
local_store_options=DEFAULT_LOCAL_STORE_OPTIONS,
include_trace_on_error=include_trace_on_error,
)
return scheduler.new_session(
build_id="buildid_for_test",
max_workunit_level=max_workunit_verbosity,
)
def mk_scheduler(
self,
rules,
include_trace_on_error: bool = True,
) -> SchedulerSession:
"""Creates a SchedulerSession for a Scheduler with the given Rules installed."""
work_dir = self._create_work_dir()
build_root = os.path.join(work_dir, "build_root")
os.makedirs(build_root)
local_store_dir = os.path.realpath(safe_mkdtemp())
local_execution_root_dir = os.path.realpath(safe_mkdtemp())
named_caches_dir = os.path.realpath(safe_mkdtemp())
scheduler = Scheduler(
native=self._native,
ignore_patterns=[],
use_gitignore=False,
build_root=build_root,
local_store_dir=local_store_dir,
local_execution_root_dir=local_execution_root_dir,
named_caches_dir=named_caches_dir,
ca_certs_path=None,
rules=rules,
union_membership=UnionMembership({}),
executor=self._executor,
execution_options=DEFAULT_EXECUTION_OPTIONS,
include_trace_on_error=include_trace_on_error,
)
return scheduler.new_session(build_id="buildid_for_test", )
def mk_scheduler(
self,
rules=None,
union_rules=None,
project_tree=None,
work_dir=None,
include_trace_on_error=True,
should_report_workunits=False,
execution_options=None,
):
"""Creates a SchedulerSession for a Scheduler with the given Rules installed."""
rules = rules or []
work_dir = work_dir or self._create_work_dir()
project_tree = project_tree or self.mk_fs_tree(work_dir=work_dir)
local_store_dir = os.path.realpath(safe_mkdtemp())
if execution_options is not None:
eo = asdict(DEFAULT_EXECUTION_OPTIONS)
eo.update(execution_options)
execution_options = ExecutionOptions(**eo)
scheduler = Scheduler(
native=self._native,
ignore_patterns=project_tree.ignore_patterns,
use_gitignore=False,
build_root=project_tree.build_root,
local_store_dir=local_store_dir,
rules=rules,
union_rules=union_rules,
execution_options=execution_options or DEFAULT_EXECUTION_OPTIONS,
include_trace_on_error=include_trace_on_error,
)
return scheduler.new_session(
zipkin_trace_v2=False,
build_id="buildid_for_test",
should_report_workunits=should_report_workunits,
)
def _set_new_session(self, scheduler: Scheduler) -> None:
self.scheduler = scheduler.new_session(
build_id="buildid_for_test",
session_values=SessionValues({
OptionsBootstrapper: self.options_bootstrapper,
CompleteEnvironment: self.environment,
**self.extra_session_values,
}),
max_workunit_level=self.max_workunit_verbosity,
)
def __init__(
self,
scheduler: Scheduler,
period_secs=10,
lease_extension_interval_secs=(30 * 60),
gc_interval_secs=(4 * 60 * 60),
):
super().__init__()
self._scheduler_session = scheduler.new_session(
build_id="store_gc_service_session")
self._logger = logging.getLogger(__name__)
self._period_secs = period_secs
self._lease_extension_interval_secs = lease_extension_interval_secs
self._gc_interval_secs = gc_interval_secs
self._set_next_gc()
self._set_next_lease_extension()
def __init__(
self, watchman: Watchman, scheduler: Scheduler, build_root: str,
):
"""
:param watchman: The Watchman instance as provided by the WatchmanLauncher subsystem.
:param session: A SchedulerSession to invalidate for.
:param build_root: The current build root.
"""
super().__init__()
self._logger = logging.getLogger(__name__)
self._watchman = watchman
self._build_root = os.path.realpath(build_root)
self._watchman_is_running = threading.Event()
self._scheduler_session = scheduler.new_session(
zipkin_trace_v2=False, build_id="fs_event_service_session"
)
self._handler = Watchman.EventHandler(
name=self.PANTS_ALL_FILES_SUBSCRIPTION_NAME,
metadata=dict(
fields=["name"],
# Request events for all file types.
# NB: Touching a file invalidates its parent directory due to:
# https://github.com/facebook/watchman/issues/305
# ...but if we were to skip watching directories, we'd still have to invalidate
# the parents of any changed files, and we wouldn't see creation/deletion of
# empty directories.
expression=[
"allof", # All of the below rules must be true to match.
["not", ["dirname", "dist", self.ZERO_DEPTH]], # Exclude the ./dist dir.
# N.B. 'wholename' ensures we match against the absolute ('x/y/z') vs base path ('z').
[
"not",
["pcre", r"^\..*", "wholename"],
], # Exclude files in hidden dirs (.pants.d etc).
["not", ["match", "*.pyc"]] # Exclude .pyc files.
# TODO(kwlzn): Make exclusions here optionable.
# Related: https://github.com/pantsbuild/pants/issues/2956
],
),
# NB: We stream events from Watchman in `self.run`, so we don't need a callback.
callback=lambda: None,
)
def __init__(
self,
scheduler: Scheduler,
period_secs: float = 10,
lease_extension_interval_secs: float = (float(LOCAL_STORE_LEASE_TIME_SECS) / 100),
gc_interval_secs: float = (1 * 60 * 60),
local_store_options: LocalStoreOptions = DEFAULT_LOCAL_STORE_OPTIONS,
):
super().__init__()
self._scheduler_session = scheduler.new_session(build_id="store_gc_service_session")
self._logger = logging.getLogger(__name__)
self._period_secs = period_secs
self._lease_extension_interval_secs = lease_extension_interval_secs
self._gc_interval_secs = gc_interval_secs
self._target_size_bytes = local_store_options.target_total_size_bytes()
self._set_next_gc()
self._set_next_lease_extension()
def mk_scheduler(
self,
rules=None,
project_tree=None,
work_dir=None,
include_trace_on_error=True,
execution_options=None,
ca_certs_path=None,
) -> SchedulerSession:
"""Creates a SchedulerSession for a Scheduler with the given Rules installed."""
rules = rules or []
work_dir = work_dir or self._create_work_dir()
project_tree = project_tree or self.mk_fs_tree(work_dir=work_dir)
local_store_dir = os.path.realpath(safe_mkdtemp())
local_execution_root_dir = os.path.realpath(safe_mkdtemp())
named_caches_dir = os.path.realpath(safe_mkdtemp())
if execution_options is not None:
eo = asdict(DEFAULT_EXECUTION_OPTIONS)
eo.update(execution_options)
execution_options = ExecutionOptions(**eo)
scheduler = Scheduler(
native=self._native,
ignore_patterns=project_tree.ignore_patterns,
use_gitignore=False,
build_root=project_tree.build_root,
local_store_dir=local_store_dir,
local_execution_root_dir=local_execution_root_dir,
named_caches_dir=named_caches_dir,
ca_certs_path=ca_certs_path,
rules=rules,
union_membership=UnionMembership({}),
executor=self._executor,
execution_options=execution_options or DEFAULT_EXECUTION_OPTIONS,
include_trace_on_error=include_trace_on_error,
)
return scheduler.new_session(build_id="buildid_for_test", )
