def mk_scheduler( self, tmp_path: Path, rules, include_trace_on_error: bool = True, max_workunit_verbosity: LogLevel = LogLevel.DEBUG, ) -> SchedulerSession: """Creates a SchedulerSession for a Scheduler with the given Rules installed.""" build_root = tmp_path / "build_root" build_root.mkdir(parents=True, exist_ok=True) local_execution_root_dir = os.path.realpath(safe_mkdtemp()) named_caches_dir = os.path.realpath(safe_mkdtemp()) scheduler = Scheduler( ignore_patterns=[], use_gitignore=False, build_root=build_root.as_posix(), local_execution_root_dir=local_execution_root_dir, named_caches_dir=named_caches_dir, ca_certs_path=None, rules=rules, union_membership=UnionMembership({}), executor=self._executor, execution_options=DEFAULT_EXECUTION_OPTIONS, local_store_options=DEFAULT_LOCAL_STORE_OPTIONS, include_trace_on_error=include_trace_on_error, ) return scheduler.new_session( build_id="buildid_for_test", max_workunit_level=max_workunit_verbosity, )
def mk_scheduler( self, rules, include_trace_on_error: bool = True, ) -> SchedulerSession: """Creates a SchedulerSession for a Scheduler with the given Rules installed.""" work_dir = self._create_work_dir() build_root = os.path.join(work_dir, "build_root") os.makedirs(build_root) local_store_dir = os.path.realpath(safe_mkdtemp()) local_execution_root_dir = os.path.realpath(safe_mkdtemp()) named_caches_dir = os.path.realpath(safe_mkdtemp()) scheduler = Scheduler( native=self._native, ignore_patterns=[], use_gitignore=False, build_root=build_root, local_store_dir=local_store_dir, local_execution_root_dir=local_execution_root_dir, named_caches_dir=named_caches_dir, ca_certs_path=None, rules=rules, union_membership=UnionMembership({}), executor=self._executor, execution_options=DEFAULT_EXECUTION_OPTIONS, include_trace_on_error=include_trace_on_error, ) return scheduler.new_session(build_id="buildid_for_test", )
def mk_scheduler( self, rules=None, union_rules=None, project_tree=None, work_dir=None, include_trace_on_error=True, should_report_workunits=False, execution_options=None, ): """Creates a SchedulerSession for a Scheduler with the given Rules installed.""" rules = rules or [] work_dir = work_dir or self._create_work_dir() project_tree = project_tree or self.mk_fs_tree(work_dir=work_dir) local_store_dir = os.path.realpath(safe_mkdtemp()) if execution_options is not None: eo = asdict(DEFAULT_EXECUTION_OPTIONS) eo.update(execution_options) execution_options = ExecutionOptions(**eo) scheduler = Scheduler( native=self._native, ignore_patterns=project_tree.ignore_patterns, use_gitignore=False, build_root=project_tree.build_root, local_store_dir=local_store_dir, rules=rules, union_rules=union_rules, execution_options=execution_options or DEFAULT_EXECUTION_OPTIONS, include_trace_on_error=include_trace_on_error, ) return scheduler.new_session( zipkin_trace_v2=False, build_id="buildid_for_test", should_report_workunits=should_report_workunits, )
def _set_new_session(self, scheduler: Scheduler) -> None: self.scheduler = scheduler.new_session( build_id="buildid_for_test", session_values=SessionValues({ OptionsBootstrapper: self.options_bootstrapper, CompleteEnvironment: self.environment, **self.extra_session_values, }), max_workunit_level=self.max_workunit_verbosity, )
def __init__( self, scheduler: Scheduler, period_secs=10, lease_extension_interval_secs=(30 * 60), gc_interval_secs=(4 * 60 * 60), ): super().__init__() self._scheduler_session = scheduler.new_session( build_id="store_gc_service_session") self._logger = logging.getLogger(__name__) self._period_secs = period_secs self._lease_extension_interval_secs = lease_extension_interval_secs self._gc_interval_secs = gc_interval_secs self._set_next_gc() self._set_next_lease_extension()
def __init__( self, watchman: Watchman, scheduler: Scheduler, build_root: str, ): """ :param watchman: The Watchman instance as provided by the WatchmanLauncher subsystem. :param session: A SchedulerSession to invalidate for. :param build_root: The current build root. """ super().__init__() self._logger = logging.getLogger(__name__) self._watchman = watchman self._build_root = os.path.realpath(build_root) self._watchman_is_running = threading.Event() self._scheduler_session = scheduler.new_session( zipkin_trace_v2=False, build_id="fs_event_service_session" ) self._handler = Watchman.EventHandler( name=self.PANTS_ALL_FILES_SUBSCRIPTION_NAME, metadata=dict( fields=["name"], # Request events for all file types. # NB: Touching a file invalidates its parent directory due to: # https://github.com/facebook/watchman/issues/305 # ...but if we were to skip watching directories, we'd still have to invalidate # the parents of any changed files, and we wouldn't see creation/deletion of # empty directories. expression=[ "allof", # All of the below rules must be true to match. ["not", ["dirname", "dist", self.ZERO_DEPTH]], # Exclude the ./dist dir. # N.B. 'wholename' ensures we match against the absolute ('x/y/z') vs base path ('z'). [ "not", ["pcre", r"^\..*", "wholename"], ], # Exclude files in hidden dirs (.pants.d etc). ["not", ["match", "*.pyc"]] # Exclude .pyc files. # TODO(kwlzn): Make exclusions here optionable. # Related: https://github.com/pantsbuild/pants/issues/2956 ], ), # NB: We stream events from Watchman in `self.run`, so we don't need a callback. callback=lambda: None, )
def __init__( self, scheduler: Scheduler, period_secs: float = 10, lease_extension_interval_secs: float = (float(LOCAL_STORE_LEASE_TIME_SECS) / 100), gc_interval_secs: float = (1 * 60 * 60), local_store_options: LocalStoreOptions = DEFAULT_LOCAL_STORE_OPTIONS, ): super().__init__() self._scheduler_session = scheduler.new_session(build_id="store_gc_service_session") self._logger = logging.getLogger(__name__) self._period_secs = period_secs self._lease_extension_interval_secs = lease_extension_interval_secs self._gc_interval_secs = gc_interval_secs self._target_size_bytes = local_store_options.target_total_size_bytes() self._set_next_gc() self._set_next_lease_extension()
def mk_scheduler( self, rules=None, project_tree=None, work_dir=None, include_trace_on_error=True, execution_options=None, ca_certs_path=None, ) -> SchedulerSession: """Creates a SchedulerSession for a Scheduler with the given Rules installed.""" rules = rules or [] work_dir = work_dir or self._create_work_dir() project_tree = project_tree or self.mk_fs_tree(work_dir=work_dir) local_store_dir = os.path.realpath(safe_mkdtemp()) local_execution_root_dir = os.path.realpath(safe_mkdtemp()) named_caches_dir = os.path.realpath(safe_mkdtemp()) if execution_options is not None: eo = asdict(DEFAULT_EXECUTION_OPTIONS) eo.update(execution_options) execution_options = ExecutionOptions(**eo) scheduler = Scheduler( native=self._native, ignore_patterns=project_tree.ignore_patterns, use_gitignore=False, build_root=project_tree.build_root, local_store_dir=local_store_dir, local_execution_root_dir=local_execution_root_dir, named_caches_dir=named_caches_dir, ca_certs_path=ca_certs_path, rules=rules, union_membership=UnionMembership({}), executor=self._executor, execution_options=execution_options or DEFAULT_EXECUTION_OPTIONS, include_trace_on_error=include_trace_on_error, ) return scheduler.new_session(build_id="buildid_for_test", )