def _init_graph_session( options_bootstrapper: OptionsBootstrapper, build_config: BuildConfiguration, options: Options, ) -> LegacyGraphSession: native = Native() native.set_panic_handler() graph_scheduler_helper = EngineInitializer.setup_legacy_graph( native, options_bootstrapper, build_config) v2_ui = options.for_global_scope().get("v2_ui", False) zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2 # TODO(#8658) This should_report_workunits flag must be set to True for # StreamingWorkunitHandler to receive WorkUnits. It should eventually # be merged with the zipkin_trace_v2 flag, since they both involve most # of the same engine functionality, but for now is separate to avoid # breaking functionality associated with zipkin tracing while iterating on streaming workunit reporting. stream_workunits = len( options.for_global_scope().streaming_workunits_handlers) != 0 return graph_scheduler_helper.new_session( zipkin_trace_v2, RunTracker.global_instance().run_id, v2_ui, should_report_workunits=stream_workunits, )
def _run_pants(self, sock, arguments, environment): """Execute a given run with a pants runner.""" # For the pants run, we want to log to stderr. # TODO Might be worth to make contextmanagers for this? Native().override_thread_logging_destination_to_just_stderr() self.server.runner_factory(sock, arguments, environment).run() Native().override_thread_logging_destination_to_just_pantsd()
def _maybe_init_graph_session(graph_session, options_bootstrapper, build_config, global_options): if graph_session: return graph_session native = Native() native.set_panic_handler() graph_scheduler_helper = EngineInitializer.setup_legacy_graph( native, options_bootstrapper, build_config) return graph_scheduler_helper.new_session(global_options.v2_ui)
def create_native_stderr_log_handler( log_level: LogLevel, native: Native, stream: Optional[TextIO] = None) -> NativeHandler: try: native.setup_stderr_logger(log_level.level) except Exception as e: print(f"Error setting up pantsd logger: {e!r}", file=sys.stderr) raise e return NativeHandler(log_level, native, stream)
def _maybe_init_graph_session(graph_session, options_bootstrapper, build_config, options): if graph_session: return graph_session native = Native() native.set_panic_handler() graph_scheduler_helper = EngineInitializer.setup_legacy_graph( native, options_bootstrapper, build_config) v2_ui = options.for_global_scope().v2_ui zipkin_trace_v2 = options.for_scope('reporting').zipkin_trace_v2 return graph_scheduler_helper.new_session(zipkin_trace_v2, v2_ui)
def _maybe_init_graph_session(graph_session, options_bootstrapper,build_config, options): if graph_session: return graph_session native = Native() native.set_panic_handler() graph_scheduler_helper = EngineInitializer.setup_legacy_graph( native, options_bootstrapper, build_config ) v2_ui = options.for_global_scope().v2_ui zipkin_trace_v2 = options.for_scope('reporting').zipkin_trace_v2 return graph_scheduler_helper.new_session(zipkin_trace_v2, v2_ui)
def process_request_thread(self, request, client_address): """Override of ThreadingMixIn.process_request_thread() that delegates to the request handler.""" # Instantiate the request handler. Native().override_thread_logging_destination_to_just_pantsd() handler = self.RequestHandlerClass(request, client_address, self) _, _, _, environment = handler.parsed_request() try: with self.ensure_request_is_exclusive(environment, request): # Attempt to handle a request with the handler. handler.handle_request() self.request_complete_callback() except BrokenPipeError as e: # The client has closed the connection, most likely from a SIGINT self.logger.error( f"Request {request} abruptly closed with {type(e)}, probably because the client crashed or was sent a SIGINT." ) except Exception as e: # If that fails, (synchronously) handle the error with the error handler sans-fork. try: self.logger.error( f"Request {request} errored with {type(e)}({e!r})") handler.handle_error(e) finally: # Shutdown the socket since we don't expect a fork() in the exception context. self.shutdown_request(request) else: # At this point, we expect a fork() has taken place - the parent side will return, and so we # close the request here from the parent without explicitly shutting down the socket. The # child half of this will perform an os._exit() before it gets to this point and is also # responsible for shutdown and closing of the socket when its execution is complete. self.close_request(request)
def create(cls, bootstrap_options=None): """ :param Options bootstrap_options: The bootstrap options, if available. """ bootstrap_options = bootstrap_options or cls._parse_bootstrap_options() bootstrap_options_values = bootstrap_options.for_global_scope() build_root = get_buildroot() native = Native.create(bootstrap_options_values) # TODO: https://github.com/pantsbuild/pants/issues/3479 watchman = WatchmanLauncher.create(bootstrap_options_values).watchman legacy_graph_helper = cls._setup_legacy_graph_helper(native, bootstrap_options_values) services, port_map = cls._setup_services( build_root, bootstrap_options_values, legacy_graph_helper, watchman ) return PantsDaemon( native, build_root, bootstrap_options_values.pants_workdir, bootstrap_options_values.level.upper(), services, port_map, bootstrap_options_values.pants_subprocessdir, bootstrap_options )
def create(cls, bootstrap_options=None): """ :param Options bootstrap_options: The bootstrap options, if available. """ bootstrap_options = bootstrap_options or cls._parse_bootstrap_options( ) bootstrap_options_values = bootstrap_options.for_global_scope() build_root = get_buildroot() native = Native.create(bootstrap_options_values) # TODO: https://github.com/pantsbuild/pants/issues/3479 watchman = WatchmanLauncher.create( bootstrap_options_values).watchman legacy_graph_helper = cls._setup_legacy_graph_helper( native, bootstrap_options_values) services, port_map = cls._setup_services(build_root, bootstrap_options_values, legacy_graph_helper, watchman) return PantsDaemon(native, build_root, bootstrap_options_values.pants_workdir, bootstrap_options_values.level.upper(), services, port_map, bootstrap_options_values.pants_subprocessdir, bootstrap_options)
def _init_graph(self, target_roots, graph_helper, exclude_target_regexps, tags): """Determine the BuildGraph, AddressMapper and spec_roots for a given run. :param TargetRoots target_roots: The existing `TargetRoots` object, if any. :param LegacyGraphSession graph_helper: A LegacyGraphSession to use for graph construction, if available. This would usually come from the daemon. :returns: A tuple of (BuildGraph, AddressMapper, SchedulerSession, TargetRoots). """ # The daemon may provide a `graph_helper`. If that's present, use it for graph construction. if not graph_helper: native = Native.create(self._global_options) native.set_panic_handler() graph_scheduler_helper = EngineInitializer.setup_legacy_graph(native, self._global_options, self._build_config) graph_helper = graph_scheduler_helper.new_session() target_roots = target_roots or TargetRootsCalculator.create( options=self._options, session=graph_helper.scheduler_session, build_root=self._root_dir, symbol_table=graph_helper.symbol_table, exclude_patterns=tuple(exclude_target_regexps), tags=tuple(tags) ) graph, address_mapper = graph_helper.create_build_graph(target_roots, self._root_dir) return graph, address_mapper, graph_helper.scheduler_session, target_roots
def globs_matches( paths: Iterable[str], patterns: Iterable[str], exclude_patterns: Iterable[str], ) -> bool: path_globs = PathGlobs(include=patterns, exclude=exclude_patterns) return Native().match_path_globs(path_globs, paths)
def __init__(self, bootstrap_options, engine_initializer=None): """ :param Options bootstrap_options: An Options object containing the bootstrap options. :param class engine_initializer: The class representing the EngineInitializer. Only required for startup. """ self._bootstrap_options = bootstrap_options self._engine_initializer = engine_initializer self._pailgun_host = bootstrap_options.pantsd_pailgun_host self._pailgun_port = bootstrap_options.pantsd_pailgun_port self._log_dir = bootstrap_options.pantsd_log_dir self._fs_event_workers = bootstrap_options.pantsd_fs_event_workers self._pants_workdir = bootstrap_options.pants_workdir self._log_level = bootstrap_options.level.upper() self._pants_ignore_patterns = bootstrap_options.pants_ignore self._build_ignore_patterns = bootstrap_options.build_ignore self._exclude_target_regexp = bootstrap_options.exclude_target_regexp self._subproject_roots = bootstrap_options.subproject_roots self._metadata_base_dir = bootstrap_options.pants_subprocessdir # TODO: https://github.com/pantsbuild/pants/issues/3479 self._build_root = get_buildroot() self._native = Native.create(bootstrap_options) self._logger = logging.getLogger(__name__) self._lock = OwnerPrintingInterProcessFileLock( os.path.join(self._build_root, '.pantsd.startup'))
def __init__(self, build_root, engine_initializer, options): """ :param str build_root: The path of the build root. :param class engine_initializer: The class representing the EngineInitializer. """ self._build_root = build_root self._engine_initializer = engine_initializer # The options we register directly. self._pailgun_host = options.pailgun_host self._pailgun_port = options.pailgun_port self._log_dir = options.log_dir self._fs_event_workers = options.fs_event_workers # Values derived from global options (which our scoped options inherit). self._pants_workdir = options.pants_workdir self._log_level = options.level.upper() self._pants_ignore_patterns = options.pants_ignore self._build_ignore_patterns = options.build_ignore self._exclude_target_regexp = options.exclude_target_regexp self._subproject_roots = options.subproject_roots # Native.create() reads global options, which, thanks to inheritance, it can # read them via our scoped options. self._native = Native.create(options) # TODO(kwlzn): Thread filesystem path ignores here to Watchman's subscription registration. lock_location = os.path.join(self._build_root, '.pantsd.startup') self._lock = OwnerPrintingInterProcessFileLock(lock_location) self._logger = logging.getLogger(__name__)
def __init__(self, build_root, engine_initializer, options): """ :param str build_root: The path of the build root. :param class engine_initializer: The class representing the EngineInitializer. """ self._build_root = build_root self._engine_initializer = engine_initializer # The options we register directly. self._pailgun_host = options.pailgun_host self._pailgun_port = options.pailgun_port self._log_dir = options.log_dir self._fs_event_workers = options.fs_event_workers # Values derived from global options (which our scoped options inherit). self._pants_workdir = options.pants_workdir self._log_level = options.level.upper() self._pants_ignore_patterns = options.pants_ignore self._build_ignore_patterns = options.build_ignore self._exclude_target_regexp = options.exclude_target_regexp self._subproject_roots = options.subproject_roots # Native.create() reads global options, which, thanks to inheritance, it can # read them via our scoped options. self._native = Native.create(options) # TODO(kwlzn): Thread filesystem path ignores here to Watchman's subscription registration. lock_location = os.path.join(self._build_root, '.pantsd.startup') self._lock = OwnerPrintingInterProcessFileLock(lock_location) self._logger = logging.getLogger(__name__)
def globs_matches( paths: Iterable[str], patterns: Iterable[str], exclude_patterns: Iterable[str], ) -> bool: path_globs = PathGlobs(globs=(*patterns, *(f"!{e}" for e in exclude_patterns))) return Native().match_path_globs(path_globs, paths)
def setup_legacy_graph(pants_ignore_patterns, workdir, build_root=None, native=None, symbol_table_cls=None, build_ignore_patterns=None, exclude_target_regexps=None, subproject_roots=None, include_trace_on_error=True): """Construct and return the components necessary for LegacyBuildGraph construction. :param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree, usually taken from the '--pants-ignore' global option. :param str workdir: The pants workdir. :param str build_root: A path to be used as the build root. If None, then default is used. :param Native native: An instance of the native-engine subsystem. :param SymbolTable symbol_table_cls: A SymbolTable class to use for build file parsing, or None to use the default. :param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD files, usually taken from the '--build-ignore' global option. :param list exclude_target_regexps: A list of regular expressions for excluding targets. :param list subproject_roots: Paths that correspond with embedded build roots under the current build root. :param bool include_trace_on_error: If True, when an error occurs, the error message will include the graph trace. :returns: A tuple of (scheduler, engine, symbol_table_cls, build_graph_cls). """ build_root = build_root or get_buildroot() scm = get_scm() symbol_table_cls = symbol_table_cls or LegacySymbolTable project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns) # Register "literal" subjects required for these tasks. # TODO: Replace with `Subsystems`. address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, parser_cls=LegacyPythonCallbacksParser, build_ignore_patterns=build_ignore_patterns, exclude_target_regexps=exclude_target_regexps, subproject_roots=subproject_roots) # Load the native backend. native = native or Native.create() # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The # LegacyBuildGraph will explicitly request the products it needs. tasks = ( create_legacy_graph_tasks(symbol_table_cls) + create_fs_rules() + create_graph_rules(address_mapper, symbol_table_cls) ) # TODO: Do not use the cache yet, as it incurs a high overhead. scheduler = LocalScheduler(workdir, dict(), tasks, project_tree, native, include_trace_on_error=include_trace_on_error) change_calculator = EngineChangeCalculator(scheduler, symbol_table_cls, scm) if scm else None return LegacyGraphHelper(scheduler, symbol_table_cls, change_calculator)
def _maybe_init_graph_session(graph_session, global_options, build_config): if graph_session: return graph_session native = Native.create(global_options) native.set_panic_handler() graph_scheduler_helper = EngineInitializer.setup_legacy_graph( native, global_options, build_config) return graph_scheduler_helper.new_session()
def setup_logging_from_options(bootstrap_options): # N.B. quiet help says 'Squelches all console output apart from errors'. level = 'ERROR' if bootstrap_options.quiet else bootstrap_options.level.upper( ) native = Native() return setup_logging(level, console_stream=sys.stderr, log_dir=bootstrap_options.logdir, native=native)
def _init_graph(self, use_engine, pants_ignore_patterns, build_ignore_patterns, exclude_target_regexps, target_specs, workdir, graph_helper=None, subproject_build_roots=None): """Determine the BuildGraph, AddressMapper and spec_roots for a given run. :param bool use_engine: Whether or not to use the v2 engine to construct the BuildGraph. :param list pants_ignore_patterns: The pants ignore patterns from '--pants-ignore'. :param list build_ignore_patterns: The build ignore patterns from '--build-ignore', applied during BUILD file searching. :param str workdir: The pants workdir. :param list exclude_target_regexps: Regular expressions for targets to be excluded. :param list target_specs: The original target specs. :param LegacyGraphHelper graph_helper: A LegacyGraphHelper to use for graph construction, if available. This would usually come from the daemon. :returns: A tuple of (BuildGraph, AddressMapper, opt Scheduler, spec_roots). """ # N.B. Use of the daemon implies use of the v2 engine. if graph_helper or use_engine: # The daemon may provide a `graph_helper`. If that's present, use it for graph construction. if not graph_helper: native = Native.create(self._global_options) native.set_panic_handler() graph_helper = EngineInitializer.setup_legacy_graph( pants_ignore_patterns, workdir, self._global_options.build_file_imports, native=native, build_file_aliases=self._build_config.registered_aliases(), build_ignore_patterns=build_ignore_patterns, exclude_target_regexps=exclude_target_regexps, subproject_roots=subproject_build_roots, include_trace_on_error=self._options.for_global_scope( ).print_exception_stacktrace) target_roots = TargetRoots.create( options=self._options, build_root=self._root_dir, change_calculator=graph_helper.change_calculator) graph, address_mapper = graph_helper.create_build_graph( target_roots, self._root_dir) return graph, address_mapper, graph_helper.scheduler, target_roots.as_specs( ) else: spec_roots = TargetRoots.parse_specs(target_specs, self._root_dir) address_mapper = BuildFileAddressMapper( self._build_file_parser, get_project_tree(self._global_options), build_ignore_patterns, exclude_target_regexps, subproject_build_roots) return MutableBuildGraph( address_mapper), address_mapper, None, spec_roots
def setup_logging_to_stderr(python_logger, level): """We setup logging as loose as possible from the Python side, and let Rust do the filtering.""" native = Native() levelno = get_numeric_level(level) handler = create_native_stderr_log_handler(levelno, native, stream=sys.stderr) python_logger.addHandler(handler) # Let the rust side filter levels; try to have the python side send everything to the rust logger. python_logger.setLevel("TRACE")
def setup_logging_to_stderr(python_logger: Logger, log_level: LogLevel) -> None: """We setup logging as loose as possible from the Python side, and let Rust do the filtering.""" native = Native() handler = create_native_stderr_log_handler(log_level, native, stream=sys.stderr) python_logger.addHandler(handler) # Let the rust side filter levels; try to have the python side send everything to the rust logger. LogLevel.TRACE.set_level_for(python_logger)
def _maybe_init_graph_session(graph_session, options_bootstrapper,build_config, options): if not graph_session: native = Native() native.set_panic_handler() graph_scheduler_helper = EngineInitializer.setup_legacy_graph( native, options_bootstrapper, build_config ) v2_ui = options.for_global_scope().v2_ui zipkin_trace_v2 = options.for_scope('reporting').zipkin_trace_v2 #TODO(gregorys) This should_report_workunits flag must be set to True for # AsyncWorkunitHandler to receive WorkUnits. It should eventually # be merged with the zipkin_trace_v2 flag, since they both involve most # of the same engine functionality, but for now is separate to avoid # breaking functionality associated with zipkin tracing while iterating on async workunit reporting. should_report_workunits = False graph_session = graph_scheduler_helper.new_session(zipkin_trace_v2, RunTracker.global_instance().run_id, v2_ui, should_report_workunits) return graph_session, graph_session.scheduler_session
def _maybe_init_graph_session(graph_session, options_bootstrapper, build_config): if graph_session: return graph_session native = Native.create( options_bootstrapper.bootstrap_options.for_global_scope()) native.set_panic_handler() graph_scheduler_helper = EngineInitializer.setup_legacy_graph( native, options_bootstrapper, build_config) return graph_scheduler_helper.new_session()
def _maybe_init_graph_session(graph_session, global_options, build_config): if graph_session: return graph_session native = Native.create(global_options) native.set_panic_handler() graph_scheduler_helper = EngineInitializer.setup_legacy_graph( native, global_options, build_config ) return graph_scheduler_helper.new_session()
def __init__( self, log_level: LogLevel, stream: Optional[TextIO] = None, native_filename: Optional[str] = None, ): if stream is not None and native_filename is not None: raise RuntimeError("NativeHandler must output to either a stream or a file, not both") super().__init__(stream) self.native = Native() self.native_filename = native_filename self.setLevel(log_level.level) if stream: try: self.native.setup_stderr_logger(log_level.level) except Exception as e: print(f"Error setting up pantsd logger: {e!r}", file=sys.stderr) raise e
def setup_logging_from_options(bootstrap_options): # N.B. quiet help says 'Squelches all console output apart from errors'. level = ("ERROR" if getattr(bootstrap_options, "quiet", False) else bootstrap_options.level.upper()) native = Native() return setup_logging( level, console_stream=sys.stderr, log_dir=bootstrap_options.logdir, native=native, warnings_filter_regexes=bootstrap_options.ignore_pants_warnings, )
def setup_logging_to_file( level: LogLevel, *, log_dir: str, log_filename: str = "pants.log", warnings_filter_regexes: Optional[List[str]] = None, ) -> NativeHandler: native = Native() logger = logging.getLogger(None) _common_logging_setup(level, warnings_filter_regexes) safe_mkdir(log_dir) log_path = os.path.join(log_dir, log_filename) fd = native.setup_pantsd_logger(log_path, level.level) ExceptionSink.reset_interactive_output_stream(os.fdopen(os.dup(fd), "a")) native_handler = NativeHandler(level, native_filename=log_path) logger.addHandler(native_handler) return native_handler
def setup_logging_from_options( bootstrap_options: OptionValueContainer, ) -> Optional[FileLoggingSetupResult]: # N.B. quiet help says 'Squelches all console output apart from errors'. level = (LogLevel.ERROR if getattr(bootstrap_options, "quiet", False) else bootstrap_options.level) native = Native() log_dir: Optional[str] = bootstrap_options.logdir return setup_logging( level, native=native, log_dir=log_dir, console_stream=sys.stderr, warnings_filter_regexes=bootstrap_options.ignore_pants_warnings, )
def _init_graph(self, pants_ignore_patterns, build_ignore_patterns, exclude_target_regexps, target_specs, target_roots, workdir, graph_helper, subproject_build_roots): """Determine the BuildGraph, AddressMapper and spec_roots for a given run. :param list pants_ignore_patterns: The pants ignore patterns from '--pants-ignore'. :param list build_ignore_patterns: The build ignore patterns from '--build-ignore', applied during BUILD file searching. :param str workdir: The pants workdir. :param list exclude_target_regexps: Regular expressions for targets to be excluded. :param list target_specs: The original target specs. :param TargetRoots target_roots: The existing `TargetRoots` object, if any. :param LegacyGraphHelper graph_helper: A LegacyGraphHelper to use for graph construction, if available. This would usually come from the daemon. :returns: A tuple of (BuildGraph, AddressMapper, opt Scheduler, TargetRoots). """ # The daemon may provide a `graph_helper`. If that's present, use it for graph construction. if not graph_helper: native = Native.create(self._global_options) native.set_panic_handler() graph_helper = EngineInitializer.setup_legacy_graph( pants_ignore_patterns, workdir, self._global_options.build_file_imports, native=native, build_file_aliases=self._build_config.registered_aliases(), rules=self._build_config.rules(), build_ignore_patterns=build_ignore_patterns, exclude_target_regexps=exclude_target_regexps, subproject_roots=subproject_build_roots, include_trace_on_error=self._options.for_global_scope().print_exception_stacktrace ) target_roots = target_roots or TargetRootsCalculator.create( options=self._options, build_root=self._root_dir, change_calculator=graph_helper.change_calculator ) graph, address_mapper = graph_helper.create_build_graph(target_roots, self._root_dir) return graph, address_mapper, graph_helper.scheduler, target_roots
def _init_graph(self, use_engine, pants_ignore_patterns, build_ignore_patterns, exclude_target_regexps, target_specs, workdir, graph_helper=None, subproject_build_roots=None): """Determine the BuildGraph, AddressMapper and spec_roots for a given run. :param bool use_engine: Whether or not to use the v2 engine to construct the BuildGraph. :param list pants_ignore_patterns: The pants ignore patterns from '--pants-ignore'. :param list build_ignore_patterns: The build ignore patterns from '--build-ignore', applied during BUILD file searching. :param str workdir: The pants workdir. :param list exclude_target_regexps: Regular expressions for targets to be excluded. :param list target_specs: The original target specs. :param LegacyGraphHelper graph_helper: A LegacyGraphHelper to use for graph construction, if available. This would usually come from the daemon. :returns: A tuple of (BuildGraph, AddressMapper, spec_roots). """ # N.B. Use of the daemon implies use of the v2 engine. if graph_helper or use_engine: # The daemon may provide a `graph_helper`. If that's present, use it for graph construction. if not graph_helper: native = Native.create(self._global_options) native.set_panic_handler() graph_helper = EngineInitializer.setup_legacy_graph( pants_ignore_patterns, workdir, native=native, build_ignore_patterns=build_ignore_patterns, exclude_target_regexps=exclude_target_regexps, subproject_roots=subproject_build_roots, include_trace_on_error=self._options.for_global_scope().print_exception_stacktrace ) target_roots = TargetRoots.create(options=self._options, build_root=self._root_dir, change_calculator=graph_helper.change_calculator) graph, address_mapper = graph_helper.create_build_graph(target_roots, self._root_dir) return graph, address_mapper, target_roots.as_specs() else: spec_roots = TargetRoots.parse_specs(target_specs, self._root_dir) address_mapper = BuildFileAddressMapper(self._build_file_parser, get_project_tree(self._global_options), build_ignore_patterns, exclude_target_regexps, subproject_build_roots) return MutableBuildGraph(address_mapper), address_mapper, spec_roots
def create(cls, options_bootstrapper, full_init=True) -> "PantsDaemon": """ :param OptionsBootstrapper options_bootstrapper: The bootstrap options. :param bool full_init: Whether or not to fully initialize an engine et al for the purposes of spawning a new daemon. `full_init=False` is intended primarily for lightweight lifecycle checks (since there is a ~1s overhead to initialize the engine). See the impl of `maybe_launch` for an example of the intended usage. """ bootstrap_options = options_bootstrapper.bootstrap_options bootstrap_options_values = bootstrap_options.for_global_scope() # TODO: https://github.com/pantsbuild/pants/issues/3479 watchman = WatchmanLauncher.create( bootstrap_options_values).watchman native: Optional[Native] = None build_root: Optional[str] = None if full_init: build_root = get_buildroot() native = Native() build_config = BuildConfigInitializer.get(options_bootstrapper) legacy_graph_scheduler = EngineInitializer.setup_legacy_graph( native, options_bootstrapper, build_config) services = cls._setup_services( build_root, bootstrap_options_values, legacy_graph_scheduler, watchman, union_membership=UnionMembership( build_config.union_rules()), ) else: services = PantsServices() return PantsDaemon( native=native, build_root=build_root, work_dir=bootstrap_options_values.pants_workdir, log_level=bootstrap_options_values.level, services=services, metadata_base_dir=bootstrap_options_values.pants_subprocessdir, bootstrap_options=bootstrap_options, )
def create(cls, bootstrap_options=None, full_init=True): """ :param Options bootstrap_options: The bootstrap options, if available. :param bool full_init: Whether or not to fully initialize an engine et al for the purposes of spawning a new daemon. `full_init=False` is intended primarily for lightweight lifecycle checks (since there is a ~1s overhead to initialize the engine). See the impl of `maybe_launch` for an example of the intended usage. """ bootstrap_options = bootstrap_options or cls._parse_bootstrap_options() bootstrap_options_values = bootstrap_options.for_global_scope() # TODO: https://github.com/pantsbuild/pants/issues/3479 watchman = WatchmanLauncher.create(bootstrap_options_values).watchman native = None build_root = None services = None port_map = None if full_init: build_root = get_buildroot() native = Native.create(bootstrap_options_values) options_bootstrapper = OptionsBootstrapper() build_config = BuildConfigInitializer.get(options_bootstrapper) legacy_graph_scheduler = EngineInitializer.setup_legacy_graph(native, bootstrap_options_values, build_config) services, port_map = cls._setup_services( build_root, bootstrap_options_values, legacy_graph_scheduler, watchman ) return PantsDaemon( native=native, build_root=build_root, work_dir=bootstrap_options_values.pants_workdir, log_level=bootstrap_options_values.level.upper(), services=services, socket_map=port_map, metadata_base_dir=bootstrap_options_values.pants_subprocessdir, bootstrap_options=bootstrap_options )
def create(cls, bootstrap_options=None, full_init=True): """ :param Options bootstrap_options: The bootstrap options, if available. :param bool full_init: Whether or not to fully initialize an engine et al for the purposes of spawning a new daemon. `full_init=False` is intended primarily for lightweight lifecycle checks (since there is a ~1s overhead to initialize the engine). See the impl of `maybe_launch` for an example of the intended usage. """ bootstrap_options = bootstrap_options or cls._parse_bootstrap_options() bootstrap_options_values = bootstrap_options.for_global_scope() # TODO: https://github.com/pantsbuild/pants/issues/3479 watchman = WatchmanLauncher.create(bootstrap_options_values).watchman native = None build_root = None services = None port_map = None if full_init: build_root = get_buildroot() native = Native.create(bootstrap_options_values) options_bootstrapper = OptionsBootstrapper() build_config = BuildConfigInitializer.get(options_bootstrapper) legacy_graph_scheduler = EngineInitializer.setup_legacy_graph(native, bootstrap_options_values, build_config) services, port_map = cls._setup_services( build_root, bootstrap_options_values, legacy_graph_scheduler, watchman ) return PantsDaemon( native=native, build_root=build_root, work_dir=bootstrap_options_values.pants_workdir, log_level=bootstrap_options_values.level.upper(), services=services, socket_map=port_map, metadata_base_dir=bootstrap_options_values.pants_subprocessdir, bootstrap_options=bootstrap_options )
def _init_graph(self, pants_ignore_patterns, build_ignore_patterns, exclude_target_regexps, target_specs, target_roots, workdir, graph_helper, subproject_build_roots): """Determine the BuildGraph, AddressMapper and spec_roots for a given run. :param list pants_ignore_patterns: The pants ignore patterns from '--pants-ignore'. :param list build_ignore_patterns: The build ignore patterns from '--build-ignore', applied during BUILD file searching. :param str workdir: The pants workdir. :param list exclude_target_regexps: Regular expressions for targets to be excluded. :param list target_specs: The original target specs. :param TargetRoots target_roots: The existing `TargetRoots` object, if any. :param LegacyGraphSession graph_helper: A LegacyGraphSession to use for graph construction, if available. This would usually come from the daemon. :returns: A tuple of (BuildGraph, AddressMapper, SchedulerSession, TargetRoots). """ # The daemon may provide a `graph_helper`. If that's present, use it for graph construction. if not graph_helper: native = Native.create(self._global_options) native.set_panic_handler() graph_helper = EngineInitializer.setup_legacy_graph( pants_ignore_patterns, workdir, self._global_options.build_file_imports, native=native, build_file_aliases=self._build_config.registered_aliases(), rules=self._build_config.rules(), build_ignore_patterns=build_ignore_patterns, exclude_target_regexps=exclude_target_regexps, subproject_roots=subproject_build_roots, include_trace_on_error=self._options.for_global_scope( ).print_exception_stacktrace).new_session() target_roots = target_roots or TargetRootsCalculator.create( options=self._options, build_root=self._root_dir, change_calculator=graph_helper.change_calculator) graph, address_mapper = graph_helper.create_build_graph( target_roots, self._root_dir) return graph, address_mapper, graph_helper.scheduler_session, target_roots
def process_request_thread(self, request, client_address): """Override of ThreadingMixIn.process_request_thread() that delegates to the request handler.""" # Instantiate the request handler. Native().override_thread_logging_destination_to_just_stderr() handler = self.RequestHandlerClass(request, client_address, self) try: # Attempt to handle a request with the handler. handler.handle_request() self.request_complete_callback() except Exception as e: # If that fails, (synchronously) handle the error with the error handler sans-fork. try: handler.handle_error(e) finally: # Shutdown the socket since we don't expect a fork() in the exception context. self.shutdown_request(request) else: # At this point, we expect a fork() has taken place - the parent side will return, and so we # close the request here from the parent without explicitly shutting down the socket. The # child half of this will perform an os._exit() before it gets to this point and is also # responsible for shutdown and closing of the socket when its execution is complete. self.close_request(request)
class NativeHandler(StreamHandler): """This class is installed as a Python logging module handler (using the logging.addHandler method) and proxies logs to the Rust logging infrastructure.""" def __init__( self, log_level: LogLevel, stream: Optional[TextIO] = None, native_filename: Optional[str] = None, ): if stream is not None and native_filename is not None: raise RuntimeError("NativeHandler must output to either a stream or a file, not both") super().__init__(stream) self.native = Native() self.native_filename = native_filename self.setLevel(log_level.level) if stream: try: self.native.setup_stderr_logger(log_level.level) except Exception as e: print(f"Error setting up pantsd logger: {e!r}", file=sys.stderr) raise e def emit(self, record: LogRecord) -> None: self.native.write_log( self.format(record), record.levelno, f"{record.name}:pid={os.getpid()}" ) def flush(self) -> None: self.native.flush_log() def __repr__(self) -> str: return ( f"NativeHandler(id={id(self)}, level={self.level}, filename={self.native_filename}, " f"stream={self.stream})" )
def init_native(): """Initialize and return a `Native` instance.""" init_subsystem(BinaryUtilPrivate.Factory) opts = create_options_for_optionables([]) return Native.create(opts.for_global_scope())
def setup_legacy_graph_extended( pants_ignore_patterns, workdir, build_file_imports_behavior, build_configuration, build_root=None, native=None, glob_match_error_behavior=None, rules=None, build_ignore_patterns=None, exclude_target_regexps=None, subproject_roots=None, include_trace_on_error=True, remote_store_server=None, remote_execution_server=None ): """Construct and return the components necessary for LegacyBuildGraph construction. :param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree, usually taken from the '--pants-ignore' global option. :param str workdir: The pants workdir. :param build_file_imports_behavior: How to behave if a BUILD file being parsed tries to use import statements. Valid values: "allow", "warn", "error". :type build_file_imports_behavior: string :param str build_root: A path to be used as the build root. If None, then default is used. :param Native native: An instance of the native-engine subsystem. :param build_configuration: The `BuildConfiguration` object to get build file aliases from. :type build_configuration: :class:`pants.build_graph.build_configuration.BuildConfiguration` :param glob_match_error_behavior: How to behave if a glob specified for a target's sources or bundles does not expand to anything. :type glob_match_error_behavior: :class:`pants.option.global_options.GlobMatchErrorBehavior` :param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD files, usually taken from the '--build-ignore' global option. :param list exclude_target_regexps: A list of regular expressions for excluding targets. :param list subproject_roots: Paths that correspond with embedded build roots under the current build root. :param bool include_trace_on_error: If True, when an error occurs, the error message will include the graph trace. :returns: A LegacyGraphScheduler. """ build_root = build_root or get_buildroot() build_configuration = build_configuration or BuildConfigInitializer.get(OptionsBootstrapper()) build_file_aliases = build_configuration.registered_aliases() rules = rules or build_configuration.rules() or [] symbol_table = LegacySymbolTable(build_file_aliases) project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns) # Register "literal" subjects required for these rules. parser = LegacyPythonCallbacksParser( symbol_table, build_file_aliases, build_file_imports_behavior ) address_mapper = AddressMapper(parser=parser, build_ignore_patterns=build_ignore_patterns, exclude_target_regexps=exclude_target_regexps, subproject_roots=subproject_roots) # Load the native backend. native = native or Native.create() # Create a Scheduler containing graph and filesystem rules, with no installed goals. The # LegacyBuildGraph will explicitly request the products it needs. rules = ( [ SingletonRule.from_instance(GlobMatchErrorBehavior.create(glob_match_error_behavior)), SingletonRule.from_instance(build_configuration), ] + create_legacy_graph_tasks(symbol_table) + create_fs_rules() + create_process_rules() + create_graph_rules(address_mapper, symbol_table) + create_options_parsing_rules() + rules ) scheduler = Scheduler( native, project_tree, workdir, rules, remote_store_server, remote_execution_server, include_trace_on_error=include_trace_on_error, ) return LegacyGraphScheduler(scheduler, symbol_table)
def init_native(): """Initialize and return a `Native` instance.""" init_subsystem(BinaryUtil.Factory) opts = create_options_for_optionables([]) return Native.create(opts.for_global_scope())
def setup_legacy_graph(pants_ignore_patterns, workdir, build_file_imports_behavior, build_root=None, native=None, build_file_aliases=None, rules=None, build_ignore_patterns=None, exclude_target_regexps=None, subproject_roots=None, include_trace_on_error=True): """Construct and return the components necessary for LegacyBuildGraph construction. :param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree, usually taken from the '--pants-ignore' global option. :param str workdir: The pants workdir. :param build_file_imports_behavior: How to behave if a BUILD file being parsed tries to use import statements. Valid values: "allow", "warn", "error". :type build_file_imports_behavior: string :param str build_root: A path to be used as the build root. If None, then default is used. :param Native native: An instance of the native-engine subsystem. :param build_file_aliases: BuildFileAliases to register. :type build_file_aliases: :class:`pants.build_graph.build_file_aliases.BuildFileAliases` :param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD files, usually taken from the '--build-ignore' global option. :param list exclude_target_regexps: A list of regular expressions for excluding targets. :param list subproject_roots: Paths that correspond with embedded build roots under the current build root. :param bool include_trace_on_error: If True, when an error occurs, the error message will include the graph trace. :returns: A tuple of (scheduler, engine, symbol_table, build_graph_cls). """ build_root = build_root or get_buildroot() scm = get_scm() if not build_file_aliases: build_file_aliases = EngineInitializer.get_default_build_file_aliases() if not rules: rules = [] symbol_table = LegacySymbolTable(build_file_aliases) project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns) # Register "literal" subjects required for these tasks. parser = LegacyPythonCallbacksParser( symbol_table, build_file_aliases, build_file_imports_behavior ) address_mapper = AddressMapper(parser=parser, build_ignore_patterns=build_ignore_patterns, exclude_target_regexps=exclude_target_regexps, subproject_roots=subproject_roots) # Load the native backend. native = native or Native.create() # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The # LegacyBuildGraph will explicitly request the products it needs. tasks = ( create_legacy_graph_tasks(symbol_table) + create_fs_rules() + create_graph_rules(address_mapper, symbol_table) + create_process_rules() + rules ) scheduler = LocalScheduler(workdir, dict(), tasks, project_tree, native, include_trace_on_error=include_trace_on_error) change_calculator = EngineChangeCalculator(scheduler, symbol_table, scm) if scm else None return LegacyGraphHelper(scheduler, symbol_table, change_calculator)
def init_rust_logger(level, log_show_rust_3rdparty): native = Native() levelno = get_numeric_level(level) native.init_rust_logging(levelno, log_show_rust_3rdparty)