def _init_graph_session(
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
options: Options,
) -> LegacyGraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native, options_bootstrapper, build_config)
v2_ui = options.for_global_scope().get("v2_ui", False)
zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2
# TODO(#8658) This should_report_workunits flag must be set to True for
# StreamingWorkunitHandler to receive WorkUnits. It should eventually
# be merged with the zipkin_trace_v2 flag, since they both involve most
# of the same engine functionality, but for now is separate to avoid
# breaking functionality associated with zipkin tracing while iterating on streaming workunit reporting.
stream_workunits = len(
options.for_global_scope().streaming_workunits_handlers) != 0
return graph_scheduler_helper.new_session(
zipkin_trace_v2,
RunTracker.global_instance().run_id,
v2_ui,
should_report_workunits=stream_workunits,
)
def _run_pants(self, sock, arguments, environment):
"""Execute a given run with a pants runner."""
# For the pants run, we want to log to stderr.
# TODO Might be worth to make contextmanagers for this?
Native().override_thread_logging_destination_to_just_stderr()
self.server.runner_factory(sock, arguments, environment).run()
Native().override_thread_logging_destination_to_just_pantsd()
def _maybe_init_graph_session(graph_session, options_bootstrapper,
build_config, global_options):
if graph_session:
return graph_session
native = Native()
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native, options_bootstrapper, build_config)
return graph_scheduler_helper.new_session(global_options.v2_ui)
def create_native_stderr_log_handler(
log_level: LogLevel,
native: Native,
stream: Optional[TextIO] = None) -> NativeHandler:
try:
native.setup_stderr_logger(log_level.level)
except Exception as e:
print(f"Error setting up pantsd logger: {e!r}", file=sys.stderr)
raise e
return NativeHandler(log_level, native, stream)
def _maybe_init_graph_session(graph_session, options_bootstrapper,
build_config, options):
if graph_session:
return graph_session
native = Native()
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native, options_bootstrapper, build_config)
v2_ui = options.for_global_scope().v2_ui
zipkin_trace_v2 = options.for_scope('reporting').zipkin_trace_v2
return graph_scheduler_helper.new_session(zipkin_trace_v2, v2_ui)
def _maybe_init_graph_session(graph_session, options_bootstrapper,build_config, options):
if graph_session:
return graph_session
native = Native()
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native,
options_bootstrapper,
build_config
)
v2_ui = options.for_global_scope().v2_ui
zipkin_trace_v2 = options.for_scope('reporting').zipkin_trace_v2
return graph_scheduler_helper.new_session(zipkin_trace_v2, v2_ui)
def process_request_thread(self, request, client_address):
"""Override of ThreadingMixIn.process_request_thread() that delegates to the request
handler."""
# Instantiate the request handler.
Native().override_thread_logging_destination_to_just_pantsd()
handler = self.RequestHandlerClass(request, client_address, self)
_, _, _, environment = handler.parsed_request()
try:
with self.ensure_request_is_exclusive(environment, request):
# Attempt to handle a request with the handler.
handler.handle_request()
self.request_complete_callback()
except BrokenPipeError as e:
# The client has closed the connection, most likely from a SIGINT
self.logger.error(
f"Request {request} abruptly closed with {type(e)}, probably because the client crashed or was sent a SIGINT."
)
except Exception as e:
# If that fails, (synchronously) handle the error with the error handler sans-fork.
try:
self.logger.error(
f"Request {request} errored with {type(e)}({e!r})")
handler.handle_error(e)
finally:
# Shutdown the socket since we don't expect a fork() in the exception context.
self.shutdown_request(request)
else:
# At this point, we expect a fork() has taken place - the parent side will return, and so we
# close the request here from the parent without explicitly shutting down the socket. The
# child half of this will perform an os._exit() before it gets to this point and is also
# responsible for shutdown and closing of the socket when its execution is complete.
self.close_request(request)
def create(cls, bootstrap_options=None):
"""
:param Options bootstrap_options: The bootstrap options, if available.
"""
bootstrap_options = bootstrap_options or cls._parse_bootstrap_options()
bootstrap_options_values = bootstrap_options.for_global_scope()
build_root = get_buildroot()
native = Native.create(bootstrap_options_values)
# TODO: https://github.com/pantsbuild/pants/issues/3479
watchman = WatchmanLauncher.create(bootstrap_options_values).watchman
legacy_graph_helper = cls._setup_legacy_graph_helper(native, bootstrap_options_values)
services, port_map = cls._setup_services(
build_root,
bootstrap_options_values,
legacy_graph_helper,
watchman
)
return PantsDaemon(
native,
build_root,
bootstrap_options_values.pants_workdir,
bootstrap_options_values.level.upper(),
services,
port_map,
bootstrap_options_values.pants_subprocessdir,
bootstrap_options
)
def create(cls, bootstrap_options=None):
"""
:param Options bootstrap_options: The bootstrap options, if available.
"""
bootstrap_options = bootstrap_options or cls._parse_bootstrap_options(
)
bootstrap_options_values = bootstrap_options.for_global_scope()
build_root = get_buildroot()
native = Native.create(bootstrap_options_values)
# TODO: https://github.com/pantsbuild/pants/issues/3479
watchman = WatchmanLauncher.create(
bootstrap_options_values).watchman
legacy_graph_helper = cls._setup_legacy_graph_helper(
native, bootstrap_options_values)
services, port_map = cls._setup_services(build_root,
bootstrap_options_values,
legacy_graph_helper,
watchman)
return PantsDaemon(native, build_root,
bootstrap_options_values.pants_workdir,
bootstrap_options_values.level.upper(),
services, port_map,
bootstrap_options_values.pants_subprocessdir,
bootstrap_options)
def _init_graph(self, target_roots, graph_helper, exclude_target_regexps, tags):
"""Determine the BuildGraph, AddressMapper and spec_roots for a given run.
:param TargetRoots target_roots: The existing `TargetRoots` object, if any.
:param LegacyGraphSession graph_helper: A LegacyGraphSession to use for graph construction,
if available. This would usually come from the daemon.
:returns: A tuple of (BuildGraph, AddressMapper, SchedulerSession, TargetRoots).
"""
# The daemon may provide a `graph_helper`. If that's present, use it for graph construction.
if not graph_helper:
native = Native.create(self._global_options)
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(native,
self._global_options,
self._build_config)
graph_helper = graph_scheduler_helper.new_session()
target_roots = target_roots or TargetRootsCalculator.create(
options=self._options,
session=graph_helper.scheduler_session,
build_root=self._root_dir,
symbol_table=graph_helper.symbol_table,
exclude_patterns=tuple(exclude_target_regexps),
tags=tuple(tags)
)
graph, address_mapper = graph_helper.create_build_graph(target_roots,
self._root_dir)
return graph, address_mapper, graph_helper.scheduler_session, target_roots
def globs_matches(
paths: Iterable[str],
patterns: Iterable[str],
exclude_patterns: Iterable[str],
) -> bool:
path_globs = PathGlobs(include=patterns, exclude=exclude_patterns)
return Native().match_path_globs(path_globs, paths)
def __init__(self, bootstrap_options, engine_initializer=None):
"""
:param Options bootstrap_options: An Options object containing the bootstrap options.
:param class engine_initializer: The class representing the EngineInitializer. Only required
for startup.
"""
self._bootstrap_options = bootstrap_options
self._engine_initializer = engine_initializer
self._pailgun_host = bootstrap_options.pantsd_pailgun_host
self._pailgun_port = bootstrap_options.pantsd_pailgun_port
self._log_dir = bootstrap_options.pantsd_log_dir
self._fs_event_workers = bootstrap_options.pantsd_fs_event_workers
self._pants_workdir = bootstrap_options.pants_workdir
self._log_level = bootstrap_options.level.upper()
self._pants_ignore_patterns = bootstrap_options.pants_ignore
self._build_ignore_patterns = bootstrap_options.build_ignore
self._exclude_target_regexp = bootstrap_options.exclude_target_regexp
self._subproject_roots = bootstrap_options.subproject_roots
self._metadata_base_dir = bootstrap_options.pants_subprocessdir
# TODO: https://github.com/pantsbuild/pants/issues/3479
self._build_root = get_buildroot()
self._native = Native.create(bootstrap_options)
self._logger = logging.getLogger(__name__)
self._lock = OwnerPrintingInterProcessFileLock(
os.path.join(self._build_root, '.pantsd.startup'))
def __init__(self,
build_root,
engine_initializer,
options):
"""
:param str build_root: The path of the build root.
:param class engine_initializer: The class representing the EngineInitializer.
"""
self._build_root = build_root
self._engine_initializer = engine_initializer
# The options we register directly.
self._pailgun_host = options.pailgun_host
self._pailgun_port = options.pailgun_port
self._log_dir = options.log_dir
self._fs_event_workers = options.fs_event_workers
# Values derived from global options (which our scoped options inherit).
self._pants_workdir = options.pants_workdir
self._log_level = options.level.upper()
self._pants_ignore_patterns = options.pants_ignore
self._build_ignore_patterns = options.build_ignore
self._exclude_target_regexp = options.exclude_target_regexp
self._subproject_roots = options.subproject_roots
# Native.create() reads global options, which, thanks to inheritance, it can
# read them via our scoped options.
self._native = Native.create(options)
# TODO(kwlzn): Thread filesystem path ignores here to Watchman's subscription registration.
lock_location = os.path.join(self._build_root, '.pantsd.startup')
self._lock = OwnerPrintingInterProcessFileLock(lock_location)
self._logger = logging.getLogger(__name__)
def __init__(self,
build_root,
engine_initializer,
options):
"""
:param str build_root: The path of the build root.
:param class engine_initializer: The class representing the EngineInitializer.
"""
self._build_root = build_root
self._engine_initializer = engine_initializer
# The options we register directly.
self._pailgun_host = options.pailgun_host
self._pailgun_port = options.pailgun_port
self._log_dir = options.log_dir
self._fs_event_workers = options.fs_event_workers
# Values derived from global options (which our scoped options inherit).
self._pants_workdir = options.pants_workdir
self._log_level = options.level.upper()
self._pants_ignore_patterns = options.pants_ignore
self._build_ignore_patterns = options.build_ignore
self._exclude_target_regexp = options.exclude_target_regexp
self._subproject_roots = options.subproject_roots
# Native.create() reads global options, which, thanks to inheritance, it can
# read them via our scoped options.
self._native = Native.create(options)
# TODO(kwlzn): Thread filesystem path ignores here to Watchman's subscription registration.
lock_location = os.path.join(self._build_root, '.pantsd.startup')
self._lock = OwnerPrintingInterProcessFileLock(lock_location)
self._logger = logging.getLogger(__name__)
def globs_matches(
paths: Iterable[str],
patterns: Iterable[str],
exclude_patterns: Iterable[str],
) -> bool:
path_globs = PathGlobs(globs=(*patterns, *(f"!{e}"
for e in exclude_patterns)))
return Native().match_path_globs(path_globs, paths)
def setup_legacy_graph(pants_ignore_patterns,
workdir,
build_root=None,
native=None,
symbol_table_cls=None,
build_ignore_patterns=None,
exclude_target_regexps=None,
subproject_roots=None,
include_trace_on_error=True):
"""Construct and return the components necessary for LegacyBuildGraph construction.
:param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree,
usually taken from the '--pants-ignore' global option.
:param str workdir: The pants workdir.
:param str build_root: A path to be used as the build root. If None, then default is used.
:param Native native: An instance of the native-engine subsystem.
:param SymbolTable symbol_table_cls: A SymbolTable class to use for build file parsing, or
None to use the default.
:param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD
files, usually taken from the '--build-ignore' global option.
:param list exclude_target_regexps: A list of regular expressions for excluding targets.
:param list subproject_roots: Paths that correspond with embedded build roots
under the current build root.
:param bool include_trace_on_error: If True, when an error occurs, the error message will
include the graph trace.
:returns: A tuple of (scheduler, engine, symbol_table_cls, build_graph_cls).
"""
build_root = build_root or get_buildroot()
scm = get_scm()
symbol_table_cls = symbol_table_cls or LegacySymbolTable
project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns)
# Register "literal" subjects required for these tasks.
# TODO: Replace with `Subsystems`.
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
parser_cls=LegacyPythonCallbacksParser,
build_ignore_patterns=build_ignore_patterns,
exclude_target_regexps=exclude_target_regexps,
subproject_roots=subproject_roots)
# Load the native backend.
native = native or Native.create()
# Create a Scheduler containing graph and filesystem tasks, with no installed goals. The
# LegacyBuildGraph will explicitly request the products it needs.
tasks = (
create_legacy_graph_tasks(symbol_table_cls) +
create_fs_rules() +
create_graph_rules(address_mapper, symbol_table_cls)
)
# TODO: Do not use the cache yet, as it incurs a high overhead.
scheduler = LocalScheduler(workdir, dict(), tasks, project_tree, native, include_trace_on_error=include_trace_on_error)
change_calculator = EngineChangeCalculator(scheduler, symbol_table_cls, scm) if scm else None
return LegacyGraphHelper(scheduler, symbol_table_cls, change_calculator)
def _maybe_init_graph_session(graph_session, global_options, build_config):
if graph_session:
return graph_session
native = Native.create(global_options)
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native, global_options, build_config)
return graph_scheduler_helper.new_session()
def setup_logging_from_options(bootstrap_options):
# N.B. quiet help says 'Squelches all console output apart from errors'.
level = 'ERROR' if bootstrap_options.quiet else bootstrap_options.level.upper(
)
native = Native()
return setup_logging(level,
console_stream=sys.stderr,
log_dir=bootstrap_options.logdir,
native=native)
def _init_graph(self,
use_engine,
pants_ignore_patterns,
build_ignore_patterns,
exclude_target_regexps,
target_specs,
workdir,
graph_helper=None,
subproject_build_roots=None):
"""Determine the BuildGraph, AddressMapper and spec_roots for a given run.
:param bool use_engine: Whether or not to use the v2 engine to construct the BuildGraph.
:param list pants_ignore_patterns: The pants ignore patterns from '--pants-ignore'.
:param list build_ignore_patterns: The build ignore patterns from '--build-ignore',
applied during BUILD file searching.
:param str workdir: The pants workdir.
:param list exclude_target_regexps: Regular expressions for targets to be excluded.
:param list target_specs: The original target specs.
:param LegacyGraphHelper graph_helper: A LegacyGraphHelper to use for graph construction,
if available. This would usually come from the daemon.
:returns: A tuple of (BuildGraph, AddressMapper, opt Scheduler, spec_roots).
"""
# N.B. Use of the daemon implies use of the v2 engine.
if graph_helper or use_engine:
# The daemon may provide a `graph_helper`. If that's present, use it for graph construction.
if not graph_helper:
native = Native.create(self._global_options)
native.set_panic_handler()
graph_helper = EngineInitializer.setup_legacy_graph(
pants_ignore_patterns,
workdir,
self._global_options.build_file_imports,
native=native,
build_file_aliases=self._build_config.registered_aliases(),
build_ignore_patterns=build_ignore_patterns,
exclude_target_regexps=exclude_target_regexps,
subproject_roots=subproject_build_roots,
include_trace_on_error=self._options.for_global_scope(
).print_exception_stacktrace)
target_roots = TargetRoots.create(
options=self._options,
build_root=self._root_dir,
change_calculator=graph_helper.change_calculator)
graph, address_mapper = graph_helper.create_build_graph(
target_roots, self._root_dir)
return graph, address_mapper, graph_helper.scheduler, target_roots.as_specs(
)
else:
spec_roots = TargetRoots.parse_specs(target_specs, self._root_dir)
address_mapper = BuildFileAddressMapper(
self._build_file_parser,
get_project_tree(self._global_options), build_ignore_patterns,
exclude_target_regexps, subproject_build_roots)
return MutableBuildGraph(
address_mapper), address_mapper, None, spec_roots
def setup_logging_to_stderr(python_logger, level):
"""We setup logging as loose as possible from the Python side, and let Rust do the filtering."""
native = Native()
levelno = get_numeric_level(level)
handler = create_native_stderr_log_handler(levelno,
native,
stream=sys.stderr)
python_logger.addHandler(handler)
# Let the rust side filter levels; try to have the python side send everything to the rust logger.
python_logger.setLevel("TRACE")
def setup_logging_to_stderr(python_logger: Logger,
log_level: LogLevel) -> None:
"""We setup logging as loose as possible from the Python side, and let Rust do the filtering."""
native = Native()
handler = create_native_stderr_log_handler(log_level,
native,
stream=sys.stderr)
python_logger.addHandler(handler)
# Let the rust side filter levels; try to have the python side send everything to the rust logger.
LogLevel.TRACE.set_level_for(python_logger)
def _maybe_init_graph_session(graph_session, options_bootstrapper,build_config, options):
if not graph_session:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native,
options_bootstrapper,
build_config
)
v2_ui = options.for_global_scope().v2_ui
zipkin_trace_v2 = options.for_scope('reporting').zipkin_trace_v2
#TODO(gregorys) This should_report_workunits flag must be set to True for
# AsyncWorkunitHandler to receive WorkUnits. It should eventually
# be merged with the zipkin_trace_v2 flag, since they both involve most
# of the same engine functionality, but for now is separate to avoid
# breaking functionality associated with zipkin tracing while iterating on async workunit reporting.
should_report_workunits = False
graph_session = graph_scheduler_helper.new_session(zipkin_trace_v2, RunTracker.global_instance().run_id, v2_ui, should_report_workunits)
return graph_session, graph_session.scheduler_session
def _maybe_init_graph_session(graph_session, options_bootstrapper,
build_config):
if graph_session:
return graph_session
native = Native.create(
options_bootstrapper.bootstrap_options.for_global_scope())
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native, options_bootstrapper, build_config)
return graph_scheduler_helper.new_session()
def _maybe_init_graph_session(graph_session, global_options, build_config):
if graph_session:
return graph_session
native = Native.create(global_options)
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native,
global_options,
build_config
)
return graph_scheduler_helper.new_session()
def __init__(
self,
log_level: LogLevel,
stream: Optional[TextIO] = None,
native_filename: Optional[str] = None,
):
if stream is not None and native_filename is not None:
raise RuntimeError("NativeHandler must output to either a stream or a file, not both")
super().__init__(stream)
self.native = Native()
self.native_filename = native_filename
self.setLevel(log_level.level)
if stream:
try:
self.native.setup_stderr_logger(log_level.level)
except Exception as e:
print(f"Error setting up pantsd logger: {e!r}", file=sys.stderr)
raise e
def setup_logging_from_options(bootstrap_options):
# N.B. quiet help says 'Squelches all console output apart from errors'.
level = ("ERROR" if getattr(bootstrap_options, "quiet", False) else
bootstrap_options.level.upper())
native = Native()
return setup_logging(
level,
console_stream=sys.stderr,
log_dir=bootstrap_options.logdir,
native=native,
warnings_filter_regexes=bootstrap_options.ignore_pants_warnings,
)
def setup_logging_to_file(
level: LogLevel,
*,
log_dir: str,
log_filename: str = "pants.log",
warnings_filter_regexes: Optional[List[str]] = None,
) -> NativeHandler:
native = Native()
logger = logging.getLogger(None)
_common_logging_setup(level, warnings_filter_regexes)
safe_mkdir(log_dir)
log_path = os.path.join(log_dir, log_filename)
fd = native.setup_pantsd_logger(log_path, level.level)
ExceptionSink.reset_interactive_output_stream(os.fdopen(os.dup(fd), "a"))
native_handler = NativeHandler(level, native_filename=log_path)
logger.addHandler(native_handler)
return native_handler
def setup_logging_from_options(
bootstrap_options: OptionValueContainer,
) -> Optional[FileLoggingSetupResult]:
# N.B. quiet help says 'Squelches all console output apart from errors'.
level = (LogLevel.ERROR if getattr(bootstrap_options, "quiet", False) else
bootstrap_options.level)
native = Native()
log_dir: Optional[str] = bootstrap_options.logdir
return setup_logging(
level,
native=native,
log_dir=log_dir,
console_stream=sys.stderr,
warnings_filter_regexes=bootstrap_options.ignore_pants_warnings,
)
def _init_graph(self,
pants_ignore_patterns,
build_ignore_patterns,
exclude_target_regexps,
target_specs,
target_roots,
workdir,
graph_helper,
subproject_build_roots):
"""Determine the BuildGraph, AddressMapper and spec_roots for a given run.
:param list pants_ignore_patterns: The pants ignore patterns from '--pants-ignore'.
:param list build_ignore_patterns: The build ignore patterns from '--build-ignore',
applied during BUILD file searching.
:param str workdir: The pants workdir.
:param list exclude_target_regexps: Regular expressions for targets to be excluded.
:param list target_specs: The original target specs.
:param TargetRoots target_roots: The existing `TargetRoots` object, if any.
:param LegacyGraphHelper graph_helper: A LegacyGraphHelper to use for graph construction,
if available. This would usually come from the daemon.
:returns: A tuple of (BuildGraph, AddressMapper, opt Scheduler, TargetRoots).
"""
# The daemon may provide a `graph_helper`. If that's present, use it for graph construction.
if not graph_helper:
native = Native.create(self._global_options)
native.set_panic_handler()
graph_helper = EngineInitializer.setup_legacy_graph(
pants_ignore_patterns,
workdir,
self._global_options.build_file_imports,
native=native,
build_file_aliases=self._build_config.registered_aliases(),
rules=self._build_config.rules(),
build_ignore_patterns=build_ignore_patterns,
exclude_target_regexps=exclude_target_regexps,
subproject_roots=subproject_build_roots,
include_trace_on_error=self._options.for_global_scope().print_exception_stacktrace
)
target_roots = target_roots or TargetRootsCalculator.create(
options=self._options,
build_root=self._root_dir,
change_calculator=graph_helper.change_calculator
)
graph, address_mapper = graph_helper.create_build_graph(target_roots, self._root_dir)
return graph, address_mapper, graph_helper.scheduler, target_roots
def _init_graph(self, use_engine, pants_ignore_patterns, build_ignore_patterns,
exclude_target_regexps, target_specs, workdir, graph_helper=None,
subproject_build_roots=None):
"""Determine the BuildGraph, AddressMapper and spec_roots for a given run.
:param bool use_engine: Whether or not to use the v2 engine to construct the BuildGraph.
:param list pants_ignore_patterns: The pants ignore patterns from '--pants-ignore'.
:param list build_ignore_patterns: The build ignore patterns from '--build-ignore',
applied during BUILD file searching.
:param str workdir: The pants workdir.
:param list exclude_target_regexps: Regular expressions for targets to be excluded.
:param list target_specs: The original target specs.
:param LegacyGraphHelper graph_helper: A LegacyGraphHelper to use for graph construction,
if available. This would usually come from the daemon.
:returns: A tuple of (BuildGraph, AddressMapper, spec_roots).
"""
# N.B. Use of the daemon implies use of the v2 engine.
if graph_helper or use_engine:
# The daemon may provide a `graph_helper`. If that's present, use it for graph construction.
if not graph_helper:
native = Native.create(self._global_options)
native.set_panic_handler()
graph_helper = EngineInitializer.setup_legacy_graph(
pants_ignore_patterns,
workdir,
native=native,
build_ignore_patterns=build_ignore_patterns,
exclude_target_regexps=exclude_target_regexps,
subproject_roots=subproject_build_roots,
include_trace_on_error=self._options.for_global_scope().print_exception_stacktrace
)
target_roots = TargetRoots.create(options=self._options,
build_root=self._root_dir,
change_calculator=graph_helper.change_calculator)
graph, address_mapper = graph_helper.create_build_graph(target_roots, self._root_dir)
return graph, address_mapper, target_roots.as_specs()
else:
spec_roots = TargetRoots.parse_specs(target_specs, self._root_dir)
address_mapper = BuildFileAddressMapper(self._build_file_parser,
get_project_tree(self._global_options),
build_ignore_patterns,
exclude_target_regexps,
subproject_build_roots)
return MutableBuildGraph(address_mapper), address_mapper, spec_roots
def create(cls, options_bootstrapper, full_init=True) -> "PantsDaemon":
"""
:param OptionsBootstrapper options_bootstrapper: The bootstrap options.
:param bool full_init: Whether or not to fully initialize an engine et al for the purposes
of spawning a new daemon. `full_init=False` is intended primarily
for lightweight lifecycle checks (since there is a ~1s overhead to
initialize the engine). See the impl of `maybe_launch` for an example
of the intended usage.
"""
bootstrap_options = options_bootstrapper.bootstrap_options
bootstrap_options_values = bootstrap_options.for_global_scope()
# TODO: https://github.com/pantsbuild/pants/issues/3479
watchman = WatchmanLauncher.create(
bootstrap_options_values).watchman
native: Optional[Native] = None
build_root: Optional[str] = None
if full_init:
build_root = get_buildroot()
native = Native()
build_config = BuildConfigInitializer.get(options_bootstrapper)
legacy_graph_scheduler = EngineInitializer.setup_legacy_graph(
native, options_bootstrapper, build_config)
services = cls._setup_services(
build_root,
bootstrap_options_values,
legacy_graph_scheduler,
watchman,
union_membership=UnionMembership(
build_config.union_rules()),
)
else:
services = PantsServices()
return PantsDaemon(
native=native,
build_root=build_root,
work_dir=bootstrap_options_values.pants_workdir,
log_level=bootstrap_options_values.level,
services=services,
metadata_base_dir=bootstrap_options_values.pants_subprocessdir,
bootstrap_options=bootstrap_options,
)
def create(cls, bootstrap_options=None, full_init=True):
"""
:param Options bootstrap_options: The bootstrap options, if available.
:param bool full_init: Whether or not to fully initialize an engine et al for the purposes
of spawning a new daemon. `full_init=False` is intended primarily
for lightweight lifecycle checks (since there is a ~1s overhead to
initialize the engine). See the impl of `maybe_launch` for an example
of the intended usage.
"""
bootstrap_options = bootstrap_options or cls._parse_bootstrap_options()
bootstrap_options_values = bootstrap_options.for_global_scope()
# TODO: https://github.com/pantsbuild/pants/issues/3479
watchman = WatchmanLauncher.create(bootstrap_options_values).watchman
native = None
build_root = None
services = None
port_map = None
if full_init:
build_root = get_buildroot()
native = Native.create(bootstrap_options_values)
options_bootstrapper = OptionsBootstrapper()
build_config = BuildConfigInitializer.get(options_bootstrapper)
legacy_graph_scheduler = EngineInitializer.setup_legacy_graph(native,
bootstrap_options_values,
build_config)
services, port_map = cls._setup_services(
build_root,
bootstrap_options_values,
legacy_graph_scheduler,
watchman
)
return PantsDaemon(
native=native,
build_root=build_root,
work_dir=bootstrap_options_values.pants_workdir,
log_level=bootstrap_options_values.level.upper(),
services=services,
socket_map=port_map,
metadata_base_dir=bootstrap_options_values.pants_subprocessdir,
bootstrap_options=bootstrap_options
)
def create(cls, bootstrap_options=None, full_init=True):
"""
:param Options bootstrap_options: The bootstrap options, if available.
:param bool full_init: Whether or not to fully initialize an engine et al for the purposes
of spawning a new daemon. `full_init=False` is intended primarily
for lightweight lifecycle checks (since there is a ~1s overhead to
initialize the engine). See the impl of `maybe_launch` for an example
of the intended usage.
"""
bootstrap_options = bootstrap_options or cls._parse_bootstrap_options()
bootstrap_options_values = bootstrap_options.for_global_scope()
# TODO: https://github.com/pantsbuild/pants/issues/3479
watchman = WatchmanLauncher.create(bootstrap_options_values).watchman
native = None
build_root = None
services = None
port_map = None
if full_init:
build_root = get_buildroot()
native = Native.create(bootstrap_options_values)
options_bootstrapper = OptionsBootstrapper()
build_config = BuildConfigInitializer.get(options_bootstrapper)
legacy_graph_scheduler = EngineInitializer.setup_legacy_graph(native,
bootstrap_options_values,
build_config)
services, port_map = cls._setup_services(
build_root,
bootstrap_options_values,
legacy_graph_scheduler,
watchman
)
return PantsDaemon(
native=native,
build_root=build_root,
work_dir=bootstrap_options_values.pants_workdir,
log_level=bootstrap_options_values.level.upper(),
services=services,
socket_map=port_map,
metadata_base_dir=bootstrap_options_values.pants_subprocessdir,
bootstrap_options=bootstrap_options
)
def _init_graph(self, pants_ignore_patterns, build_ignore_patterns,
exclude_target_regexps, target_specs, target_roots,
workdir, graph_helper, subproject_build_roots):
"""Determine the BuildGraph, AddressMapper and spec_roots for a given run.
:param list pants_ignore_patterns: The pants ignore patterns from '--pants-ignore'.
:param list build_ignore_patterns: The build ignore patterns from '--build-ignore',
applied during BUILD file searching.
:param str workdir: The pants workdir.
:param list exclude_target_regexps: Regular expressions for targets to be excluded.
:param list target_specs: The original target specs.
:param TargetRoots target_roots: The existing `TargetRoots` object, if any.
:param LegacyGraphSession graph_helper: A LegacyGraphSession to use for graph construction,
if available. This would usually come from the daemon.
:returns: A tuple of (BuildGraph, AddressMapper, SchedulerSession, TargetRoots).
"""
# The daemon may provide a `graph_helper`. If that's present, use it for graph construction.
if not graph_helper:
native = Native.create(self._global_options)
native.set_panic_handler()
graph_helper = EngineInitializer.setup_legacy_graph(
pants_ignore_patterns,
workdir,
self._global_options.build_file_imports,
native=native,
build_file_aliases=self._build_config.registered_aliases(),
rules=self._build_config.rules(),
build_ignore_patterns=build_ignore_patterns,
exclude_target_regexps=exclude_target_regexps,
subproject_roots=subproject_build_roots,
include_trace_on_error=self._options.for_global_scope(
).print_exception_stacktrace).new_session()
target_roots = target_roots or TargetRootsCalculator.create(
options=self._options,
build_root=self._root_dir,
change_calculator=graph_helper.change_calculator)
graph, address_mapper = graph_helper.create_build_graph(
target_roots, self._root_dir)
return graph, address_mapper, graph_helper.scheduler_session, target_roots
def process_request_thread(self, request, client_address):
"""Override of ThreadingMixIn.process_request_thread() that delegates to the request handler."""
# Instantiate the request handler.
Native().override_thread_logging_destination_to_just_stderr()
handler = self.RequestHandlerClass(request, client_address, self)
try:
# Attempt to handle a request with the handler.
handler.handle_request()
self.request_complete_callback()
except Exception as e:
# If that fails, (synchronously) handle the error with the error handler sans-fork.
try:
handler.handle_error(e)
finally:
# Shutdown the socket since we don't expect a fork() in the exception context.
self.shutdown_request(request)
else:
# At this point, we expect a fork() has taken place - the parent side will return, and so we
# close the request here from the parent without explicitly shutting down the socket. The
# child half of this will perform an os._exit() before it gets to this point and is also
# responsible for shutdown and closing of the socket when its execution is complete.
self.close_request(request)
class NativeHandler(StreamHandler):
"""This class is installed as a Python logging module handler (using the logging.addHandler
method) and proxies logs to the Rust logging infrastructure."""
def __init__(
self,
log_level: LogLevel,
stream: Optional[TextIO] = None,
native_filename: Optional[str] = None,
):
if stream is not None and native_filename is not None:
raise RuntimeError("NativeHandler must output to either a stream or a file, not both")
super().__init__(stream)
self.native = Native()
self.native_filename = native_filename
self.setLevel(log_level.level)
if stream:
try:
self.native.setup_stderr_logger(log_level.level)
except Exception as e:
print(f"Error setting up pantsd logger: {e!r}", file=sys.stderr)
raise e
def emit(self, record: LogRecord) -> None:
self.native.write_log(
self.format(record), record.levelno, f"{record.name}:pid={os.getpid()}"
)
def flush(self) -> None:
self.native.flush_log()
def __repr__(self) -> str:
return (
f"NativeHandler(id={id(self)}, level={self.level}, filename={self.native_filename}, "
f"stream={self.stream})"
)
def init_native():
"""Initialize and return a `Native` instance."""
init_subsystem(BinaryUtilPrivate.Factory)
opts = create_options_for_optionables([])
return Native.create(opts.for_global_scope())
def setup_legacy_graph_extended(
pants_ignore_patterns,
workdir,
build_file_imports_behavior,
build_configuration,
build_root=None,
native=None,
glob_match_error_behavior=None,
rules=None,
build_ignore_patterns=None,
exclude_target_regexps=None,
subproject_roots=None,
include_trace_on_error=True,
remote_store_server=None,
remote_execution_server=None
):
"""Construct and return the components necessary for LegacyBuildGraph construction.
:param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree,
usually taken from the '--pants-ignore' global option.
:param str workdir: The pants workdir.
:param build_file_imports_behavior: How to behave if a BUILD file being parsed tries to use
import statements. Valid values: "allow", "warn", "error".
:type build_file_imports_behavior: string
:param str build_root: A path to be used as the build root. If None, then default is used.
:param Native native: An instance of the native-engine subsystem.
:param build_configuration: The `BuildConfiguration` object to get build file aliases from.
:type build_configuration: :class:`pants.build_graph.build_configuration.BuildConfiguration`
:param glob_match_error_behavior: How to behave if a glob specified for a target's sources or
bundles does not expand to anything.
:type glob_match_error_behavior: :class:`pants.option.global_options.GlobMatchErrorBehavior`
:param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD
files, usually taken from the '--build-ignore' global option.
:param list exclude_target_regexps: A list of regular expressions for excluding targets.
:param list subproject_roots: Paths that correspond with embedded build roots
under the current build root.
:param bool include_trace_on_error: If True, when an error occurs, the error message will
include the graph trace.
:returns: A LegacyGraphScheduler.
"""
build_root = build_root or get_buildroot()
build_configuration = build_configuration or BuildConfigInitializer.get(OptionsBootstrapper())
build_file_aliases = build_configuration.registered_aliases()
rules = rules or build_configuration.rules() or []
symbol_table = LegacySymbolTable(build_file_aliases)
project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns)
# Register "literal" subjects required for these rules.
parser = LegacyPythonCallbacksParser(
symbol_table,
build_file_aliases,
build_file_imports_behavior
)
address_mapper = AddressMapper(parser=parser,
build_ignore_patterns=build_ignore_patterns,
exclude_target_regexps=exclude_target_regexps,
subproject_roots=subproject_roots)
# Load the native backend.
native = native or Native.create()
# Create a Scheduler containing graph and filesystem rules, with no installed goals. The
# LegacyBuildGraph will explicitly request the products it needs.
rules = (
[
SingletonRule.from_instance(GlobMatchErrorBehavior.create(glob_match_error_behavior)),
SingletonRule.from_instance(build_configuration),
] +
create_legacy_graph_tasks(symbol_table) +
create_fs_rules() +
create_process_rules() +
create_graph_rules(address_mapper, symbol_table) +
create_options_parsing_rules() +
rules
)
scheduler = Scheduler(
native,
project_tree,
workdir,
rules,
remote_store_server,
remote_execution_server,
include_trace_on_error=include_trace_on_error,
)
return LegacyGraphScheduler(scheduler, symbol_table)
def init_native(): """Initialize and return a `Native` instance.""" init_subsystem(BinaryUtil.Factory) opts = create_options_for_optionables([]) return Native.create(opts.for_global_scope())
def setup_legacy_graph(pants_ignore_patterns,
workdir,
build_file_imports_behavior,
build_root=None,
native=None,
build_file_aliases=None,
rules=None,
build_ignore_patterns=None,
exclude_target_regexps=None,
subproject_roots=None,
include_trace_on_error=True):
"""Construct and return the components necessary for LegacyBuildGraph construction.
:param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree,
usually taken from the '--pants-ignore' global option.
:param str workdir: The pants workdir.
:param build_file_imports_behavior: How to behave if a BUILD file being parsed tries to use
import statements. Valid values: "allow", "warn", "error".
:type build_file_imports_behavior: string
:param str build_root: A path to be used as the build root. If None, then default is used.
:param Native native: An instance of the native-engine subsystem.
:param build_file_aliases: BuildFileAliases to register.
:type build_file_aliases: :class:`pants.build_graph.build_file_aliases.BuildFileAliases`
:param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD
files, usually taken from the '--build-ignore' global option.
:param list exclude_target_regexps: A list of regular expressions for excluding targets.
:param list subproject_roots: Paths that correspond with embedded build roots
under the current build root.
:param bool include_trace_on_error: If True, when an error occurs, the error message will
include the graph trace.
:returns: A tuple of (scheduler, engine, symbol_table, build_graph_cls).
"""
build_root = build_root or get_buildroot()
scm = get_scm()
if not build_file_aliases:
build_file_aliases = EngineInitializer.get_default_build_file_aliases()
if not rules:
rules = []
symbol_table = LegacySymbolTable(build_file_aliases)
project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns)
# Register "literal" subjects required for these tasks.
parser = LegacyPythonCallbacksParser(
symbol_table,
build_file_aliases,
build_file_imports_behavior
)
address_mapper = AddressMapper(parser=parser,
build_ignore_patterns=build_ignore_patterns,
exclude_target_regexps=exclude_target_regexps,
subproject_roots=subproject_roots)
# Load the native backend.
native = native or Native.create()
# Create a Scheduler containing graph and filesystem tasks, with no installed goals. The
# LegacyBuildGraph will explicitly request the products it needs.
tasks = (
create_legacy_graph_tasks(symbol_table) +
create_fs_rules() +
create_graph_rules(address_mapper, symbol_table) +
create_process_rules() +
rules
)
scheduler = LocalScheduler(workdir, dict(), tasks, project_tree, native, include_trace_on_error=include_trace_on_error)
change_calculator = EngineChangeCalculator(scheduler, symbol_table, scm) if scm else None
return LegacyGraphHelper(scheduler, symbol_table, change_calculator)
def init_rust_logger(level, log_show_rust_3rdparty): native = Native() levelno = get_numeric_level(level) native.init_rust_logging(levelno, log_show_rust_3rdparty)
