def mk_scheduler(
self,
rules=None,
union_rules=None,
project_tree=None,
work_dir=None,
include_trace_on_error=True,
should_report_workunits=False,
):
"""Creates a SchedulerSession for a Scheduler with the given Rules installed."""
rules = rules or []
work_dir = work_dir or self._create_work_dir()
project_tree = project_tree or self.mk_fs_tree(work_dir=work_dir)
local_store_dir = os.path.realpath(safe_mkdtemp())
scheduler = Scheduler(self._native,
project_tree,
local_store_dir,
rules,
union_rules,
DEFAULT_EXECUTION_OPTIONS,
include_trace_on_error=include_trace_on_error)
return scheduler.new_session(
zipkin_trace_v2=False,
build_id="buildid_for_test",
should_report_workunits=should_report_workunits)
def mk_scheduler(
self,
rules=None,
union_rules=None,
project_tree=None,
work_dir=None,
include_trace_on_error=True,
should_report_workunits=False,
execution_options=None,
):
"""Creates a SchedulerSession for a Scheduler with the given Rules installed."""
rules = rules or []
work_dir = work_dir or self._create_work_dir()
project_tree = project_tree or self.mk_fs_tree(work_dir=work_dir)
local_store_dir = os.path.realpath(safe_mkdtemp())
if execution_options is not None:
eo = asdict(DEFAULT_EXECUTION_OPTIONS)
eo.update(execution_options)
execution_options = ExecutionOptions(**eo)
scheduler = Scheduler(
native=self._native,
ignore_patterns=project_tree.ignore_patterns,
use_gitignore=False,
build_root=project_tree.build_root,
local_store_dir=local_store_dir,
rules=rules,
union_rules=union_rules,
execution_options=execution_options or DEFAULT_EXECUTION_OPTIONS,
include_trace_on_error=include_trace_on_error,
)
return scheduler.new_session(
zipkin_trace_v2=False,
build_id="buildid_for_test",
should_report_workunits=should_report_workunits,
)
def setup_json_scheduler(build_root, native):
"""Return a build graph and scheduler configured for BLD.json files under the given build root.
:rtype :class:`pants.engine.scheduler.SchedulerSession`
"""
symbol_table = ExampleTable()
# Register "literal" subjects required for these rules.
address_mapper = AddressMapper(build_patterns=('BLD.json',),
parser=JsonParser(symbol_table))
work_dir = os_path_join(build_root, '.pants.d')
project_tree = FileSystemProjectTree(build_root)
rules = [
# Codegen
GenGoal.rule(),
gen_apache_java_thrift,
gen_apache_python_thrift,
gen_scrooge_scala_thrift,
gen_scrooge_java_thrift,
SingletonRule(Scrooge, Scrooge(Address.parse('src/scala/scrooge')))
] + [
# scala dependency inference
reify_scala_sources,
select_package_address,
calculate_package_search_path,
SingletonRule(SourceRoots, SourceRoots(('src/java','src/scala'))),
] + [
# Remote dependency resolution
ivy_resolve,
select_rev,
] + [
# Compilers
isolate_resources,
write_name_file,
javac,
scalac,
] + (
create_graph_rules(address_mapper, symbol_table)
) + (
create_fs_rules()
)
scheduler = Scheduler(native,
project_tree,
work_dir,
rules,
DEFAULT_EXECUTION_OPTIONS,
None,
None)
return scheduler.new_session()
def setup_json_scheduler(build_root, native):
"""Return a build graph and scheduler configured for BLD.json files under the given build root.
:rtype :class:`pants.engine.scheduler.SchedulerSession`
"""
symbol_table = ExampleTable()
# Register "literal" subjects required for these rules.
address_mapper = AddressMapper(build_patterns=('BLD.json',),
parser=JsonParser(symbol_table))
work_dir = os_path_join(build_root, '.pants.d')
project_tree = FileSystemProjectTree(build_root)
rules = [
# Codegen
GenGoal.rule(),
gen_apache_java_thrift,
gen_apache_python_thrift,
gen_scrooge_scala_thrift,
gen_scrooge_java_thrift,
SingletonRule(Scrooge, Scrooge(Address.parse('src/scala/scrooge')))
] + [
# scala dependency inference
reify_scala_sources,
select_package_address,
calculate_package_search_path,
SingletonRule(SourceRoots, SourceRoots(('src/java','src/scala'))),
] + [
# Remote dependency resolution
ivy_resolve,
select_rev,
] + [
# Compilers
isolate_resources,
write_name_file,
javac,
scalac,
] + (
create_graph_rules(address_mapper, symbol_table)
) + (
create_fs_rules()
)
scheduler = Scheduler(native,
project_tree,
work_dir,
rules,
DEFAULT_EXECUTION_OPTIONS,
None,
None)
return scheduler.new_session()
def mk_scheduler(self,
rules=None,
project_tree=None,
work_dir=None,
include_trace_on_error=True):
"""Creates a SchedulerSession for a Scheduler with the given Rules installed."""
rules = rules or []
work_dir = work_dir or self._create_work_dir()
project_tree = project_tree or self.mk_fs_tree(work_dir=work_dir)
scheduler = Scheduler(self._native,
project_tree,
work_dir,
rules,
include_trace_on_error=include_trace_on_error)
return scheduler.new_session()
def mk_scheduler(self,
rules=None,
project_tree=None,
work_dir=None,
include_trace_on_error=True):
"""Creates a SchedulerSession for a Scheduler with the given Rules installed."""
rules = rules or []
work_dir = work_dir or self._create_work_dir()
project_tree = project_tree or self.mk_fs_tree(work_dir=work_dir)
scheduler = Scheduler(self._native,
project_tree,
work_dir,
rules,
DEFAULT_EXECUTION_OPTIONS,
include_trace_on_error=include_trace_on_error)
return scheduler.new_session()
def mk_scheduler(self,
rules=None,
project_tree=None,
work_dir=None,
include_trace_on_error=True):
"""Creates a SchedulerSession for a Scheduler with the given Rules installed."""
rules = rules or []
work_dir = work_dir or self._create_work_dir()
project_tree = project_tree or self.mk_fs_tree(work_dir=work_dir)
local_store_dir = os.path.realpath(safe_mkdtemp())
scheduler = Scheduler(self._native,
project_tree,
work_dir,
local_store_dir,
rules,
DEFAULT_EXECUTION_OPTIONS,
include_trace_on_error=include_trace_on_error)
return scheduler.new_session()
def create_scheduler(rules, validate=True, native=None):
"""Create a Scheduler."""
native = native or init_native()
return Scheduler(
native,
FileSystemProjectTree(os.getcwd()),
'./.pants.d',
rules,
execution_options=DEFAULT_EXECUTION_OPTIONS,
validate=validate,
)
def mk_scheduler(self,
rules=None,
union_rules=None,
project_tree=None,
work_dir=None,
include_trace_on_error=True):
"""Creates a SchedulerSession for a Scheduler with the given Rules installed."""
rules = rules or []
work_dir = work_dir or self._create_work_dir()
project_tree = project_tree or self.mk_fs_tree(work_dir=work_dir)
local_store_dir = os.path.realpath(safe_mkdtemp())
scheduler = Scheduler(self._native,
project_tree,
work_dir,
local_store_dir,
rules,
union_rules,
DEFAULT_EXECUTION_OPTIONS,
include_trace_on_error=include_trace_on_error)
return scheduler.new_session(zipkin_trace_v2=False)
def create_scheduler(rules, validate=True):
"""Create a Scheduler."""
native = init_native()
return Scheduler(
native,
FileSystemProjectTree(os.getcwd()),
'./.pants.d',
rules,
remote_store_server=None,
remote_execution_server=None,
validate=validate,
)
def create_scheduler(rules, union_rules=None, validate=True, native=None):
"""Create a Scheduler."""
native = native or init_native()
tree = FileSystemProjectTree(os.getcwd())
return Scheduler(
native=native,
ignore_patterns=tree.ignore_patterns,
build_root=tree.build_root,
local_store_dir="./.pants.d",
rules=rules,
union_rules=union_rules,
execution_options=DEFAULT_EXECUTION_OPTIONS,
validate=validate,
)
def setup_legacy_graph_extended(
pants_ignore_patterns,
workdir,
local_store_dir,
build_file_imports_behavior,
options_bootstrapper,
build_configuration,
build_root=None,
native=None,
glob_match_error_behavior=None,
build_ignore_patterns=None,
exclude_target_regexps=None,
subproject_roots=None,
include_trace_on_error=True,
execution_options=None,
):
"""Construct and return the components necessary for LegacyBuildGraph construction.
:param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree,
usually taken from the '--pants-ignore' global option.
:param str workdir: The pants workdir.
:param local_store_dir: The directory to use for storing the engine's LMDB store in.
:param build_file_imports_behavior: How to behave if a BUILD file being parsed tries to use
import statements. Valid values: "allow", "warn", "error".
:type build_file_imports_behavior: string
:param str build_root: A path to be used as the build root. If None, then default is used.
:param Native native: An instance of the native-engine subsystem.
:param options_bootstrapper: A `OptionsBootstrapper` object containing bootstrap options.
:type options_bootstrapper: :class:`pants.options.options_bootstrapper.OptionsBootstrapper`
:param build_configuration: The `BuildConfiguration` object to get build file aliases from.
:type build_configuration: :class:`pants.build_graph.build_configuration.BuildConfiguration`
:param glob_match_error_behavior: How to behave if a glob specified for a target's sources or
bundles does not expand to anything.
:type glob_match_error_behavior: :class:`pants.option.global_options.GlobMatchErrorBehavior`
:param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD
files, usually taken from the '--build-ignore' global option.
:param list exclude_target_regexps: A list of regular expressions for excluding targets.
:param list subproject_roots: Paths that correspond with embedded build roots
under the current build root.
:param bool include_trace_on_error: If True, when an error occurs, the error message will
include the graph trace.
:param execution_options: Option values for (remote) process execution.
:type execution_options: :class:`pants.option.global_options.ExecutionOptions`
:returns: A LegacyGraphScheduler.
"""
build_root = build_root or get_buildroot()
build_configuration = build_configuration or BuildConfigInitializer.get(
options_bootstrapper)
build_file_aliases = build_configuration.registered_aliases()
rules = build_configuration.rules()
console = Console()
symbol_table = LegacySymbolTable(build_file_aliases)
project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns)
execution_options = execution_options or DEFAULT_EXECUTION_OPTIONS
# Register "literal" subjects required for these rules.
parser = LegacyPythonCallbacksParser(symbol_table, build_file_aliases,
build_file_imports_behavior)
address_mapper = AddressMapper(
parser=parser,
build_ignore_patterns=build_ignore_patterns,
exclude_target_regexps=exclude_target_regexps,
subproject_roots=subproject_roots)
# Load the native backend.
native = native or Native.create()
# Create a Scheduler containing graph and filesystem rules, with no installed goals. The
# LegacyBuildGraph will explicitly request the products it needs.
rules = (
[
SingletonRule.from_instance(console),
SingletonRule.from_instance(
GlobMatchErrorBehavior.create(glob_match_error_behavior)),
SingletonRule.from_instance(build_configuration),
] + create_legacy_graph_tasks(symbol_table) + create_fs_rules() +
create_process_rules() +
create_graph_rules(address_mapper, symbol_table) +
create_options_parsing_rules() + create_core_rules() +
# TODO: This should happen automatically, but most tests (e.g. tests/python/pants_test/auth) fail if it's not here:
[run_python_test] + rules)
goal_map = EngineInitializer._make_goal_map_from_rules(rules)
scheduler = Scheduler(
native,
project_tree,
workdir,
local_store_dir,
rules,
execution_options,
include_trace_on_error=include_trace_on_error,
)
return LegacyGraphScheduler(scheduler, symbol_table, console, goal_map)
def setup_legacy_graph_extended(
pants_ignore_patterns: List[str],
local_store_dir,
build_file_imports_behavior: BuildFileImportsBehavior,
options_bootstrapper: OptionsBootstrapper,
build_configuration: BuildConfiguration,
build_root: Optional[str] = None,
native: Optional[Native] = None,
glob_match_error_behavior:
GlobMatchErrorBehavior = GlobMatchErrorBehavior.warn,
build_ignore_patterns=None,
exclude_target_regexps=None,
subproject_roots=None,
include_trace_on_error: bool = True,
execution_options: Optional[ExecutionOptions] = None,
) -> LegacyGraphScheduler:
"""Construct and return the components necessary for LegacyBuildGraph construction.
:param local_store_dir: The directory to use for storing the engine's LMDB store in.
:param build_file_imports_behavior: How to behave if a BUILD file being parsed tries to use
import statements.
:param build_root: A path to be used as the build root. If None, then default is used.
:param native: An instance of the native-engine subsystem.
:param options_bootstrapper: A `OptionsBootstrapper` object containing bootstrap options.
:param build_configuration: The `BuildConfiguration` object to get build file aliases from.
:param glob_match_error_behavior: How to behave if a glob specified for a target's sources or
bundles does not expand to anything.
:param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD
files, usually taken from the '--build-ignore' global option.
:param list exclude_target_regexps: A list of regular expressions for excluding targets.
:param list subproject_roots: Paths that correspond with embedded build roots
under the current build root.
:param include_trace_on_error: If True, when an error occurs, the error message will include
the graph trace.
:param execution_options: Option values for (remote) process execution.
"""
build_root = build_root or get_buildroot()
build_configuration = build_configuration or BuildConfigInitializer.get(
options_bootstrapper)
bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope(
)
build_file_aliases = build_configuration.registered_aliases()
rules = build_configuration.rules()
symbol_table = _legacy_symbol_table(build_file_aliases)
execution_options = execution_options or DEFAULT_EXECUTION_OPTIONS
# Register "literal" subjects required for these rules.
parser = LegacyPythonCallbacksParser(symbol_table, build_file_aliases,
build_file_imports_behavior)
address_mapper = AddressMapper(
parser=parser,
build_ignore_patterns=build_ignore_patterns,
exclude_target_regexps=exclude_target_regexps,
subproject_roots=subproject_roots,
)
@rule
def glob_match_error_behavior_singleton() -> GlobMatchErrorBehavior:
return glob_match_error_behavior
@rule
def build_configuration_singleton() -> BuildConfiguration:
return build_configuration
@rule
def symbol_table_singleton() -> SymbolTable:
return symbol_table
@rule
def union_membership_singleton() -> UnionMembership:
return UnionMembership(build_configuration.union_rules())
@rule
def build_root_singleton() -> BuildRoot:
return cast(BuildRoot, BuildRoot.instance)
# Create a Scheduler containing graph and filesystem rules, with no installed goals. The
# LegacyBuildGraph will explicitly request the products it needs.
rules = (
RootRule(Console),
glob_match_error_behavior_singleton,
build_configuration_singleton,
symbol_table_singleton,
union_membership_singleton,
build_root_singleton,
*create_legacy_graph_tasks(),
*create_fs_rules(),
*create_interactive_runner_rules(),
*create_process_rules(),
*create_platform_rules(),
*create_graph_rules(address_mapper),
*create_options_parsing_rules(),
*structs_rules(),
*changed_rules(),
*binary_tool_rules(),
*binary_util_rules(),
*rules,
)
goal_map = EngineInitializer._make_goal_map_from_rules(rules)
union_rules = build_configuration.union_rules()
scheduler = Scheduler(
native=native,
ignore_patterns=pants_ignore_patterns,
build_root=build_root,
local_store_dir=local_store_dir,
rules=rules,
union_rules=union_rules,
execution_options=execution_options,
include_trace_on_error=include_trace_on_error,
visualize_to_dir=bootstrap_options.native_engine_visualize_to,
)
return LegacyGraphScheduler(scheduler, build_file_aliases, goal_map)
def setup_legacy_graph_extended(
pants_ignore_patterns,
local_store_dir,
build_file_imports_behavior,
options_bootstrapper,
build_configuration,
build_root=None,
native=None,
glob_match_error_behavior=None,
build_ignore_patterns=None,
exclude_target_regexps=None,
subproject_roots=None,
include_trace_on_error=True,
execution_options=None,
):
"""Construct and return the components necessary for LegacyBuildGraph construction.
:param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree,
usually taken from the '--pants-ignore' global option.
:param local_store_dir: The directory to use for storing the engine's LMDB store in.
:param build_file_imports_behavior: How to behave if a BUILD file being parsed tries to use
import statements. Valid values: "allow", "warn", "error".
:type build_file_imports_behavior: string
:param str build_root: A path to be used as the build root. If None, then default is used.
:param Native native: An instance of the native-engine subsystem.
:param options_bootstrapper: A `OptionsBootstrapper` object containing bootstrap options.
:type options_bootstrapper: :class:`pants.options.options_bootstrapper.OptionsBootstrapper`
:param build_configuration: The `BuildConfiguration` object to get build file aliases from.
:type build_configuration: :class:`pants.build_graph.build_configuration.BuildConfiguration`
:param glob_match_error_behavior: How to behave if a glob specified for a target's sources or
bundles does not expand to anything.
:type glob_match_error_behavior: :class:`pants.option.global_options.GlobMatchErrorBehavior`
:param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD
files, usually taken from the '--build-ignore' global option.
:param list exclude_target_regexps: A list of regular expressions for excluding targets.
:param list subproject_roots: Paths that correspond with embedded build roots
under the current build root.
:param bool include_trace_on_error: If True, when an error occurs, the error message will
include the graph trace.
:param execution_options: Option values for (remote) process execution.
:type execution_options: :class:`pants.option.global_options.ExecutionOptions`
:returns: A LegacyGraphScheduler.
"""
build_root = build_root or get_buildroot()
build_configuration = build_configuration or BuildConfigInitializer.get(options_bootstrapper)
bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope()
build_file_aliases = build_configuration.registered_aliases()
rules = build_configuration.rules()
symbol_table = _legacy_symbol_table(build_file_aliases)
project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns)
execution_options = execution_options or DEFAULT_EXECUTION_OPTIONS
# Register "literal" subjects required for these rules.
parser = LegacyPythonCallbacksParser(
symbol_table,
build_file_aliases,
build_file_imports_behavior
)
address_mapper = AddressMapper(parser=parser,
build_ignore_patterns=build_ignore_patterns,
exclude_target_regexps=exclude_target_regexps,
subproject_roots=subproject_roots)
@rule
def glob_match_error_behavior_singleton() -> GlobMatchErrorBehavior:
return glob_match_error_behavior or GlobMatchErrorBehavior.ignore
@rule
def build_configuration_singleton() -> BuildConfiguration:
return build_configuration
@rule
def symbol_table_singleton() -> SymbolTable:
return symbol_table
@rule
def union_membership_singleton() -> UnionMembership:
return UnionMembership(build_configuration.union_rules())
@rule
def build_root_singleton() -> BuildRoot:
return BuildRoot.instance
@rule
async def single_build_file_address(specs: Specs) -> BuildFileAddress:
build_file_addresses = await Get(BuildFileAddresses, Specs, specs)
if len(build_file_addresses.dependencies) == 0:
raise ResolveError("No targets were matched")
if len(build_file_addresses.dependencies) > 1:
potential_addresses = await Get(BuildFileAddresses, Specs, specs)
targets = [bfa.to_address() for bfa in potential_addresses]
output = '\n '.join(str(target) for target in targets)
raise ResolveError(
"Expected a single target, but was given multiple targets:\n"
f"Did you mean one of:\n {output}"
)
return build_file_addresses.dependencies[0]
# Create a Scheduler containing graph and filesystem rules, with no installed goals. The
# LegacyBuildGraph will explicitly request the products it needs.
rules = (
[
RootRule(Console),
glob_match_error_behavior_singleton,
build_configuration_singleton,
symbol_table_singleton,
union_membership_singleton,
build_root_singleton,
single_build_file_address,
] +
create_legacy_graph_tasks() +
create_fs_rules() +
create_interactive_runner_rules() +
create_process_rules() +
create_platform_rules() +
create_graph_rules(address_mapper) +
create_options_parsing_rules() +
structs_rules() +
rules
)
goal_map = EngineInitializer._make_goal_map_from_rules(rules)
union_rules = build_configuration.union_rules()
scheduler = Scheduler(
native,
project_tree,
local_store_dir,
rules,
union_rules,
execution_options,
include_trace_on_error=include_trace_on_error,
visualize_to_dir=bootstrap_options.native_engine_visualize_to,
)
return LegacyGraphScheduler(scheduler, build_file_aliases, goal_map)
def setup_legacy_graph_extended(pants_ignore_patterns,
workdir,
build_file_imports_behavior,
build_file_aliases,
build_root=None,
native=None,
glob_match_error_behavior=None,
rules=None,
build_ignore_patterns=None,
exclude_target_regexps=None,
subproject_roots=None,
include_trace_on_error=True,
remote_store_server=None,
remote_execution_server=None):
"""Construct and return the components necessary for LegacyBuildGraph construction.
:param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree,
usually taken from the '--pants-ignore' global option.
:param str workdir: The pants workdir.
:param build_file_imports_behavior: How to behave if a BUILD file being parsed tries to use
import statements. Valid values: "allow", "warn", "error".
:type build_file_imports_behavior: string
:param str build_root: A path to be used as the build root. If None, then default is used.
:param Native native: An instance of the native-engine subsystem.
:param build_file_aliases: BuildFileAliases to register.
:type build_file_aliases: :class:`pants.build_graph.build_file_aliases.BuildFileAliases`
:param glob_match_error_behavior: How to behave if a glob specified for a target's sources or
bundles does not expand to anything.
:type glob_match_error_behavior: :class:`pants.option.global_options.GlobMatchErrorBehavior`
:param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD
files, usually taken from the '--build-ignore' global option.
:param list exclude_target_regexps: A list of regular expressions for excluding targets.
:param list subproject_roots: Paths that correspond with embedded build roots
under the current build root.
:param bool include_trace_on_error: If True, when an error occurs, the error message will
include the graph trace.
:returns: A LegacyGraphScheduler.
"""
build_root = build_root or get_buildroot()
if not rules:
rules = []
symbol_table = LegacySymbolTable(build_file_aliases)
project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns)
# Register "literal" subjects required for these rules.
parser = LegacyPythonCallbacksParser(symbol_table, build_file_aliases,
build_file_imports_behavior)
address_mapper = AddressMapper(
parser=parser,
build_ignore_patterns=build_ignore_patterns,
exclude_target_regexps=exclude_target_regexps,
subproject_roots=subproject_roots)
# Load the native backend.
native = native or Native.create()
# Create a Scheduler containing graph and filesystem rules, with no installed goals. The
# LegacyBuildGraph will explicitly request the products it needs.
rules = (
create_legacy_graph_tasks(symbol_table) + create_fs_rules() +
create_process_rules() +
create_graph_rules(address_mapper, symbol_table) + [
SingletonRule(
GlobMatchErrorBehavior,
GlobMatchErrorBehavior.create(glob_match_error_behavior))
] + rules)
scheduler = Scheduler(
native,
project_tree,
workdir,
rules,
remote_store_server,
remote_execution_server,
include_trace_on_error=include_trace_on_error,
)
return LegacyGraphScheduler(scheduler, symbol_table)
def create_scheduler(rules, validate=True): """Create a Scheduler.""" native = init_native() return Scheduler(native, FileSystemProjectTree(os.getcwd()), './.pants.d', rules, validate=validate)