def test_load_valid_partial_goals(self):
def register_goals():
Goal.by_name("jack").install(TaskRegistrar("jill", DummyTask))
with self.create_register(
register_goals=register_goals) as backend_package:
Goal.clear()
self.assertEqual(0, len(Goal.all()))
load_backend(self.build_configuration,
backend_package,
is_v1_backend=False)
self.assertEqual(0, len(Goal.all()))
load_backend(self.build_configuration,
backend_package,
is_v1_backend=True)
self.assert_empty_aliases()
self.assertEqual(1, len(Goal.all()))
task_names = Goal.by_name("jack").ordered_task_names()
self.assertEqual(1, len(task_names))
task_name = task_names[0]
self.assertEqual("jill", task_name)
def setup(self):
options_bootstrapper = OptionsBootstrapper()
# Force config into the cache so we (and plugin/backend loading code) can use it.
# TODO: Plumb options in explicitly.
options_bootstrapper.get_bootstrap_options()
self.config = Config.from_cache()
# Add any extra paths to python path (eg for loading extra source backends)
extra_paths = self.config.getlist('backends', 'python-path', [])
if extra_paths:
sys.path.extend(extra_paths)
# Load plugins and backends.
backend_packages = self.config.getlist('backends', 'packages', [])
plugins = self.config.getlist('backends', 'plugins', [])
build_configuration = load_plugins_and_backends(plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scopes = ['']
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scopes=known_scopes)
self.register_options()
self.run_tracker = RunTracker.from_config(self.config)
report = initial_reporting(self.config, self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: %s' % url)
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]):
build_file = BuildFile.from_cache(root_dir=self.root_dir, relpath=path)
# TODO(pl): This is an unfortunate interface leak, but I don't think
# in the long run that we should be relying on "bootstrap" BUILD files
# that do nothing except modify global state. That type of behavior
# (e.g. source roots, goal registration) should instead happen in
# project plugins, or specialized configuration files.
self.build_file_parser.parse_build_file_family(build_file)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
self._expand_goals_and_specs()
def _print_goals_help(self) -> None:
print(
f"\nUse `{self.bin_name} help $goal` to get help for a particular goal.\n"
)
global_options = self._options.for_global_scope()
goal_descriptions: Dict[str, str] = {}
if global_options.v2:
goal_scope_infos = [
scope_info
for scope_info in self._options.known_scope_to_info.values()
if scope_info.category == ScopeInfo.GOAL
]
for scope_info in goal_scope_infos:
optionable_cls = scope_info.optionable_cls
if optionable_cls is None or not issubclass(
optionable_cls, GoalSubsystem):
continue
is_implemented = self._union_membership.has_members_for_all(
optionable_cls.required_union_implementations)
if not is_implemented:
continue
description = scope_info.description or "<no description>"
goal_descriptions[scope_info.scope] = description
if global_options.v1:
goal_descriptions.update({
goal.name: goal.description_first_line
for goal in Goal.all() if goal.description
})
max_width = max(
len(name)
for name in goal_descriptions.keys()) if goal_descriptions else 0
for name, description in sorted(goal_descriptions.items()):
print(f" {name.rjust(max_width)}: {description}")
print()
def _print_help(self):
"""Print a help screen, followed by an optional message.
Note: Ony useful if called after options have been registered.
"""
show_all_help = self._help_request and self._help_request.all_scopes
goals = (Goal.all() if show_all_help else [Goal.by_name(goal_name) for goal_name in self.goals])
if goals:
for goal in goals:
if not goal.ordered_task_names():
print('\nUnknown goal: {}'.format(goal.name))
else:
print('\n{0}: {1}\n'.format(goal.name, goal.description))
for scope_info in goal.known_scope_infos():
help_str = self._format_help_for_scope(scope_info.scope)
if help_str:
print(help_str)
else:
print(pants_release())
print('\nUsage:')
print(' ./pants [option ...] [goal ...] [target...] Attempt the specified goals.')
print(' ./pants help Get help.')
print(' ./pants help [goal] Get help for a goal.')
print(' ./pants help-advanced [goal] Get help for a goal\'s advanced options.')
print(' ./pants help-all Get help for all goals.')
print(' ./pants goals List all installed goals.')
print('')
print(' [target] accepts two special forms:')
print(' dir: to include all targets in the specified directory.')
print(' dir:: to include all targets found recursively under the directory.')
print('\nFriendly docs:\n http://pantsbuild.github.io/')
if show_all_help or not goals:
print(self.get_parser(GLOBAL_SCOPE).format_help('Global', self._help_request.advanced))
def gen_tasks_goals_reference_data():
"""Generate the template data for the goals reference rst doc."""
goal_dict = {}
goal_names = []
for goal in Goal.all():
tasks = []
for task_name in goal.ordered_task_names():
task_type = goal.task_type_by_name(task_name)
doc_rst = indent_docstring_by_n(task_type.__doc__ or '', 2)
doc_html = rst_to_html(dedent_docstring(task_type.__doc__))
option_parser = Parser(env={}, config={}, scope='', parent_parser=None)
task_type.register_options(option_parser.register)
argparser = option_parser._help_argparser
scope = Goal.scope(goal.name, task_name)
# task_type may actually be a synthetic subclass of the authored class from the source code.
# We want to display the authored class's name in the docs (but note that we must use the
# subclass for registering options above)
for authored_task_type in task_type.mro():
if authored_task_type.__module__ != 'abc':
break
impl = '{0}.{1}'.format(authored_task_type.__module__, authored_task_type.__name__)
tasks.append(TemplateData(
impl=impl,
doc_html=doc_html,
doc_rst=doc_rst,
ogroup=gref_template_data_from_options(scope, argparser)))
goal_dict[goal.name] = TemplateData(goal=goal, tasks=tasks)
goal_names.append(goal.name)
goals = [goal_dict[name] for name in sorted(goal_names, key=lambda x: x.lower())]
return goals
def _install_options(self, options_bootstrapper, build_configuration):
"""Parse and register options.
:returns: An Options object representing the full set of runtime options.
"""
# TODO: This inline import is currently necessary to resolve a ~legitimate cycle between
# `GoalRunner`->`EngineInitializer`->`OptionsInitializer`->`GoalRunner`.
from pants.bin.goal_runner import GoalRunner
# Now that plugins and backends are loaded, we can gather the known scopes.
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems via a union of all known subsystem sets.
subsystems = Subsystem.closure(
GoalRunner.subsystems() | Goal.subsystems() | build_configuration.subsystems()
)
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
options = options_bootstrapper.get_full_options(known_scope_infos)
self._register_options(subsystems, options)
# Make the options values available to all subsystems.
Subsystem.set_options(options)
return options
def _print_help(self):
"""Print a help screen, followed by an optional message.
Note: Ony useful if called after options have been registered.
"""
show_all_help = self._help_request and self._help_request.all_scopes
goals = (Goal.all() if show_all_help else [Goal.by_name(goal_name) for goal_name in self.goals])
if goals:
for goal in goals:
if not goal.ordered_task_names():
print('\nUnknown goal: {}'.format(goal.name))
else:
print('\n{0}: {1}\n'.format(goal.name, goal.description))
for scope_info in goal.known_scope_infos():
help_str = self._format_help_for_scope(scope_info.scope)
if help_str:
print(help_str)
else:
print(pants_release())
print('\nUsage:')
print(' ./pants [option ...] [goal ...] [target...] Attempt the specified goals.')
print(' ./pants help Get help.')
print(' ./pants help [goal] Get help for a goal.')
print(' ./pants help-advanced [goal] Get help for a goal\'s advanced options.')
print(' ./pants help-all Get help for all goals.')
print(' ./pants goals List all installed goals.')
print('')
print(' [target] accepts two special forms:')
print(' dir: to include all targets in the specified directory.')
print(' dir:: to include all targets found recursively under the directory.')
print('\nFriendly docs:\n http://pantsbuild.github.io/')
if show_all_help or not goals:
print(self.get_parser(GLOBAL_SCOPE).format_help('Global', self._help_request.advanced))
def gen_tasks_options_reference_data(options):
"""Generate the template data for the options reference rst doc."""
goal_dict = {}
goal_names = []
def fill_template(options, task_type):
for authored_task_type in task_type.mro():
if authored_task_type.__module__ != 'abc':
break
doc_rst = indent_docstring_by_n(authored_task_type.__doc__ or '', 2)
doc_html = rst_to_html(dedent_docstring(authored_task_type.__doc__))
parser = options.get_parser(task_type.options_scope)
oschi = HelpInfoExtracter.get_option_scope_help_info_from_parser(
parser)
impl = '{0}.{1}'.format(authored_task_type.__module__,
authored_task_type.__name__)
return TemplateData(impl=impl,
doc_html=doc_html,
doc_rst=doc_rst,
ogroup=oref_template_data_from_help_info(oschi))
for goal in Goal.all():
tasks = {}
for task_name, task_type in goal.task_items():
# task_type may actually be a synthetic subclass of the authored class from the source code.
# We want to display the authored class's name in the docs.
tasks[task_name] = fill_template(options, task_type)
sorted_tasks = [tasks[k] for k in sorted(tasks.keys())]
goal_dict[goal.name] = TemplateData(goal=goal, tasks=sorted_tasks)
goal_names.append(goal.name)
goals = [
goal_dict[name] for name in sorted(goal_names, key=lambda x: x.lower())
]
return goals
def gen_tasks_options_reference_data():
"""Generate the template data for the options reference rst doc."""
goal_dict = {}
goal_names = []
for goal in Goal.all():
tasks = []
for task_name in goal.ordered_task_names():
task_type = goal.task_type_by_name(task_name)
doc_rst = indent_docstring_by_n(task_type.__doc__ or '', 2)
doc_html = rst_to_html(dedent_docstring(task_type.__doc__))
option_parser = Parser(env={}, config={}, scope='', help_request=None, parent_parser=None)
def register(*args, **kwargs):
option_parser.register(*args, **kwargs)
register.bootstrap = bootstrap_option_values()
task_type.register_options(register)
argparser = option_parser._help_argparser
scope = Goal.scope(goal.name, task_name)
# task_type may actually be a synthetic subclass of the authored class from the source code.
# We want to display the authored class's name in the docs (but note that we must use the
# subclass for registering options above)
for authored_task_type in task_type.mro():
if authored_task_type.__module__ != 'abc':
break
impl = '{0}.{1}'.format(authored_task_type.__module__, authored_task_type.__name__)
tasks.append(TemplateData(
impl=impl,
doc_html=doc_html,
doc_rst=doc_rst,
ogroup=oref_template_data_from_options(scope, argparser)))
goal_dict[goal.name] = TemplateData(goal=goal, tasks=tasks)
goal_names.append(goal.name)
goals = [goal_dict[name] for name in sorted(goal_names, key=lambda x: x.lower())]
return goals
def gen_tasks_options_reference_data(options):
"""Generate the template data for the options reference rst doc."""
goal_dict = {}
goal_names = []
for goal in Goal.all():
tasks = []
for task_name in goal.ordered_task_names():
task_type = goal.task_type_by_name(task_name)
# task_type may actually be a synthetic subclass of the authored class from the source code.
# We want to display the authored class's name in the docs.
for authored_task_type in task_type.mro():
if authored_task_type.__module__ != 'abc':
break
doc_rst = indent_docstring_by_n(authored_task_type.__doc__ or '', 2)
doc_html = rst_to_html(dedent_docstring(authored_task_type.__doc__))
parser = options.get_parser(task_type.options_scope)
oschi = HelpInfoExtracter.get_option_scope_help_info_from_parser(parser)
impl = '{0}.{1}'.format(authored_task_type.__module__, authored_task_type.__name__)
tasks.append(TemplateData(
impl=impl,
doc_html=doc_html,
doc_rst=doc_rst,
ogroup=oref_template_data_from_help_info(oschi)))
goal_dict[goal.name] = TemplateData(goal=goal, tasks=tasks)
goal_names.append(goal.name)
goals = [goal_dict[name] for name in sorted(goal_names, key=lambda x: x.lower())]
return goals
def test_load_valid_partial_goals(self):
def register_goals():
Goal.by_name('jack').install(TaskRegistrar('jill', DummyTask))
with self.create_register(register_goals=register_goals) as backend_package:
Goal.clear()
self.assertEqual(0, len(Goal.all()))
load_backend(self.build_configuration, backend_package)
self.assert_empty_aliases()
self.assertEqual(1, len(Goal.all()))
task_names = Goal.by_name('jack').ordered_task_names()
self.assertEqual(1, len(task_names))
task_name = task_names[0]
self.assertEqual('jill', task_name)
def _index_products():
producer_info_by_product_type = defaultdict(set)
for goal in Goal.all():
for task_type in goal.task_types():
for product_type in task_type.product_types():
producer_info = ProducerInfo(product_type, task_type, goal)
producer_info_by_product_type[product_type].add(producer_info)
return producer_info_by_product_type
def test_load_valid_partial_goals(self):
def register_goals():
Goal.by_name('jack').install(TaskRegistrar('jill', lambda: 42))
with self.create_register(register_goals=register_goals) as backend_package:
Goal.clear()
self.assertEqual(0, len(Goal.all()))
load_backend(self.build_configuration, backend_package)
self.assert_empty_aliases()
self.assertEqual(1, len(Goal.all()))
task_names = Goal.by_name('jack').ordered_task_names()
self.assertEqual(1, len(task_names))
task_name = task_names[0]
self.assertEqual('jill', task_name)
def _index_products():
producer_info_by_product_type = defaultdict(set)
for goal in Goal.all():
for task_type in goal.task_types():
for product_type in task_type.product_types():
producer_info = ProducerInfo(product_type, task_type, goal)
producer_info_by_product_type[product_type].add(producer_info)
return producer_info_by_product_type
def _print_goals_help(self) -> None:
global_options = self._options.for_global_scope()
goal_descriptions: Dict[str, str] = {}
if global_options.v2:
goal_scope_infos = [
scope_info
for scope_info in self._options.known_scope_to_info.values()
if scope_info.category == ScopeInfo.GOAL
]
for scope_info in goal_scope_infos:
optionable_cls = scope_info.optionable_cls
if optionable_cls is None or not issubclass(
optionable_cls, GoalSubsystem):
continue
is_implemented = self._union_membership.has_members_for_all(
optionable_cls.required_union_implementations)
if not is_implemented:
continue
description = scope_info.description or "<no description>"
goal_descriptions[scope_info.scope] = description
if global_options.v1:
goal_descriptions.update({
goal.name: goal.description_first_line
for goal in Goal.all() if goal.description
})
title_text = "Goals"
title = f"{title_text}\n{'-' * len(title_text)}"
if self._use_color:
title = green(title)
max_width = max(len(name) for name in goal_descriptions.keys())
chars_before_description = max_width + 2
def format_goal(name: str, description: str) -> str:
name = name.ljust(chars_before_description)
if self._use_color:
name = cyan(name)
description_lines = textwrap.wrap(description,
80 - chars_before_description)
if len(description_lines) > 1:
description_lines = [
description_lines[0],
*(f"{' ' * chars_before_description}{line}"
for line in description_lines[1:]),
]
description = "\n".join(description_lines)
return f"{name}{description}\n"
lines = [
f"\n{title}\n",
f"Use `{self.bin_name} help $goal` to get help for a particular goal.",
"\n",
*(format_goal(name, description)
for name, description in sorted(goal_descriptions.items())),
]
print("\n".join(lines))
def register_options(self):
# Add a 'bootstrap' attribute to the register function, so that register_global can
# access the bootstrap option values.
def register_global(*args, **kwargs):
return self.options.register_global(*args, **kwargs)
register_global.bootstrap = self.options.bootstrap_option_values()
register_global_options(register_global)
for goal in Goal.all():
goal.register_options(self.options)
def __init__(self, *args, **kwargs):
self.targets = []
known_scopes = ['']
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
self.new_options = create_bootstrapped_options(known_scopes=known_scopes)
self.config = Config.from_cache() # Get the bootstrapped version.
super(GoalRunner, self).__init__(*args, needs_old_options=False, **kwargs)
def register_options(self):
# Add a 'bootstrap' attribute to the register function, so that register_global can
# access the bootstrap option values.
def register_global(*args, **kwargs):
return self.new_options.register_global(*args, **kwargs)
register_global.bootstrap = self.new_options.bootstrap_option_values()
register_global_options(register_global)
for goal in Goal.all():
goal.register_options(self.new_options)
def _print_help(self):
"""Print a help screen, followed by an optional message.
Note: Ony useful if called after options have been registered.
"""
def _maybe_help(scope):
s = self._format_help_for_scope(scope)
if s != '': # Avoid printing scope name for scope with empty options.
print(scope)
for line in s.split('\n'):
if line != '': # Avoid superfluous blank lines for empty strings.
print(' {0}'.format(line))
show_all_help = self._help_request and self._help_request.all_scopes
goals = (Goal.all() if show_all_help else
[Goal.by_name(goal_name) for goal_name in self.goals])
if goals:
for goal in goals:
if not goal.ordered_task_names():
print('\nUnknown goal: %s' % goal.name)
else:
print('\n{0}: {1}\n'.format(goal.name, goal.description))
for scope in goal.known_scopes():
_maybe_help(scope)
else:
print(pants_release())
print('\nUsage:')
print(
' ./pants [option ...] [goal ...] [target...] Attempt the specified goals.'
)
print(' ./pants help Get help.')
print(
' ./pants help [goal] Get help for a goal.'
)
print(
' ./pants help-advanced [goal] Get help for a goal\'s advanced options.'
)
print(
' ./pants help-all Get help for all goals.'
)
print(
' ./pants goals List all installed goals.'
)
print('')
print(' [target] accepts two special forms:')
print(
' dir: to include all targets in the specified directory.')
print(
' dir:: to include all targets found recursively under the directory.'
)
print('\nFriendly docs:\n http://pantsbuild.github.io/')
if show_all_help or not goals:
print('\nGlobal options:')
print(self.get_global_parser().format_help())
def graph():
def get_cluster_name(goal):
return 'cluster_%s' % goal.name.replace('-', '_')
def get_node_name(goal, task_name):
name = '%s_%s' % (goal.name, task_name)
return name.replace('-', '_')
yield '\n'.join([
'digraph G {',
' rankdir=LR;',
' graph [compound=true];',
])
for goal in Goal.all():
yield '\n'.join([
' subgraph %s {' % get_cluster_name(goal),
' node [style=filled];',
' color = blue;',
' label = "%s";' % goal.name,
])
for name in goal.ordered_task_names():
yield ' %s [label="%s"];' % (get_node_name(goal,
name), name)
yield ' }'
edges = set()
for goal in Goal.all():
tail_task_name = goal.ordered_task_names()[-1]
for dep in goal.dependencies:
edge = 'ltail=%s lhead=%s' % (get_cluster_name(goal),
get_cluster_name(dep))
if edge not in edges:
# We display edges between clusters (representing goals), but dot still requires
# us to specify them between nodes (representing tasks) and then add ltail, lhead
# annotations. We connect the last task in the dependee to the first task in
# the dependency, as this leads to the neatest-looking graph.
yield ' %s -> %s [%s];' % (
get_node_name(goal, tail_task_name),
get_node_name(dep,
dep.ordered_task_names()[0]), edge)
edges.add(edge)
yield '}'
def _get_all_cmd_line_scopes():
"""Return all scopes that may be explicitly specified on the cmd line, in no particular order.
Note that this includes only task scope, and not, say, subsystem scopes,
as those aren't specifiable on the cmd line.
"""
all_scopes = {GLOBAL_SCOPE}
for goal in Goal.all():
for task in goal.task_types():
all_scopes.add(task.get_scope_info().scope)
return all_scopes
def _get_all_cmd_line_scopes():
"""Return all scopes that may be explicitly specified on the cmd line, in no particular order.
Note that this includes only task scope, and not, say, subsystem scopes,
as those aren't specifiable on the cmd line.
"""
all_scopes ={GLOBAL_SCOPE}
for goal in Goal.all():
for task in goal.task_types():
all_scopes.add(task.get_scope_info().scope)
return all_scopes
def get_all_cmd_line_scopes(self):
"""Return all scopes that may be explicitly specified on the cmd line, in no particular order.
Note that this includes only task scope, and not, say, subsystem scopes,
as those aren't specifiable on the cmd line.
"""
all_scopes = set([''])
for goal in Goal.all():
for scope_info in goal.known_scope_infos():
if scope_info.category == ScopeInfo.TASK:
all_scopes.add(scope_info.scope)
return all_scopes
def get_all_cmd_line_scopes(self):
"""Return all scopes that may be explicitly specified on the cmd line, in no particular order.
Note that this includes only goal and task scopes, and not, say, subsystem scopes,
as those aren't specifiable on the cmd line.
"""
all_scopes = set([''])
for goal in Goal.all():
for scope_info in goal.known_scope_infos():
if scope_info.category in [ScopeInfo.GOAL, ScopeInfo.TASK]:
all_scopes.add(scope_info.scope)
return all_scopes
def __init__(self, *args, **kwargs):
self.targets = []
known_scopes = ['']
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
self.new_options = create_bootstrapped_options(
known_scopes=known_scopes)
self.config = Config.from_cache() # Get the bootstrapped version.
super(GoalRunner, self).__init__(*args,
needs_old_options=False,
**kwargs)
def register_options(self, subsystems):
# Standalone global options.
GlobalOptionsRegistrar.register_options_on_scope(self.options)
# Options for subsystems.
for subsystem in subsystems:
subsystem.register_options_on_scope(self.options)
# TODO(benjy): Should Goals be subsystems? Or should the entire goal-running mechanism
# be a subsystem?
for goal in Goal.all():
# Register task options.
goal.register_options(self.options)
def _register_options(self, subsystems, options):
"""Registers global options."""
# Standalone global options.
GlobalOptionsRegistrar.register_options_on_scope(options)
# Options for subsystems.
for subsystem in subsystems:
subsystem.register_options_on_scope(options)
# TODO(benjy): Should Goals or the entire goal-running mechanism be a Subsystem?
for goal in Goal.all():
# Register task options.
goal.register_options(options)
def _register_options(self, subsystems, options):
"""Registers global options."""
# Standalone global options.
GlobalOptionsRegistrar.register_options_on_scope(options)
# Options for subsystems.
for subsystem in subsystems:
subsystem.register_options_on_scope(options)
# TODO(benjy): Should Goals or the entire goal-running mechanism be a Subsystem?
for goal in Goal.all():
# Register task options.
goal.register_options(options)
def register_subsystem_options(self, global_subsystems):
# Standalone global options.
register_global_options(self.options.registration_function_for_global_scope())
# Options for global-level subsystems.
for subsystem_type in global_subsystems:
subsystem_type.register_options_on_scope(self.options, Options.GLOBAL_SCOPE)
# TODO(benjy): Should Goals be subsystems? Or should the entire goal-running mechanism
# be a subsystem?
for goal in Goal.all():
# Register task options (including per-task subsystem options).
goal.register_options(self.options)
def _setup_options(self, options_bootstrapper, working_set):
# TODO: This inline import is currently necessary to resolve a ~legitimate cycle between
# `GoalRunner`->`EngineInitializer`->`OptionsInitializer`->`GoalRunner`.
from pants.bin.goal_runner import GoalRunner
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
if global_bootstrap_options.pants_version != pants_version():
raise BuildConfigurationError(
'Version mismatch: Requested version was {}, our version is {}.'
.format(global_bootstrap_options.pants_version,
pants_version()))
# Get logging setup prior to loading backends so that they can log as needed.
if self._init_logging:
self._setup_logging(global_bootstrap_options)
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in global_bootstrap_options.pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = global_bootstrap_options.plugins
backend_packages = global_bootstrap_options.backend_packages
build_configuration = load_backends_and_plugins(
plugins, working_set, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems via a union of all known subsystem sets.
subsystems = Subsystem.closure(GoalRunner.subsystems()
| Goal.subsystems()
| build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
options = options_bootstrapper.get_full_options(known_scope_infos)
self._register_options(subsystems, options)
# Make the options values available to all subsystems.
Subsystem.set_options(options)
return options, build_configuration
def _setup_options(self, options_bootstrapper, working_set):
# TODO: This inline import is currently necessary to resolve a ~legitimate cycle between
# `GoalRunner`->`EngineInitializer`->`OptionsInitializer`->`GoalRunner`.
from pants.bin.goal_runner import GoalRunner
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
if global_bootstrap_options.pants_version != pants_version():
raise BuildConfigurationError(
'Version mismatch: Requested version was {}, our version is {}.'.format(
global_bootstrap_options.pants_version, pants_version()
)
)
# Get logging setup prior to loading backends so that they can log as needed.
if self._init_logging:
self._setup_logging(global_bootstrap_options)
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in global_bootstrap_options.pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = global_bootstrap_options.plugins
backend_packages = global_bootstrap_options.backend_packages
build_configuration = load_plugins_and_backends(plugins, working_set, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems via a union of all known subsystem sets.
subsystems = Subsystem.closure(
GoalRunner.subsystems() | Goal.subsystems() | build_configuration.subsystems()
)
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
options = options_bootstrapper.get_full_options(known_scope_infos)
self._register_options(subsystems, options)
# Make the options values available to all subsystems.
Subsystem.set_options(options)
return options, build_configuration
def graph():
def get_cluster_name(goal):
return 'cluster_%s' % goal.name.replace('-', '_')
def get_node_name(goal, task_name):
name = '%s_%s' % (goal.name, task_name)
return name.replace('-', '_')
yield '\n'.join([
'digraph G {',
' rankdir=LR;',
' graph [compound=true];',
])
for goal in Goal.all():
yield '\n'.join([
' subgraph %s {' % get_cluster_name(goal),
' node [style=filled];',
' color = blue;',
' label = "%s";' % goal.name,
])
for name in goal.ordered_task_names():
yield ' %s [label="%s"];' % (get_node_name(goal, name), name)
yield ' }'
edges = set()
for goal in Goal.all():
tail_task_name = goal.ordered_task_names()[-1]
for dep in goal.dependencies:
edge = 'ltail=%s lhead=%s' % (get_cluster_name(goal), get_cluster_name(dep))
if edge not in edges:
# We display edges between clusters (representing goals), but dot still requires
# us to specify them between nodes (representing tasks) and then add ltail, lhead
# annotations. We connect the last task in the dependee to the first task in
# the dependency, as this leads to the neatest-looking graph.
yield ' %s -> %s [%s];' % (get_node_name(goal, tail_task_name),
get_node_name(dep, dep.ordered_task_names()[0]), edge)
edges.add(edge)
yield '}'
def parse_args(args):
goals = OrderedSet()
specs = OrderedSet()
explicit_multi = False
logger = logging.getLogger(__name__)
has_double_dash = u'--' in args
goal_names = [goal.name for goal in Goal.all()]
if not goal_names:
raise GoalError(
'Arguments cannot be parsed before the list of goals from Goal.all() is populated.'
)
def is_spec(spec):
if os.sep in spec or ':' in spec:
return True # Definitely not a goal.
if not (spec in goal_names):
return True # Definitely not a (known) goal.
if has_double_dash:
# This means that we're parsing the half of the expression before a --, so assume it's a
# goal without warning.
return False
# Here, it's possible we have a goal and target with the same name. For now, always give
# priority to the goal, but give a warning if they might have meant the target (if the BUILD
# file exists).
if BuildFile.from_cache(get_buildroot(), spec,
must_exist=False).exists():
logger.warning(
' Command-line argument "{spec}" is ambiguous, and was assumed to be a '
'goal. If this is incorrect, disambiguate it with the "--" argument to '
'separate goals from targets.'.format(spec=spec))
return False
for i, arg in enumerate(args):
if not arg.startswith('-'):
specs.add(arg) if is_spec(arg) else goals.add(arg)
elif '--' == arg:
if specs:
raise GoalRunner.IntermixedArgumentsError(
'Cannot intermix targets with goals when using --. Targets should appear on the right'
)
explicit_multi = True
del args[i]
break
if explicit_multi:
specs.update(arg for arg in args[len(goals):]
if not arg.startswith('-'))
return goals, specs
def console_output(self, targets):
if targets:
raise TaskError('This task does not accept any target addresses.')
options = self.context.options
goals = Goal.all()
all_scopes = set()
option_strings_by_scope = defaultdict(set)
def record(scope, option_strings):
option_strings_by_scope[scope] |= self.expand_option_strings(option_strings)
for goal in goals:
for scope in goal.known_scopes():
all_scopes.add(scope)
option_strings_for_scope = set()
parser = options.get_parser(scope).get_help_argparser()
if parser:
option_strings = self.option_strings_from_parser(parser) if parser else set()
record(scope, option_strings)
scoped_option_strings = self.generate_scoped_option_strings(option_strings, scope)
record(scope, scoped_option_strings)
if '.' in scope:
outer_scope = scope.partition('.')[0]
record(outer_scope, scoped_option_strings)
# TODO: This does not currently handle subsystem-specific options.
global_argparser = options.get_parser(GLOBAL_SCOPE).get_help_argparser()
global_option_strings = self.option_strings_from_parser(global_argparser) or []
global_option_strings_set = self.expand_option_strings(global_option_strings)
options_text = '\n'.join([
"__pants_options_for_{}='{}'".format(self.bash_scope_name(scope), ' '.join(sorted(list(option_strings))))
for scope, option_strings in sorted(option_strings_by_scope.items(), key=lambda x: x[0])
])
generator = Generator(
resource_string(__name__, os.path.join('templates', 'bash_completion', 'autocomplete.sh.mustache')),
scopes_text=' '.join(sorted(list(all_scopes))),
options_text=options_text,
global_options=' '.join(sorted(list(global_option_strings_set)))
)
for line in generator.render().split('\n'):
yield line
def _print_goals_help(self):
print('\nUse `pants help $goal` to get help for a particular goal.\n')
goal_descriptions = {}
for scope_info in self._options.known_scope_to_info.values():
if scope_info.category not in (ScopeInfo.GOAL, ScopeInfo.GOAL_V1):
continue
goal_descriptions[scope_info.scope] = scope_info.description
goal_descriptions.update({goal.name: goal.description_first_line
for goal in Goal.all()
if goal.description})
max_width = max(len(name) for name in goal_descriptions.keys())
for name, description in sorted(goal_descriptions.items()):
print(' {}: {}'.format(name.rjust(max_width), description))
print()
def _print_goals_help(self):
print('\nUse `pants help $goal` to get help for a particular goal.\n')
goal_descriptions = {}
for scope_info in self._options.known_scope_to_info.values():
if scope_info.category not in (ScopeInfo.GOAL, ScopeInfo.GOAL_V1):
continue
goal_descriptions[scope_info.scope] = scope_info.description
goal_descriptions.update({
goal.name: goal.description_first_line
for goal in Goal.all() if goal.description
})
max_width = max(len(name) for name in goal_descriptions.keys())
for name, description in sorted(goal_descriptions.items()):
print(' {}: {}'.format(name.rjust(max_width), description))
print()
def __init__(self, *args, **kwargs):
self.targets = []
self.config = Config.load()
known_scopes = ['']
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
# Annoying but temporary hack to get the parser. We can't use self.parser because
# that only gets set up in the superclass ctor, and we can't call that until we have
# self.new_options set up because the superclass ctor calls our register_options().
# Fortunately this will all go away once we're fully off the old "Command" mechanism.
legacy_parser = args[2] if len(args) > 2 else kwargs['parser']
self.new_options = Options(os.environ.copy(), self.config, known_scopes, args=sys.argv,
legacy_parser=legacy_parser)
super(GoalRunner, self).__init__(*args, **kwargs)
def parse_args(args):
goals = OrderedSet()
specs = OrderedSet()
explicit_multi = False
fail_fast = False
logger = logging.getLogger(__name__)
has_double_dash = u'--' in args
goal_names = [goal.name for goal in Goal.all()]
if not goal_names:
raise GoalError(
'Arguments cannot be parsed before the list of goals from Goal.all() is populated.')
def is_spec(spec):
if os.sep in spec or ':' in spec:
return True # Definitely not a goal.
if not (spec in goal_names):
return True # Definitely not a (known) goal.
if has_double_dash:
# This means that we're parsing the half of the expression before a --, so assume it's a
# goal without warning.
return False
# Here, it's possible we have a goal and target with the same name. For now, always give
# priority to the goal, but give a warning if they might have meant the target (if the BUILD
# file exists).
if BuildFile.from_cache(get_buildroot(), spec, must_exist=False).exists():
logger.warning(' Command-line argument "{spec}" is ambiguous, and was assumed to be a '
'goal. If this is incorrect, disambiguate it with the "--" argument to '
'separate goals from targets.'.format(spec=spec))
return False
for i, arg in enumerate(args):
if not arg.startswith('-'):
specs.add(arg) if is_spec(arg) else goals.add(arg)
elif '--' == arg:
if specs:
raise GoalRunner.IntermixedArgumentsError(
'Cannot intermix targets with goals when using --. Targets should appear on the right')
explicit_multi = True
del args[i]
break
elif '--fail-fast' == arg.lower():
fail_fast = True
if explicit_multi:
specs.update(arg for arg in args[len(goals):] if not arg.startswith('-'))
return goals, specs, fail_fast
def _setup_options(self, options_bootstrapper, working_set):
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
# The pants_version may be set in pants.ini for bootstrapping, so we make sure the user actually
# requested the version on the command line before deciding to print the version and exit.
if global_bootstrap_options.is_flagged('pants_version'):
print(global_bootstrap_options.pants_version)
self._exiter(0)
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(global_bootstrap_options)
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in global_bootstrap_options.pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = global_bootstrap_options.plugins
backend_packages = global_bootstrap_options.backend_packages
build_configuration = load_plugins_and_backends(
plugins, working_set, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems via a union of all known subsystem sets.
subsystems = Subsystem.closure(GoalRunner.subsystems()
| Goal.subsystems()
| build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
options = options_bootstrapper.get_full_options(known_scope_infos)
self._register_options(subsystems, options)
# Make the options values available to all subsystems.
Subsystem.set_options(options)
return options, build_configuration
def report():
yield 'Installed goals:'
documented_rows = []
undocumented = []
max_width = 0
for goal in Goal.all():
if goal.description:
documented_rows.append((goal.name, goal.description))
max_width = max(max_width, len(goal.name))
elif self.get_options().all:
undocumented.append(goal.name)
for name, description in documented_rows:
yield ' {}: {}'.format(name.rjust(max_width), description)
if undocumented:
yield ''
yield 'Undocumented goals:'
yield ' {}'.format(' '.join(undocumented))
def report():
yield 'Installed goals:'
documented_rows = []
undocumented = []
max_width = 0
for goal in Goal.all():
if goal.description:
documented_rows.append((goal.name, goal.description))
max_width = max(max_width, len(goal.name))
elif self.context.options.goal_list_all:
undocumented.append(goal.name)
for name, description in documented_rows:
yield ' %s: %s' % (name.rjust(max_width), description)
if undocumented:
yield ''
yield 'Undocumented goals:'
yield ' %s' % ' '.join(undocumented)
def report():
yield 'Installed goals:'
documented_rows = []
undocumented = []
max_width = 0
for goal in Goal.all():
if goal.description:
documented_rows.append((goal.name, goal.description))
max_width = max(max_width, len(goal.name))
elif self.context.options.goal_list_all:
undocumented.append(goal.name)
for name, description in documented_rows:
yield ' %s: %s' % (name.rjust(max_width), description)
if undocumented:
yield ''
yield 'Undocumented goals:'
yield ' %s' % ' '.join(undocumented)
def _setup_options(self, options_bootstrapper, working_set):
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
# The pants_version may be set in pants.ini for bootstrapping, so we make sure the user actually
# requested the version on the command line before deciding to print the version and exit.
if global_bootstrap_options.is_flagged('pants_version'):
print(global_bootstrap_options.pants_version)
self._exiter(0)
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(global_bootstrap_options)
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in global_bootstrap_options.pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = global_bootstrap_options.plugins
backend_packages = global_bootstrap_options.backend_packages
build_configuration = load_plugins_and_backends(plugins, working_set, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems via a union of all known subsystem sets.
subsystems = Subsystem.closure(
GoalRunner.subsystems() | Goal.subsystems() | build_configuration.subsystems()
)
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
options = options_bootstrapper.get_full_options(known_scope_infos)
self._register_options(subsystems, options)
# Make the options values available to all subsystems.
Subsystem.set_options(options)
return options, build_configuration
def register_options(self):
# Add a 'bootstrap' attribute to the register function, so that register_global can
# access the bootstrap option values.
def register_global(*args, **kwargs):
return self.options.register_global(*args, **kwargs)
register_global.bootstrap = self.options.bootstrap_option_values()
register_global_options(register_global)
# This is the first case we have of non-task, non-global options.
# The current implementation special-cases RunTracker, and is temporary.
# In the near future it will be replaced with a 'Subsystem' abstraction.
# But for now this is useful for kicking the tires.
def register_run_tracker(*args, **kwargs):
self.options.register('run-tracker', *args, **kwargs)
RunTracker.register_options(register_run_tracker)
for goal in Goal.all():
goal.register_options(self.options)
def report():
yield 'Installed goals:'
documented_rows = []
undocumented = []
max_width = 0
for goal in Goal.all():
desc = goal.description
if desc:
first_sentence = desc.partition('\n')[0]
documented_rows.append((goal.name, first_sentence))
max_width = max(max_width, len(goal.name))
elif self.get_options().all:
undocumented.append(goal.name)
for name, description in documented_rows:
yield ' {}: {}'.format(name.rjust(max_width), description)
if undocumented:
yield ''
yield 'Undocumented goals:'
yield ' {}'.format(' '.join(undocumented))
def report():
yield 'Installed goals:'
documented_rows = []
undocumented = []
max_width = 0
for goal in Goal.all():
desc = goal.description
if desc:
first_sentence = desc.partition('\n')[0]
documented_rows.append((goal.name, first_sentence))
max_width = max(max_width, len(goal.name))
elif self.get_options().all:
undocumented.append(goal.name)
for name, description in documented_rows:
yield ' {}: {}'.format(name.rjust(max_width), description)
if undocumented:
yield ''
yield 'Undocumented goals:'
yield ' {}'.format(' '.join(undocumented))
def gen_tasks_goals_reference_data():
"""Generate the template data for the goals reference rst doc."""
goal_dict = {}
goal_names = []
for goal in Goal.all():
parser = optparse.OptionParser(add_help_option=False)
Goal.setup_parser(parser, [], [goal])
options_by_title = defaultdict(lambda: None)
for group in parser.option_groups:
options_by_title[group.title] = group
found_option_groups = set()
tasks = []
for task_name in goal.ordered_task_names():
task_type = goal.task_type_by_name(task_name)
doc_rst = indent_docstring_by_n(task_type.__doc__ or '', 2)
doc_html = rst_to_html(dedent_docstring(task_type.__doc__))
options_title = Goal.scope(goal.name, task_name)
og = options_by_title[options_title]
if og:
found_option_groups.add(options_title)
impl = '{0}.{1}'.format(task_type.__module__, task_type.__name__)
tasks.append(TemplateData(
impl=impl,
doc_html=doc_html,
doc_rst=doc_rst,
ogroup=gref_template_data_from_options(og)))
leftover_option_groups = []
for group in parser.option_groups:
if group.title in found_option_groups: continue
leftover_option_groups.append(gref_template_data_from_options(group))
leftover_options = []
for option in parser.option_list:
leftover_options.append(TemplateData(st=str(option)))
goal_dict[goal.name] = TemplateData(goal=goal,
tasks=tasks,
leftover_opts=leftover_options,
leftover_ogs=leftover_option_groups)
goal_names.append(goal.name)
goals = [goal_dict[name] for name in sorted(goal_names, key=lambda x: x.lower())]
return goals
def parse_all_tasks_and_help(self):
"""Loads all goals, and mines the options & help text for each one.
Returns: a set of all_scopes, options_text string, a set of all option strings
"""
options = self.context.options
goals = Goal.all()
all_scopes = set()
option_strings_by_scope = defaultdict(set)
def record(scope, option_strings):
option_strings_by_scope[scope] |= self.expand_option_strings(option_strings)
for goal in goals:
for scope in goal.known_scopes():
all_scopes.add(scope)
option_strings_for_scope = set()
parser = options.get_parser(scope).get_help_argparser()
if parser:
option_strings = self.option_strings_from_parser(parser) if parser else set()
record(scope, option_strings)
scoped_option_strings = self.generate_scoped_option_strings(option_strings, scope)
record(scope, scoped_option_strings)
if '.' in scope:
outer_scope = scope.partition('.')[0]
record(outer_scope, scoped_option_strings)
# TODO: This does not currently handle subsystem-specific options.
global_argparser = options.get_parser(GLOBAL_SCOPE).get_help_argparser()
global_option_strings = self.option_strings_from_parser(global_argparser) or []
global_option_strings_set = self.expand_option_strings(global_option_strings)
options_text = '\n'.join([
"__pants_options_for_{}='{}'".format(self.bash_scope_name(scope), ' '.join(sorted(list(option_strings))))
for scope, option_strings in sorted(option_strings_by_scope.items(), key=lambda x: x[0])
])
return all_scopes, options_text, global_option_strings_set
def register_options(self):
# Add a 'bootstrap' attribute to the register function, so that register_global can
# access the bootstrap option values.
def register_global(*args, **kwargs):
return self.options.register_global(*args, **kwargs)
register_global.bootstrap = self.options.bootstrap_option_values()
register_global_options(register_global)
# This is the first case we have of non-task, non-global options.
# The current implementation special-cases RunTracker, and is temporary.
# In the near future it will be replaced with a 'Subsystem' abstraction.
# But for now this is useful for kicking the tires.
def register_run_tracker(*args, **kwargs):
self.options.register('run-tracker', *args, **kwargs)
RunTracker.register_options(register_run_tracker)
for goal in Goal.all():
goal.register_options(self.options)
def _print_help(self):
"""Print a help screen, followed by an optional message.
Note: Ony useful if called after options have been registered.
"""
def _maybe_help(scope):
s = self._format_help_for_scope(scope)
if s != '': # Avoid printing scope name for scope with empty options.
print(scope)
for line in s.split('\n'):
if line != '': # Avoid superfluous blank lines for empty strings.
print(' {0}'.format(line))
show_all_help = self._help_request and self._help_request.all_scopes
goals = (Goal.all() if show_all_help else [Goal.by_name(goal_name) for goal_name in self.goals])
if goals:
for goal in goals:
if not goal.ordered_task_names():
print('\nUnknown goal: {}'.format(goal.name))
else:
print('\n{0}: {1}\n'.format(goal.name, goal.description))
for scope in goal.known_scopes():
_maybe_help(scope)
else:
print(pants_release())
print('\nUsage:')
print(' ./pants [option ...] [goal ...] [target...] Attempt the specified goals.')
print(' ./pants help Get help.')
print(' ./pants help [goal] Get help for a goal.')
print(' ./pants help-advanced [goal] Get help for a goal\'s advanced options.')
print(' ./pants help-all Get help for all goals.')
print(' ./pants goals List all installed goals.')
print('')
print(' [target] accepts two special forms:')
print(' dir: to include all targets in the specified directory.')
print(' dir:: to include all targets found recursively under the directory.')
print('\nFriendly docs:\n http://pantsbuild.github.io/')
if show_all_help or not goals:
print('\nGlobal options:')
print(self.get_global_parser().format_help())
def _print_goals_help(self):
print(
f'\nUse `{self.bin_name} help $goal` to get help for a particular goal.\n'
)
global_options = self._options.for_global_scope()
goal_descriptions = {}
if global_options.v2:
for scope_info in self._options.known_scope_to_info.values():
if scope_info.category == ScopeInfo.GOAL:
description = scope_info.description or "<no description>"
goal_descriptions[scope_info.scope] = description
if global_options.v1:
goal_descriptions.update({
goal.name: goal.description_first_line
for goal in Goal.all() if goal.description
})
max_width = max(len(name) for name in goal_descriptions.keys())
for name, description in sorted(goal_descriptions.items()):
print(' {}: {}'.format(name.rjust(max_width), description))
print()
def gen_tasks_goals_reference_data():
"""Generate the template data for the goals reference rst doc."""
goal_dict = {}
goal_names = []
for goal in Goal.all():
parser = optparse.OptionParser(add_help_option=False)
Goal.setup_parser(parser, [], [goal])
options_by_title = defaultdict(lambda: None)
for group in parser.option_groups:
options_by_title[group.title] = group
found_option_groups = set()
tasks = []
for task_name in goal.ordered_task_names():
task_type = goal.task_type_by_name(task_name)
doc = indent_docstring_by_n(task_type.__doc__ or '', 2)
options_title = Goal.option_group_title(goal, task_name)
og = options_by_title[options_title]
if og:
found_option_groups.add(options_title)
impl = '{0}.{1}'.format(task_type.__module__, task_type.__name__)
tasks.append(TemplateData(
impl=impl,
doc=doc,
ogroup=gref_template_data_from_options(og)))
leftover_option_groups = []
for group in parser.option_groups:
if group.title in found_option_groups: continue
leftover_option_groups.append(gref_template_data_from_options(group))
leftover_options = []
for option in parser.option_list:
leftover_options.append(TemplateData(st=str(option)))
goal_dict[goal.name] = TemplateData(goal=goal,
tasks=tasks,
leftover_opts=leftover_options,
leftover_ogs=leftover_option_groups)
goal_names.append(goal.name)
goals = [goal_dict[name] for name in sorted(goal_names, key=_lower)]
return goals
def _find_gather_sources():
for goal in Goal.all():
for task_name, task_type in goal.task_items():
if issubclass(task_type, GatherSources):
yield goal.name, task_name
def setup(self):
options_bootstrapper = OptionsBootstrapper()
bootstrap_options = options_bootstrapper.get_bootstrap_options()
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(bootstrap_options.for_global_scope())
# Add any extra paths to python path (eg for loading extra source backends)
for path in bootstrap_options.for_global_scope().pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = bootstrap_options.for_global_scope().plugins
backend_packages = bootstrap_options.for_global_scope().backend_packages
build_configuration = load_plugins_and_backends(plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scope_infos = [ScopeInfo.for_global_scope()]
# Add scopes for all needed subsystems.
subsystems = (set(self.subsystems) | Goal.subsystems() | build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(ScopeInfo(subsystem.options_scope, ScopeInfo.GLOBAL_SUBSYSTEM))
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scope_infos)
self.register_options(subsystems)
# Make the options values available to all subsystems.
Subsystem._options = self.options
# Now that we have options we can instantiate subsystems.
self.run_tracker = RunTracker.global_instance()
self.reporting = Reporting.global_instance()
report = self.reporting.initial_reporting(self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
rev = self.options.for_global_scope().build_file_rev
if rev:
ScmBuildFile.set_rev(rev)
ScmBuildFile.set_scm(get_scm())
build_file_type = ScmBuildFile
else:
build_file_type = FilesystemBuildFile
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
# TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
source_root_bootstrapper = SourceRootBootstrapper.global_instance()
source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser)
self._expand_goals_and_specs()
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
def _checkstyle_enabled():
for goal in Goal.all():
if goal.has_task_of_type(Checkstyle):
return True
return False
def _checkstyle_enabled():
for goal in Goal.all():
if goal.has_task_of_type(Checkstyle):
return True
return False
def register_options(self):
register_global_options(self.new_options.register_global)
for goal in Goal.all():
goal.register_options(self.new_options)
def setup(self):
options_bootstrapper = OptionsBootstrapper()
# Force config into the cache so we (and plugin/backend loading code) can use it.
# TODO: Plumb options in explicitly.
bootstrap_options = options_bootstrapper.get_bootstrap_options()
self.config = Config.from_cache()
# Add any extra paths to python path (eg for loading extra source backends)
for path in bootstrap_options.for_global_scope().pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
backend_packages = self.config.getlist('backends', 'packages', [])
plugins = self.config.getlist('backends', 'plugins', [])
build_configuration = load_plugins_and_backends(
plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
# TODO: Create a 'Subsystem' abstraction instead of special-casing run-tracker here
# and in register_options().
known_scopes = ['', 'run-tracker']
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(
known_scopes=known_scopes)
self.register_options()
# TODO(Eric Ayers) We are missing log messages. Set the log level earlier
# Enable standard python logging for code with no handle to a context/work-unit.
self._setup_logging() # NB: self.options are needed for this call.
self.run_tracker = RunTracker.from_options(self.options)
report = initial_reporting(self.config, self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: %s' % url)
else:
self.run_tracker.log(
Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(
build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
with self.run_tracker.new_workunit(name='bootstrap',
labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in self.config.getlist('goals',
'bootstrap_buildfiles',
default=[]):
build_file = BuildFile.from_cache(root_dir=self.root_dir,
relpath=path)
# TODO(pl): This is an unfortunate interface leak, but I don't think
# in the long run that we should be relying on "bootstrap" BUILD files
# that do nothing except modify global state. That type of behavior
# (e.g. source roots, goal registration) should instead happen in
# project plugins, or specialized configuration files.
self.build_file_parser.parse_build_file_family(build_file)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
self._expand_goals_and_specs()
