def setUp(self):
super(EngineTestBase, self).setUp()
# TODO(John Sirois): Now that the BuildFileParser controls goal registration by iterating
# over plugin callbacks a PhaseRegistry can be constructed by it and handed to all these
# callbacks in place of having a global Phase registry. Remove the Phase static cling.
Phase.clear()
def execute(cls, context, *names):
parser = OptionParser()
add_global_options(parser)
phases = [Phase(name) for name in names]
Phase.setup_parser(parser, [], phases)
options, _ = parser.parse_args([])
context = Context(context.config, options, context.run_tracker, context.target_roots,
requested_goals=list(names))
return cls._execute(context, phases, print_timing=False)
def execute(self, targets):
goal = self.context.options.help_goal
if goal is None:
return self.list_goals('You must supply a goal name to provide help for.')
phase = Phase(goal)
if not phase.goals():
self.list_goals('Goal %s is unknown.' % goal)
parser = setup_parser_for_phase_help(phase)
parser.parse_args(['--help'])
def execute(self, targets):
goal = self.context.options.help_goal
if goal is None:
return self.list_goals('You must supply a goal name to provide help for.')
phase = Phase(goal)
if not phase.goals():
return self.list_goals('Goal %s is unknown.' % goal)
parser = OptionParser()
parser.set_usage('%s goal %s ([target]...)' % (sys.argv[0], goal))
parser.epilog = phase.description
Goal.add_global_options(parser)
Phase.setup_parser(parser, [], [phase])
parser.parse_args(['--help'])
def test_load_valid_partial_goals(self):
def register_goals():
Phase('jack').install(Goal('jill', lambda: 42))
with self.create_register(register_goals=register_goals) as backend_package:
Phase.clear()
self.assertEqual(0, len(Phase.all()))
load_backend(self.build_configuration, backend_package)
self.assert_empty_aliases()
self.assertEqual(1, len(Phase.all()))
goals = Phase('jack').goals()
self.assertEqual(1, len(goals))
goal = goals[0]
self.assertEqual('jill', goal.name)
def graph():
def get_cluster_name(phase):
return 'cluster_%s' % phase.name.replace('-', '_')
def get_goal_name(phase, goal):
name = '%s_%s' % (phase.name, goal.name)
return name.replace('-', '_')
phase_by_phasename = {}
for phase, goals in Phase.all():
phase_by_phasename[phase.name] = phase
yield '\n'.join([
'digraph G {',
' rankdir=LR;',
' graph [compound=true];',
])
for phase, installed_goals in Phase.all():
yield '\n'.join([
' subgraph %s {' % get_cluster_name(phase),
' node [style=filled];',
' color = blue;',
' label = "%s";' % phase.name,
])
for installed_goal in installed_goals:
yield ' %s [label="%s"];' % (get_goal_name(
phase, installed_goal), installed_goal.name)
yield ' }'
edges = set()
for phase, installed_goals in Phase.all():
for installed_goal in installed_goals:
for dependency in installed_goal.dependencies:
tail_goal = phase_by_phasename.get(
dependency.name).goals()[-1]
edge = 'ltail=%s lhead=%s' % (get_cluster_name(phase),
get_cluster_name(
Phase.of(tail_goal)))
if edge not in edges:
yield ' %s -> %s [%s];' % (
get_goal_name(phase, installed_goal),
get_goal_name(Phase.of(tail_goal),
tail_goal), edge)
edges.add(edge)
yield '}'
def graph():
def get_cluster_name(phase):
return 'cluster_%s' % phase.name.replace('-', '_')
def get_goal_name(phase, goal):
name = '%s_%s' % (phase.name, goal.name)
return name.replace('-', '_')
phase_by_phasename = {}
for phase, goals in Phase.all():
phase_by_phasename[phase.name] = phase
yield '\n'.join([
'digraph G {',
' rankdir=LR;',
' graph [compound=true];',
])
for phase, installed_goals in Phase.all():
yield '\n'.join([
' subgraph %s {' % get_cluster_name(phase),
' node [style=filled];',
' color = blue;',
' label = "%s";' % phase.name,
])
for installed_goal in installed_goals:
yield ' %s [label="%s"];' % (get_goal_name(phase, installed_goal),
installed_goal.name)
yield ' }'
edges = set()
for phase, installed_goals in Phase.all():
for installed_goal in installed_goals:
for dependency in installed_goal.dependencies:
tail_goal = phase_by_phasename.get(dependency.name).goals()[-1]
edge = 'ltail=%s lhead=%s' % (get_cluster_name(phase),
get_cluster_name(Phase.of(tail_goal)))
if edge not in edges:
yield ' %s -> %s [%s];' % (get_goal_name(phase, installed_goal),
get_goal_name(Phase.of(tail_goal), tail_goal),
edge)
edges.add(edge)
yield '}'
def register_goals():
changed = Phase('changed').with_description('Print the targets changed since some prior commit.')
changed.install(Goal(name='changed', action=ScmWhatChanged))
# We always want compile to finish with a checkstyle
compile = Phase('compile')
compile.install(Goal(name='checkstyle', action=Checkstyle,
dependencies=['gen', 'resolve']))
def _record(self, goal, elapsed):
phase = Phase.of(goal)
phase_timings = self._timings.get(phase)
if phase_timings is None:
phase_timings = OrderedDict(())
self._timings[phase] = phase_timings
goal_timings = phase_timings.get(goal)
if goal_timings is None:
goal_timings = []
phase_timings[goal] = goal_timings
goal_timings.append(elapsed)
def parse_args(args):
goals = OrderedSet()
specs = OrderedSet()
explicit_multi = False
logger = logging.getLogger(__name__)
has_double_dash = u'--' in args
goal_names = [phase.name for phase, goal in Phase.all()]
if not goal_names:
raise GoalError(
'Arguments cannot be parsed before the list of goals from Phase.all() is populated.')
def is_spec(spec):
if os.sep in spec or ':' in spec:
return True # Definitely not a goal.
if not (spec in goal_names):
return True # Definitely not a (known) goal.
if has_double_dash:
# This means that we're parsing the half of the expression before a --, so assume it's a
# goal without warning.
return False
# Here, it's possible we have a goal and target with the same name. For now, always give
# priority to the goal, but give a warning if they might have meant the target (if the BUILD
# file exists).
try:
BuildFile(get_buildroot(), spec)
msg = (' Command-line argument "{spec}" is ambiguous, and was assumed to be a goal.'
' If this is incorrect, disambiguate it with the "--" argument to separate goals'
' from targets.')
logger.warning(msg.format(spec=spec))
except IOError: pass # Awesome, it's unambiguous.
return False
for i, arg in enumerate(args):
if not arg.startswith('-'):
specs.add(arg) if is_spec(arg) else goals.add(arg)
elif '--' == arg:
if specs:
raise Goal.IntermixedArgumentsError('Cannot intermix targets with goals when using --. '
'Targets should appear on the right')
explicit_multi = True
del args[i]
break
if explicit_multi:
specs.update(arg for arg in args[len(goals):] if not arg.startswith('-'))
return goals, specs
def report():
yield 'Installed goals:'
documented_rows = []
undocumented = []
max_width = 0
for phase, _ in Phase.all():
if phase.description:
documented_rows.append((phase.name, phase.description))
max_width = max(max_width, len(phase.name))
elif self.context.options.goal_list_all:
undocumented.append(phase.name)
for name, description in documented_rows:
yield ' %s: %s' % (name.rjust(max_width), description)
if undocumented:
yield ''
yield 'Undocumented goals:'
yield ' %s' % ' '.join(undocumented)
def report():
yield 'Installed goals:'
documented_rows = []
undocumented = []
max_width = 0
for phase, _ in Phase.all():
if phase.description:
documented_rows.append((phase.name, phase.description))
max_width = max(max_width, len(phase.name))
elif self.context.options.goal_list_all:
undocumented.append(phase.name)
for name, description in documented_rows:
yield ' %s: %s' % (name.rjust(max_width), description)
if undocumented:
yield ''
yield 'Undocumented goals:'
yield ' %s' % ' '.join(undocumented)
def setup_parser(self, parser, args):
self.config = Config.load()
add_global_options(parser)
# We support attempting zero or more goals. Multiple goals must be delimited from further
# options and non goal args with a '--'. The key permutations we need to support:
# ./pants goal => goals
# ./pants goal goals => goals
# ./pants goal compile src/java/... => compile
# ./pants goal compile -x src/java/... => compile
# ./pants goal compile src/java/... -x => compile
# ./pants goal compile run -- src/java/... => compile, run
# ./pants goal compile run -- src/java/... -x => compile, run
# ./pants goal compile run -- -x src/java/... => compile, run
if not args:
args.append('help')
help_flags = set(['-h', '--help', 'help'])
show_help = len(help_flags.intersection(args)) > 0
args = filter(lambda f: f not in help_flags, args)
goals, specs = Goal.parse_args(args)
if show_help:
print_help(goals)
sys.exit(0)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup', labels=[WorkUnit.SETUP]):
# Bootstrap user goals by loading any BUILD files implied by targets.
spec_parser = SpecParser(self.root_dir, self.build_file_parser)
with self.run_tracker.new_workunit(name='parse', labels=[WorkUnit.SETUP]):
for spec in specs:
for address in spec_parser.parse_addresses(spec):
self.build_file_parser.inject_spec_closure_into_build_graph(address.spec,
self.build_graph)
self.targets.append(self.build_graph.get_target(address))
self.phases = [Phase(goal) for goal in goals]
rcfiles = self.config.getdefault('rcfiles', type=list,
default=['/etc/pantsrc', '~/.pants.rc'])
if rcfiles:
rcfile = RcFile(rcfiles, default_prepend=False, process_default=True)
# Break down the goals specified on the command line to the full set that will be run so we
# can apply default flags to inner goal nodes. Also break down goals by Task subclass and
# register the task class hierarchy fully qualified names so we can apply defaults to
# baseclasses.
sections = OrderedSet()
for phase in Engine.execution_order(self.phases):
for goal in phase.goals():
sections.add(goal.name)
for clazz in goal.task_type.mro():
if clazz == Task:
break
sections.add('%s.%s' % (clazz.__module__, clazz.__name__))
augmented_args = rcfile.apply_defaults(sections, args)
if augmented_args != args:
del args[:]
args.extend(augmented_args)
sys.stderr.write("(using pantsrc expansion: pants goal %s)\n" % ' '.join(augmented_args))
Phase.setup_parser(parser, args, self.phases)
def setup_parser(self, parser, args):
self.config = Config.load()
add_global_options(parser)
# We support attempting zero or more goals. Multiple goals must be delimited from further
# options and non goal args with a '--'. The key permutations we need to support:
# ./pants goal => goals
# ./pants goal goals => goals
# ./pants goal compile src/java/... => compile
# ./pants goal compile -x src/java/... => compile
# ./pants goal compile src/java/... -x => compile
# ./pants goal compile run -- src/java/... => compile, run
# ./pants goal compile run -- src/java/... -x => compile, run
# ./pants goal compile run -- -x src/java/... => compile, run
if not args:
args.append('goals')
if len(args) == 1 and args[0] in set(['-h', '--help', 'help']):
def format_usage(usages):
left_colwidth = 0
for left, right in usages:
left_colwidth = max(left_colwidth, len(left))
lines = []
for left, right in usages:
lines.append(' %s%s%s' % (left, ' ' * (left_colwidth - len(left) + 1), right))
return '\n'.join(lines)
usages = [
("%prog goal goals ([spec]...)", Phase('goals').description),
("%prog goal help [goal] ([spec]...)", Phase('help').description),
("%prog goal [goal] [spec]...", "Attempt goal against one or more targets."),
("%prog goal [goal] ([goal]...) -- [spec]...", "Attempts all the specified goals."),
]
parser.set_usage("\n%s" % format_usage(usages))
parser.epilog = ("Either lists all installed goals, provides extra help for a goal or else "
"attempts to achieve the specified goal for the listed targets.")
parser.print_help()
# Add some text that we can't put in the epilog, because that formats away newlines.
print(textwrap.dedent("""
Note that target specs accept two special forms:
[dir]: to include all targets in the specified directory
[dir]:: to include all targets found recursively under the directory"""))
sys.exit(0)
else:
goals, specs = Goal.parse_args(args)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup', labels=[WorkUnit.SETUP]):
# Bootstrap goals by loading any configured bootstrap BUILD files
with self.check_errors('The following bootstrap_buildfiles cannot be loaded:') as error:
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
for path in self.config.getlist('goals', 'bootstrap_buildfiles', default = []):
try:
buildfile = BuildFile(get_buildroot(), os.path.relpath(path, get_buildroot()))
ParseContext(buildfile).parse()
except (TypeError, ImportError, TaskError, GoalError):
error(path, include_traceback=True)
except (IOError, SyntaxError):
error(path)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
# Bootstrap user goals by loading any BUILD files implied by targets.
spec_parser = SpecParser(self.root_dir)
with self.check_errors('The following targets could not be loaded:') as error:
with self.run_tracker.new_workunit(name='parse', labels=[WorkUnit.SETUP]):
for spec in specs:
try:
for target, address in spec_parser.parse(spec):
if target:
self.targets.append(target)
# Force early BUILD file loading if this target is an alias that expands
# to others.
unused = list(target.resolve())
else:
siblings = Target.get_all_addresses(address.buildfile)
prompt = 'did you mean' if len(siblings) == 1 else 'maybe you meant one of these'
error('%s => %s?:\n %s' % (address, prompt,
'\n '.join(str(a) for a in siblings)))
except (TypeError, ImportError, TaskError, GoalError):
error(spec, include_traceback=True)
except (IOError, SyntaxError, TargetDefinitionException):
error(spec)
self.phases = [Phase(goal) for goal in goals]
rcfiles = self.config.getdefault('rcfiles', type=list,
default=['/etc/pantsrc', '~/.pants.rc'])
if rcfiles:
rcfile = RcFile(rcfiles, default_prepend=False, process_default=True)
# Break down the goals specified on the command line to the full set that will be run so we
# can apply default flags to inner goal nodes. Also break down goals by Task subclass and
# register the task class hierarchy fully qualified names so we can apply defaults to
# baseclasses.
sections = OrderedSet()
for phase in Engine.execution_order(self.phases):
for goal in phase.goals():
sections.add(goal.name)
for clazz in goal.task_type.mro():
if clazz == Task:
break
sections.add('%s.%s' % (clazz.__module__, clazz.__name__))
augmented_args = rcfile.apply_defaults(sections, args)
if augmented_args != args:
del args[:]
args.extend(augmented_args)
sys.stderr.write("(using pantsrc expansion: pants goal %s)\n" % ' '.join(augmented_args))
Phase.setup_parser(parser, args, self.phases)
def tearDown(self): Phase.clear()
def register_goals():
resources = Phase('resources')
resources.install(Goal(name='args-apt', action=ResourceMapper,
dependencies=['compile']))
def tearDown(self):
Phase.clear()
super(EngineTestBase, self).tearDown()
def as_phase(cls, phase_name): """Returns a ``Phase`` object of the given name""" return Phase(cls._namespace(phase_name))
def setup_parser(self, parser, args):
self.config = Config.load()
add_global_options(parser)
# We support attempting zero or more goals. Multiple goals must be delimited from further
# options and non goal args with a '--'. The key permutations we need to support:
# ./pants goal => goals
# ./pants goal goals => goals
# ./pants goal compile src/java/... => compile
# ./pants goal compile -x src/java/... => compile
# ./pants goal compile src/java/... -x => compile
# ./pants goal compile run -- src/java/... => compile, run
# ./pants goal compile run -- src/java/... -x => compile, run
# ./pants goal compile run -- -x src/java/... => compile, run
if not args:
args.append('help')
help_flags = set(['-h', '--help', 'help'])
show_help = len(help_flags.intersection(args)) > 0
args = filter(lambda f: f not in help_flags, args)
goals, specs = Goal.parse_args(args)
if show_help:
print_help(goals)
sys.exit(0)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup',
labels=[WorkUnit.SETUP]):
# Bootstrap user goals by loading any BUILD files implied by targets.
spec_parser = SpecParser(self.root_dir, self.build_file_parser)
with self.run_tracker.new_workunit(name='parse',
labels=[WorkUnit.SETUP]):
for spec in specs:
for address in spec_parser.parse_addresses(spec):
self.build_file_parser.inject_spec_closure_into_build_graph(
address.spec, self.build_graph)
self.targets.append(
self.build_graph.get_target(address))
self.phases = [Phase(goal) for goal in goals]
rcfiles = self.config.getdefault(
'rcfiles', type=list, default=['/etc/pantsrc', '~/.pants.rc'])
if rcfiles:
rcfile = RcFile(rcfiles,
default_prepend=False,
process_default=True)
# Break down the goals specified on the command line to the full set that will be run so we
# can apply default flags to inner goal nodes. Also break down goals by Task subclass and
# register the task class hierarchy fully qualified names so we can apply defaults to
# baseclasses.
sections = OrderedSet()
for phase in Engine.execution_order(self.phases):
for goal in phase.goals():
sections.add(goal.name)
for clazz in goal.task_type.mro():
if clazz == Task:
break
sections.add('%s.%s' %
(clazz.__module__, clazz.__name__))
augmented_args = rcfile.apply_defaults(sections, args)
if augmented_args != args:
del args[:]
args.extend(augmented_args)
sys.stderr.write("(using pantsrc expansion: pants goal %s)\n" %
' '.join(augmented_args))
Phase.setup_parser(parser, args, self.phases)