class CmdLineSpecParserBadBuildTest(BaseTest):
def setUp(self):
super(CmdLineSpecParserBadBuildTest, self).setUp()
self.add_to_build_file("bad/a", "a_is_bad")
self.add_to_build_file("bad/b", "b_is_bad")
self.spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper)
self.NO_FAIL_FAST_RE = """^Exception message: name 'a_is_bad' is not defined
while executing BUILD file (/[^/]+)*/bad/a/BUILD
Loading addresses from 'bad/a' failed\.
Exception message: name 'b_is_bad' is not defined
while executing BUILD file (/[^/]+)*T/bad/b/BUILD
Loading addresses from 'bad/b' failed\.
Invalid BUILD files for \[::\]$"""
self.FAIL_FAST_RE = """^name 'a_is_bad' is not defined
while executing BUILD file (/[^/]+)*/bad/a/BUILD
Loading addresses from 'bad/a' failed.$"""
def test_bad_build_files(self):
with self.assertRaisesRegexp(self.spec_parser.BadSpecError, self.NO_FAIL_FAST_RE):
list(self.spec_parser.parse_addresses("::"))
def test_bad_build_files_fail_fast(self):
with self.assertRaisesRegexp(self.spec_parser.BadSpecError, self.FAIL_FAST_RE):
list(self.spec_parser.parse_addresses("::", True))
class CmdLineSpecParserBadBuildTest(BaseTest):
def setUp(self):
super(CmdLineSpecParserBadBuildTest, self).setUp()
self.add_to_build_file('bad/a', 'a_is_bad')
self.add_to_build_file('bad/b', 'b_is_bad')
self.spec_parser = CmdLineSpecParser(self.build_root,
self.address_mapper)
self.NO_FAIL_FAST_RE = re.compile(
r"""^--------------------
.*
Exception message: name 'a_is_bad' is not defined
while executing BUILD file (/[^/]+)*/bad/a/BUILD
Loading addresses from 'bad/a' failed\.
.*
Exception message: name 'b_is_bad' is not defined
while executing BUILD file (/[^/]+)*T/bad/b/BUILD
Loading addresses from 'bad/b' failed\.
Invalid BUILD files for \[::\]$""", re.DOTALL)
self.FAIL_FAST_RE = """^name 'a_is_bad' is not defined
while executing BUILD file (/[^/]+)*/bad/a/BUILD
Loading addresses from 'bad/a' failed.$"""
def test_bad_build_files(self):
with self.assertRaisesRegexp(self.spec_parser.BadSpecError,
self.NO_FAIL_FAST_RE):
list(self.spec_parser.parse_addresses('::'))
def test_bad_build_files_fail_fast(self):
with self.assertRaisesRegexp(self.spec_parser.BadSpecError,
self.FAIL_FAST_RE):
list(self.spec_parser.parse_addresses('::', True))
class CmdLineSpecParserBadBuildTest(BaseTest):
def setUp(self):
super(CmdLineSpecParserBadBuildTest, self).setUp()
self.add_to_build_file('bad/a', 'a_is_bad')
self.add_to_build_file('bad/b', 'b_is_bad')
self.spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper)
self.NO_FAIL_FAST_RE = re.compile(r"""^--------------------
.*
Exception message: name 'a_is_bad' is not defined
while executing BUILD file FilesystemBuildFile\((/[^/]+)*/bad/a/BUILD\)
Loading addresses from 'bad/a' failed\.
.*
Exception message: name 'b_is_bad' is not defined
while executing BUILD file FilesystemBuildFile\((/[^/]+)*T/bad/b/BUILD\)
Loading addresses from 'bad/b' failed\.
Invalid BUILD files for \[::\]$""", re.DOTALL)
self.FAIL_FAST_RE = """^name 'a_is_bad' is not defined
while executing BUILD file FilesystemBuildFile\((/[^/]+)*/bad/a/BUILD\)
Loading addresses from 'bad/a' failed.$"""
def test_bad_build_files(self):
with self.assertRaisesRegexp(self.spec_parser.BadSpecError, self.NO_FAIL_FAST_RE):
list(self.spec_parser.parse_addresses('::'))
def test_bad_build_files_fail_fast(self):
with self.assertRaisesRegexp(self.spec_parser.BadSpecError, self.FAIL_FAST_RE):
list(self.spec_parser.parse_addresses('::', True))
def _expand_goals_and_specs(self):
goals = self.options.goals
specs = self.options.target_specs
fail_fast = self.options.for_global_scope().fail_fast
for goal in goals:
if self.address_mapper.from_cache(get_buildroot(), goal, must_exist=False).file_exists():
logger.warning(" Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal))
if self.options.help_request:
help_printer = HelpPrinter(self.options)
help_printer.print_help()
self._exiter(0)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup', labels=[WorkUnitLabel.SETUP]):
spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper,
spec_excludes=self.spec_excludes,
exclude_target_regexps=self.global_options.exclude_target_regexp)
with self.run_tracker.new_workunit(name='parse', labels=[WorkUnitLabel.SETUP]):
def filter_for_tag(tag):
return lambda target: tag in map(str, target.tags)
tag_filter = wrap_filters(create_filters(self.global_options.tag, filter_for_tag))
for spec in specs:
for address in spec_parser.parse_addresses(spec, fail_fast):
self.build_graph.inject_address_closure(address)
tgt = self.build_graph.get_target(address)
if tag_filter(tgt):
self.targets.append(tgt)
self.goals = [Goal.by_name(goal) for goal in goals]
def _expand_goals_and_specs(self):
goals = self.options.goals
specs = self.options.target_specs
fail_fast = self.options.for_global_scope().fail_fast
for goal in goals:
if BuildFile.from_cache(get_buildroot(), goal,
must_exist=False).exists():
logger.warning(
" Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}."
.format(goal))
if self.options.print_help_if_requested():
sys.exit(0)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup',
labels=[WorkUnit.SETUP]):
spec_parser = CmdLineSpecParser(
self.root_dir,
self.address_mapper,
spec_excludes=self.spec_excludes,
exclude_target_regexps=self.global_options.
exclude_target_regexp)
with self.run_tracker.new_workunit(name='parse',
labels=[WorkUnit.SETUP]):
for spec in specs:
for address in spec_parser.parse_addresses(
spec, fail_fast):
self.build_graph.inject_address_closure(address)
self.targets.append(
self.build_graph.get_target(address))
self.goals = [Goal.by_name(goal) for goal in goals]
def _expand_goals_and_specs(self):
goals = self.options.goals
specs = self.options.target_specs
fail_fast = self.options.for_global_scope().fail_fast
for goal in goals:
if BuildFile.from_cache(get_buildroot(), goal, must_exist=False).exists():
logger.warning(" Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal))
if self.options.print_help_if_requested():
sys.exit(0)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup', labels=[WorkUnit.SETUP]):
spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper,
spec_excludes=self.spec_excludes,
exclude_target_regexps=self.global_options.exclude_target_regexp)
with self.run_tracker.new_workunit(name='parse', labels=[WorkUnit.SETUP]):
for spec in specs:
for address in spec_parser.parse_addresses(spec, fail_fast):
self.build_graph.inject_address_closure(address)
self.targets.append(self.build_graph.get_target(address))
self.goals = [Goal.by_name(goal) for goal in goals]
def setup_parser(self, parser, args):
if not args:
args.append('help')
logger = logging.getLogger(__name__)
goals = self.new_options.goals
specs = self.new_options.target_specs
fail_fast = self.new_options.for_global_scope().fail_fast
for goal in goals:
if BuildFile.from_cache(get_buildroot(), goal, must_exist=False).exists():
logger.warning(" Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal))
if self.new_options.is_help:
self.new_options.print_help(goals=goals)
sys.exit(0)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup', labels=[WorkUnit.SETUP]):
spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper,
spec_excludes=self.get_spec_excludes())
with self.run_tracker.new_workunit(name='parse', labels=[WorkUnit.SETUP]):
for spec in specs:
for address in spec_parser.parse_addresses(spec, fail_fast):
self.build_graph.inject_address_closure(address)
self.targets.append(self.build_graph.get_target(address))
self.goals = [Goal.by_name(goal) for goal in goals]
rcfiles = self.config.getdefault('rcfiles', type=list,
default=['/etc/pantsrc', '~/.pants.rc'])
if rcfiles:
rcfile = RcFile(rcfiles, default_prepend=False, process_default=True)
# Break down the goals specified on the command line to the full set that will be run so we
# can apply default flags to inner goal nodes. Also break down goals by Task subclass and
# register the task class hierarchy fully qualified names so we can apply defaults to
# baseclasses.
sections = OrderedSet()
for goal in Engine.execution_order(self.goals):
for task_name in goal.ordered_task_names():
sections.add(task_name)
task_type = goal.task_type_by_name(task_name)
for clazz in task_type.mro():
if clazz == Task:
break
sections.add('%s.%s' % (clazz.__module__, clazz.__name__))
augmented_args = rcfile.apply_defaults(sections, args)
if augmented_args != args:
# TODO(John Sirois): Cleanup this currently important mutation of the passed in args
# once the 2-layer of command -> goal is squashed into one.
args[:] = augmented_args
sys.stderr.write("(using pantsrc expansion: pants goal %s)\n" % ' '.join(augmented_args))
def _get_targets(spec):
try:
spec_parser = CmdLineSpecParser(get_buildroot(), self.context.build_file_parser)
addresses = spec_parser.parse_addresses(spec)
except (IOError, ValueError) as e:
raise TaskError('Failed to parse spec: %s: %s' % (spec, e))
matches = set(self.context.build_graph.get_target(address) for address in addresses)
if not matches:
raise TaskError('No matches for spec: %s' % spec)
return matches
def __init__(self, *args, **kwargs):
super(Build, self).__init__(*args, **kwargs)
if not self.args:
self.error("A spec argument is required")
self.config = Config.load()
interpreters = self.options.interpreters or [b'']
self.interpreter_cache = PythonInterpreterCache(self.config,
logger=self.debug)
self.interpreter_cache.setup(filters=interpreters)
interpreters = self.interpreter_cache.select_interpreter(
list(self.interpreter_cache.matches(interpreters)))
if len(interpreters) != 1:
self.error('Unable to detect suitable interpreter.')
else:
self.debug('Selected %s' % interpreters[0])
self.interpreter = interpreters[0]
try:
specs_end = self.args.index('--')
if len(self.args) > specs_end:
self.build_args = self.args[specs_end + 1:len(self.args) + 1]
else:
self.build_args = []
except ValueError:
specs_end = 1
self.build_args = self.args[1:] if len(self.args) > 1 else []
self.targets = OrderedSet()
spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper)
self.top_level_addresses = set()
specs = self.args[0:specs_end]
addresses = spec_parser.parse_addresses(specs)
for address in addresses:
self.top_level_addresses.add(address)
try:
self.build_graph.inject_address_closure(address)
target = self.build_graph.get_target(address)
except:
self.error("Problem parsing BUILD target %s: %s" %
(address, traceback.format_exc()))
if not target:
self.error("Target %s does not exist" % address)
transitive_targets = self.build_graph.transitive_subgraph_of_addresses(
[target.address])
for transitive_target in transitive_targets:
self.targets.add(transitive_target)
self.targets = [target for target in self.targets if target.is_python]
def __init__(self, *args, **kwargs):
super(Build, self).__init__(*args, **kwargs)
if not self.args:
self.error("A spec argument is required")
self._verbose = self.options.verbose
self.config = Config.load()
interpreters = self.options.interpreters or [b'']
self.interpreter_cache = PythonInterpreterCache(self.config, logger=self.debug)
self.interpreter_cache.setup(filters=interpreters)
interpreters = self.interpreter_cache.select_interpreter(
list(self.interpreter_cache.matches(interpreters)))
if len(interpreters) != 1:
self.error('Unable to detect suitable interpreter.')
else:
self.debug('Selected %s' % interpreters[0])
self.interpreter = interpreters[0]
try:
specs_end = self.args.index('--')
if len(self.args) > specs_end:
self.build_args = self.args[specs_end+1:len(self.args)+1]
else:
self.build_args = []
except ValueError:
specs_end = 1
self.build_args = self.args[1:] if len(self.args) > 1 else []
self.targets = OrderedSet()
spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper)
self.top_level_addresses = set()
specs = self.args[0:specs_end]
addresses = spec_parser.parse_addresses(specs)
for address in addresses:
self.top_level_addresses.add(address)
try:
self.build_graph.inject_address_closure(address)
target = self.build_graph.get_target(address)
except:
self.error("Problem parsing BUILD target %s: %s" % (address, traceback.format_exc()))
if not target:
self.error("Target %s does not exist" % address)
transitive_targets = self.build_graph.transitive_subgraph_of_addresses([target.address])
for transitive_target in transitive_targets:
self.targets.add(transitive_target)
self.targets = [target for target in self.targets if target.is_python]
def targets(self, spec):
"""Resolves a target spec to one or more Target objects.
spec: Either BUILD target address or else a target glob using the siblings ':' or
descendants '::' suffixes.
Returns the set of all Targets found.
"""
spec_parser = CmdLineSpecParser(self.build_root, self.build_file_parser)
addresses = list(spec_parser.parse_addresses(spec))
for address in addresses:
self.build_file_parser.inject_spec_closure_into_build_graph(address.spec, self.build_graph)
targets = [self.build_graph.get_target(address) for address in addresses]
return targets
def _get_targets(spec):
try:
spec_parser = CmdLineSpecParser(get_buildroot(), self.context.build_file_parser)
addresses = spec_parser.parse_addresses(spec)
except (IOError, ValueError) as e:
raise TaskError('Failed to parse spec: %s: %s' % (spec, e))
# filter specs may not have been parsed as part of the context: force parsing
matches = set()
for address in addresses:
self.context.build_file_parser.inject_address_into_build_graph(address, self.context.build_graph)
matches.add(self.context.build_graph.get_target(address))
if not matches:
raise TaskError('No matches for spec: %s' % spec)
return matches
def _get_targets(spec):
try:
spec_parser = CmdLineSpecParser(get_buildroot(), self.context.address_mapper)
addresses = spec_parser.parse_addresses(spec)
except AddressLookupError as e:
raise TaskError('Failed to parse address selector: {spec}\n {message}'.format(spec=spec, message=e))
# filter specs may not have been parsed as part of the context: force parsing
matches = set()
for address in addresses:
self.context.build_graph.inject_address_closure(address)
matches.add(self.context.build_graph.get_target(address))
if not matches:
raise TaskError('No matches for address selector: {spec}'.format(spec=spec))
return matches
def _get_targets(spec):
try:
spec_parser = CmdLineSpecParser(get_buildroot(), self.context.address_mapper)
addresses = spec_parser.parse_addresses(spec)
except AddressLookupError as e:
raise TaskError('Failed to parse address selector: {spec}\n {message}'.format(spec=spec, message=e))
# filter specs may not have been parsed as part of the context: force parsing
matches = set()
for address in addresses:
self.context.build_graph.inject_address_closure(address)
matches.add(self.context.build_graph.get_target(address))
if not matches:
raise TaskError('No matches for address selector: {spec}'.format(spec=spec))
return matches
def targets(self, spec):
"""Resolves a target spec to one or more Target objects.
spec: Either BUILD target address or else a target glob using the siblings ':' or
descendants '::' suffixes.
Returns the set of all Targets found.
"""
spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper)
addresses = list(spec_parser.parse_addresses(spec))
for address in addresses:
self.build_graph.inject_address_closure(address)
targets = [self.build_graph.get_target(address) for address in addresses]
return targets
class GoalRunnerFactory(object):
def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit):
"""
:param str root_dir: The root directory of the pants workspace (aka the "build root").
:param Options options: The global, pre-initialized Options instance.
:param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance.
:param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance.
:param Reporting reporting: The global, pre-initialized Reporting instance.
:param func exiter: A function that accepts an exit code value and exits (for tests, Optional).
"""
self._root_dir = root_dir
self._options = options
self._build_config = build_config
self._run_tracker = run_tracker
self._reporting = reporting
self._exiter = exiter
self._goals = []
self._targets = []
self._requested_goals = self._options.goals
self._target_specs = self._options.target_specs
self._help_request = self._options.help_request
self._global_options = options.for_global_scope()
self._tag = self._global_options.tag
self._fail_fast = self._global_options.fail_fast
self._spec_excludes = self._global_options.spec_excludes
self._explain = self._global_options.explain
self._kill_nailguns = self._global_options.kill_nailguns
self._project_tree = self._get_project_tree(self._global_options.build_file_rev)
self._build_file_parser = BuildFileParser(self._build_config, self._root_dir)
self._address_mapper = BuildFileAddressMapper(self._build_file_parser, self._project_tree)
self._build_graph = BuildGraph(self._address_mapper)
self._spec_parser = CmdLineSpecParser(
self._root_dir,
self._address_mapper,
spec_excludes=self._spec_excludes,
exclude_target_regexps=self._global_options.exclude_target_regexp,
)
def _get_project_tree(self, build_file_rev):
"""Creates the project tree for build files for use in a given pants run."""
if build_file_rev:
return ScmProjectTree(self._root_dir, get_scm(), build_file_rev)
else:
return FileSystemProjectTree(self._root_dir)
def _expand_goals(self, goals):
"""Check and populate the requested goals for a given run."""
for goal in goals:
try:
self._address_mapper.resolve_spec(goal)
logger.warning(
"Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal)
)
except AddressLookupError:
pass
if self._help_request:
help_printer = HelpPrinter(self._options)
result = help_printer.print_help()
self._exiter(result)
self._goals.extend([Goal.by_name(goal) for goal in goals])
def _expand_specs(self, specs, fail_fast):
"""Populate the BuildGraph and target list from a set of input specs."""
with self._run_tracker.new_workunit(name="parse", labels=[WorkUnitLabel.SETUP]):
def filter_for_tag(tag):
return lambda target: tag in map(str, target.tags)
tag_filter = wrap_filters(create_filters(self._tag, filter_for_tag))
for spec in specs:
for address in self._spec_parser.parse_addresses(spec, fail_fast):
self._build_graph.inject_address_closure(address)
target = self._build_graph.get_target(address)
if tag_filter(target):
self._targets.append(target)
def _maybe_launch_pantsd(self):
"""Launches pantsd if configured to do so."""
if self._global_options.enable_pantsd:
# Avoid runtracker output if pantsd is disabled. Otherwise, show up to inform the user its on.
with self._run_tracker.new_workunit(name="pantsd", labels=[WorkUnitLabel.SETUP]):
PantsDaemonLauncher.global_instance().maybe_launch()
def _is_quiet(self):
return any(goal.has_task_of_type(QuietTaskMixin) for goal in self._goals) or self._explain
def _setup_context(self):
self._maybe_launch_pantsd()
with self._run_tracker.new_workunit(name="setup", labels=[WorkUnitLabel.SETUP]):
self._expand_goals(self._requested_goals)
self._expand_specs(self._target_specs, self._fail_fast)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self._run_tracker.run_info.add_scm_info()
# Update the Reporting settings now that we have options and goal info.
invalidation_report = self._reporting.update_reporting(
self._global_options, self._is_quiet(), self._run_tracker
)
context = Context(
options=self._options,
run_tracker=self._run_tracker,
target_roots=self._targets,
requested_goals=self._requested_goals,
build_graph=self._build_graph,
build_file_parser=self._build_file_parser,
address_mapper=self._address_mapper,
spec_excludes=self._spec_excludes,
invalidation_report=invalidation_report,
)
return context, invalidation_report
def setup(self):
context, invalidation_report = self._setup_context()
return GoalRunner(
context=context,
goals=self._goals,
kill_nailguns=self._kill_nailguns,
run_tracker=self._run_tracker,
invalidation_report=invalidation_report,
exiter=self._exiter,
)
class CmdLineSpecParserTest(BaseTest):
@property
def alias_groups(self):
return BuildFileAliases.create(targets={"generic": Target})
def setUp(self):
super(CmdLineSpecParserTest, self).setUp()
def add_target(path, name):
self.add_to_build_file(path, 'generic(name="{name}")\n'.format(name=name))
add_target("BUILD", "root")
add_target("a", "a")
add_target("a", "b")
add_target("a/b", "b")
add_target("a/b", "c")
self.spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper)
def test_normal(self):
self.assert_parsed(cmdline_spec=":root", expected=[":root"])
self.assert_parsed(cmdline_spec="//:root", expected=[":root"])
self.assert_parsed(cmdline_spec="a", expected=["a"])
self.assert_parsed(cmdline_spec="a:a", expected=["a"])
self.assert_parsed(cmdline_spec="a/b", expected=["a/b"])
self.assert_parsed(cmdline_spec="a/b:b", expected=["a/b"])
self.assert_parsed(cmdline_spec="a/b:c", expected=["a/b:c"])
def test_sibling(self):
self.assert_parsed(cmdline_spec=":", expected=[":root"])
self.assert_parsed(cmdline_spec="//:", expected=[":root"])
self.assert_parsed(cmdline_spec="a:", expected=["a", "a:b"])
self.assert_parsed(cmdline_spec="//a:", expected=["a", "a:b"])
self.assert_parsed(cmdline_spec="a/b:", expected=["a/b", "a/b:c"])
self.assert_parsed(cmdline_spec="//a/b:", expected=["a/b", "a/b:c"])
def test_sibling_or_descendents(self):
self.assert_parsed(cmdline_spec="::", expected=[":root", "a", "a:b", "a/b", "a/b:c"])
self.assert_parsed(cmdline_spec="//::", expected=[":root", "a", "a:b", "a/b", "a/b:c"])
self.assert_parsed(cmdline_spec="a::", expected=["a", "a:b", "a/b", "a/b:c"])
self.assert_parsed(cmdline_spec="//a::", expected=["a", "a:b", "a/b", "a/b:c"])
self.assert_parsed(cmdline_spec="a/b::", expected=["a/b", "a/b:c"])
self.assert_parsed(cmdline_spec="//a/b::", expected=["a/b", "a/b:c"])
def test_absolute(self):
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, "a"), expected=["a"])
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, "a:a"), expected=["a"])
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, "a:"), expected=["a", "a:b"])
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, "a::"), expected=["a", "a:b", "a/b", "a/b:c"])
double_absolute = "/" + os.path.join(self.build_root, "a")
self.assertEquals("//", double_absolute[:2], "A sanity check we have a leading-// absolute spec")
with self.assertRaises(self.spec_parser.BadSpecError):
self.spec_parser.parse_addresses(double_absolute).next()
with self.assertRaises(self.spec_parser.BadSpecError):
self.spec_parser.parse_addresses("/not/the/buildroot/a").next()
def test_cmd_line_affordances(self):
self.assert_parsed(cmdline_spec="./:root", expected=[":root"])
self.assert_parsed(cmdline_spec="//./:root", expected=[":root"])
self.assert_parsed(cmdline_spec="//./a/../:root", expected=[":root"])
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, "./a/../:root"), expected=[":root"])
self.assert_parsed(cmdline_spec="a/", expected=["a"])
self.assert_parsed(cmdline_spec="./a/", expected=["a"])
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, "./a/"), expected=["a"])
self.assert_parsed(cmdline_spec="a/b/:b", expected=["a/b"])
self.assert_parsed(cmdline_spec="./a/b/:b", expected=["a/b"])
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, "./a/b/:b"), expected=["a/b"])
def test_cmd_line_spec_list(self):
self.assert_parsed_list(cmdline_spec_list=["a", "a/b"], expected=["a", "a/b"])
self.assert_parsed_list(cmdline_spec_list=["a", "a/b", "^a/b"], expected=["a"])
self.assert_parsed_list(cmdline_spec_list=["^a/b", "a", "a/b"], expected=["a"])
self.assert_parsed_list(cmdline_spec_list=["::"], expected=[":root", "a", "a:b", "a/b", "a/b:c"])
self.assert_parsed_list(cmdline_spec_list=["::", "^a/b::"], expected=[":root", "a", "a:b"])
self.assert_parsed_list(cmdline_spec_list=["^a/b::", "::"], expected=[":root", "a", "a:b"])
def test_does_not_exist(self):
with self.assertRaises(self.spec_parser.BadSpecError):
self.spec_parser.parse_addresses("c").next()
with self.assertRaises(self.spec_parser.BadSpecError):
self.spec_parser.parse_addresses("c:").next()
with self.assertRaises(self.spec_parser.BadSpecError):
self.spec_parser.parse_addresses("c::").next()
def assert_parsed(self, cmdline_spec, expected):
def sort(addresses):
return sorted(addresses, key=lambda address: address.spec)
self.assertEqual(
sort(SyntheticAddress.parse(addr) for addr in expected),
sort(self.spec_parser.parse_addresses(cmdline_spec)),
)
def assert_parsed_list(self, cmdline_spec_list, expected):
def sort(addresses):
return sorted(addresses, key=lambda address: address.spec)
self.assertEqual(
sort(SyntheticAddress.parse(addr) for addr in expected),
sort(self.spec_parser.parse_addresses(cmdline_spec_list)),
)
class GoalRunnerFactory(object):
def __init__(self,
root_dir,
options,
build_config,
run_tracker,
reporting,
exiter=sys.exit):
"""
:param str root_dir: The root directory of the pants workspace (aka the "build root").
:param Options options: The global, pre-initialized Options instance.
:param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance.
:param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance.
:param Reporting reporting: The global, pre-initialized Reporting instance.
:param func exiter: A function that accepts an exit code value and exits (for tests, Optional).
"""
self._root_dir = root_dir
self._options = options
self._build_config = build_config
self._run_tracker = run_tracker
self._reporting = reporting
self._exiter = exiter
self._goals = []
self._targets = []
self._requested_goals = self._options.goals
self._target_specs = self._options.target_specs
self._help_request = self._options.help_request
self._global_options = options.for_global_scope()
self._tag = self._global_options.tag
self._fail_fast = self._global_options.fail_fast
self._spec_excludes = self._global_options.spec_excludes
self._explain = self._global_options.explain
self._kill_nailguns = self._global_options.kill_nailguns
self._project_tree = self._get_project_tree(
self._global_options.build_file_rev)
self._build_file_parser = BuildFileParser(self._build_config,
self._root_dir)
self._address_mapper = BuildFileAddressMapper(self._build_file_parser,
self._project_tree)
self._build_graph = BuildGraph(self._address_mapper)
self._spec_parser = CmdLineSpecParser(
self._root_dir,
self._address_mapper,
spec_excludes=self._spec_excludes,
exclude_target_regexps=self._global_options.exclude_target_regexp)
def _get_project_tree(self, build_file_rev):
"""Creates the project tree for build files for use in a given pants run."""
if build_file_rev:
return ScmProjectTree(self._root_dir, get_scm(), build_file_rev)
else:
return FileSystemProjectTree(self._root_dir)
def _expand_goals(self, goals):
"""Check and populate the requested goals for a given run."""
for goal in goals:
try:
self._address_mapper.resolve_spec(goal)
logger.warning(
"Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}."
.format(goal))
except AddressLookupError:
pass
if self._help_request:
help_printer = HelpPrinter(self._options)
result = help_printer.print_help()
self._exiter(result)
self._goals.extend([Goal.by_name(goal) for goal in goals])
def _expand_specs(self, specs, fail_fast):
"""Populate the BuildGraph and target list from a set of input specs."""
with self._run_tracker.new_workunit(name='parse',
labels=[WorkUnitLabel.SETUP]):
def filter_for_tag(tag):
return lambda target: tag in map(str, target.tags)
tag_filter = wrap_filters(create_filters(self._tag,
filter_for_tag))
for spec in specs:
for address in self._spec_parser.parse_addresses(
spec, fail_fast):
self._build_graph.inject_address_closure(address)
target = self._build_graph.get_target(address)
if tag_filter(target):
self._targets.append(target)
def _maybe_launch_pantsd(self):
"""Launches pantsd if configured to do so."""
if self._global_options.enable_pantsd:
# Avoid runtracker output if pantsd is disabled. Otherwise, show up to inform the user its on.
with self._run_tracker.new_workunit(name='pantsd',
labels=[WorkUnitLabel.SETUP]):
PantsDaemonLauncher.global_instance().maybe_launch()
def _is_quiet(self):
return any(
goal.has_task_of_type(QuietTaskMixin)
for goal in self._goals) or self._explain
def _setup_context(self):
self._maybe_launch_pantsd()
with self._run_tracker.new_workunit(name='setup',
labels=[WorkUnitLabel.SETUP]):
self._expand_goals(self._requested_goals)
self._expand_specs(self._target_specs, self._fail_fast)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self._run_tracker.run_info.add_scm_info()
# Update the Reporting settings now that we have options and goal info.
invalidation_report = self._reporting.update_reporting(
self._global_options, self._is_quiet(), self._run_tracker)
context = Context(options=self._options,
run_tracker=self._run_tracker,
target_roots=self._targets,
requested_goals=self._requested_goals,
build_graph=self._build_graph,
build_file_parser=self._build_file_parser,
address_mapper=self._address_mapper,
spec_excludes=self._spec_excludes,
invalidation_report=invalidation_report)
return context, invalidation_report
def setup(self):
context, invalidation_report = self._setup_context()
return GoalRunner(context=context,
goals=self._goals,
kill_nailguns=self._kill_nailguns,
run_tracker=self._run_tracker,
invalidation_report=invalidation_report,
exiter=self._exiter)
class GoalRunnerFactory(object):
def __init__(self,
root_dir,
options,
build_config,
run_tracker,
reporting,
exiter=sys.exit):
"""
:param str root_dir: The root directory of the pants workspace (aka the "build root").
:param Options options: The global, pre-initialized Options instance.
:param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance.
:param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance.
:param Reporting reporting: The global, pre-initialized Reporting instance.
:param func exiter: A function that accepts an exit code value and exits (for tests, Optional).
"""
self._root_dir = root_dir
self._options = options
self._build_config = build_config
self._run_tracker = run_tracker
self._reporting = reporting
self._exiter = exiter
self._goals = []
self._targets = []
self._requested_goals = self._options.goals
self._target_specs = self._options.target_specs
self._help_request = self._options.help_request
self._global_options = options.for_global_scope()
self._tag = self._global_options.tag
self._fail_fast = self._global_options.fail_fast
self._spec_excludes = self._global_options.spec_excludes
self._explain = self._global_options.explain
self._kill_nailguns = self._global_options.kill_nailguns
self._build_file_type = self._get_buildfile_type(
self._global_options.build_file_rev)
self._build_file_parser = BuildFileParser(self._build_config,
self._root_dir)
self._address_mapper = BuildFileAddressMapper(self._build_file_parser,
self._build_file_type)
self._build_graph = BuildGraph(self._address_mapper)
self._spec_parser = CmdLineSpecParser(
self._root_dir,
self._address_mapper,
spec_excludes=self._spec_excludes,
exclude_target_regexps=self._global_options.exclude_target_regexp)
def _get_buildfile_type(self, build_file_rev):
"""Selects the BuildFile type for use in a given pants run."""
if build_file_rev:
ScmBuildFile.set_rev(build_file_rev)
ScmBuildFile.set_scm(get_scm())
return ScmBuildFile
else:
return FilesystemBuildFile
def _expand_goals(self, goals):
"""Check and populate the requested goals for a given run."""
for goal in goals:
if self._address_mapper.from_cache(self._root_dir,
goal,
must_exist=False).file_exists():
logger.warning(
"Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}."
.format(goal))
if self._help_request:
help_printer = HelpPrinter(self._options)
help_printer.print_help()
self._exiter(0)
self._goals.extend([Goal.by_name(goal) for goal in goals])
def _expand_specs(self, specs, fail_fast):
"""Populate the BuildGraph and target list from a set of input specs."""
with self._run_tracker.new_workunit(name='parse',
labels=[WorkUnitLabel.SETUP]):
def filter_for_tag(tag):
return lambda target: tag in map(str, target.tags)
tag_filter = wrap_filters(create_filters(self._tag,
filter_for_tag))
for spec in specs:
for address in self._spec_parser.parse_addresses(
spec, fail_fast):
self._build_graph.inject_address_closure(address)
target = self._build_graph.get_target(address)
if tag_filter(target):
self._targets.append(target)
def _is_quiet(self):
return any(
goal.has_task_of_type(QuietTaskMixin)
for goal in self._goals) or self._explain
def _setup_context(self):
# TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
with self._run_tracker.new_workunit(name='bootstrap',
labels=[WorkUnitLabel.SETUP]):
source_root_bootstrapper = SourceRootBootstrapper.global_instance()
source_root_bootstrapper.bootstrap(self._address_mapper,
self._build_file_parser)
with self._run_tracker.new_workunit(name='setup',
labels=[WorkUnitLabel.SETUP]):
self._expand_goals(self._requested_goals)
self._expand_specs(self._target_specs, self._fail_fast)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self._run_tracker.run_info.add_scm_info()
# Update the Reporting settings now that we have options and goal info.
invalidation_report = self._reporting.update_reporting(
self._global_options, self._is_quiet(), self._run_tracker)
context = Context(options=self._options,
run_tracker=self._run_tracker,
target_roots=self._targets,
requested_goals=self._requested_goals,
build_graph=self._build_graph,
build_file_parser=self._build_file_parser,
address_mapper=self._address_mapper,
spec_excludes=self._spec_excludes,
invalidation_report=invalidation_report)
return context, invalidation_report
def setup(self):
context, invalidation_report = self._setup_context()
return GoalRunner(context=context,
goals=self._goals,
kill_nailguns=self._kill_nailguns,
run_tracker=self._run_tracker,
invalidation_report=invalidation_report)
def setup_parser(self, parser, args):
self.config = Config.load()
add_global_options(parser)
# We support attempting zero or more goals. Multiple goals must be delimited from further
# options and non goal args with a '--'. The key permutations we need to support:
# ./pants goal => goals
# ./pants goal goals => goals
# ./pants goal compile src/java/... => compile
# ./pants goal compile -x src/java/... => compile
# ./pants goal compile src/java/... -x => compile
# ./pants goal compile run -- src/java/... => compile, run
# ./pants goal compile run -- src/java/... -x => compile, run
# ./pants goal compile run -- -x src/java/... => compile, run
if not args:
args.append('help')
help_flags = set(['-h', '--help', 'help'])
show_help = len(help_flags.intersection(args)) > 0
non_help_args = filter(lambda f: f not in help_flags, args)
goals, specs = GoalRunner.parse_args(non_help_args)
if show_help:
print_help(goals)
sys.exit(0)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup',
labels=[WorkUnit.SETUP]):
# Bootstrap user goals by loading any BUILD files implied by targets.
spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper)
with self.run_tracker.new_workunit(name='parse',
labels=[WorkUnit.SETUP]):
for spec in specs:
for address in spec_parser.parse_addresses(spec):
self.build_graph.inject_address_closure(address)
self.targets.append(
self.build_graph.get_target(address))
self.goals = [Goal.by_name(goal) for goal in goals]
rcfiles = self.config.getdefault(
'rcfiles', type=list, default=['/etc/pantsrc', '~/.pants.rc'])
if rcfiles:
rcfile = RcFile(rcfiles,
default_prepend=False,
process_default=True)
# Break down the goals specified on the command line to the full set that will be run so we
# can apply default flags to inner goal nodes. Also break down goals by Task subclass and
# register the task class hierarchy fully qualified names so we can apply defaults to
# baseclasses.
sections = OrderedSet()
for goal in Engine.execution_order(self.goals):
for task_name in goal.ordered_task_names():
sections.add(task_name)
task_type = goal.task_type_by_name(task_name)
for clazz in task_type.mro():
if clazz == Task:
break
sections.add('%s.%s' %
(clazz.__module__, clazz.__name__))
augmented_args = rcfile.apply_defaults(sections, args)
if augmented_args != args:
# TODO(John Sirois): Cleanup this currently important mutation of the passed in args
# once the 2-layer of command -> goal is squashed into one.
del args[:]
args.extend(augmented_args)
sys.stderr.write("(using pantsrc expansion: pants goal %s)\n" %
' '.join(augmented_args))
Goal.setup_parser(parser, args, self.goals)
class CmdLineSpecParserTest(BaseTest):
@property
def alias_groups(self):
return BuildFileAliases(
targets={
'generic': Target
}
)
def setUp(self):
super(CmdLineSpecParserTest, self).setUp()
def add_target(path, name):
self.add_to_build_file(path, 'generic(name="{name}")\n'.format(name=name))
add_target('BUILD', 'root')
add_target('a', 'a')
add_target('a', 'b')
add_target('a/b', 'b')
add_target('a/b', 'c')
self.spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper)
def test_normal(self):
self.assert_parsed(cmdline_spec=':root', expected=[':root'])
self.assert_parsed(cmdline_spec='//:root', expected=[':root'])
self.assert_parsed(cmdline_spec='a', expected=['a'])
self.assert_parsed(cmdline_spec='a:a', expected=['a'])
self.assert_parsed(cmdline_spec='a/b', expected=['a/b'])
self.assert_parsed(cmdline_spec='a/b:b', expected=['a/b'])
self.assert_parsed(cmdline_spec='a/b:c', expected=['a/b:c'])
def test_sibling(self):
self.assert_parsed(cmdline_spec=':', expected=[':root'])
self.assert_parsed(cmdline_spec='//:', expected=[':root'])
self.assert_parsed(cmdline_spec='a:', expected=['a', 'a:b'])
self.assert_parsed(cmdline_spec='//a:', expected=['a', 'a:b'])
self.assert_parsed(cmdline_spec='a/b:', expected=['a/b', 'a/b:c'])
self.assert_parsed(cmdline_spec='//a/b:', expected=['a/b', 'a/b:c'])
def test_sibling_or_descendents(self):
self.assert_parsed(cmdline_spec='::', expected=[':root', 'a', 'a:b', 'a/b', 'a/b:c'])
self.assert_parsed(cmdline_spec='//::', expected=[':root', 'a', 'a:b', 'a/b', 'a/b:c'])
self.assert_parsed(cmdline_spec='a::', expected=['a', 'a:b', 'a/b', 'a/b:c'])
self.assert_parsed(cmdline_spec='//a::', expected=['a', 'a:b', 'a/b', 'a/b:c'])
self.assert_parsed(cmdline_spec='a/b::', expected=['a/b', 'a/b:c'])
self.assert_parsed(cmdline_spec='//a/b::', expected=['a/b', 'a/b:c'])
def test_absolute(self):
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, 'a'), expected=['a'])
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, 'a:a'), expected=['a'])
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, 'a:'), expected=['a', 'a:b'])
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, 'a::'),
expected=['a', 'a:b', 'a/b', 'a/b:c'])
double_absolute = '/' + os.path.join(self.build_root, 'a')
self.assertEquals('//', double_absolute[:2],
'A sanity check we have a leading-// absolute spec')
with self.assertRaises(self.spec_parser.BadSpecError):
self.spec_parser.parse_addresses(double_absolute).next()
with self.assertRaises(self.spec_parser.BadSpecError):
self.spec_parser.parse_addresses('/not/the/buildroot/a').next()
def test_cmd_line_affordances(self):
self.assert_parsed(cmdline_spec='./:root', expected=[':root'])
self.assert_parsed(cmdline_spec='//./:root', expected=[':root'])
self.assert_parsed(cmdline_spec='//./a/../:root', expected=[':root'])
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, './a/../:root'),
expected=[':root'])
self.assert_parsed(cmdline_spec='a/', expected=['a'])
self.assert_parsed(cmdline_spec='./a/', expected=['a'])
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, './a/'), expected=['a'])
self.assert_parsed(cmdline_spec='a/b/:b', expected=['a/b'])
self.assert_parsed(cmdline_spec='./a/b/:b', expected=['a/b'])
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, './a/b/:b'), expected=['a/b'])
def test_cmd_line_spec_list(self):
self.assert_parsed_list(cmdline_spec_list=['a', 'a/b'], expected=['a', 'a/b'])
self.assert_parsed_list(cmdline_spec_list=['::'], expected=[':root', 'a', 'a:b', 'a/b', 'a/b:c'])
def test_does_not_exist(self):
with self.assertRaises(self.spec_parser.BadSpecError):
self.spec_parser.parse_addresses('c').next()
with self.assertRaises(self.spec_parser.BadSpecError):
self.spec_parser.parse_addresses('c:').next()
with self.assertRaises(self.spec_parser.BadSpecError):
self.spec_parser.parse_addresses('c::').next()
def assert_parsed(self, cmdline_spec, expected):
def sort(addresses):
return sorted(addresses, key=lambda address: address.spec)
self.assertEqual(sort(Address.parse(addr) for addr in expected),
sort(self.spec_parser.parse_addresses(cmdline_spec)))
def assert_parsed_list(self, cmdline_spec_list, expected):
def sort(addresses):
return sorted(addresses, key=lambda address: address.spec)
self.assertEqual(sort(Address.parse(addr) for addr in expected),
sort(self.spec_parser.parse_addresses(cmdline_spec_list)))
def test_spec_excludes(self):
expected_specs = [':root', 'a', 'a:b', 'a/b', 'a/b:c']
# This bogus BUILD file gets in the way of parsing.
self.add_to_build_file('some/dir', 'COMPLETELY BOGUS BUILDFILE)\n')
with self.assertRaises(CmdLineSpecParser.BadSpecError):
self.assert_parsed_list(cmdline_spec_list=['::'], expected=expected_specs)
# Test absolute path in spec_excludes.
self.spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper,
spec_excludes=[os.path.join(self.build_root, 'some')])
self.assert_parsed_list(cmdline_spec_list=['::'], expected=expected_specs)
# Test relative path in spec_excludes.
self.spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper,
spec_excludes=['some'])
self.assert_parsed_list(cmdline_spec_list=['::'], expected=expected_specs)
def test_exclude_target_regexps(self):
expected_specs = [':root', 'a', 'a:b', 'a/b', 'a/b:c']
# This bogus BUILD file gets in the way of parsing.
self.add_to_build_file('some/dir', 'COMPLETELY BOGUS BUILDFILE)\n')
with self.assertRaises(CmdLineSpecParser.BadSpecError):
self.assert_parsed_list(cmdline_spec_list=['::'], expected=expected_specs)
self.spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper,
exclude_target_regexps=[r'.*some/dir.*'])
self.assert_parsed_list(cmdline_spec_list=['::'], expected=expected_specs)
def setup_parser(self, parser, args):
if not args:
args.append('help')
logger = logging.getLogger(__name__)
goals = self.new_options.goals
specs = self.new_options.target_specs
fail_fast = self.new_options.for_global_scope().fail_fast
for goal in goals:
if BuildFile.from_cache(get_buildroot(), goal,
must_exist=False).exists():
logger.warning(
" Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}."
.format(goal))
if self.new_options.is_help:
self.new_options.print_help(goals=goals)
sys.exit(0)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup',
labels=[WorkUnit.SETUP]):
spec_parser = CmdLineSpecParser(
self.root_dir,
self.address_mapper,
spec_excludes=self.get_spec_excludes())
with self.run_tracker.new_workunit(name='parse',
labels=[WorkUnit.SETUP]):
for spec in specs:
for address in spec_parser.parse_addresses(
spec, fail_fast):
self.build_graph.inject_address_closure(address)
self.targets.append(
self.build_graph.get_target(address))
self.goals = [Goal.by_name(goal) for goal in goals]
rcfiles = self.config.getdefault(
'rcfiles', type=list, default=['/etc/pantsrc', '~/.pants.rc'])
if rcfiles:
rcfile = RcFile(rcfiles,
default_prepend=False,
process_default=True)
# Break down the goals specified on the command line to the full set that will be run so we
# can apply default flags to inner goal nodes. Also break down goals by Task subclass and
# register the task class hierarchy fully qualified names so we can apply defaults to
# baseclasses.
sections = OrderedSet()
for goal in Engine.execution_order(self.goals):
for task_name in goal.ordered_task_names():
sections.add(task_name)
task_type = goal.task_type_by_name(task_name)
for clazz in task_type.mro():
if clazz == Task:
break
sections.add('%s.%s' %
(clazz.__module__, clazz.__name__))
augmented_args = rcfile.apply_defaults(sections, args)
if augmented_args != args:
# TODO(John Sirois): Cleanup this currently important mutation of the passed in args
# once the 2-layer of command -> goal is squashed into one.
args[:] = augmented_args
sys.stderr.write("(using pantsrc expansion: pants goal %s)\n" %
' '.join(augmented_args))
def setup_parser(self, parser, args):
self.config = Config.load()
add_global_options(parser)
# We support attempting zero or more goals. Multiple goals must be delimited from further
# options and non goal args with a '--'. The key permutations we need to support:
# ./pants goal => goals
# ./pants goal goals => goals
# ./pants goal compile src/java/... => compile
# ./pants goal compile -x src/java/... => compile
# ./pants goal compile src/java/... -x => compile
# ./pants goal compile run -- src/java/... => compile, run
# ./pants goal compile run -- src/java/... -x => compile, run
# ./pants goal compile run -- -x src/java/... => compile, run
if not args:
args.append("help")
help_flags = set(["-h", "--help", "help"])
show_help = len(help_flags.intersection(args)) > 0
non_help_args = filter(lambda f: f not in help_flags, args)
goals, specs = GoalRunner.parse_args(non_help_args)
if show_help:
print_help(goals)
sys.exit(0)
self.requested_goals = goals
with self.run_tracker.new_workunit(name="setup", labels=[WorkUnit.SETUP]):
# Bootstrap user goals by loading any BUILD files implied by targets.
spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper)
with self.run_tracker.new_workunit(name="parse", labels=[WorkUnit.SETUP]):
for spec in specs:
for address in spec_parser.parse_addresses(spec):
self.build_graph.inject_address_closure(address)
self.targets.append(self.build_graph.get_target(address))
self.goals = [Goal.by_name(goal) for goal in goals]
rcfiles = self.config.getdefault("rcfiles", type=list, default=["/etc/pantsrc", "~/.pants.rc"])
if rcfiles:
rcfile = RcFile(rcfiles, default_prepend=False, process_default=True)
# Break down the goals specified on the command line to the full set that will be run so we
# can apply default flags to inner goal nodes. Also break down goals by Task subclass and
# register the task class hierarchy fully qualified names so we can apply defaults to
# baseclasses.
sections = OrderedSet()
for goal in Engine.execution_order(self.goals):
for task_name in goal.ordered_task_names():
sections.add(task_name)
task_type = goal.task_type_by_name(task_name)
for clazz in task_type.mro():
if clazz == Task:
break
sections.add("%s.%s" % (clazz.__module__, clazz.__name__))
augmented_args = rcfile.apply_defaults(sections, args)
if augmented_args != args:
# TODO(John Sirois): Cleanup this currently important mutation of the passed in args
# once the 2-layer of command -> goal is squashed into one.
del args[:]
args.extend(augmented_args)
sys.stderr.write("(using pantsrc expansion: pants goal %s)\n" % " ".join(augmented_args))
Goal.setup_parser(parser, args, self.goals)
class GoalRunnerFactory(object):
def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit):
"""
:param str root_dir: The root directory of the pants workspace (aka the "build root").
:param Options options: The global, pre-initialized Options instance.
:param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance.
:param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance.
:param Reporting reporting: The global, pre-initialized Reporting instance.
:param func exiter: A function that accepts an exit code value and exits (for tests, Optional).
"""
self._root_dir = root_dir
self._options = options
self._build_config = build_config
self._run_tracker = run_tracker
self._reporting = reporting
self._exiter = exiter
self._goals = []
self._targets = []
self._requested_goals = self._options.goals
self._target_specs = self._options.target_specs
self._help_request = self._options.help_request
self._global_options = options.for_global_scope()
self._tag = self._global_options.tag
self._fail_fast = self._global_options.fail_fast
self._spec_excludes = self._global_options.spec_excludes
self._explain = self._global_options.explain
self._kill_nailguns = self._global_options.kill_nailguns
self._build_file_type = self._get_buildfile_type(self._global_options.build_file_rev)
self._build_file_parser = BuildFileParser(self._build_config, self._root_dir)
self._address_mapper = BuildFileAddressMapper(self._build_file_parser, self._build_file_type)
self._build_graph = BuildGraph(self._address_mapper)
self._spec_parser = CmdLineSpecParser(
self._root_dir,
self._address_mapper,
spec_excludes=self._spec_excludes,
exclude_target_regexps=self._global_options.exclude_target_regexp
)
def _get_buildfile_type(self, build_file_rev):
"""Selects the BuildFile type for use in a given pants run."""
if build_file_rev:
ScmBuildFile.set_rev(build_file_rev)
ScmBuildFile.set_scm(get_scm())
return ScmBuildFile
else:
return FilesystemBuildFile
def _expand_goals(self, goals):
"""Check and populate the requested goals for a given run."""
for goal in goals:
if self._address_mapper.from_cache(self._root_dir, goal, must_exist=False).file_exists():
logger.warning("Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal))
if self._help_request:
help_printer = HelpPrinter(self._options)
help_printer.print_help()
self._exiter(0)
self._goals.extend([Goal.by_name(goal) for goal in goals])
def _expand_specs(self, specs, fail_fast):
"""Populate the BuildGraph and target list from a set of input specs."""
with self._run_tracker.new_workunit(name='parse', labels=[WorkUnitLabel.SETUP]):
def filter_for_tag(tag):
return lambda target: tag in map(str, target.tags)
tag_filter = wrap_filters(create_filters(self._tag, filter_for_tag))
for spec in specs:
for address in self._spec_parser.parse_addresses(spec, fail_fast):
self._build_graph.inject_address_closure(address)
target = self._build_graph.get_target(address)
if tag_filter(target):
self._targets.append(target)
def _is_quiet(self):
return any(goal.has_task_of_type(QuietTaskMixin) for goal in self._goals) or self._explain
def _setup_context(self):
# TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
with self._run_tracker.new_workunit(name='bootstrap', labels=[WorkUnitLabel.SETUP]):
source_root_bootstrapper = SourceRootBootstrapper.global_instance()
source_root_bootstrapper.bootstrap(self._address_mapper, self._build_file_parser)
with self._run_tracker.new_workunit(name='setup', labels=[WorkUnitLabel.SETUP]):
self._expand_goals(self._requested_goals)
self._expand_specs(self._target_specs, self._fail_fast)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self._run_tracker.run_info.add_scm_info()
# Update the Reporting settings now that we have options and goal info.
invalidation_report = self._reporting.update_reporting(self._global_options,
self._is_quiet(),
self._run_tracker)
context = Context(options=self._options,
run_tracker=self._run_tracker,
target_roots=self._targets,
requested_goals=self._requested_goals,
build_graph=self._build_graph,
build_file_parser=self._build_file_parser,
address_mapper=self._address_mapper,
spec_excludes=self._spec_excludes,
invalidation_report=invalidation_report)
return context, invalidation_report
def setup(self):
context, invalidation_report = self._setup_context()
return GoalRunner(context=context,
goals=self._goals,
kill_nailguns=self._kill_nailguns,
run_tracker=self._run_tracker,
invalidation_report=invalidation_report)
class CmdLineSpecParserTest(BaseTest):
@property
def alias_groups(self):
return BuildFileAliases.create(targets={'generic': Target})
def setUp(self):
super(CmdLineSpecParserTest, self).setUp()
def add_target(path, name):
self.add_to_build_file(
path, 'generic(name="{name}")\n'.format(name=name))
add_target('BUILD', 'root')
add_target('a', 'a')
add_target('a', 'b')
add_target('a/b', 'b')
add_target('a/b', 'c')
self.spec_parser = CmdLineSpecParser(self.build_root,
self.address_mapper)
def test_normal(self):
self.assert_parsed(cmdline_spec=':root', expected=[':root'])
self.assert_parsed(cmdline_spec='//:root', expected=[':root'])
self.assert_parsed(cmdline_spec='a', expected=['a'])
self.assert_parsed(cmdline_spec='a:a', expected=['a'])
self.assert_parsed(cmdline_spec='a/b', expected=['a/b'])
self.assert_parsed(cmdline_spec='a/b:b', expected=['a/b'])
self.assert_parsed(cmdline_spec='a/b:c', expected=['a/b:c'])
def test_sibling(self):
self.assert_parsed(cmdline_spec=':', expected=[':root'])
self.assert_parsed(cmdline_spec='//:', expected=[':root'])
self.assert_parsed(cmdline_spec='a:', expected=['a', 'a:b'])
self.assert_parsed(cmdline_spec='//a:', expected=['a', 'a:b'])
self.assert_parsed(cmdline_spec='a/b:', expected=['a/b', 'a/b:c'])
self.assert_parsed(cmdline_spec='//a/b:', expected=['a/b', 'a/b:c'])
def test_sibling_or_descendents(self):
self.assert_parsed(cmdline_spec='::',
expected=[':root', 'a', 'a:b', 'a/b', 'a/b:c'])
self.assert_parsed(cmdline_spec='//::',
expected=[':root', 'a', 'a:b', 'a/b', 'a/b:c'])
self.assert_parsed(cmdline_spec='a::',
expected=['a', 'a:b', 'a/b', 'a/b:c'])
self.assert_parsed(cmdline_spec='//a::',
expected=['a', 'a:b', 'a/b', 'a/b:c'])
self.assert_parsed(cmdline_spec='a/b::', expected=['a/b', 'a/b:c'])
self.assert_parsed(cmdline_spec='//a/b::', expected=['a/b', 'a/b:c'])
def test_absolute(self):
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, 'a'),
expected=['a'])
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, 'a:a'),
expected=['a'])
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, 'a:'),
expected=['a', 'a:b'])
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, 'a::'),
expected=['a', 'a:b', 'a/b', 'a/b:c'])
double_absolute = '/' + os.path.join(self.build_root, 'a')
self.assertEquals('//', double_absolute[:2],
'A sanity check we have a leading-// absolute spec')
with self.assertRaises(self.spec_parser.BadSpecError):
self.spec_parser.parse_addresses(double_absolute).next()
with self.assertRaises(self.spec_parser.BadSpecError):
self.spec_parser.parse_addresses('/not/the/buildroot/a').next()
def test_cmd_line_affordances(self):
self.assert_parsed(cmdline_spec='./:root', expected=[':root'])
self.assert_parsed(cmdline_spec='//./:root', expected=[':root'])
self.assert_parsed(cmdline_spec='//./a/../:root', expected=[':root'])
self.assert_parsed(cmdline_spec=os.path.join(self.build_root,
'./a/../:root'),
expected=[':root'])
self.assert_parsed(cmdline_spec='a/', expected=['a'])
self.assert_parsed(cmdline_spec='./a/', expected=['a'])
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, './a/'),
expected=['a'])
self.assert_parsed(cmdline_spec='a/b/:b', expected=['a/b'])
self.assert_parsed(cmdline_spec='./a/b/:b', expected=['a/b'])
self.assert_parsed(cmdline_spec=os.path.join(self.build_root,
'./a/b/:b'),
expected=['a/b'])
def test_cmd_line_spec_list(self):
self.assert_parsed_list(cmdline_spec_list=['a', 'a/b'],
expected=['a', 'a/b'])
self.assert_parsed_list(cmdline_spec_list=['a', 'a/b', '^a/b'],
expected=['a'])
self.assert_parsed_list(cmdline_spec_list=['^a/b', 'a', 'a/b'],
expected=['a'])
self.assert_parsed_list(cmdline_spec_list=['::'],
expected=[':root', 'a', 'a:b', 'a/b', 'a/b:c'])
self.assert_parsed_list(cmdline_spec_list=['::', '^a/b::'],
expected=[':root', 'a', 'a:b'])
self.assert_parsed_list(cmdline_spec_list=['^a/b::', '::'],
expected=[':root', 'a', 'a:b'])
def test_does_not_exist(self):
with self.assertRaises(self.spec_parser.BadSpecError):
self.spec_parser.parse_addresses('c').next()
with self.assertRaises(self.spec_parser.BadSpecError):
self.spec_parser.parse_addresses('c:').next()
with self.assertRaises(self.spec_parser.BadSpecError):
self.spec_parser.parse_addresses('c::').next()
def assert_parsed(self, cmdline_spec, expected):
def sort(addresses):
return sorted(addresses, key=lambda address: address.spec)
self.assertEqual(
sort(SyntheticAddress.parse(addr) for addr in expected),
sort(self.spec_parser.parse_addresses(cmdline_spec)))
def assert_parsed_list(self, cmdline_spec_list, expected):
def sort(addresses):
return sorted(addresses, key=lambda address: address.spec)
self.assertEqual(
sort(SyntheticAddress.parse(addr) for addr in expected),
sort(self.spec_parser.parse_addresses(cmdline_spec_list)))
def test_pants_dot_d_excluded(self):
expected_specs = [':root', 'a', 'a:b', 'a/b', 'a/b:c']
# This bogus BUILD file gets in the way of parsing.
self.add_to_build_file('.pants.d/some/dir',
'COMPLETELY BOGUS BUILDFILE)\n')
with self.assertRaises(CmdLineSpecParser.BadSpecError):
self.assert_parsed_list(cmdline_spec_list=['::'],
expected=expected_specs)
self.spec_parser = CmdLineSpecParser(
self.build_root,
self.address_mapper,
spec_excludes=[os.path.join(self.build_root, '.pants.d')])
self.assert_parsed_list(cmdline_spec_list=['::'],
expected=expected_specs)
class CmdLineSpecParserTest(BaseTest):
@property
def alias_groups(self):
return BuildFileAliases.create(
targets={
'generic': Target
}
)
def setUp(self):
super(CmdLineSpecParserTest, self).setUp()
def add_target(path, name):
self.add_to_build_file(path, 'generic(name="{name}")\n'.format(name=name))
add_target('BUILD', 'root')
add_target('a', 'a')
add_target('a', 'b')
add_target('a/b', 'b')
add_target('a/b', 'c')
self.spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper)
def test_normal(self):
self.assert_parsed(cmdline_spec=':root', expected=[':root'])
self.assert_parsed(cmdline_spec='//:root', expected=[':root'])
self.assert_parsed(cmdline_spec='a', expected=['a'])
self.assert_parsed(cmdline_spec='a:a', expected=['a'])
self.assert_parsed(cmdline_spec='a/b', expected=['a/b'])
self.assert_parsed(cmdline_spec='a/b:b', expected=['a/b'])
self.assert_parsed(cmdline_spec='a/b:c', expected=['a/b:c'])
def test_sibling(self):
self.assert_parsed(cmdline_spec=':', expected=[':root'])
self.assert_parsed(cmdline_spec='//:', expected=[':root'])
self.assert_parsed(cmdline_spec='a:', expected=['a', 'a:b'])
self.assert_parsed(cmdline_spec='//a:', expected=['a', 'a:b'])
self.assert_parsed(cmdline_spec='a/b:', expected=['a/b', 'a/b:c'])
self.assert_parsed(cmdline_spec='//a/b:', expected=['a/b', 'a/b:c'])
def test_sibling_or_descendents(self):
self.assert_parsed(cmdline_spec='::', expected=[':root', 'a', 'a:b', 'a/b', 'a/b:c'])
self.assert_parsed(cmdline_spec='//::', expected=[':root', 'a', 'a:b', 'a/b', 'a/b:c'])
self.assert_parsed(cmdline_spec='a::', expected=['a', 'a:b', 'a/b', 'a/b:c'])
self.assert_parsed(cmdline_spec='//a::', expected=['a', 'a:b', 'a/b', 'a/b:c'])
self.assert_parsed(cmdline_spec='a/b::', expected=['a/b', 'a/b:c'])
self.assert_parsed(cmdline_spec='//a/b::', expected=['a/b', 'a/b:c'])
def test_absolute(self):
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, 'a'), expected=['a'])
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, 'a:a'), expected=['a'])
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, 'a:'), expected=['a', 'a:b'])
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, 'a::'),
expected=['a', 'a:b', 'a/b', 'a/b:c'])
double_absolute = '/' + os.path.join(self.build_root, 'a')
self.assertEquals('//', double_absolute[:2],
'A sanity check we have a leading-// absolute spec')
with self.assertRaises(self.spec_parser.BadSpecError):
self.spec_parser.parse_addresses(double_absolute).next()
with self.assertRaises(self.spec_parser.BadSpecError):
self.spec_parser.parse_addresses('/not/the/buildroot/a').next()
def test_cmd_line_affordances(self):
self.assert_parsed(cmdline_spec='./:root', expected=[':root'])
self.assert_parsed(cmdline_spec='//./:root', expected=[':root'])
self.assert_parsed(cmdline_spec='//./a/../:root', expected=[':root'])
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, './a/../:root'),
expected=[':root'])
self.assert_parsed(cmdline_spec='a/', expected=['a'])
self.assert_parsed(cmdline_spec='./a/', expected=['a'])
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, './a/'), expected=['a'])
self.assert_parsed(cmdline_spec='a/b/:b', expected=['a/b'])
self.assert_parsed(cmdline_spec='./a/b/:b', expected=['a/b'])
self.assert_parsed(cmdline_spec=os.path.join(self.build_root, './a/b/:b'), expected=['a/b'])
def test_cmd_line_spec_list(self):
self.assert_parsed_list(cmdline_spec_list=['a', 'a/b'], expected=['a', 'a/b'])
self.assert_parsed_list(cmdline_spec_list=['a', 'a/b', '^a/b'], expected=['a'])
self.assert_parsed_list(cmdline_spec_list=['^a/b', 'a', 'a/b'], expected=['a'])
self.assert_parsed_list(cmdline_spec_list=['::'], expected=[':root', 'a', 'a:b', 'a/b', 'a/b:c'])
self.assert_parsed_list(cmdline_spec_list=['::', '^a/b::'], expected=[':root', 'a', 'a:b'])
self.assert_parsed_list(cmdline_spec_list=['^a/b::', '::'], expected=[':root', 'a', 'a:b'])
def test_does_not_exist(self):
with self.assertRaises(self.spec_parser.BadSpecError):
self.spec_parser.parse_addresses('c').next()
with self.assertRaises(self.spec_parser.BadSpecError):
self.spec_parser.parse_addresses('c:').next()
with self.assertRaises(self.spec_parser.BadSpecError):
self.spec_parser.parse_addresses('c::').next()
def assert_parsed(self, cmdline_spec, expected):
def sort(addresses):
return sorted(addresses, key=lambda address: address.spec)
self.assertEqual(sort(SyntheticAddress.parse(addr) for addr in expected),
sort(self.spec_parser.parse_addresses(cmdline_spec)))
def assert_parsed_list(self, cmdline_spec_list, expected):
def sort(addresses):
return sorted(addresses, key=lambda address: address.spec)
self.assertEqual(sort(SyntheticAddress.parse(addr) for addr in expected),
sort(self.spec_parser.parse_addresses(cmdline_spec_list)))
