def _determine_goals(self, requested_goals):
"""Check and populate the requested goals for a given run."""
def is_quiet(goals):
return any(
goal.has_task_of_type(QuietTaskMixin)
for goal in goals) or self._explain
spec_parser = CmdLineSpecParser(self._root_dir)
for goal in requested_goals:
if self._address_mapper.is_valid_single_address(
spec_parser.parse_spec(goal)):
logger.warning(
"Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}."
.format(goal))
goals = [Goal.by_name(goal) for goal in requested_goals]
return goals, is_quiet(goals)
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs')
self.spec_parser = CmdLineSpecParser(build_root)
self.scheduler = setup_json_scheduler(build_root, self._native)
self.guava = Address.parse('3rdparty/jvm:guava')
self.thrift = Address.parse('src/thrift/codegen/simple')
self.java = Address.parse('src/java/codegen/simple')
self.java_simple = Address.parse('src/java/simple')
self.java_multi = Address.parse('src/java/multiple_classpath_entries')
self.no_variant_thrift = Address.parse('src/java/codegen/selector:conflict')
self.unconfigured_thrift = Address.parse('src/thrift/codegen/unconfigured')
self.resources = Address.parse('src/resources/simple')
self.consumes_resources = Address.parse('src/java/consumes_resources')
self.consumes_managed_thirdparty = Address.parse('src/java/managed_thirdparty')
self.managed_guava = Address.parse('3rdparty/jvm/managed:guava')
self.managed_hadoop = Address.parse('3rdparty/jvm/managed:hadoop-common')
self.managed_resolve_latest = Address.parse('3rdparty/jvm/managed:latest-hadoop')
self.inferred_deps = Address.parse('src/scala/inferred_deps')
def set_start_time(self, start_time: Optional[float]) -> None:
# Launch RunTracker as early as possible (before .run() is called).
self._run_tracker = RunTracker.global_instance()
# Propagates parent_build_id to pants runs that may be called from this pants run.
os.environ["PANTS_PARENT_BUILD_ID"] = self._run_tracker.run_id
self._reporting = Reporting.global_instance()
self._reporting.initialize(self._run_tracker, self.options, start_time=start_time)
spec_parser = CmdLineSpecParser(get_buildroot())
specs = [spec_parser.parse_spec(spec).to_spec_string() for spec in self.options.specs]
# Note: This will not include values from `--changed-*` flags.
self._run_tracker.run_info.add_info("specs_from_command_line", specs, stringify=False)
# Capture a repro of the 'before' state for this build, if needed.
self._repro = Reproducer.global_instance().create_repro()
if self._repro:
self._repro.capture(self._run_tracker.run_info.get_as_dict())
def parse_address_specs(
cls,
target_specs: Iterable[str],
build_root: Optional[str] = None,
exclude_patterns: Optional[Iterable[str]] = None,
tags: Optional[Iterable[str]] = None,
) -> AddressSpecs:
"""Parse string specs into unique `AddressSpec` objects."""
build_root = build_root or get_buildroot()
spec_parser = CmdLineSpecParser(build_root)
dependencies = tuple(
OrderedSet(
spec_parser.parse_address_spec(spec_str)
for spec_str in target_specs))
return AddressSpecs(
dependencies=dependencies,
exclude_patterns=exclude_patterns if exclude_patterns else tuple(),
tags=tags)
def _get_targets(spec):
try:
spec_parser = CmdLineSpecParser(get_buildroot(),
self.context.address_mapper)
addresses = spec_parser.parse_addresses(spec)
except AddressLookupError as e:
raise TaskError(
'Failed to parse address selector: {spec}\n {message}'.
format(spec=spec, message=e))
# filter specs may not have been parsed as part of the context: force parsing
matches = set()
for address in addresses:
self.context.build_graph.inject_address_closure(address)
matches.add(self.context.build_graph.get_target(address))
if not matches:
raise TaskError(
'No matches for address selector: {spec}'.format(
spec=spec))
return matches
def targets(self, spec):
"""Resolves a target spec to one or more Target objects.
:API: public
spec: Either BUILD target address or else a target glob using the siblings ':' or
descendants '::' suffixes.
Returns the set of all Targets found.
"""
spec = CmdLineSpecParser(self.build_root).parse_spec(spec)
addresses = list(self.address_mapper.scan_specs([spec]))
for address in addresses:
self.build_graph.inject_address_closure(address)
targets = [
self.build_graph.get_target(address) for address in addresses
]
return targets
def _expand_goals_and_specs(self):
goals = self.options.goals
specs = self.options.target_specs
fail_fast = self.options.for_global_scope().fail_fast
for goal in goals:
if self.address_mapper.from_cache(get_buildroot(),
goal,
must_exist=False).file_exists():
logger.warning(
" Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}."
.format(goal))
if self.options.print_help_if_requested():
self._exiter(0)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup',
labels=[WorkUnitLabel.SETUP]):
spec_parser = CmdLineSpecParser(
self.root_dir,
self.address_mapper,
spec_excludes=self.spec_excludes,
exclude_target_regexps=self.global_options.
exclude_target_regexp)
with self.run_tracker.new_workunit(name='parse',
labels=[WorkUnitLabel.SETUP]):
def filter_for_tag(tag):
return lambda target: tag in map(str, target.tags)
tag_filter = wrap_filters(
create_filters(self.global_options.tag, filter_for_tag))
for spec in specs:
for address in spec_parser.parse_addresses(
spec, fail_fast):
self.build_graph.inject_address_closure(address)
tgt = self.build_graph.get_target(address)
if tag_filter(tgt):
self.targets.append(tgt)
self.goals = [Goal.by_name(goal) for goal in goals]
def targets(self, address_spec):
"""Resolves a target spec to one or more Target objects.
:API: public
spec: Either BUILD target address or else a target glob using the siblings ':' or
descendants '::' suffixes.
Returns the set of all Targets found.
"""
address_spec = CmdLineSpecParser(
self.build_root).parse_spec(address_spec)
assert isinstance(address_spec, AddressSpec)
targets = []
for address in self.build_graph.inject_address_specs_closure(
[address_spec]):
targets.append(self.build_graph.get_target(address))
return targets
def _expand_specs(self, spec_strs, fail_fast):
"""Populate the BuildGraph and target list from a set of input specs."""
with self._run_tracker.new_workunit(name='parse', labels=[WorkUnitLabel.SETUP]):
def filter_for_tag(tag):
return lambda target: tag in map(str, target.tags)
tag_filter = wrap_filters(create_filters(self._tag, filter_for_tag))
# Parse all specs into unique Spec objects.
spec_parser = CmdLineSpecParser(self._root_dir)
specs = OrderedSet()
for spec_str in spec_strs:
specs.add(spec_parser.parse_spec(spec_str))
# Then scan them to generate unique Addresses.
for address in self._address_mapper.scan_specs(specs, fail_fast, self._spec_excludes):
self._build_graph.inject_address_closure(address)
target = self._build_graph.get_target(address)
if tag_filter(target):
self._targets.append(target)
def setUp(self):
super(CmdLineSpecParserBadBuildTest, self).setUp()
self.add_to_build_file('bad/a', 'a_is_bad')
self.add_to_build_file('bad/b', 'b_is_bad')
self.spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper)
self.NO_FAIL_FAST_RE = re.compile(r"""^--------------------
.*
Exception message: name 'a_is_bad' is not defined
while executing BUILD file FilesystemBuildFile\((/[^/]+)*/bad/a/BUILD\)
Loading addresses from 'bad/a' failed\.
.*
Exception message: name 'b_is_bad' is not defined
while executing BUILD file FilesystemBuildFile\((/[^/]+)*T/bad/b/BUILD\)
Loading addresses from 'bad/b' failed\.
Invalid BUILD files for \[::\]$""", re.DOTALL)
self.FAIL_FAST_RE = """^name 'a_is_bad' is not defined
def set_start_time(self, start_time):
# Launch RunTracker as early as possible (before .run() is called).
self._run_tracker = RunTracker.global_instance()
self._reporting = Reporting.global_instance()
self._run_start_time = start_time
self._reporting.initialize(self._run_tracker, self._options, start_time=self._run_start_time)
spec_parser = CmdLineSpecParser(get_buildroot())
target_specs = [
spec_parser.parse_spec(spec).to_spec_string()
for spec in self._options.positional_args
]
# Note: This will not include values from `--owner-of` or `--changed-*` flags.
self._run_tracker.run_info.add_info("specs_from_command_line", target_specs, stringify=False)
# Capture a repro of the 'before' state for this build, if needed.
self._repro = Reproducer.global_instance().create_repro()
if self._repro:
self._repro.capture(self._run_tracker.run_info.get_as_dict())
def parse_specs(cls,
raw_specs: Iterable[str],
*,
build_root: Optional[str] = None) -> Specs:
"""Parse raw string specs into a Specs object."""
build_root = build_root or get_buildroot()
spec_parser = CmdLineSpecParser(build_root)
address_specs: OrderedSet[AddressSpec] = OrderedSet()
filesystem_specs: OrderedSet[FilesystemSpec] = OrderedSet()
for spec_str in raw_specs:
parsed_spec = spec_parser.parse_spec(spec_str)
if isinstance(parsed_spec, AddressSpec):
address_specs.add(parsed_spec)
else:
filesystem_specs.add(parsed_spec)
return Specs(
AddressSpecs(address_specs, filter_by_global_options=True),
FilesystemSpecs(filesystem_specs),
)
def parse_specs(cls,
target_specs,
build_root=None,
exclude_patterns=None,
tags=None):
"""Parse string specs into unique `Spec` objects.
:param iterable target_specs: An iterable of string specs.
:param string build_root: The path to the build root.
:returns: A `Specs` object.
"""
build_root = build_root or get_buildroot()
spec_parser = CmdLineSpecParser(build_root)
dependencies = tuple(
OrderedSet(
spec_parser.parse_spec(spec_str) for spec_str in target_specs))
return Specs(
dependencies=dependencies,
exclude_patterns=exclude_patterns if exclude_patterns else tuple(),
tags=tags)
def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit):
"""
:param str root_dir: The root directory of the pants workspace (aka the "build root").
:param Options options: The global, pre-initialized Options instance.
:param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance.
:param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance.
:param Reporting reporting: The global, pre-initialized Reporting instance.
:param func exiter: A function that accepts an exit code value and exits (for tests, Optional).
"""
self._root_dir = root_dir
self._options = options
self._build_config = build_config
self._run_tracker = run_tracker
self._reporting = reporting
self._exiter = exiter
self._goals = []
self._targets = []
self._requested_goals = self._options.goals
self._target_specs = self._options.target_specs
self._help_request = self._options.help_request
self._global_options = options.for_global_scope()
self._tag = self._global_options.tag
self._fail_fast = self._global_options.fail_fast
self._spec_excludes = self._global_options.spec_excludes
self._explain = self._global_options.explain
self._kill_nailguns = self._global_options.kill_nailguns
self._build_file_type = self._get_buildfile_type(self._global_options.build_file_rev)
self._build_file_parser = BuildFileParser(self._build_config, self._root_dir)
self._address_mapper = BuildFileAddressMapper(self._build_file_parser, self._build_file_type)
self._build_graph = BuildGraph(self._address_mapper)
self._spec_parser = CmdLineSpecParser(
self._root_dir,
self._address_mapper,
spec_excludes=self._spec_excludes,
exclude_target_regexps=self._global_options.exclude_target_regexp
)
def _expand_goals_and_specs(self):
logger = logging.getLogger(__name__)
goals = self.options.goals
specs = self.options.target_specs
fail_fast = self.options.for_global_scope().fail_fast
for goal in goals:
if BuildFile.from_cache(get_buildroot(), goal,
must_exist=False).exists():
logger.warning(
" Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}."
.format(goal))
if self.options.is_help:
self.options.print_help(goals=goals)
sys.exit(0)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup',
labels=[WorkUnit.SETUP]):
spec_parser = CmdLineSpecParser(
self.root_dir,
self.address_mapper,
spec_excludes=self.get_spec_excludes(),
exclude_target_regexps=self.global_options.
exclude_target_regexp)
with self.run_tracker.new_workunit(name='parse',
labels=[WorkUnit.SETUP]):
for spec in specs:
for address in spec_parser.parse_addresses(
spec, fail_fast):
self.build_graph.inject_address_closure(address)
self.targets.append(
self.build_graph.get_target(address))
self.goals = [Goal.by_name(goal) for goal in goals]
def _run_and_check(self, address_specs, incremental_import=None):
"""
Invoke idea-plugin goal and check for target specs and project in the
generated project and workspace file.
:param address_specs: list of address specs
:return: n/a
"""
self.assertTrue(address_specs, "targets are empty")
spec_parser = CmdLineSpecParser(get_buildroot())
# project_path is always the directory of the first target,
# which is where intellij is going to zoom in under project view.
spec = spec_parser.parse_spec(address_specs[0])
assert isinstance(
spec, (SingleAddress, SiblingAddresses, DescendantAddresses))
project_path = spec.directory
with self.temporary_workdir() as workdir:
with temporary_file(root_dir=workdir, cleanup=True) as output_file:
args = [
'idea-plugin',
f'--output-file={output_file.name}',
'--no-open',
]
if incremental_import is not None:
args.append(f'--incremental-import={incremental_import}')
pants_run = self.run_pants_with_workdir(
args + address_specs, workdir)
self.assert_success(pants_run)
project_dir = self._get_project_dir(output_file.name)
self.assertTrue(os.path.exists(project_dir),
f"{project_dir} does not exist")
self._do_check(project_dir,
project_path,
address_specs,
incremental_import=incremental_import)
def create(cls, options=None, args=None, build_root=None, change_calculator=None):
"""
:param Options options: An `Options` instance to use, if available.
:param string args: Raw cli args to use for parsing if an `Options` instance isn't available.
:param string build_root: The build root.
:param ChangeCalculator change_calculator: A `ChangeCalculator` for calculating changes.
"""
if not options:
assert args is not None, 'must pass `args` if not passing `options`'
options, _ = OptionsInitializer(OptionsBootstrapper(args=args)).setup(init_logging=False)
# Determine the literal target roots.
cmd_line_spec_parser = CmdLineSpecParser(build_root or get_buildroot())
spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in options.target_specs]
# Determine `Changed` arguments directly from options to support pre-`Subsystem` initialization paths.
changed_options = options.for_scope('changed')
changed_request = ChangedRequest.from_options(changed_options)
logger.debug('args are: %s', args)
logger.debug('spec_roots are: %s', spec_roots)
logger.debug('changed_request is: %s', changed_request)
if change_calculator and changed_request.is_actionable():
if spec_roots:
# We've been provided spec roots (e.g. `./pants list ::`) AND a changed request. Error out.
raise InvalidSpecConstraint('cannot provide changed parameters and target specs!')
# We've been provided no spec roots (e.g. `./pants list`) AND a changed request. Compute
# alternate target roots.
changed_addresses = change_calculator.changed_target_addresses(changed_request)
logger.debug('changed addresses: %s', changed_addresses)
return ChangedTargetRoots(changed_addresses)
# If no spec roots are passed, assume `::` as the intended target.
return LiteralTargetRoots(spec_roots if spec_roots else [DescendantAddresses('')])
def main_addresses():
build_root, goals, args = pop_build_root_and_goals('[build root path] [goal]+ [address spec]*', sys.argv[1:])
cmd_line_spec_parser = CmdLineSpecParser(build_root)
spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in args]
visualize_build_request(build_root, goals, spec_roots)
def setup_parser(self, parser, args):
if not args:
args.append('help')
logger = logging.getLogger(__name__)
goals = self.new_options.goals
specs = self.new_options.target_specs
fail_fast = self.new_options.for_global_scope().fail_fast
for goal in goals:
if BuildFile.from_cache(get_buildroot(), goal,
must_exist=False).exists():
logger.warning(
" Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}."
.format(goal))
if self.new_options.is_help:
self.new_options.print_help(goals=goals)
sys.exit(0)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup',
labels=[WorkUnit.SETUP]):
spec_parser = CmdLineSpecParser(
self.root_dir,
self.address_mapper,
spec_excludes=self.get_spec_excludes())
with self.run_tracker.new_workunit(name='parse',
labels=[WorkUnit.SETUP]):
for spec in specs:
for address in spec_parser.parse_addresses(
spec, fail_fast):
self.build_graph.inject_address_closure(address)
self.targets.append(
self.build_graph.get_target(address))
self.goals = [Goal.by_name(goal) for goal in goals]
rcfiles = self.config.getdefault(
'rcfiles', type=list, default=['/etc/pantsrc', '~/.pants.rc'])
if rcfiles:
rcfile = RcFile(rcfiles,
default_prepend=False,
process_default=True)
# Break down the goals specified on the command line to the full set that will be run so we
# can apply default flags to inner goal nodes. Also break down goals by Task subclass and
# register the task class hierarchy fully qualified names so we can apply defaults to
# baseclasses.
sections = OrderedSet()
for goal in Engine.execution_order(self.goals):
for task_name in goal.ordered_task_names():
sections.add(task_name)
task_type = goal.task_type_by_name(task_name)
for clazz in task_type.mro():
if clazz == Task:
break
sections.add('%s.%s' %
(clazz.__module__, clazz.__name__))
augmented_args = rcfile.apply_defaults(sections, args)
if augmented_args != args:
# TODO(John Sirois): Cleanup this currently important mutation of the passed in args
# once the 2-layer of command -> goal is squashed into one.
args[:] = augmented_args
sys.stderr.write("(using pantsrc expansion: pants goal %s)\n" %
' '.join(augmented_args))
def setUp(self) -> None:
super().setUp()
self._spec_parser = CmdLineSpecParser(self.build_root)
def setUp(self):
super(CmdLineSpecParserTest, self).setUp()
self._spec_parser = CmdLineSpecParser(self.build_root)
def parse_commandline_to_spec_roots(options=None, args=None, build_root=None):
if not options:
options, _ = OptionsInitializer(OptionsBootstrapper(args=args)).setup(init_logging=False)
cmd_line_spec_parser = CmdLineSpecParser(build_root or get_buildroot())
spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in options.target_specs]
return spec_roots
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs')
self.scheduler = setup_json_scheduler(build_root)
self.spec_parser = CmdLineSpecParser(build_root)
self.java = Address.parse('src/java/codegen/simple')
def setup_parser(self, parser, args):
self.config = Config.load()
add_global_options(parser)
# We support attempting zero or more goals. Multiple goals must be delimited from further
# options and non goal args with a '--'. The key permutations we need to support:
# ./pants goal => goals
# ./pants goal goals => goals
# ./pants goal compile src/java/... => compile
# ./pants goal compile -x src/java/... => compile
# ./pants goal compile src/java/... -x => compile
# ./pants goal compile run -- src/java/... => compile, run
# ./pants goal compile run -- src/java/... -x => compile, run
# ./pants goal compile run -- -x src/java/... => compile, run
if not args:
args.append('help')
help_flags = set(['-h', '--help', 'help'])
show_help = len(help_flags.intersection(args)) > 0
non_help_args = filter(lambda f: f not in help_flags, args)
goals, specs = GoalRunner.parse_args(non_help_args)
if show_help:
print_help(goals)
sys.exit(0)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup',
labels=[WorkUnit.SETUP]):
# Bootstrap user goals by loading any BUILD files implied by targets.
spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper)
with self.run_tracker.new_workunit(name='parse',
labels=[WorkUnit.SETUP]):
for spec in specs:
for address in spec_parser.parse_addresses(spec):
self.build_graph.inject_address_closure(address)
self.targets.append(
self.build_graph.get_target(address))
self.goals = [Goal.by_name(goal) for goal in goals]
rcfiles = self.config.getdefault(
'rcfiles', type=list, default=['/etc/pantsrc', '~/.pants.rc'])
if rcfiles:
rcfile = RcFile(rcfiles,
default_prepend=False,
process_default=True)
# Break down the goals specified on the command line to the full set that will be run so we
# can apply default flags to inner goal nodes. Also break down goals by Task subclass and
# register the task class hierarchy fully qualified names so we can apply defaults to
# baseclasses.
sections = OrderedSet()
for goal in Engine.execution_order(self.goals):
for task_name in goal.ordered_task_names():
sections.add(task_name)
task_type = goal.task_type_by_name(task_name)
for clazz in task_type.mro():
if clazz == Task:
break
sections.add('%s.%s' %
(clazz.__module__, clazz.__name__))
augmented_args = rcfile.apply_defaults(sections, args)
if augmented_args != args:
# TODO(John Sirois): Cleanup this currently important mutation of the passed in args
# once the 2-layer of command -> goal is squashed into one.
del args[:]
args.extend(augmented_args)
sys.stderr.write("(using pantsrc expansion: pants goal %s)\n" %
' '.join(augmented_args))
Goal.setup_parser(parser, args, self.goals)
