def address_map_from_spec_path(self, spec_path):
build_file = BuildFile.from_cache(self._root_dir, spec_path)
family_address_map_by_build_file = self.parse_build_file_family(build_file)
address_map = {}
for build_file, sibling_address_map in family_address_map_by_build_file.items():
address_map.update(sibling_address_map)
return address_map
def setup(self):
options_bootstrapper = OptionsBootstrapper()
# Force config into the cache so we (and plugin/backend loading code) can use it.
# TODO: Plumb options in explicitly.
options_bootstrapper.get_bootstrap_options()
self.config = Config.from_cache()
# Add any extra paths to python path (eg for loading extra source backends)
extra_paths = self.config.getlist('backends', 'python-path', [])
if extra_paths:
sys.path.extend(extra_paths)
# Load plugins and backends.
backend_packages = self.config.getlist('backends', 'packages', [])
plugins = self.config.getlist('backends', 'plugins', [])
build_configuration = load_plugins_and_backends(plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scopes = ['']
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scopes=known_scopes)
self.register_options()
self.run_tracker = RunTracker.from_config(self.config)
report = initial_reporting(self.config, self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: %s' % url)
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]):
build_file = BuildFile.from_cache(root_dir=self.root_dir, relpath=path)
# TODO(pl): This is an unfortunate interface leak, but I don't think
# in the long run that we should be relying on "bootstrap" BUILD files
# that do nothing except modify global state. That type of behavior
# (e.g. source roots, goal registration) should instead happen in
# project plugins, or specialized configuration files.
self.build_file_parser.parse_build_file_family(build_file)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
self._expand_goals_and_specs()
def _expand_goals_and_specs(self):
goals = self.options.goals
specs = self.options.target_specs
fail_fast = self.options.for_global_scope().fail_fast
for goal in goals:
if BuildFile.from_cache(get_buildroot(), goal,
must_exist=False).exists():
logger.warning(
" Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}."
.format(goal))
if self.options.print_help_if_requested():
sys.exit(0)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup',
labels=[WorkUnit.SETUP]):
spec_parser = CmdLineSpecParser(
self.root_dir,
self.address_mapper,
spec_excludes=self.spec_excludes,
exclude_target_regexps=self.global_options.
exclude_target_regexp)
with self.run_tracker.new_workunit(name='parse',
labels=[WorkUnit.SETUP]):
for spec in specs:
for address in spec_parser.parse_addresses(
spec, fail_fast):
self.build_graph.inject_address_closure(address)
self.targets.append(
self.build_graph.get_target(address))
self.goals = [Goal.by_name(goal) for goal in goals]
def _expand_goals_and_specs(self):
goals = self.options.goals
specs = self.options.target_specs
fail_fast = self.options.for_global_scope().fail_fast
for goal in goals:
if BuildFile.from_cache(get_buildroot(), goal, must_exist=False).exists():
logger.warning(" Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal))
if self.options.print_help_if_requested():
sys.exit(0)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup', labels=[WorkUnit.SETUP]):
spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper,
spec_excludes=self.spec_excludes,
exclude_target_regexps=self.global_options.exclude_target_regexp)
with self.run_tracker.new_workunit(name='parse', labels=[WorkUnit.SETUP]):
for spec in specs:
for address in spec_parser.parse_addresses(spec, fail_fast):
self.build_graph.inject_address_closure(address)
self.targets.append(self.build_graph.get_target(address))
self.goals = [Goal.by_name(goal) for goal in goals]
def __init__(self,
run_tracker,
root_dir,
parser,
args,
build_file_parser,
address_mapper,
build_graph,
needs_old_options=True):
"""run_tracker: The (already opened) RunTracker to track this run with
root_dir: The root directory of the pants workspace
parser: an OptionParser
args: the subcommand arguments to parse"""
self.run_tracker = run_tracker
self.root_dir = root_dir
self.build_file_parser = build_file_parser
self.address_mapper = address_mapper
self.build_graph = build_graph
config = Config.from_cache()
with self.run_tracker.new_workunit(name='bootstrap',
labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in config.getlist('goals',
'bootstrap_buildfiles',
default=[]):
build_file = BuildFile.from_cache(root_dir=self.root_dir,
relpath=path)
# TODO(pl): This is an unfortunate interface leak, but I don't think
# in the long run that we should be relying on "bootstrap" BUILD files
# that do nothing except modify global state. That type of behavior
# (e.g. source roots, goal registration) should instead happen in
# project plugins, or specialized configuration files.
self.build_file_parser.parse_build_file_family(build_file)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
# Override the OptionParser's error with more useful output
def error(message=None, show_help=True):
if message:
print(message + '\n')
if show_help:
parser.print_help()
parser.exit(status=1)
parser.error = error
self.error = error
self.register_options()
self.setup_parser(parser, args)
if needs_old_options:
self.old_options, self.args = parser.parse_args(args)
else:
# Ensure a predictable error if anything under goal tries to use these.
self.old_options = None
self.args = None
def setup_parser(self, parser, args):
if not args:
args.append('help')
logger = logging.getLogger(__name__)
goals = self.new_options.goals
specs = self.new_options.target_specs
fail_fast = self.new_options.for_global_scope().fail_fast
for goal in goals:
if BuildFile.from_cache(get_buildroot(), goal, must_exist=False).exists():
logger.warning(" Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal))
if self.new_options.is_help:
self.new_options.print_help(goals=goals)
sys.exit(0)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup', labels=[WorkUnit.SETUP]):
spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper,
spec_excludes=self.get_spec_excludes())
with self.run_tracker.new_workunit(name='parse', labels=[WorkUnit.SETUP]):
for spec in specs:
for address in spec_parser.parse_addresses(spec, fail_fast):
self.build_graph.inject_address_closure(address)
self.targets.append(self.build_graph.get_target(address))
self.goals = [Goal.by_name(goal) for goal in goals]
rcfiles = self.config.getdefault('rcfiles', type=list,
default=['/etc/pantsrc', '~/.pants.rc'])
if rcfiles:
rcfile = RcFile(rcfiles, default_prepend=False, process_default=True)
# Break down the goals specified on the command line to the full set that will be run so we
# can apply default flags to inner goal nodes. Also break down goals by Task subclass and
# register the task class hierarchy fully qualified names so we can apply defaults to
# baseclasses.
sections = OrderedSet()
for goal in Engine.execution_order(self.goals):
for task_name in goal.ordered_task_names():
sections.add(task_name)
task_type = goal.task_type_by_name(task_name)
for clazz in task_type.mro():
if clazz == Task:
break
sections.add('%s.%s' % (clazz.__module__, clazz.__name__))
augmented_args = rcfile.apply_defaults(sections, args)
if augmented_args != args:
# TODO(John Sirois): Cleanup this currently important mutation of the passed in args
# once the 2-layer of command -> goal is squashed into one.
args[:] = augmented_args
sys.stderr.write("(using pantsrc expansion: pants goal %s)\n" % ' '.join(augmented_args))
def _candidate_owners(self, path):
build_file = BuildFile.from_cache(get_buildroot(),
relpath=os.path.dirname(path),
must_exist=False)
if build_file.exists():
yield build_file
for sibling in build_file.siblings():
yield sibling
for ancestor in build_file.ancestors():
yield ancestor
def _parse_spec(self, spec):
def normalize_spec_path(path):
is_abs = not path.startswith('//') and os.path.isabs(path)
if is_abs:
path = os.path.realpath(path)
if os.path.commonprefix([self._root_dir, path
]) != self._root_dir:
raise self.BadSpecError(
'Absolute address path {0} does not share build root {1}'
.format(path, self._root_dir))
else:
if path.startswith('//'):
path = path[2:]
path = os.path.join(self._root_dir, path)
normalized = os.path.relpath(path, self._root_dir)
if normalized == '.':
normalized = ''
return normalized
if spec.endswith('::'):
addresses = set()
spec_path = spec[:-len('::')]
spec_dir = normalize_spec_path(spec_path)
if not os.path.isdir(os.path.join(self._root_dir, spec_dir)):
raise self.BadSpecError(
'Can only recursive glob directories and {0} is not a valid dir'
.format(spec_dir))
try:
for build_file in BuildFile.scan_buildfiles(
self._root_dir, spec_dir):
addresses.update(
self._address_mapper.addresses_in_spec_path(
build_file.spec_path))
return addresses
except (BuildFile.BuildFileError, AddressLookupError) as e:
raise self.BadSpecError(e)
elif spec.endswith(':'):
spec_path = spec[:-len(':')]
spec_dir = normalize_spec_path(spec_path)
try:
return set(
self._address_mapper.addresses_in_spec_path(spec_dir))
except AddressLookupError as e:
raise self.BadSpecError(e)
else:
spec_parts = spec.rsplit(':', 1)
spec_parts[0] = normalize_spec_path(spec_parts[0])
spec_path, target_name = parse_spec(':'.join(spec_parts))
try:
build_file = BuildFile.from_cache(self._root_dir, spec_path)
return set([BuildFileAddress(build_file, target_name)])
except BuildFile.BuildFileError as e:
raise self.BadSpecError(e)
def is_spec(spec):
if os.sep in spec or ':' in spec:
return True # Definitely not a goal.
if not (spec in goal_names):
return True # Definitely not a (known) goal.
if has_double_dash:
# This means that we're parsing the half of the expression before a --, so assume it's a
# goal without warning.
return False
# Here, it's possible we have a goal and target with the same name. For now, always give
# priority to the goal, but give a warning if they might have meant the target (if the BUILD
# file exists).
try:
BuildFile.from_cache(get_buildroot(), spec)
msg = (' Command-line argument "{spec}" is ambiguous, and was assumed to be a goal.'
' If this is incorrect, disambiguate it with the "--" argument to separate goals'
' from targets.')
logger.warning(msg.format(spec=spec))
except IOError: pass # Awesome, it's unambiguous.
return False
def __init__(self,
run_tracker,
root_dir,
parser,
args,
build_file_parser,
address_mapper,
build_graph,
needs_old_options=True):
"""run_tracker: The (already opened) RunTracker to track this run with
root_dir: The root directory of the pants workspace
parser: an OptionParser
args: the subcommand arguments to parse"""
self.run_tracker = run_tracker
self.root_dir = root_dir
self.build_file_parser = build_file_parser
self.address_mapper = address_mapper
self.build_graph = build_graph
config = Config.from_cache()
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in config.getlist('goals', 'bootstrap_buildfiles', default=[]):
build_file = BuildFile.from_cache(root_dir=self.root_dir, relpath=path)
# TODO(pl): This is an unfortunate interface leak, but I don't think
# in the long run that we should be relying on "bootstrap" BUILD files
# that do nothing except modify global state. That type of behavior
# (e.g. source roots, goal registration) should instead happen in
# project plugins, or specialized configuration files.
self.build_file_parser.parse_build_file_family(build_file)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
# Override the OptionParser's error with more useful output
def error(message=None, show_help=True):
if message:
print(message + '\n')
if show_help:
parser.print_help()
parser.exit(status=1)
parser.error = error
self.error = error
self.register_options()
self.setup_parser(parser, args)
if needs_old_options:
self.old_options, self.args = parser.parse_args(args)
else:
# Ensure a predictable error if anything under goal tries to use these.
self.old_options = None
self.args = None
def address_map_from_spec_path(self, spec_path):
try:
build_file = BuildFile.from_cache(self._root_dir, spec_path)
except BuildFile.BuildFileError as e:
raise self.BuildFileScanError("{message}\n searching {spec_path}"
.format(message=e,
spec_path=spec_path))
family_address_map_by_build_file = self.parse_build_file_family(build_file)
address_map = {}
for build_file, sibling_address_map in family_address_map_by_build_file.items():
address_map.update(sibling_address_map)
return address_map
def resolve(self, address):
"""Maps an address in the virtual address space to an object.
:param Address address: the address to lookup in a BUILD file
:raises AddressLookupError: if the path to the address is not found.
:returns: Addressable from a build file specified by address
"""
address_map = self.address_map_from_spec_path(address.spec_path)
if address not in address_map:
build_file = BuildFile.from_cache(self.root_dir, address.spec_path, must_exist=False)
self._raise_incorrect_address_error(build_file, address.target_name, address_map)
else:
return address_map[address]
def resolve(self, address):
"""Maps an address in the virtual address space to an object.
:param Address address: the address to lookup in a BUILD file
:raises AddressLookupError: if the path to the address is not found.
:returns: Addressable from a build file specified by address
"""
address_map = self.address_map_from_spec_path(address.spec_path)
if address not in address_map:
build_file = BuildFile.from_cache(self.root_dir, address.spec_path, must_exist=False)
self._raise_incorrect_address_error(build_file, address.target_name, address_map)
else:
return address_map[address]
def spec_to_address(self, spec, relative_to=''):
"""A helper method for mapping a spec to the correct BuildFileAddress.
:param spec: a spec to lookup in the map.
:raises AddressLookupError: if the BUILD file cannot be found in the path specified by the spec
:returns a new BuildFileAddress instanace
"""
spec_path, name = parse_spec(spec, relative_to=relative_to)
try:
build_file = BuildFile.from_cache(self.root_dir, spec_path)
except BuildFile.BuildFileError as e:
raise self.InvalidBuildFileReference('{message}\n when translating spec {spec}'
.format(message=e, spec=spec))
return BuildFileAddress(build_file, name)
def spec_to_address(self, spec, relative_to=''):
"""A helper method for mapping a spec to the correct BuildFileAddress.
:param spec: a spec to lookup in the map.
:raises AddressLookupError: if the BUILD file cannot be found in the path specified by the spec
:returns a new BuildFileAddress instanace
"""
spec_path, name = parse_spec(spec, relative_to=relative_to)
try:
build_file = BuildFile.from_cache(self.root_dir, spec_path)
except BuildFile.BuildFileError as e:
raise self.InvalidBuildFileReference('{message}\n when translating spec {spec}'
.format(message=e, spec=spec))
return BuildFileAddress(build_file, name)
def _parse_spec(self, spec):
def normalize_spec_path(path):
is_abs = not path.startswith('//') and os.path.isabs(path)
if is_abs:
path = os.path.realpath(path)
if os.path.commonprefix([self._root_dir, path]) != self._root_dir:
raise self.BadSpecError('Absolute spec path {0} does not share build root {1}'
.format(path, self._root_dir))
else:
if path.startswith('//'):
path = path[2:]
path = os.path.join(self._root_dir, path)
normalized = os.path.relpath(path, self._root_dir)
if normalized == '.':
normalized = ''
return normalized
if spec.endswith('::'):
addresses = set()
spec_path = spec[:-len('::')]
spec_dir = normalize_spec_path(spec_path)
if not os.path.isdir(os.path.join(self._root_dir, spec_dir)):
raise self.BadSpecError('Can only recursive glob directories and {0} is not a valid dir'
.format(spec_dir))
try:
for build_file in BuildFile.scan_buildfiles(self._root_dir, spec_dir):
addresses.update(self._address_mapper.addresses_in_spec_path(build_file.spec_path))
return addresses
except (IOError, BuildFile.MissingBuildFileError, AddressLookupError) as e:
raise self.BadSpecError(e)
elif spec.endswith(':'):
spec_path = spec[:-len(':')]
spec_dir = normalize_spec_path(spec_path)
try:
return set(self._address_mapper.addresses_in_spec_path(spec_dir))
except (IOError, BuildFile.MissingBuildFileError, AddressLookupError) as e:
raise self.BadSpecError(e)
else:
spec_parts = spec.rsplit(':', 1)
spec_parts[0] = normalize_spec_path(spec_parts[0])
spec_path, target_name = parse_spec(':'.join(spec_parts))
try:
build_file = BuildFile.from_cache(self._root_dir, spec_path)
return set([BuildFileAddress(build_file, target_name)])
except (IOError, BuildFile.MissingBuildFileError) as e:
raise self.BadSpecError(e)
def is_spec(spec):
if os.sep in spec or ':' in spec:
return True # Definitely not a goal.
if not (spec in goal_names):
return True # Definitely not a (known) goal.
if has_double_dash:
# This means that we're parsing the half of the expression before a --, so assume it's a
# goal without warning.
return False
# Here, it's possible we have a goal and target with the same name. For now, always give
# priority to the goal, but give a warning if they might have meant the target (if the BUILD
# file exists).
if BuildFile.from_cache(get_buildroot(), spec,
must_exist=False).exists():
logger.warning(
' Command-line argument "{spec}" is ambiguous, and was assumed to be a '
'goal. If this is incorrect, disambiguate it with the "--" argument to '
'separate goals from targets.'.format(spec=spec))
return False
def is_spec(spec):
if os.sep in spec or ":" in spec:
return True # Definitely not a goal.
if not (spec in goal_names):
return True # Definitely not a (known) goal.
if has_double_dash:
# This means that we're parsing the half of the expression before a --, so assume it's a
# goal without warning.
return False
# Here, it's possible we have a goal and target with the same name. For now, always give
# priority to the goal, but give a warning if they might have meant the target (if the BUILD
# file exists).
if BuildFile.from_cache(get_buildroot(), spec, must_exist=False).exists():
logger.warning(
' Command-line argument "{spec}" is ambiguous, and was assumed to be a '
'goal. If this is incorrect, disambiguate it with the "--" argument to '
"separate goals from targets.".format(spec=spec)
)
return False
def _find_owners(self, source):
"""Searches for BUILD files adjacent or above a source in the file hierarchy.
- Walks up the directory tree until it reaches a previously searched path.
- Stops after looking in the buildroot.
If self._stop_after_match is set, stops searching once a source is mapped, even if the parent
has yet to be searched. See class docstring for discussion.
:param str source: The source at which to start the search.
"""
# Bail instantly if a source has already been searched
if source in self._searched_sources:
return
self._searched_sources.add(source)
root = get_buildroot()
path = os.path.dirname(source)
# a top-level source has empty dirname, so do/while instead of straight while loop.
walking = True
while walking:
# It is possible
if path not in self._mapped_paths:
candidate = BuildFile.from_cache(root_dir=root,
relpath=path,
must_exist=False)
if candidate.exists():
self._map_sources_from_family(candidate.family())
self._mapped_paths.add(path)
elif not self._stop_after_match:
# If not in stop-after-match mode, once a path is seen visited, all parents can be assumed.
return
# See class docstring
if self._stop_after_match and source in self._source_to_address:
return
walking = bool(path)
path = os.path.dirname(path)
def _find_owners(self, source):
"""Searches for BUILD files adjacent or above a source in the file hierarchy.
- Walks up the directory tree until it reaches a previously searched path.
- Stops after looking in the buildroot.
If self._stop_after_match is set, stops searching once a source is mapped, even if the parent
has yet to be searched. See class docstring for discussion.
:param str source: The source at which to start the search.
"""
# Bail instantly if a source has already been searched
if source in self._searched_sources:
return
self._searched_sources.add(source)
root = get_buildroot()
path = os.path.dirname(source)
# a top-level source has empty dirname, so do/while instead of straight while loop.
walking = True
while walking:
# It is possible
if path not in self._mapped_paths:
candidate = BuildFile.from_cache(root_dir=root, relpath=path, must_exist=False)
if candidate.exists():
self._map_sources_from_family(candidate.family())
self._mapped_paths.add(path)
elif not self._stop_after_match:
# If not in stop-after-match mode, once a path is seen visited, all parents can be assumed.
return
# See class docstring
if self._stop_after_match and source in self._source_to_address:
return
walking = bool(path)
path = os.path.dirname(path)
def spec_to_address(self, spec, relative_to=''): """A helper method for mapping a spec to the correct BuildFileAddress.""" spec_path, name = parse_spec(spec, relative_to=relative_to) build_file = BuildFile.from_cache(self.root_dir, spec_path) return BuildFileAddress(build_file, name)
def __init__(self, *args, **kwargs):
super(Py, self).__init__(*args, **kwargs)
self.binary = None
self.targets = []
self.extra_requirements = []
self.config = Config.load()
interpreters = self.options.interpreters or [b'']
self.interpreter_cache = PythonInterpreterCache(self.config,
logger=self.debug)
self.interpreter_cache.setup(filters=interpreters)
interpreters = self.interpreter_cache.select_interpreter(
list(self.interpreter_cache.matches(interpreters)))
if len(interpreters) != 1:
self.error('Unable to detect suitable interpreter.')
self.interpreter = interpreters[0]
for req in self.options.extra_requirements:
self.extra_requirements.append(
PythonRequirement(req, use_2to3=True))
# We parse each arg in the context of the cli usage:
# ./pants command (options) [spec] (build args)
# ./pants command (options) [spec]... -- (build args)
# Our command token and our options are parsed out so we see args of the form:
# [spec] (build args)
# [spec]... -- (build args)
for k in range(len(self.args)):
arg = self.args.pop(0)
if arg == '--':
break
def not_a_target(debug_msg):
self.debug('Not a target, assuming option: %s.' % debug_msg)
# We failed to parse the arg as a target or else it was in valid address format but did not
# correspond to a real target. Assume this is the 1st of the build args and terminate
# processing args for target addresses.
self.args.insert(0, arg)
try:
print(self.root_dir, arg, file=sys.stderr)
self.build_graph.inject_spec_closure(arg)
spec_path, target_name = parse_spec(arg)
build_file = BuildFile.from_cache(self.root_dir, spec_path)
address = BuildFileAddress(build_file, target_name)
target = self.build_graph.get_target(address)
if target is None:
not_a_target(debug_msg='Unrecognized target')
break
except Exception as e:
not_a_target(debug_msg=e)
break
if isinstance(target, PythonBinary):
if self.binary:
self.error(
'Can only process 1 binary target. Found %s and %s.' %
(self.binary, target))
else:
self.binary = target
self.targets.append(target)
if not self.targets:
self.error('No valid targets specified!')
def setup(self):
options_bootstrapper = OptionsBootstrapper()
# Force config into the cache so we (and plugin/backend loading code) can use it.
# TODO: Plumb options in explicitly.
bootstrap_options = options_bootstrapper.get_bootstrap_options()
self.config = Config.from_cache()
# Add any extra paths to python path (eg for loading extra source backends)
for path in bootstrap_options.for_global_scope().pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
backend_packages = self.config.getlist('backends', 'packages', [])
plugins = self.config.getlist('backends', 'plugins', [])
build_configuration = load_plugins_and_backends(
plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
# TODO: Create a 'Subsystem' abstraction instead of special-casing run-tracker here
# and in register_options().
known_scopes = ['', 'run-tracker']
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(
known_scopes=known_scopes)
self.register_options()
# TODO(Eric Ayers) We are missing log messages. Set the log level earlier
# Enable standard python logging for code with no handle to a context/work-unit.
self._setup_logging() # NB: self.options are needed for this call.
self.run_tracker = RunTracker.from_options(self.options)
report = initial_reporting(self.config, self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: %s' % url)
else:
self.run_tracker.log(
Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(
build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
with self.run_tracker.new_workunit(name='bootstrap',
labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in self.config.getlist('goals',
'bootstrap_buildfiles',
default=[]):
build_file = BuildFile.from_cache(root_dir=self.root_dir,
relpath=path)
# TODO(pl): This is an unfortunate interface leak, but I don't think
# in the long run that we should be relying on "bootstrap" BUILD files
# that do nothing except modify global state. That type of behavior
# (e.g. source roots, goal registration) should instead happen in
# project plugins, or specialized configuration files.
self.build_file_parser.parse_build_file_family(build_file)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
self._expand_goals_and_specs()
def setup_parser(self, parser, args):
if not args:
args.append('help')
logger = logging.getLogger(__name__)
goals = self.new_options.goals
specs = self.new_options.target_specs
fail_fast = self.new_options.for_global_scope().fail_fast
for goal in goals:
if BuildFile.from_cache(get_buildroot(), goal,
must_exist=False).exists():
logger.warning(
" Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}."
.format(goal))
if self.new_options.is_help:
self.new_options.print_help(goals=goals)
sys.exit(0)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup',
labels=[WorkUnit.SETUP]):
spec_parser = CmdLineSpecParser(
self.root_dir,
self.address_mapper,
spec_excludes=self.get_spec_excludes())
with self.run_tracker.new_workunit(name='parse',
labels=[WorkUnit.SETUP]):
for spec in specs:
for address in spec_parser.parse_addresses(
spec, fail_fast):
self.build_graph.inject_address_closure(address)
self.targets.append(
self.build_graph.get_target(address))
self.goals = [Goal.by_name(goal) for goal in goals]
rcfiles = self.config.getdefault(
'rcfiles', type=list, default=['/etc/pantsrc', '~/.pants.rc'])
if rcfiles:
rcfile = RcFile(rcfiles,
default_prepend=False,
process_default=True)
# Break down the goals specified on the command line to the full set that will be run so we
# can apply default flags to inner goal nodes. Also break down goals by Task subclass and
# register the task class hierarchy fully qualified names so we can apply defaults to
# baseclasses.
sections = OrderedSet()
for goal in Engine.execution_order(self.goals):
for task_name in goal.ordered_task_names():
sections.add(task_name)
task_type = goal.task_type_by_name(task_name)
for clazz in task_type.mro():
if clazz == Task:
break
sections.add('%s.%s' %
(clazz.__module__, clazz.__name__))
augmented_args = rcfile.apply_defaults(sections, args)
if augmented_args != args:
# TODO(John Sirois): Cleanup this currently important mutation of the passed in args
# once the 2-layer of command -> goal is squashed into one.
args[:] = augmented_args
sys.stderr.write("(using pantsrc expansion: pants goal %s)\n" %
' '.join(augmented_args))
def _parse_spec(self, spec, fail_fast=False):
def normalize_spec_path(path):
is_abs = not path.startswith('//') and os.path.isabs(path)
if is_abs:
path = os.path.realpath(path)
if os.path.commonprefix([self._root_dir, path
]) != self._root_dir:
raise self.BadSpecError(
'Absolute address path {0} does not share build root {1}'
.format(path, self._root_dir))
else:
if path.startswith('//'):
path = path[2:]
path = os.path.join(self._root_dir, path)
normalized = os.path.relpath(path, self._root_dir)
if normalized == '.':
normalized = ''
return normalized
errored_out = []
if spec.endswith('::'):
addresses = set()
spec_path = spec[:-len('::')]
spec_dir = normalize_spec_path(spec_path)
if not os.path.isdir(os.path.join(self._root_dir, spec_dir)):
raise self.BadSpecError(
'Can only recursive glob directories and {0} is not a valid dir'
.format(spec_dir))
try:
build_files = BuildFile.scan_buildfiles(
self._root_dir,
spec_dir,
spec_excludes=self._spec_excludes)
except (BuildFile.BuildFileError, AddressLookupError) as e:
raise self.BadSpecError(e)
for build_file in build_files:
try:
# This attempts to filter out broken BUILD files before we parse them.
if self._not_excluded_spec(build_file.spec_path):
addresses.update(
self._address_mapper.addresses_in_spec_path(
build_file.spec_path))
except (BuildFile.BuildFileError, AddressLookupError) as e:
if fail_fast:
raise self.BadSpecError(e)
errored_out.append('--------------------')
errored_out.append(traceback.format_exc())
errored_out.append('Exception message: {0}'.format(
e.message))
if errored_out:
error_msg = '\n'.join(
errored_out +
["Invalid BUILD files for [{0}]".format(spec)])
raise self.BadSpecError(error_msg)
return addresses
elif spec.endswith(':'):
spec_path = spec[:-len(':')]
spec_dir = normalize_spec_path(spec_path)
try:
return set(
self._address_mapper.addresses_in_spec_path(spec_dir))
except AddressLookupError as e:
raise self.BadSpecError(e)
else:
spec_parts = spec.rsplit(':', 1)
spec_parts[0] = normalize_spec_path(spec_parts[0])
spec_path, target_name = parse_spec(':'.join(spec_parts))
try:
build_file = BuildFile.from_cache(self._root_dir, spec_path)
return set([BuildFileAddress(build_file, target_name)])
except BuildFile.BuildFileError as e:
raise self.BadSpecError(e)
def _parse_spec(self, spec, fail_fast=False):
def normalize_spec_path(path):
is_abs = not path.startswith('//') and os.path.isabs(path)
if is_abs:
path = os.path.realpath(path)
if os.path.commonprefix([self._root_dir, path]) != self._root_dir:
raise self.BadSpecError('Absolute address path {0} does not share build root {1}'
.format(path, self._root_dir))
else:
if path.startswith('//'):
path = path[2:]
path = os.path.join(self._root_dir, path)
normalized = os.path.relpath(path, self._root_dir)
if normalized == '.':
normalized = ''
return normalized
errored_out = []
if spec.endswith('::'):
addresses = set()
spec_path = spec[:-len('::')]
spec_dir = normalize_spec_path(spec_path)
if not os.path.isdir(os.path.join(self._root_dir, spec_dir)):
raise self.BadSpecError('Can only recursive glob directories and {0} is not a valid dir'
.format(spec_dir))
try:
build_files = BuildFile.scan_buildfiles(self._root_dir, spec_dir, spec_excludes=self._spec_excludes)
except (BuildFile.BuildFileError, AddressLookupError) as e:
raise self.BadSpecError(e)
for build_file in build_files:
try:
# This attempts to filter out broken BUILD files before we parse them.
if self._not_excluded_spec(build_file.spec_path):
addresses.update(self._address_mapper.addresses_in_spec_path(build_file.spec_path))
except (BuildFile.BuildFileError, AddressLookupError) as e:
if fail_fast:
raise self.BadSpecError(e)
errored_out.append('--------------------')
errored_out.append(traceback.format_exc())
errored_out.append('Exception message: {0}'.format(e.message))
if errored_out:
error_msg = '\n'.join(errored_out + ["Invalid BUILD files for [{0}]".format(spec)])
raise self.BadSpecError(error_msg)
return addresses
elif spec.endswith(':'):
spec_path = spec[:-len(':')]
spec_dir = normalize_spec_path(spec_path)
try:
return set(self._address_mapper.addresses_in_spec_path(spec_dir))
except AddressLookupError as e:
raise self.BadSpecError(e)
else:
spec_parts = spec.rsplit(':', 1)
spec_parts[0] = normalize_spec_path(spec_parts[0])
spec_path, target_name = parse_spec(':'.join(spec_parts))
try:
build_file = BuildFile.from_cache(self._root_dir, spec_path)
return set([BuildFileAddress(build_file, target_name)])
except BuildFile.BuildFileError as e:
raise self.BadSpecError(e)
def __init__(self, *args, **kwargs):
super(Py, self).__init__(*args, **kwargs)
self.binary = None
self.targets = []
self.extra_requirements = []
self.config = Config.from_cache()
interpreters = self.old_options.interpreters or [b'']
self.interpreter_cache = PythonInterpreterCache(self.config, logger=self.debug)
self.interpreter_cache.setup(filters=interpreters)
interpreters = self.interpreter_cache.select_interpreter(
list(self.interpreter_cache.matches(interpreters)))
if len(interpreters) != 1:
self.error('Unable to detect suitable interpreter.')
self.interpreter = interpreters[0]
for req in self.old_options.extra_requirements:
self.extra_requirements.append(PythonRequirement(req, use_2to3=True))
# We parse each arg in the context of the cli usage:
# ./pants command (options) [spec] (build args)
# ./pants command (options) [spec]... -- (build args)
# Our command token and our options are parsed out so we see args of the form:
# [spec] (build args)
# [spec]... -- (build args)
for k in range(len(self.args)):
arg = self.args.pop(0)
if arg == '--':
break
def not_a_target(debug_msg):
self.debug('Not a target, assuming option: %s.' % debug_msg)
# We failed to parse the arg as a target or else it was in valid address format but did not
# correspond to a real target. Assume this is the 1st of the build args and terminate
# processing args for target addresses.
self.args.insert(0, arg)
try:
print(self.root_dir, arg, file=sys.stderr)
self.build_graph.inject_spec_closure(arg)
spec_path, target_name = parse_spec(arg)
build_file = BuildFile.from_cache(self.root_dir, spec_path)
address = BuildFileAddress(build_file, target_name)
target = self.build_graph.get_target(address)
if target is None:
not_a_target(debug_msg='Unrecognized target')
break
except Exception as e:
not_a_target(debug_msg=e)
break
if isinstance(target, PythonBinary):
if self.binary:
self.error('Can only process 1 binary target. Found %s and %s.' % (self.binary, target))
else:
self.binary = target
self.targets.append(target)
if not self.targets:
self.error('No valid targets specified!')
