def _parse_addresses(self, spec):
if spec.endswith('::'):
dir = self._get_dir(spec[:-len('::')])
for buildfile in BuildFile.scan_buildfiles(self._root_dir, os.path.join(self._root_dir, dir)):
for address in Target.get_all_addresses(buildfile):
yield address
elif spec.endswith(':'):
dir = self._get_dir(spec[:-len(':')])
for address in Target.get_all_addresses(BuildFile(self._root_dir, dir)):
yield address
else:
yield Address.parse(self._root_dir, spec)
def add_target_recursive(self, *specs):
with self.check_errors('There was a problem scanning the '
'following directories for targets:') as error:
for spec in specs:
dir = self.get_dir(spec)
for buildfile in BuildFile.scan_buildfiles(self.root_dir, dir):
self.add_targets(error, dir, buildfile)
def create_buildfile(root_dir, relpath, content=''):
path = os.path.join(root_dir, relpath)
os.makedirs(path)
buildfile = os.path.join(path, 'BUILD')
with open(buildfile, 'a') as f:
f.write(content)
return BuildFile(root_dir, relpath)
def _addresses(self):
if self.context.target_roots:
return (target.address for target in self.context.target_roots)
else:
return (address
for buildfile in BuildFile.scan_buildfiles(self._root_dir)
for address in Target.get_all_addresses(buildfile))
def execute(self):
for buildfile in BuildFile.scan_buildfiles(self.root_dir):
for address in Target.get_all_addresses(buildfile):
target = Target.get(address)
if hasattr(target, 'sources') and target.sources is not None:
for sourcefile in target.sources:
print sourcefile, address
def execute(self):
for buildfile in BuildFile.scan_buildfiles(self.root_dir):
for address in Target.get_all_addresses(buildfile):
target = Target.get(address)
if hasattr(target, 'sources') and target.sources is not None:
for sourcefile in target.sources:
print sourcefile, address
def add_target_recursive(self, *specs):
with self.check_errors('There was a problem scanning the '
'following directories for targets:') as error:
for spec in specs:
dir = self.get_dir(spec)
for buildfile in BuildFile.scan_buildfiles(self.root_dir, dir):
self.add_targets(error, dir, buildfile)
def create_buildfile(root_dir, relpath, name='BUILD', content=''):
path = os.path.join(root_dir, relpath)
safe_mkdir(path)
buildfile = os.path.join(path, name)
with open(buildfile, 'a') as f:
f.write(content)
return BuildFile(root_dir, relpath)
def _addresses(self):
if self.context.target_roots:
for target in self.context.target_roots:
yield target.address
else:
for buildfile in BuildFile.scan_buildfiles(self._root_dir):
for address in Target.get_all_addresses(buildfile):
yield address
def scan_addresses(root_dir, base_path = None):
"""Parses all targets available in BUILD files under base_path and returns their addresses. If no
base_path is specified, root_dir is assumed to be the base_path"""
addresses = OrderedSet()
for buildfile in BuildFile.scan_buildfiles(root_dir, base_path):
addresses.update(Target.get_all_addresses(buildfile))
return addresses
def _addresses(self):
if self.context.target_roots:
for target in self.context.target_roots:
yield target.address
else:
for buildfile in BuildFile.scan_buildfiles(self._root_dir):
for address in Target.get_all_addresses(buildfile):
yield address
def _find_targets(self):
if len(self.context.target_roots) > 0:
for target in self.context.target_roots:
yield target
else:
for buildfile in BuildFile.scan_buildfiles(get_buildroot()):
target_addresses = Target.get_all_addresses(buildfile)
for target_address in target_addresses:
yield Target.get(target_address)
def _find_targets(self):
if len(self.context.target_roots) > 0:
for target in self.context.target_roots:
yield target
else:
for buildfile in BuildFile.scan_buildfiles(get_buildroot()):
target_addresses = Target.get_all_addresses(buildfile)
for target_address in target_addresses:
yield Target.get(target_address)
def add_target_directory(self, *specs):
with self.check_errors("There was a problem loading targets "
"from the following directory's BUILD files") as error:
for spec in specs:
dir = self.get_dir(spec)
try:
self.add_targets(error, dir, BuildFile(self.root_dir, dir))
except IOError:
error(dir)
def temp(basedir=None):
"""Activates a temporary parse context in the given basedir relative to the build root or else
in the build root dir itself if no basedir is specified."""
context = ParseContext(
BuildFile(get_buildroot(),
basedir or 'BUILD.temp',
must_exist=False))
with ParseContext.activate(context):
yield
def scan_addresses(root_dir, base_path=None):
"""Parses all targets available in BUILD files under base_path and
returns their addresses. If no base_path is specified, root_dir is
assumed to be the base_path"""
addresses = OrderedSet()
for buildfile in BuildFile.scan_buildfiles(root_dir, base_path):
addresses.update(Target.get_all_addresses(buildfile))
return addresses
def execute(self):
def add_targets(dir, buildfile):
try:
self.targets.extend(Target.get(addr) for addr in Target.get_all_addresses(buildfile))
except (TypeError, ImportError):
error(dir, include_traceback=True)
except (IOError, SyntaxError):
error(dir)
if self.options.recursive_directory:
with self.check_errors('There was a problem scanning the '
'following directories for targets:') as error:
for dir in self.options.recursive_directory:
for buildfile in BuildFile.scan_buildfiles(self.root_dir, dir):
add_targets(dir, buildfile)
if self.options.target_directory:
with self.check_errors("There was a problem loading targets "
"from the following directory's BUILD files") as error:
for dir in self.options.target_directory:
add_targets(dir, BuildFile(self.root_dir, dir))
timer = None
if self.options.time:
class Timer(object):
def now(self):
return time.time()
def log(self, message):
print(message)
timer = Timer()
logger = None
if self.options.log or self.options.log_level:
from twitter.common.log import init
from twitter.common.log.options import LogOptions
LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper())
logdir = self.config.get('goals', 'logdir')
if logdir:
safe_mkdir(logdir)
LogOptions.set_log_dir(logdir)
init('goals')
logger = log
context = Context(self.config, self.options, self.targets, log=logger)
unknown = []
for phase in self.phases:
if not phase.goals():
unknown.append(phase)
if unknown:
print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown))
print()
return Phase.execute(context, 'goals')
return Phase.attempt(context, self.phases, timer=timer)
def execute(self, expanded_target_addresses):
buildroot = get_buildroot()
if len(self.context.target_roots) > 0:
for target in self.context.target_roots:
self._execute_target(target, buildroot)
else:
for buildfile in BuildFile.scan_buildfiles(buildroot):
target_addresses = Target.get_all_addresses(buildfile)
for target_address in target_addresses:
target = Target.get(target_address)
self._execute_target(target, buildroot)
def execute(self, expanded_target_addresses):
buildroot = get_buildroot()
if len(self.context.target_roots) > 0:
for target in self.context.target_roots:
self._execute_target(target, buildroot)
else:
for buildfile in BuildFile.scan_buildfiles(buildroot):
target_addresses = Target.get_all_addresses(buildfile)
for target_address in target_addresses:
target = Target.get(target_address)
self._execute_target(target, buildroot)
def _parse_addresses(self, spec):
if spec.endswith('::'):
dir = self._get_dir(spec[:-len('::')])
for buildfile in BuildFile.scan_buildfiles(self._root_dir, os.path.join(self._root_dir, dir)):
for address in Target.get_all_addresses(buildfile):
yield address
elif spec.endswith(':'):
dir = self._get_dir(spec[:-len(':')])
for address in Target.get_all_addresses(BuildFile(self._root_dir, dir)):
yield address
else:
yield Address.parse(self._root_dir, spec)
def _parse_buildfiles(self):
buildfiles = OrderedSet()
for spec in self.args:
try:
if self.options.is_directory_list:
for buildfile in BuildFile.scan_buildfiles(self.root_dir, spec):
buildfiles.add(buildfile)
else:
buildfiles.add(Address.parse(self.root_dir, spec).buildfile)
except:
self.error("Problem parsing spec %s: %s" % (spec, traceback.format_exc()))
return buildfiles
def __init__(self, context):
ConsoleTask.__init__(self, context)
self._print_uptodate = context.options.check_deps_print_uptodate
self.repos = context.config.getdict('jar-publish', 'repos')
self._artifacts_to_targets = {}
all_addresses = (address for buildfile in BuildFile.scan_buildfiles(get_buildroot())
for address in Target.get_all_addresses(buildfile))
for address in all_addresses:
target = Target.get(address)
if target.is_exported:
provided_jar, _, _ = target.get_artifact_info()
artifact = (provided_jar.org, provided_jar.name)
if not artifact in self._artifacts_to_targets:
self._artifacts_to_targets[artifact] = target
def buildfile(cls, path):
return BuildFile(BuildFileTest.root_dir, path)
def __init__(self, root_dir, parser, argv):
Command.__init__(self, root_dir, parser, argv)
self.buildfiles = self._parse_buildfiles() if self.args else BuildFile.scan_buildfiles(root_dir)
def setup_parser(self, parser, args):
self.config = Config.load()
Goal.add_global_options(parser)
# We support attempting zero or more goals. Multiple goals must be delimited from further
# options and non goal args with a '--'. The key permutations we need to support:
# ./pants goal => goals
# ./pants goal goals => goals
# ./pants goal compile src/java/... => compile
# ./pants goal compile -x src/java/... => compile
# ./pants goal compile src/java/... -x => compile
# ./pants goal compile run -- src/java/... => compile, run
# ./pants goal compile run -- src/java/... -x => compile, run
# ./pants goal compile run -- -x src/java/... => compile, run
if not args:
args.append('goals')
if len(args) == 1 and args[0] in set(['-h', '--help', 'help']):
def format_usage(usages):
left_colwidth = 0
for left, right in usages:
left_colwidth = max(left_colwidth, len(left))
lines = []
for left, right in usages:
lines.append(' %s%s%s' %
(left, ' ' *
(left_colwidth - len(left) + 1), right))
return '\n'.join(lines)
usages = [
("%prog goal goals ([spec]...)", Phase('goals').description),
("%prog goal help [goal] ([spec]...)",
Phase('help').description),
("%prog goal [goal] [spec]...",
"Attempt goal against one or more targets."),
("%prog goal [goal] ([goal]...) -- [spec]...",
"Attempts all the specified goals."),
]
parser.set_usage("\n%s" % format_usage(usages))
parser.epilog = (
"Either lists all installed goals, provides extra help for a goal or else "
"attempts to achieve the specified goal for the listed targets."
"""
Note that target specs accept two special forms:
[dir]: to include all targets in the specified directory
[dir]:: to include all targets found in all BUILD files recursively under
the directory""")
parser.print_help()
sys.exit(0)
else:
goals, specs = Goal.parse_args(args)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup',
labels=[WorkUnit.SETUP]):
# Bootstrap goals by loading any configured bootstrap BUILD files
with self.check_errors(
'The following bootstrap_buildfiles cannot be loaded:'
) as error:
with self.run_tracker.new_workunit(name='bootstrap',
labels=[WorkUnit.SETUP
]):
for path in self.config.getlist('goals',
'bootstrap_buildfiles',
default=[]):
try:
buildfile = BuildFile(
get_buildroot(),
os.path.relpath(path, get_buildroot()))
ParseContext(buildfile).parse()
except (TypeError, ImportError, TaskError,
GoalError):
error(path, include_traceback=True)
except (IOError, SyntaxError):
error(path)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
# Bootstrap user goals by loading any BUILD files implied by targets.
spec_parser = SpecParser(self.root_dir)
with self.check_errors(
'The following targets could not be loaded:') as error:
with self.run_tracker.new_workunit(name='parse',
labels=[WorkUnit.SETUP
]):
for spec in specs:
try:
for target, address in spec_parser.parse(spec):
if target:
self.targets.append(target)
# Force early BUILD file loading if this target is an alias that expands
# to others.
unused = list(target.resolve())
else:
siblings = Target.get_all_addresses(
address.buildfile)
prompt = 'did you mean' if len(
siblings
) == 1 else 'maybe you meant one of these'
error('%s => %s?:\n %s' %
(address, prompt, '\n '.join(
str(a) for a in siblings)))
except (TypeError, ImportError, TaskError,
GoalError):
error(spec, include_traceback=True)
except (IOError, SyntaxError,
TargetDefinitionException):
error(spec)
self.phases = [Phase(goal) for goal in goals]
rcfiles = self.config.getdefault('rcfiles', type=list, default=[])
if rcfiles:
rcfile = RcFile(rcfiles,
default_prepend=False,
process_default=True)
# Break down the goals specified on the command line to the full set that will be run so we
# can apply default flags to inner goal nodes. Also break down goals by Task subclass and
# register the task class hierarchy fully qualified names so we can apply defaults to
# baseclasses.
sections = OrderedSet()
for phase in Engine.execution_order(self.phases):
for goal in phase.goals():
sections.add(goal.name)
for clazz in goal.task_type.mro():
if clazz == Task:
break
sections.add('%s.%s' %
(clazz.__module__, clazz.__name__))
augmented_args = rcfile.apply_defaults(sections, args)
if augmented_args != args:
del args[:]
args.extend(augmented_args)
sys.stderr.write(
"(using pantsrc expansion: pants goal %s)\n" %
' '.join(augmented_args))
Phase.setup_parser(parser, args, self.phases)
def setup_parser(self, parser, args):
self.config = Config.load()
Goal.add_global_options(parser)
# We support attempting zero or more goals. Multiple goals must be delimited from further
# options and non goal args with a '--'. The key permutations we need to support:
# ./pants goal => goals
# ./pants goal goals => goals
# ./pants goal compile src/java/... => compile
# ./pants goal compile -x src/java/... => compile
# ./pants goal compile src/java/... -x => compile
# ./pants goal compile run -- src/java/... => compile, run
# ./pants goal compile run -- src/java/... -x => compile, run
# ./pants goal compile run -- -x src/java/... => compile, run
if not args:
args.append('goals')
if len(args) == 1 and args[0] in set(['-h', '--help', 'help']):
def format_usage(usages):
left_colwidth = 0
for left, right in usages:
left_colwidth = max(left_colwidth, len(left))
lines = []
for left, right in usages:
lines.append(' %s%s%s' % (left, ' ' * (left_colwidth - len(left) + 1), right))
return '\n'.join(lines)
usages = [
("%prog goal goals ([spec]...)", Phase('goals').description),
("%prog goal help [goal] ([spec]...)", Phase('help').description),
("%prog goal [goal] [spec]...", "Attempt goal against one or more targets."),
("%prog goal [goal] ([goal]...) -- [spec]...", "Attempts all the specified goals."),
]
parser.set_usage("\n%s" % format_usage(usages))
parser.epilog = ("Either lists all installed goals, provides extra help for a goal or else "
"attempts to achieve the specified goal for the listed targets." """
Note that target specs accept two special forms:
[dir]: to include all targets in the specified directory
[dir]:: to include all targets found in all BUILD files recursively under
the directory""")
parser.print_help()
sys.exit(0)
else:
goals, specs = Goal.parse_args(args)
# TODO(John Sirois): kill PANTS_NEW and its usages when pants.new is rolled out
ParseContext.enable_pantsnew()
# Bootstrap goals by loading any configured bootstrap BUILD files
with self.check_errors('The following bootstrap_buildfiles cannot be loaded:') as error:
with self.timer.timing('parse:bootstrap'):
for path in self.config.getlist('goals', 'bootstrap_buildfiles', default = []):
try:
buildfile = BuildFile(get_buildroot(), os.path.relpath(path, get_buildroot()))
ParseContext(buildfile).parse()
except (TypeError, ImportError, TaskError, GoalError):
error(path, include_traceback=True)
except (IOError, SyntaxError):
error(path)
# Bootstrap user goals by loading any BUILD files implied by targets
with self.check_errors('The following targets could not be loaded:') as error:
with self.timer.timing('parse:BUILD'):
for spec in specs:
self.parse_spec(error, spec)
self.phases = [Phase(goal) for goal in goals]
rcfiles = self.config.getdefault('rcfiles', type=list, default=[])
if rcfiles:
rcfile = RcFile(rcfiles, default_prepend=False, process_default=True)
# Break down the goals specified on the command line to the full set that will be run so we
# can apply default flags to inner goal nodes. Also break down goals by Task subclass and
# register the task class hierarchy fully qualified names so we can apply defaults to
# baseclasses.
all_goals = Phase.execution_order(Phase(goal) for goal in goals)
sections = OrderedSet()
for goal in all_goals:
sections.add(goal.name)
for clazz in goal.task_type.mro():
if clazz == Task:
break
sections.add('%s.%s' % (clazz.__module__, clazz.__name__))
augmented_args = rcfile.apply_defaults(sections, args)
if augmented_args != args:
del args[:]
args.extend(augmented_args)
print("(using pantsrc expansion: pants goal %s)" % ' '.join(augmented_args))
Phase.setup_parser(parser, args, self.phases)
def _addresses(self):
if self.context.target_roots:
return (target.address for target in self.context.target_roots)
else:
return (address for buildfile in BuildFile.scan_buildfiles(self._root_dir)
for address in Target.get_all_addresses(buildfile))
