def __init__(self, root_dir, parser, args): self.targets = [] # Note that we can't gate this on the self.options.time flag, because self.options is # only set up in Command.__init__, and only after it calls setup_parser(), which uses the timer. self.timer = Timer() Command.__init__(self, root_dir, parser, args)
def __init__(self, root_dir, parser, args): self.targets = [] # Note that we can't gate this on the self.options.time flag, because self.options is # only set up in Command.__init__, and only after it calls setup_parser(), which uses the timer. self.timer = Timer() Command.__init__(self, root_dir, parser, args)
class Goal(Command):
"""Lists installed goals or else executes a named goal."""
__command__ = 'goal'
GLOBAL_OPTIONS = [
Option("-x", "--time", action="store_true", dest="time", default=False,
help="Times goal phases and outputs a report."),
Option("-v", "--log", action="store_true", dest="log", default=False,
help="[%default] Logs extra build output."),
Option("-d", "--logdir", dest="logdir",
help="[%default] Forks logs to files under this directory."),
Option("-l", "--level", dest="log_level", type="choice", choices=['debug', 'info', 'warn'],
help="[info] Sets the logging level to one of 'debug', 'info' or 'warn', implies -v "
"if set."),
Option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
help="Print the commands that would be run, without actually running them."),
Option("--read-from-artifact-cache", "--no-read-from-artifact-cache", action="callback",
callback=_set_bool, dest="read_from_artifact_cache", default=False,
help="Whether to read artifacts from cache instead of building them, when possible."),
Option("--write-to-artifact-cache", "--no-write-to-artifact-cache", action="callback",
callback=_set_bool, dest="write_to_artifact_cache", default=False,
help="Whether to write artifacts to cache ."),
Option("--verify-artifact-cache", "--no-verify-artifact-cache", action="callback",
callback=_set_bool, dest="verify_artifact_cache", default=False,
help="Whether to verify that cached artifacts are identical after rebuilding them."),
Option("--all", dest="target_directory", action="append",
help="DEPRECATED: Use [dir]: with no flag in a normal target position on the command "
"line. (Adds all targets found in the given directory's BUILD file. Can be "
"specified more than once.)"),
Option("--all-recursive", dest="recursive_directory", action="append",
help="DEPRECATED: Use [dir]:: with no flag in a normal target position on the command "
"line. (Adds all targets found recursively under the given directory. Can be "
"specified more than once to add more than one root target directory to scan.)"),
]
@staticmethod
def add_global_options(parser):
for option in Goal.GLOBAL_OPTIONS:
parser.add_option(option)
@staticmethod
def parse_args(args):
goals = OrderedSet()
specs = OrderedSet()
help = False
explicit_multi = False
def is_spec(spec):
return os.sep in spec or ':' in spec
for i, arg in enumerate(args):
help = help or 'help' == arg
if not arg.startswith('-'):
specs.add(arg) if is_spec(arg) else goals.add(arg)
elif '--' == arg:
if specs:
raise GoalError('Cannot intermix targets with goals when using --. Targets should '
'appear on the right')
explicit_multi = True
del args[i]
break
if explicit_multi:
spec_offset = len(goals) + 1 if help else len(goals)
specs.update(arg for arg in args[spec_offset:] if not arg.startswith('-'))
return goals, specs
# TODO(John Sirois): revisit wholesale locking when we move py support into pants new
@classmethod
def serialized(cls):
# Goal serialization is now handled in goal execution during group processing.
# The goal command doesn't need to hold the serialization lock; individual goals will
# acquire the lock if they need to be serialized.
return False
def __init__(self, root_dir, parser, args):
self.targets = []
# Note that we can't gate this on the self.options.time flag, because self.options is
# only set up in Command.__init__, and only after it calls setup_parser(), which uses the timer.
self.timer = Timer()
Command.__init__(self, root_dir, parser, args)
@contextmanager
def check_errors(self, banner):
errors = {}
def error(key, include_traceback=False):
exc_type, exc_value, _ = sys.exc_info()
msg = StringIO()
if include_traceback:
frame = inspect.trace()[-1]
filename = frame[1]
lineno = frame[2]
funcname = frame[3]
code = ''.join(frame[4])
traceback.print_list([(filename, lineno, funcname, code)], file=msg)
if exc_type:
msg.write(''.join(traceback.format_exception_only(exc_type, exc_value)))
errors[key] = msg.getvalue()
sys.exc_clear()
yield error
if errors:
msg = StringIO()
msg.write(banner)
invalid_keys = [key for key, exc in errors.items() if not exc]
if invalid_keys:
msg.write('\n %s' % '\n '.join(invalid_keys))
for key, exc in errors.items():
if exc:
msg.write('\n %s =>\n %s' % (key, '\n '.join(exc.splitlines())))
# The help message for goal is extremely verbose, and will obscure the
# actual error message, so we don't show it in this case.
self.error(msg.getvalue(), show_help = False)
def add_targets(self, error, dir, buildfile):
try:
self.targets.extend(Target.get(addr) for addr in Target.get_all_addresses(buildfile))
except (TypeError, ImportError):
error(dir, include_traceback=True)
except (IOError, SyntaxError):
error(dir)
def get_dir(self, spec):
path = spec.split(':', 1)[0]
if os.path.isdir(path):
return path
else:
if os.path.isfile(path):
return os.path.dirname(path)
else:
return spec
def add_target_recursive(self, *specs):
with self.check_errors('There was a problem scanning the '
'following directories for targets:') as error:
for spec in specs:
dir = self.get_dir(spec)
for buildfile in BuildFile.scan_buildfiles(self.root_dir, dir):
self.add_targets(error, dir, buildfile)
def add_target_directory(self, *specs):
with self.check_errors("There was a problem loading targets "
"from the following directory's BUILD files") as error:
for spec in specs:
dir = self.get_dir(spec)
try:
self.add_targets(error, dir, BuildFile(self.root_dir, dir))
except IOError:
error(dir)
def parse_spec(self, error, spec):
if spec.endswith('::'):
self.add_target_recursive(spec[:-len('::')])
elif spec.endswith(':'):
self.add_target_directory(spec[:-len(':')])
else:
try:
address = Address.parse(get_buildroot(), spec)
ParseContext(address.buildfile).parse()
target = Target.get(address)
if target:
self.targets.append(target)
else:
siblings = Target.get_all_addresses(address.buildfile)
prompt = 'did you mean' if len(siblings) == 1 else 'maybe you meant one of these'
error('%s => %s?:\n %s' % (address, prompt,
'\n '.join(str(a) for a in siblings)))
except (TypeError, ImportError, TaskError, GoalError):
error(spec, include_traceback=True)
except (IOError, SyntaxError):
error(spec)
def setup_parser(self, parser, args):
self.config = Config.load()
Goal.add_global_options(parser)
# We support attempting zero or more goals. Multiple goals must be delimited from further
# options and non goal args with a '--'. The key permutations we need to support:
# ./pants goal => goals
# ./pants goal goals => goals
# ./pants goal compile src/java/... => compile
# ./pants goal compile -x src/java/... => compile
# ./pants goal compile src/java/... -x => compile
# ./pants goal compile run -- src/java/... => compile, run
# ./pants goal compile run -- src/java/... -x => compile, run
# ./pants goal compile run -- -x src/java/... => compile, run
if not args:
args.append('goals')
if len(args) == 1 and args[0] in set(['-h', '--help', 'help']):
def format_usage(usages):
left_colwidth = 0
for left, right in usages:
left_colwidth = max(left_colwidth, len(left))
lines = []
for left, right in usages:
lines.append(' %s%s%s' % (left, ' ' * (left_colwidth - len(left) + 1), right))
return '\n'.join(lines)
usages = [
("%prog goal goals ([spec]...)", Phase('goals').description),
("%prog goal help [goal] ([spec]...)", Phase('help').description),
("%prog goal [goal] [spec]...", "Attempt goal against one or more targets."),
("%prog goal [goal] ([goal]...) -- [spec]...", "Attempts all the specified goals."),
]
parser.set_usage("\n%s" % format_usage(usages))
parser.epilog = ("Either lists all installed goals, provides extra help for a goal or else "
"attempts to achieve the specified goal for the listed targets." """
Note that target specs accept two special forms:
[dir]: to include all targets in the specified directory
[dir]:: to include all targets found in all BUILD files recursively under
the directory""")
parser.print_help()
sys.exit(0)
else:
goals, specs = Goal.parse_args(args)
self.requested_goals = goals
# TODO(John Sirois): kill PANTS_NEW and its usages when pants.new is rolled out
ParseContext.enable_pantsnew()
# Bootstrap goals by loading any configured bootstrap BUILD files
with self.check_errors('The following bootstrap_buildfiles cannot be loaded:') as error:
with self.timer.timing('parse:bootstrap'):
for path in self.config.getlist('goals', 'bootstrap_buildfiles', default = []):
try:
buildfile = BuildFile(get_buildroot(), os.path.relpath(path, get_buildroot()))
ParseContext(buildfile).parse()
except (TypeError, ImportError, TaskError, GoalError):
error(path, include_traceback=True)
except (IOError, SyntaxError):
error(path)
# Bootstrap user goals by loading any BUILD files implied by targets
with self.check_errors('The following targets could not be loaded:') as error:
with self.timer.timing('parse:BUILD'):
for spec in specs:
self.parse_spec(error, spec)
self.phases = [Phase(goal) for goal in goals]
rcfiles = self.config.getdefault('rcfiles', type=list, default=[])
if rcfiles:
rcfile = RcFile(rcfiles, default_prepend=False, process_default=True)
# Break down the goals specified on the command line to the full set that will be run so we
# can apply default flags to inner goal nodes. Also break down goals by Task subclass and
# register the task class hierarchy fully qualified names so we can apply defaults to
# baseclasses.
all_goals = Phase.execution_order(Phase(goal) for goal in goals)
sections = OrderedSet()
for goal in all_goals:
sections.add(goal.name)
for clazz in goal.task_type.mro():
if clazz == Task:
break
sections.add('%s.%s' % (clazz.__module__, clazz.__name__))
augmented_args = rcfile.apply_defaults(sections, args)
if augmented_args != args:
del args[:]
args.extend(augmented_args)
print("(using pantsrc expansion: pants goal %s)" % ' '.join(augmented_args))
Phase.setup_parser(parser, args, self.phases)
def run(self, lock):
if self.options.dry_run:
print '****** Dry Run ******'
logger = None
if self.options.log or self.options.log_level:
from twitter.common.log import init
from twitter.common.log.options import LogOptions
LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper())
logdir = self.options.logdir or self.config.get('goals', 'logdir', default=None)
if logdir:
safe_mkdir(logdir)
LogOptions.set_log_dir(logdir)
init('goals')
else:
init()
logger = log
if self.options.recursive_directory:
log.warn('--all-recursive is deprecated, use a target spec with the form [dir]:: instead')
for dir in self.options.recursive_directory:
self.add_target_recursive(dir)
if self.options.target_directory:
log.warn('--all is deprecated, use a target spec with the form [dir]: instead')
for dir in self.options.target_directory:
self.add_target_directory(dir)
context = Context(
self.config,
self.options,
self.targets,
requested_goals=self.requested_goals,
lock=lock,
log=logger,
timer=self.timer if self.options.time else None)
unknown = []
for phase in self.phases:
if not phase.goals():
unknown.append(phase)
if unknown:
print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown))
print('')
return Phase.execute(context, 'goals')
if logger:
logger.debug('Operating on targets: %s', self.targets)
ret = Phase.attempt(context, self.phases)
if self.options.time:
print('Timing report')
print('=============')
self.timer.print_timings()
return ret
def cleanup(self):
# TODO: Make this more selective? Only kill nailguns that affect state? E.g., checkstyle
# may not need to be killed.
if NailgunTask.killall:
NailgunTask.killall(log)
sys.exit(1)
class Goal(Command):
"""Lists installed goals or else executes a named goal."""
__command__ = 'goal'
GLOBAL_OPTIONS = [
Option("-x", "--time", action="store_true", dest="time", default=False,
help="Times goal phases and outputs a report."),
Option("-v", "--log", action="store_true", dest="log", default=False,
help="[%default] Logs extra build output."),
Option("-d", "--logdir", dest="logdir",
help="[%default] Forks logs to files under this directory."),
Option("-l", "--level", dest="log_level", type="choice", choices=['debug', 'info', 'warn'],
help="[info] Sets the logging level to one of 'debug', 'info' or 'warn', implies -v "
"if set."),
Option("--read-from-artifact-cache", "--no-read-from-artifact-cache", action="callback",
callback=_set_bool, dest="read_from_artifact_cache", default=False,
help="Whether to read artifacts from cache instead of building them, when possible."),
Option("--write-to-artifact-cache", "--no-write-to-artifact-cache", action="callback",
callback=_set_bool, dest="write_to_artifact_cache", default=False,
help="Whether to write artifacts to cache ."),
Option("--verify-artifact-cache", "--no-verify-artifact-cache", action="callback",
callback=_set_bool, dest="verify_artifact_cache", default=False,
help="Whether to verify that cached artifacts are identical after rebuilding them."),
Option("--all", dest="target_directory", action="append",
help="DEPRECATED: Use [dir]: with no flag in a normal target position on the command "
"line. (Adds all targets found in the given directory's BUILD file. Can be "
"specified more than once.)"),
Option("--all-recursive", dest="recursive_directory", action="append",
help="DEPRECATED: Use [dir]:: with no flag in a normal target position on the command "
"line. (Adds all targets found recursively under the given directory. Can be "
"specified more than once to add more than one root target directory to scan.)"),
]
@staticmethod
def add_global_options(parser):
for option in Goal.GLOBAL_OPTIONS:
parser.add_option(option)
@staticmethod
def parse_args(args):
goals = OrderedSet()
specs = OrderedSet()
help = False
explicit_multi = False
def is_spec(spec):
return os.sep in spec or ':' in spec
for i, arg in enumerate(args):
help = help or 'help' == arg
if not arg.startswith('-'):
specs.add(arg) if is_spec(arg) else goals.add(arg)
elif '--' == arg:
if specs:
raise GoalError('Cannot intermix targets with goals when using --. Targets should '
'appear on the right')
explicit_multi = True
del args[i]
break
if explicit_multi:
spec_offset = len(goals) + 1 if help else len(goals)
specs.update(arg for arg in args[spec_offset:] if not arg.startswith('-'))
return goals, specs
# TODO(John Sirois): revisit wholesale locking when we move py support into pants new
@classmethod
def serialized(cls):
return True
def __init__(self, root_dir, parser, args):
self.targets = []
# Note that we can't gate this on the self.options.time flag, because self.options is
# only set up in Command.__init__, and only after it calls setup_parser(), which uses the timer.
self.timer = Timer()
Command.__init__(self, root_dir, parser, args)
@contextmanager
def check_errors(self, banner):
errors = {}
def error(key, include_traceback=False):
exc_type, exc_value, _ = sys.exc_info()
msg = StringIO()
if include_traceback:
frame = inspect.trace()[-1]
filename = frame[1]
lineno = frame[2]
funcname = frame[3]
code = ''.join(frame[4])
traceback.print_list([(filename, lineno, funcname, code)], file=msg)
if exc_type:
msg.write(''.join(traceback.format_exception_only(exc_type, exc_value)))
errors[key] = msg.getvalue()
sys.exc_clear()
yield error
if errors:
msg = StringIO()
msg.write(banner)
invalid_keys = [key for key, exc in errors.items() if not exc]
if invalid_keys:
msg.write('\n %s' % '\n '.join(invalid_keys))
for key, exc in errors.items():
if exc:
msg.write('\n %s =>\n %s' % (key, '\n '.join(exc.splitlines())))
# The help message for goal is extremely verbose, and will obscure the
# actual error message, so we don't show it in this case.
self.error(msg.getvalue(), show_help = False)
def add_targets(self, error, dir, buildfile):
try:
self.targets.extend(Target.get(addr) for addr in Target.get_all_addresses(buildfile))
except (TypeError, ImportError):
error(dir, include_traceback=True)
except (IOError, SyntaxError):
error(dir)
def get_dir(self, spec):
path = spec.split(':', 1)[0]
if os.path.isdir(path):
return path
else:
if os.path.isfile(path):
return os.path.dirname(path)
else:
return spec
def add_target_recursive(self, *specs):
with self.check_errors('There was a problem scanning the '
'following directories for targets:') as error:
for spec in specs:
dir = self.get_dir(spec)
for buildfile in BuildFile.scan_buildfiles(self.root_dir, dir):
self.add_targets(error, dir, buildfile)
def add_target_directory(self, *specs):
with self.check_errors("There was a problem loading targets "
"from the following directory's BUILD files") as error:
for spec in specs:
dir = self.get_dir(spec)
try:
self.add_targets(error, dir, BuildFile(self.root_dir, dir))
except IOError:
error(dir)
def parse_spec(self, error, spec):
if spec.endswith('::'):
self.add_target_recursive(spec[:-len('::')])
elif spec.endswith(':'):
self.add_target_directory(spec[:-len(':')])
else:
try:
address = Address.parse(get_buildroot(), spec)
ParseContext(address.buildfile).parse()
target = Target.get(address)
if target:
self.targets.append(target)
else:
siblings = Target.get_all_addresses(address.buildfile)
prompt = 'did you mean' if len(siblings) == 1 else 'maybe you meant one of these'
error('%s => %s?:\n %s' % (address, prompt,
'\n '.join(str(a) for a in siblings)))
except (TypeError, ImportError, TaskError, GoalError):
error(spec, include_traceback=True)
except (IOError, SyntaxError):
error(spec)
def setup_parser(self, parser, args):
self.config = Config.load()
Goal.add_global_options(parser)
# We support attempting zero or more goals. Multiple goals must be delimited from further
# options and non goal args with a '--'. The key permutations we need to support:
# ./pants goal => goals
# ./pants goal goals => goals
# ./pants goal compile src/java/... => compile
# ./pants goal compile -x src/java/... => compile
# ./pants goal compile src/java/... -x => compile
# ./pants goal compile run -- src/java/... => compile, run
# ./pants goal compile run -- src/java/... -x => compile, run
# ./pants goal compile run -- -x src/java/... => compile, run
if not args:
args.append('goals')
if len(args) == 1 and args[0] in set(['-h', '--help', 'help']):
def format_usage(usages):
left_colwidth = 0
for left, right in usages:
left_colwidth = max(left_colwidth, len(left))
lines = []
for left, right in usages:
lines.append(' %s%s%s' % (left, ' ' * (left_colwidth - len(left) + 1), right))
return '\n'.join(lines)
usages = [
("%prog goal goals ([spec]...)", Phase('goals').description),
("%prog goal help [goal] ([spec]...)", Phase('help').description),
("%prog goal [goal] [spec]...", "Attempt goal against one or more targets."),
("%prog goal [goal] ([goal]...) -- [spec]...", "Attempts all the specified goals."),
]
parser.set_usage("\n%s" % format_usage(usages))
parser.epilog = ("Either lists all installed goals, provides extra help for a goal or else "
"attempts to achieve the specified goal for the listed targets." """
Note that target specs accept two special forms:
[dir]: to include all targets in the specified directory
[dir]:: to include all targets found in all BUILD files recursively under
the directory""")
parser.print_help()
sys.exit(0)
else:
goals, specs = Goal.parse_args(args)
# TODO(John Sirois): kill PANTS_NEW and its usages when pants.new is rolled out
ParseContext.enable_pantsnew()
# Bootstrap goals by loading any configured bootstrap BUILD files
with self.check_errors('The following bootstrap_buildfiles cannot be loaded:') as error:
with self.timer.timing('parse:bootstrap'):
for path in self.config.getlist('goals', 'bootstrap_buildfiles', default = []):
try:
buildfile = BuildFile(get_buildroot(), os.path.relpath(path, get_buildroot()))
ParseContext(buildfile).parse()
except (TypeError, ImportError, TaskError, GoalError):
error(path, include_traceback=True)
except (IOError, SyntaxError):
error(path)
# Bootstrap user goals by loading any BUILD files implied by targets
with self.check_errors('The following targets could not be loaded:') as error:
with self.timer.timing('parse:BUILD'):
for spec in specs:
self.parse_spec(error, spec)
self.phases = [Phase(goal) for goal in goals]
rcfiles = self.config.getdefault('rcfiles', type=list, default=[])
if rcfiles:
rcfile = RcFile(rcfiles, default_prepend=False, process_default=True)
# Break down the goals specified on the command line to the full set that will be run so we
# can apply default flags to inner goal nodes. Also break down goals by Task subclass and
# register the task class hierarchy fully qualified names so we can apply defaults to
# baseclasses.
all_goals = Phase.execution_order(Phase(goal) for goal in goals)
sections = OrderedSet()
for goal in all_goals:
sections.add(goal.name)
for clazz in goal.task_type.mro():
if clazz == Task:
break
sections.add('%s.%s' % (clazz.__module__, clazz.__name__))
augmented_args = rcfile.apply_defaults(sections, args)
if augmented_args != args:
del args[:]
args.extend(augmented_args)
print("(using pantsrc expansion: pants goal %s)" % ' '.join(augmented_args))
Phase.setup_parser(parser, args, self.phases)
def run(self, lock):
with self.check_errors("Target contains a dependency cycle") as error:
with self.timer.timing('parse:check_cycles'):
for target in self.targets:
try:
InternalTarget.check_cycles(target)
except InternalTarget.CycleException as e:
error(target.id)
logger = None
if self.options.log or self.options.log_level:
from twitter.common.log import init
from twitter.common.log.options import LogOptions
LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper())
logdir = self.options.logdir or self.config.get('goals', 'logdir', default=None)
if logdir:
safe_mkdir(logdir)
LogOptions.set_log_dir(logdir)
init('goals')
else:
init()
logger = log
if self.options.recursive_directory:
log.warn('--all-recursive is deprecated, use a target spec with the form [dir]:: instead')
for dir in self.options.recursive_directory:
self.add_target_recursive(dir)
if self.options.target_directory:
log.warn('--all is deprecated, use a target spec with the form [dir]: instead')
for dir in self.options.target_directory:
self.add_target_directory(dir)
context = Context(self.config, self.options, self.targets, lock=lock, log=logger)
unknown = []
for phase in self.phases:
if not phase.goals():
unknown.append(phase)
if unknown:
print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown))
print('')
return Phase.execute(context, 'goals')
if logger:
logger.debug('Operating on targets: %s', self.targets)
ret = Phase.attempt(context, self.phases, timer=self.timer if self.options.time else None)
if self.options.time:
print('Timing report')
print('=============')
self.timer.print_timings()
return ret
def cleanup(self):
# TODO: Make this more selective? Only kill nailguns that affect state? E.g., checkstyle
# may not need to be killed.
if NailgunTask.killall:
NailgunTask.killall(log)
sys.exit(1)
