def run(self, lock):
if self.options.dry_run:
print "****** Dry Run ******"
logger = None
if self.options.log or self.options.log_level:
from twitter.common.log import init
from twitter.common.log.options import LogOptions
LogOptions.set_stderr_log_level((self.options.log_level or "info").upper())
logdir = self.options.logdir or self.config.get("goals", "logdir", default=None)
if logdir:
safe_mkdir(logdir)
LogOptions.set_log_dir(logdir)
init("goals")
else:
init()
logger = log
if self.options.recursive_directory:
log.warn("--all-recursive is deprecated, use a target spec with the form [dir]:: instead")
for dir in self.options.recursive_directory:
self.add_target_recursive(dir)
if self.options.target_directory:
log.warn("--all is deprecated, use a target spec with the form [dir]: instead")
for dir in self.options.target_directory:
self.add_target_directory(dir)
context = Context(
self.config,
self.options,
self.targets,
requested_goals=self.requested_goals,
lock=lock,
log=logger,
timer=self.timer if self.options.time else None,
)
unknown = []
for phase in self.phases:
if not phase.goals():
unknown.append(phase)
if unknown:
print ("Unknown goal(s): %s" % " ".join(phase.name for phase in unknown))
print ("")
return Phase.execute(context, "goals")
if logger:
logger.debug("Operating on targets: %s", self.targets)
ret = Phase.attempt(context, self.phases)
if self.options.time:
print ("Timing report")
print ("=============")
self.timer.print_timings()
return ret
def execute(cls, context, *names):
parser = OptionParser()
cls.add_global_options(parser)
phases = [Phase(name) for name in names]
Phase.setup_parser(parser, [], phases)
options, _ = parser.parse_args([])
context = Context(context.config, options, context.run_tracker, context.target_roots,
requested_goals=list(names))
return cls._execute(context, phases, print_timing=False)
def run(self, lock):
with self.check_errors("Target contains a dependency cycle") as error:
for target in self.targets:
try:
InternalTarget.check_cycles(target)
except InternalTarget.CycleException as e:
error(target.id)
timer = None
if self.options.time:
class Timer(object):
def now(self):
return time.time()
def log(self, message):
print(message)
timer = Timer()
logger = None
if self.options.log or self.options.log_level:
from twitter.common.log import init
from twitter.common.log.options import LogOptions
LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper())
logdir = self.options.logdir or self.config.get('goals', 'logdir', default=None)
if logdir:
safe_mkdir(logdir)
LogOptions.set_log_dir(logdir)
init('goals')
else:
init()
logger = log
if self.options.recursive_directory:
log.warn('--all-recursive is deprecated, use a target spec with the form [dir]:: instead')
for dir in self.options.recursive_directory:
self.add_target_recursive(dir)
if self.options.target_directory:
log.warn('--all is deprecated, use a target spec with the form [dir]: instead')
for dir in self.options.target_directory:
self.add_target_directory(dir)
context = Context(self.config, self.options, self.targets, lock=lock, log=logger)
unknown = []
for phase in self.phases:
if not phase.goals():
unknown.append(phase)
if unknown:
print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown))
print('')
return Phase.execute(context, 'goals')
if logger:
logger.debug('Operating on targets: %s', self.targets)
return Phase.attempt(context, self.phases, timer=timer)
def execute(cls, context, *names):
parser = OptionParser()
cls.add_global_options(parser)
phases = [Phase(name) for name in names]
Phase.setup_parser(parser, [], phases)
options, _ = parser.parse_args([])
context = Context(context.config, options, context.run_tracker, context.target_roots,
requested_goals=list(names))
return cls._execute(context, phases, print_timing=False)
def run(self, lock):
# Update the reporting settings, now that we have flags etc.
def is_console_task():
for phase in self.phases:
for goal in phase.goals():
if issubclass(goal.task_type, ConsoleTask):
return True
return False
update_reporting(self.options, is_console_task(), self.run_tracker)
if self.options.dry_run:
print '****** Dry Run ******'
context = Context(self.config,
self.options,
self.run_tracker,
self.targets,
requested_goals=self.requested_goals,
lock=lock)
# TODO: Time to get rid of this hack.
if self.options.recursive_directory:
context.log.warn(
'--all-recursive is deprecated, use a target spec with the form [dir]:: instead'
)
for dir in self.options.recursive_directory:
self.add_target_recursive(dir)
if self.options.target_directory:
context.log.warn(
'--all is deprecated, use a target spec with the form [dir]: instead'
)
for dir in self.options.target_directory:
self.add_target_directory(dir)
unknown = []
for phase in self.phases:
if not phase.goals():
unknown.append(phase)
if unknown:
print('Unknown goal(s): %s' % ' '.join(phase.name
for phase in unknown))
print('')
return Phase.execute(context, 'goals')
ret = Phase.attempt(context, self.phases)
if self.options.cleanup_nailguns or self.config.get(
'nailgun', 'autokill', default=False):
if log:
log.debug('auto-killing nailguns')
if NailgunTask.killall:
NailgunTask.killall(log)
return ret
def execute(self):
def add_targets(dir, buildfile):
try:
self.targets.extend(Target.get(addr) for addr in Target.get_all_addresses(buildfile))
except (TypeError, ImportError):
error(dir, include_traceback=True)
except (IOError, SyntaxError):
error(dir)
if self.options.recursive_directory:
with self.check_errors('There was a problem scanning the '
'following directories for targets:') as error:
for dir in self.options.recursive_directory:
for buildfile in BuildFile.scan_buildfiles(self.root_dir, dir):
add_targets(dir, buildfile)
if self.options.target_directory:
with self.check_errors("There was a problem loading targets "
"from the following directory's BUILD files") as error:
for dir in self.options.target_directory:
add_targets(dir, BuildFile(self.root_dir, dir))
timer = None
if self.options.time:
class Timer(object):
def now(self):
return time.time()
def log(self, message):
print(message)
timer = Timer()
logger = None
if self.options.log or self.options.log_level:
from twitter.common.log import init
from twitter.common.log.options import LogOptions
LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper())
logdir = self.config.get('goals', 'logdir')
if logdir:
safe_mkdir(logdir)
LogOptions.set_log_dir(logdir)
init('goals')
logger = log
context = Context(self.config, self.options, self.targets, log=logger)
unknown = []
for phase in self.phases:
if not phase.goals():
unknown.append(phase)
if unknown:
print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown))
print()
return Phase.execute(context, 'goals')
return Phase.attempt(context, self.phases, timer=timer)
def run(self, lock):
with self.check_errors("Target contains a dependency cycle") as error:
with self.timer.timing("parse:check_cycles"):
for target in self.targets:
try:
InternalTarget.check_cycles(target)
except InternalTarget.CycleException as e:
error(target.id)
logger = None
if self.options.log or self.options.log_level:
from twitter.common.log import init
from twitter.common.log.options import LogOptions
LogOptions.set_stderr_log_level((self.options.log_level or "info").upper())
logdir = self.options.logdir or self.config.get("goals", "logdir", default=None)
if logdir:
safe_mkdir(logdir)
LogOptions.set_log_dir(logdir)
init("goals")
else:
init()
logger = log
if self.options.recursive_directory:
log.warn("--all-recursive is deprecated, use a target spec with the form [dir]:: instead")
for dir in self.options.recursive_directory:
self.add_target_recursive(dir)
if self.options.target_directory:
log.warn("--all is deprecated, use a target spec with the form [dir]: instead")
for dir in self.options.target_directory:
self.add_target_directory(dir)
context = Context(self.config, self.options, self.targets, lock=lock, log=logger)
unknown = []
for phase in self.phases:
if not phase.goals():
unknown.append(phase)
if unknown:
print("Unknown goal(s): %s" % " ".join(phase.name for phase in unknown))
print("")
return Phase.execute(context, "goals")
if logger:
logger.debug("Operating on targets: %s", self.targets)
ret = Phase.attempt(context, self.phases, timer=self.timer if self.options.time else None)
if self.options.time:
print("Timing report")
print("=============")
self.timer.print_timings()
return ret
def run(self, lock):
if self.options.dry_run:
print '****** Dry Run ******'
logger = None
if self.options.log or self.options.log_level:
from twitter.common.log import init
from twitter.common.log.options import LogOptions
LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper())
logdir = self.options.logdir or self.config.get('goals', 'logdir', default=None)
if logdir:
safe_mkdir(logdir)
LogOptions.set_log_dir(logdir)
init('goals')
else:
init()
logger = log
if self.options.recursive_directory:
log.warn('--all-recursive is deprecated, use a target spec with the form [dir]:: instead')
for dir in self.options.recursive_directory:
self.add_target_recursive(dir)
if self.options.target_directory:
log.warn('--all is deprecated, use a target spec with the form [dir]: instead')
for dir in self.options.target_directory:
self.add_target_directory(dir)
context = Context(
self.config,
self.options,
self.targets,
lock=lock,
log=logger,
timer=self.timer if self.options.time else None)
unknown = []
for phase in self.phases:
if not phase.goals():
unknown.append(phase)
if unknown:
print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown))
print('')
return Phase.execute(context, 'goals')
if logger:
logger.debug('Operating on targets: %s', self.targets)
ret = Phase.attempt(context, self.phases)
if self.options.time:
print('Timing report')
print('=============')
self.timer.print_timings()
return ret
def run(self, lock):
with self.check_errors("Target contains a dependency cycle") as error:
with self.timer.timing('parse:check_cycles'):
for target in self.targets:
try:
InternalTarget.check_cycles(target)
except InternalTarget.CycleException as e:
error(target.id)
logger = None
if self.options.log or self.options.log_level:
from twitter.common.log import init
from twitter.common.log.options import LogOptions
LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper())
logdir = self.options.logdir or self.config.get('goals', 'logdir', default=None)
if logdir:
safe_mkdir(logdir)
LogOptions.set_log_dir(logdir)
init('goals')
else:
init()
logger = log
if self.options.recursive_directory:
log.warn('--all-recursive is deprecated, use a target spec with the form [dir]:: instead')
for dir in self.options.recursive_directory:
self.add_target_recursive(dir)
if self.options.target_directory:
log.warn('--all is deprecated, use a target spec with the form [dir]: instead')
for dir in self.options.target_directory:
self.add_target_directory(dir)
context = Context(self.config, self.options, self.targets, lock=lock, log=logger)
unknown = []
for phase in self.phases:
if not phase.goals():
unknown.append(phase)
if unknown:
print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown))
print('')
return Phase.execute(context, 'goals')
if logger:
logger.debug('Operating on targets: %s', self.targets)
ret = Phase.attempt(context, self.phases, timer=self.timer if self.options.time else None)
if self.options.time:
print('Timing report')
print('=============')
self.timer.print_timings()
return ret
def run(self, lock):
# Update the reporting settings, now that we have flags etc.
def is_console_task():
for phase in self.phases:
for goal in phase.goals():
if issubclass(goal.task_type, ConsoleTask):
return True
return False
update_reporting(self.options, is_console_task(), self.run_tracker)
if self.options.dry_run:
print '****** Dry Run ******'
context = Context(
self.config,
self.options,
self.run_tracker,
self.targets,
requested_goals=self.requested_goals,
lock=lock)
# TODO: Time to get rid of this hack.
if self.options.recursive_directory:
context.log.warn(
'--all-recursive is deprecated, use a target spec with the form [dir]:: instead')
for dir in self.options.recursive_directory:
self.add_target_recursive(dir)
if self.options.target_directory:
context.log.warn('--all is deprecated, use a target spec with the form [dir]: instead')
for dir in self.options.target_directory:
self.add_target_directory(dir)
unknown = []
for phase in self.phases:
if not phase.goals():
unknown.append(phase)
if unknown:
print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown))
print('')
return Phase.execute(context, 'goals')
ret = Phase.attempt(context, self.phases)
if self.options.cleanup_nailguns or self.config.get('nailgun', 'autokill', default = False):
if log:
log.debug('auto-killing nailguns')
if NailgunTask.killall:
NailgunTask.killall(log)
return ret
def execute(self, targets):
goal = self.context.options.help_goal
if goal is None:
return self.list_goals('You must supply a goal name to provide help for.')
phase = Phase(goal)
if not phase.goals():
return self.list_goals('Goal %s is unknown.' % goal)
parser = Help.parser()
parser.set_usage('%s goal %s ([target]...)' % (sys.argv[0], goal))
parser.epilog = phase.description
Phase.setup_parser(parser, [], [phase])
parser.parse_args(['--help'])
def execute(self, targets):
goal = self.context.options.help_goal
if goal is None:
return self.list_goals('You must supply a goal name to provide help for.')
phase = Phase(goal)
if not phase.goals():
return self.list_goals('Goal %s is unknown.' % goal)
parser = OptionParser()
parser.set_usage('%s goal %s ([target]...)' % (sys.argv[0], goal))
parser.epilog = phase.description
Goal.add_global_options(parser)
Phase.setup_parser(parser, [], [phase])
parser.parse_args(['--help'])
def execute(self, targets):
goal = self.context.options.help_goal
if goal is None:
return self.list_goals("You must supply a goal name to provide help for.")
phase = Phase(goal)
if not phase.goals():
return self.list_goals("Goal %s is unknown." % goal)
parser = OptionParser()
parser.set_usage("%s goal %s ([target]...)" % (sys.argv[0], goal))
parser.epilog = phase.description
Goal.add_global_options(parser)
Phase.setup_parser(parser, [], [phase])
parser.parse_args(["--help"])
def graph():
def get_cluster_name(phase):
return 'cluster_%s' % phase.name.replace('-', '_')
def get_goal_name(phase, goal):
name = '%s_%s' % (phase.name, goal.name)
return name.replace('-', '_')
phase_by_phasename = {}
for phase, goals in Phase.all():
phase_by_phasename[phase.name] = phase
yield '\n'.join([
'digraph G {',
' rankdir=LR;',
' graph [compound=true];',
])
for phase, installed_goals in Phase.all():
yield '\n'.join([
' subgraph %s {' % get_cluster_name(phase),
' node [style=filled];',
' color = blue;',
' label = "%s";' % phase.name,
])
for installed_goal in installed_goals:
yield ' %s [label="%s"];' % (get_goal_name(phase, installed_goal),
installed_goal.name)
yield ' }'
edges = set()
for phase, installed_goals in Phase.all():
for installed_goal in installed_goals:
for dependency in installed_goal.dependencies:
tail_goal = phase_by_phasename.get(dependency.name).goals()[-1]
edge = 'ltail=%s lhead=%s' % (get_cluster_name(phase),
get_cluster_name(Phase.of(tail_goal)))
if edge not in edges:
yield ' %s -> %s [%s];' % (get_goal_name(phase, installed_goal),
get_goal_name(Phase.of(tail_goal), tail_goal),
edge)
edges.add(edge)
yield '}'
def _record(self, goal, elapsed):
phase = Phase.of(goal)
phase_timings = self._timings.get(phase)
if phase_timings is None:
phase_timings = OrderedDict(())
self._timings[phase] = phase_timings
goal_timings = phase_timings.get(goal)
if goal_timings is None:
goal_timings = []
phase_timings[goal] = goal_timings
goal_timings.append(elapsed)
def execute(self, targets):
print('Installed goals:')
documented_rows = []
undocumented = []
max_width = 0
for phase, _ in Phase.all():
if phase.description:
documented_rows.append((phase.name, phase.description))
max_width = max(max_width, len(phase.name))
elif self.context.options.goal_list_all:
undocumented.append(phase.name)
for name, description in documented_rows:
print(' %s: %s' % (name.rjust(max_width), description))
if undocumented:
print('\nUndocumented goals:\n %s' % ' '.join(undocumented))
def execute(self, targets):
print('Installed goals:')
documented_rows = []
undocumented = []
max_width = 0
for phase, _ in Phase.all():
if phase.description:
documented_rows.append((phase.name, phase.description))
max_width = max(max_width, len(phase.name))
elif self.context.options.goal_list_all:
undocumented.append(phase.name)
for name, description in documented_rows:
print(' %s: %s' % (name.rjust(max_width), description))
if undocumented:
print('\nUndocumented goals:\n %s' % ' '.join(undocumented))
def console_output(self, targets):
yield "Installed goals:"
documented_rows = []
undocumented = []
max_width = 0
for phase, _ in Phase.all():
if phase.description:
documented_rows.append((phase.name, phase.description))
max_width = max(max_width, len(phase.name))
elif self.context.options.goal_list_all:
undocumented.append(phase.name)
for name, description in documented_rows:
yield " %s: %s" % (name.rjust(max_width), description)
if undocumented:
yield ""
yield "Undocumented goals:"
yield " %s" % " ".join(undocumented)
def report():
yield 'Installed goals:'
documented_rows = []
undocumented = []
max_width = 0
for phase, _ in Phase.all():
if phase.description:
documented_rows.append((phase.name, phase.description))
max_width = max(max_width, len(phase.name))
elif self.context.options.goal_list_all:
undocumented.append(phase.name)
for name, description in documented_rows:
yield ' %s: %s' % (name.rjust(max_width), description)
if undocumented:
yield ''
yield 'Undocumented goals:'
yield ' %s' % ' '.join(undocumented)
def console_output(self, targets):
yield 'Installed goals:'
documented_rows = []
undocumented = []
max_width = 0
for phase, _ in Phase.all():
if phase.description:
documented_rows.append((phase.name, phase.description))
max_width = max(max_width, len(phase.name))
elif self.context.options.goal_list_all:
undocumented.append(phase.name)
for name, description in documented_rows:
yield ' %s: %s' % (name.rjust(max_width), description)
if undocumented:
yield ''
yield 'Undocumented goals:'
yield ' %s' % ' '.join(undocumented)
def run(self, lock):
# Update the reporting settings, now that we have flags etc.
log_level = Report.log_level_from_string(self.options.log_level or 'info')
color = not self.options.no_color
timing = self.options.time
cache_stats = self.options.time # TODO: Separate flag for this?
settings_updates_map = {
'console': {
'log_level': log_level, 'color': color, 'timing': timing, 'cache_stats': cache_stats
},
'html': {
'log_level': log_level
}
}
self.run_tracker.update_report_settings(settings_updates_map)
# TODO: Do something useful with --logdir.
if self.options.dry_run:
print '****** Dry Run ******'
context = Context(
self.config,
self.options,
self.run_tracker,
self.targets,
requested_goals=self.requested_goals,
lock=lock)
# TODO: Time to get rid of this hack.
if self.options.recursive_directory:
context.log.warn(
'--all-recursive is deprecated, use a target spec with the form [dir]:: instead')
for dir in self.options.recursive_directory:
self.add_target_recursive(dir)
if self.options.target_directory:
context.log.warn('--all is deprecated, use a target spec with the form [dir]: instead')
for dir in self.options.target_directory:
self.add_target_directory(dir)
unknown = []
for phase in self.phases:
if not phase.goals():
unknown.append(phase)
if unknown:
print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown))
print('')
return Phase.execute(context, 'goals')
ret = Phase.attempt(context, self.phases)
if self.options.cleanup_nailguns or self.config.get('nailgun', 'autokill', default = False):
if log:
log.debug('auto-killing nailguns')
if NailgunTask.killall:
NailgunTask.killall(log)
return ret
def setup_parser(self, parser, args):
self.config = Config.load()
Goal.add_global_options(parser)
# We support attempting zero or more goals. Multiple goals must be delimited from further
# options and non goal args with a '--'. The key permutations we need to support:
# ./pants goal => goals
# ./pants goal goals => goals
# ./pants goal compile src/java/... => compile
# ./pants goal compile -x src/java/... => compile
# ./pants goal compile src/java/... -x => compile
# ./pants goal compile run -- src/java/... => compile, run
# ./pants goal compile run -- src/java/... -x => compile, run
# ./pants goal compile run -- -x src/java/... => compile, run
if not args:
args.append('goals')
if len(args) == 1 and args[0] in set(['-h', '--help', 'help']):
def format_usage(usages):
left_colwidth = 0
for left, right in usages:
left_colwidth = max(left_colwidth, len(left))
lines = []
for left, right in usages:
lines.append(' %s%s%s' % (left, ' ' * (left_colwidth - len(left) + 1), right))
return '\n'.join(lines)
usages = [
("%prog goal goals ([spec]...)", Phase('goals').description),
("%prog goal help [goal] ([spec]...)", Phase('help').description),
("%prog goal [goal] [spec]...", "Attempt goal against one or more targets."),
("%prog goal [goal] ([goal]...) -- [spec]...", "Attempts all the specified goals."),
]
parser.set_usage("\n%s" % format_usage(usages))
parser.epilog = ("Either lists all installed goals, provides extra help for a goal or else "
"attempts to achieve the specified goal for the listed targets." """
Note that target specs accept two special forms:
[dir]: to include all targets in the specified directory
[dir]:: to include all targets found in all BUILD files recursively under
the directory""")
parser.print_help()
sys.exit(0)
else:
goals, specs = Goal.parse_args(args)
# TODO(John Sirois): kill PANTS_NEW and its usages when pants.new is rolled out
ParseContext.enable_pantsnew()
# Bootstrap goals by loading any configured bootstrap BUILD files
with self.check_errors('The following bootstrap_buildfiles cannot be loaded:') as error:
for path in self.config.getlist('goals', 'bootstrap_buildfiles', default = []):
try:
buildfile = BuildFile(get_buildroot(), os.path.relpath(path, get_buildroot()))
ParseContext(buildfile).parse()
except (TypeError, ImportError, TaskError, GoalError):
error(path, include_traceback=True)
except (IOError, SyntaxError):
error(path)
# Bootstrap user goals by loading any BUILD files implied by targets
with self.check_errors('The following targets could not be loaded:') as error:
for spec in specs:
self.parse_spec(error, spec)
self.phases = [Phase(goal) for goal in goals]
rcfiles = self.config.getdefault('rcfiles', type=list, default=[])
if rcfiles:
rcfile = RcFile(rcfiles, default_prepend=False, process_default=True)
# Break down the goals specified on the command line to the full set that will be run so we
# can apply default flags to inner goal nodes. Also break down goals by Task subclass and
# register the task class hierarchy fully qualified names so we can apply defaults to
# baseclasses.
all_goals = Phase.execution_order(Phase(goal) for goal in goals)
sections = OrderedSet()
for goal in all_goals:
sections.add(goal.name)
for clazz in goal.task_type.mro():
if clazz == Task:
break
sections.add('%s.%s' % (clazz.__module__, clazz.__name__))
augmented_args = rcfile.apply_defaults(sections, args)
if augmented_args != args:
del args[:]
args.extend(augmented_args)
print("(using pantsrc expansion: pants goal %s)" % ' '.join(augmented_args))
Phase.setup_parser(parser, args, self.phases)
def setup_parser(self, parser, args):
self.config = Config.load()
Goal.add_global_options(parser)
# We support attempting zero or more goals. Multiple goals must be delimited from further
# options and non goal args with a '--'. The key permutations we need to support:
# ./pants goal => goals
# ./pants goal goals => goals
# ./pants goal compile src/java/... => compile
# ./pants goal compile -x src/java/... => compile
# ./pants goal compile src/java/... -x => compile
# ./pants goal compile run -- src/java/... => compile, run
# ./pants goal compile run -- src/java/... -x => compile, run
# ./pants goal compile run -- -x src/java/... => compile, run
if not args:
args.append('goals')
if len(args) == 1 and args[0] in set(['-h', '--help', 'help']):
def format_usage(usages):
left_colwidth = 0
for left, right in usages:
left_colwidth = max(left_colwidth, len(left))
lines = []
for left, right in usages:
lines.append(' %s%s%s' % (left, ' ' * (left_colwidth - len(left) + 1), right))
return '\n'.join(lines)
usages = [
("%prog goal goals ([spec]...)", Phase('goals').description),
("%prog goal help [goal] ([spec]...)", Phase('help').description),
("%prog goal [goal] [spec]...", "Attempt goal against one or more targets."),
("%prog goal [goal] ([goal]...) -- [spec]...", "Attempts all the specified goals."),
]
parser.set_usage("\n%s" % format_usage(usages))
parser.epilog = ("Either lists all installed goals, provides extra help for a goal or else "
"attempts to achieve the specified goal for the listed targets." """
Note that target specs accept two special forms:
[dir]: to include all targets in the specified directory
[dir]:: to include all targets found in all BUILD files recursively under
the directory""")
parser.print_help()
sys.exit(0)
else:
goals, specs = Goal.parse_args(args)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup', labels=[WorkUnit.SETUP]):
# Bootstrap goals by loading any configured bootstrap BUILD files
with self.check_errors('The following bootstrap_buildfiles cannot be loaded:') as error:
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
for path in self.config.getlist('goals', 'bootstrap_buildfiles', default = []):
try:
buildfile = BuildFile(get_buildroot(), os.path.relpath(path, get_buildroot()))
ParseContext(buildfile).parse()
except (TypeError, ImportError, TaskError, GoalError):
error(path, include_traceback=True)
except (IOError, SyntaxError):
error(path)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
# Bootstrap user goals by loading any BUILD files implied by targets.
spec_parser = SpecParser(self.root_dir)
with self.check_errors('The following targets could not be loaded:') as error:
with self.run_tracker.new_workunit(name='parse', labels=[WorkUnit.SETUP]):
for spec in specs:
try:
for target, address in spec_parser.parse(spec):
if target:
self.targets.append(target)
# Force early BUILD file loading if this target is an alias that expands
# to others.
unused = list(target.resolve())
else:
siblings = Target.get_all_addresses(address.buildfile)
prompt = 'did you mean' if len(siblings) == 1 else 'maybe you meant one of these'
error('%s => %s?:\n %s' % (address, prompt,
'\n '.join(str(a) for a in siblings)))
except (TypeError, ImportError, TaskError, GoalError):
error(spec, include_traceback=True)
except (IOError, SyntaxError, TargetDefinitionException):
error(spec)
self.phases = [Phase(goal) for goal in goals]
rcfiles = self.config.getdefault('rcfiles', type=list,
default=['/etc/pantsrc', '~/.pants.rc'])
if rcfiles:
rcfile = RcFile(rcfiles, default_prepend=False, process_default=True)
# Break down the goals specified on the command line to the full set that will be run so we
# can apply default flags to inner goal nodes. Also break down goals by Task subclass and
# register the task class hierarchy fully qualified names so we can apply defaults to
# baseclasses.
sections = OrderedSet()
for phase in Engine.execution_order(self.phases):
for goal in phase.goals():
sections.add(goal.name)
for clazz in goal.task_type.mro():
if clazz == Task:
break
sections.add('%s.%s' % (clazz.__module__, clazz.__name__))
augmented_args = rcfile.apply_defaults(sections, args)
if augmented_args != args:
del args[:]
args.extend(augmented_args)
sys.stderr.write("(using pantsrc expansion: pants goal %s)\n" % ' '.join(augmented_args))
Phase.setup_parser(parser, args, self.phases)
def run(self, lock):
timer = None
if self.options.time:
class Timer(object):
def now(self):
return time.time()
def log(self, message):
print(message)
timer = Timer()
logger = None
if self.options.log or self.options.log_level:
from twitter.common.log import init
from twitter.common.log.options import LogOptions
LogOptions.set_stderr_log_level((self.options.log_level
or 'info').upper())
logdir = self.options.logdir or self.config.get(
'goals', 'logdir', default=None)
if logdir:
safe_mkdir(logdir)
LogOptions.set_log_dir(logdir)
init('goals')
else:
init()
logger = log
if self.options.recursive_directory:
log.warn(
'--all-recursive is deprecated, use a target spec with the form [dir]:: instead'
)
for dir in self.options.recursive_directory:
self.add_target_recursive(dir)
if self.options.target_directory:
log.warn(
'--all is deprecated, use a target spec with the form [dir]: instead'
)
for dir in self.options.target_directory:
self.add_target_directory(dir)
context = Context(self.config,
self.options,
self.targets,
lock=lock,
log=logger)
unknown = []
for phase in self.phases:
if not phase.goals():
unknown.append(phase)
if unknown:
print('Unknown goal(s): %s' % ' '.join(phase.name
for phase in unknown))
print('')
return Phase.execute(context, 'goals')
if logger:
logger.debug('Operating on targets: %s', self.targets)
return Phase.attempt(context, self.phases, timer=timer)
def _list_goals(context, message): """Show all installed goals.""" context.log.error(message) # Execute as if the user had run "./pants goals". return Phase.execute(context, 'goals')
def setup_parser(self, parser, args):
self.config = Config.load()
Goal.add_global_options(parser)
# We support attempting zero or more goals. Multiple goals must be delimited from further
# options and non goal args with a '--'. The key permutations we need to support:
# ./pants goal => goals
# ./pants goal goals => goals
# ./pants goal compile src/java/... => compile
# ./pants goal compile -x src/java/... => compile
# ./pants goal compile src/java/... -x => compile
# ./pants goal compile run -- src/java/... => compile, run
# ./pants goal compile run -- src/java/... -x => compile, run
# ./pants goal compile run -- -x src/java/... => compile, run
if not args:
args.append('goals')
if len(args) == 1 and args[0] in set(['-h', '--help', 'help']):
def format_usage(usages):
left_colwidth = 0
for left, right in usages:
left_colwidth = max(left_colwidth, len(left))
lines = []
for left, right in usages:
lines.append(' %s%s%s' %
(left, ' ' *
(left_colwidth - len(left) + 1), right))
return '\n'.join(lines)
usages = [
("%prog goal goals ([spec]...)", Phase('goals').description),
("%prog goal help [goal] ([spec]...)",
Phase('help').description),
("%prog goal [goal] [spec]...",
"Attempt goal against one or more targets."),
("%prog goal [goal] ([goal]...) -- [spec]...",
"Attempts all the specified goals."),
]
parser.set_usage("\n%s" % format_usage(usages))
parser.epilog = (
"Either lists all installed goals, provides extra help for a goal or else "
"attempts to achieve the specified goal for the listed targets."
"""
Note that target specs accept two special forms:
[dir]: to include all targets in the specified directory
[dir]:: to include all targets found in all BUILD files recursively under
the directory""")
parser.print_help()
sys.exit(0)
else:
goals, specs = Goal.parse_args(args)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup',
labels=[WorkUnit.SETUP]):
# Bootstrap goals by loading any configured bootstrap BUILD files
with self.check_errors(
'The following bootstrap_buildfiles cannot be loaded:'
) as error:
with self.run_tracker.new_workunit(name='bootstrap',
labels=[WorkUnit.SETUP
]):
for path in self.config.getlist('goals',
'bootstrap_buildfiles',
default=[]):
try:
buildfile = BuildFile(
get_buildroot(),
os.path.relpath(path, get_buildroot()))
ParseContext(buildfile).parse()
except (TypeError, ImportError, TaskError,
GoalError):
error(path, include_traceback=True)
except (IOError, SyntaxError):
error(path)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
# Bootstrap user goals by loading any BUILD files implied by targets.
spec_parser = SpecParser(self.root_dir)
with self.check_errors(
'The following targets could not be loaded:') as error:
with self.run_tracker.new_workunit(name='parse',
labels=[WorkUnit.SETUP
]):
for spec in specs:
try:
for target, address in spec_parser.parse(spec):
if target:
self.targets.append(target)
# Force early BUILD file loading if this target is an alias that expands
# to others.
unused = list(target.resolve())
else:
siblings = Target.get_all_addresses(
address.buildfile)
prompt = 'did you mean' if len(
siblings
) == 1 else 'maybe you meant one of these'
error('%s => %s?:\n %s' %
(address, prompt, '\n '.join(
str(a) for a in siblings)))
except (TypeError, ImportError, TaskError,
GoalError):
error(spec, include_traceback=True)
except (IOError, SyntaxError,
TargetDefinitionException):
error(spec)
self.phases = [Phase(goal) for goal in goals]
rcfiles = self.config.getdefault('rcfiles', type=list, default=[])
if rcfiles:
rcfile = RcFile(rcfiles,
default_prepend=False,
process_default=True)
# Break down the goals specified on the command line to the full set that will be run so we
# can apply default flags to inner goal nodes. Also break down goals by Task subclass and
# register the task class hierarchy fully qualified names so we can apply defaults to
# baseclasses.
sections = OrderedSet()
for phase in Engine.execution_order(self.phases):
for goal in phase.goals():
sections.add(goal.name)
for clazz in goal.task_type.mro():
if clazz == Task:
break
sections.add('%s.%s' %
(clazz.__module__, clazz.__name__))
augmented_args = rcfile.apply_defaults(sections, args)
if augmented_args != args:
del args[:]
args.extend(augmented_args)
sys.stderr.write(
"(using pantsrc expansion: pants goal %s)\n" %
' '.join(augmented_args))
Phase.setup_parser(parser, args, self.phases)
def setup_parser(self, parser, args):
self.config = Config.load()
parser.add_option("-x", "--time", action="store_true", dest = "time", default = False,
help = "Times goal phases and outputs a report.")
parser.add_option("-v", "--log", action="store_true", dest = "log", default = False,
help = "[%default] Logs extra build output.")
parser.add_option("-l", "--level", dest = "log_level",
type="choice", choices=['debug', 'info', 'warn'],
help = "[info] Sets the logging level to one of 'debug', 'info' or 'warn', "
"implies -v if set.")
parser.add_option("--all", dest="target_directory", action="append",
help = "Adds all targets found in the given directory's BUILD file. Can "
"be specified more than once.")
parser.add_option("--all-recursive", dest="recursive_directory", action="append",
help = "Adds all targets found recursively under the given directory. Can "
"be specified more than once to add more than one root target "
"directory to scan.")
# We support attempting zero or more goals. Multiple goals must be delimited from further
# options and non goal args with a '--'. The key permutations we need to support:
# ./pants goal => goals
# ./pants goal goals => goals
# ./pants goal compile src/java/... => compile
# ./pants goal compile -x src/java/... => compile
# ./pants goal compile src/java/... -x => compile
# ./pants goal compile run -- src/java/... => compile, run
# ./pants goal compile run -- src/java/... -x => compile, run
# ./pants goal compile run -- -x src/java/... => compile, run
if not args:
args.append('goals')
if len(args) == 1 and args[0] in set(['-h', '--help', 'help']):
def format_usage(usages):
left_colwidth = 0
for left, right in usages:
left_colwidth = max(left_colwidth, len(left))
lines = []
for left, right in usages:
lines.append(' %s%s%s' % (left, ' ' * (left_colwidth - len(left) + 1), right))
return '\n'.join(lines)
usages = [
("%prog goal goals ([spec]...)", Phase('goals').description),
("%prog goal help [goal] ([spec]...)", Phase('help').description),
("%prog goal [goal] [spec]...", "Attempt goal against one or more targets."),
("%prog goal [goal] ([goal]...) -- [spec]...", "Attempts all the specified goals."),
]
parser.set_usage("\n%s" % format_usage(usages))
parser.epilog = ("Either lists all installed goals, provides extra help for a goal or else "
"attempts to achieve the specified goal for the listed targets.")
parser.print_help()
sys.exit(0)
else:
goals = []
help = False
multi = False
for i, arg in enumerate(args):
help = help or 'help' == arg
goals.append(arg)
if '--' == arg:
multi = True
del args[i]
goals.pop()
break
if arg.startswith('-'):
break
if not multi:
goals = [goals[0]]
spec_offset = len(goals) + 1 if help else len(goals)
specs = [arg for arg in args[spec_offset:] if not arg.startswith('-')]
def parse_build(buildfile):
# TODO(John Sirois): kill PANTS_NEW and its usages when pants.new is rolled out
ParseContext(buildfile).parse(PANTS_NEW=True)
# Bootstrap goals by loading any configured bootstrap BUILD files
with self.check_errors('The following bootstrap_buildfiles cannot be loaded:') as error:
for path in self.config.getlist('goals', 'bootstrap_buildfiles', default = []):
try:
buildfile = BuildFile(get_buildroot(), os.path.relpath(path, get_buildroot()))
parse_build(buildfile)
except (TypeError, ImportError, TaskError, GoalError):
error(path, include_traceback=True)
except (IOError, SyntaxError):
error(path)
# Bootstrap user goals by loading any BUILD files implied by targets
self.targets = []
with self.check_errors('The following targets could not be loaded:') as error:
for spec in specs:
try:
address = Address.parse(get_buildroot(), spec)
parse_build(address.buildfile)
target = Target.get(address)
if target:
self.targets.append(target)
else:
siblings = Target.get_all_addresses(address.buildfile)
prompt = 'did you mean' if len(siblings) == 1 else 'maybe you meant one of these'
error('%s => %s?:\n %s' % (address, prompt,
'\n '.join(str(a) for a in siblings)))
except (TypeError, ImportError, TaskError, GoalError):
error(spec, include_traceback=True)
except (IOError, SyntaxError):
error(spec)
self.phases = [Phase(goal) for goal in goals]
Phase.setup_parser(parser, args, self.phases)
def setup_parser(self, parser, args):
self.config = Config.load()
Goal.add_global_options(parser)
# We support attempting zero or more goals. Multiple goals must be delimited from further
# options and non goal args with a '--'. The key permutations we need to support:
# ./pants goal => goals
# ./pants goal goals => goals
# ./pants goal compile src/java/... => compile
# ./pants goal compile -x src/java/... => compile
# ./pants goal compile src/java/... -x => compile
# ./pants goal compile run -- src/java/... => compile, run
# ./pants goal compile run -- src/java/... -x => compile, run
# ./pants goal compile run -- -x src/java/... => compile, run
if not args:
args.append('goals')
if len(args) == 1 and args[0] in set(['-h', '--help', 'help']):
def format_usage(usages):
left_colwidth = 0
for left, right in usages:
left_colwidth = max(left_colwidth, len(left))
lines = []
for left, right in usages:
lines.append(' %s%s%s' % (left, ' ' * (left_colwidth - len(left) + 1), right))
return '\n'.join(lines)
usages = [
("%prog goal goals ([spec]...)", Phase('goals').description),
("%prog goal help [goal] ([spec]...)", Phase('help').description),
("%prog goal [goal] [spec]...", "Attempt goal against one or more targets."),
("%prog goal [goal] ([goal]...) -- [spec]...", "Attempts all the specified goals."),
]
parser.set_usage("\n%s" % format_usage(usages))
parser.epilog = ("Either lists all installed goals, provides extra help for a goal or else "
"attempts to achieve the specified goal for the listed targets." """
Note that target specs accept two special forms:
[dir]: to include all targets in the specified directory
[dir]:: to include all targets found in all BUILD files recursively under
the directory""")
parser.print_help()
sys.exit(0)
else:
goals, specs = Goal.parse_args(args)
# TODO(John Sirois): kill PANTS_NEW and its usages when pants.new is rolled out
ParseContext.enable_pantsnew()
# Bootstrap goals by loading any configured bootstrap BUILD files
with self.check_errors('The following bootstrap_buildfiles cannot be loaded:') as error:
with self.timer.timing('parse:bootstrap'):
for path in self.config.getlist('goals', 'bootstrap_buildfiles', default = []):
try:
buildfile = BuildFile(get_buildroot(), os.path.relpath(path, get_buildroot()))
ParseContext(buildfile).parse()
except (TypeError, ImportError, TaskError, GoalError):
error(path, include_traceback=True)
except (IOError, SyntaxError):
error(path)
# Bootstrap user goals by loading any BUILD files implied by targets
with self.check_errors('The following targets could not be loaded:') as error:
with self.timer.timing('parse:BUILD'):
for spec in specs:
self.parse_spec(error, spec)
self.phases = [Phase(goal) for goal in goals]
rcfiles = self.config.getdefault('rcfiles', type=list, default=[])
if rcfiles:
rcfile = RcFile(rcfiles, default_prepend=False, process_default=True)
# Break down the goals specified on the command line to the full set that will be run so we
# can apply default flags to inner goal nodes. Also break down goals by Task subclass and
# register the task class hierarchy fully qualified names so we can apply defaults to
# baseclasses.
all_goals = Phase.execution_order(Phase(goal) for goal in goals)
sections = OrderedSet()
for goal in all_goals:
sections.add(goal.name)
for clazz in goal.task_type.mro():
if clazz == Task:
break
sections.add('%s.%s' % (clazz.__module__, clazz.__name__))
augmented_args = rcfile.apply_defaults(sections, args)
if augmented_args != args:
del args[:]
args.extend(augmented_args)
print("(using pantsrc expansion: pants goal %s)" % ' '.join(augmented_args))
Phase.setup_parser(parser, args, self.phases)
def run(self, lock):
if self.options.dry_run:
print '****** Dry Run ******'
logger = None
if self.options.log or self.options.log_level:
from twitter.common.log import init
from twitter.common.log.options import LogOptions
LogOptions.set_stderr_log_level((self.options.log_level
or 'info').upper())
logdir = self.options.logdir or self.config.get(
'goals', 'logdir', default=None)
if logdir:
safe_mkdir(logdir)
LogOptions.set_log_dir(logdir)
init('goals')
else:
init()
logger = log
if self.options.recursive_directory:
log.warn(
'--all-recursive is deprecated, use a target spec with the form [dir]:: instead'
)
for dir in self.options.recursive_directory:
self.add_target_recursive(dir)
if self.options.target_directory:
log.warn(
'--all is deprecated, use a target spec with the form [dir]: instead'
)
for dir in self.options.target_directory:
self.add_target_directory(dir)
context = Context(self.config,
self.options,
self.targets,
requested_goals=self.requested_goals,
lock=lock,
log=logger,
timer=self.timer if self.options.time else None)
unknown = []
for phase in self.phases:
if not phase.goals():
unknown.append(phase)
if unknown:
print('Unknown goal(s): %s' % ' '.join(phase.name
for phase in unknown))
print('')
return Phase.execute(context, 'goals')
if logger:
logger.debug('Operating on targets: %s', self.targets)
ret = Phase.attempt(context, self.phases)
if self.options.cleanup_nailguns or self.config.get(
'nailgun', 'autokill', default=False):
if log:
log.debug('auto-killing nailguns')
if NailgunTask.killall:
NailgunTask.killall(log)
if self.options.time:
print('Timing report')
print('=============')
self.timer.print_timings()
return ret
def _list_goals(context, message):
"""Show all installed goals."""
context.log.error(message)
# Execute as if the user had run "./pants goals".
return Phase.execute(context, 'goals')
def run(self, lock):
if self.options.dry_run:
print('****** Dry Run ******')
logger = None
if self.options.log or self.options.log_level:
log.LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper())
logdir = self.options.logdir or self.config.get('goals', 'logdir', default=None)
if logdir:
safe_mkdir(logdir)
log.LogOptions.set_log_dir(logdir)
log.init('goals')
else:
log.init()
logger = log
if not self.options.no_color:
def colorwrapper(func, clrname):
@wraps(func)
def wrapper(msg, *args, **kwargs):
return func(clrname(msg), *args, **kwargs)
return wrapper
log.info = colorwrapper(log.info, green)
log.warn = colorwrapper(log.warn, yellow)
log.debug = colorwrapper(log.debug, cyan)
if self.options.recursive_directory:
log.warn('--all-recursive is deprecated, use a target spec with the form [dir]:: instead')
for dir in self.options.recursive_directory:
self.add_target_recursive(dir)
if self.options.target_directory:
log.warn('--all is deprecated, use a target spec with the form [dir]: instead')
for dir in self.options.target_directory:
self.add_target_directory(dir)
context = Context(
self.config,
self.options,
self.targets,
requested_goals=self.requested_goals,
lock=lock,
log=logger,
timer=self.timer if self.options.time else None)
unknown = []
for phase in self.phases:
if not phase.goals():
unknown.append(phase)
if unknown:
print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown))
print('')
return Phase.execute(context, 'goals')
if logger:
logger.debug('Operating on targets: %s' % self.targets)
ret = Phase.attempt(context, self.phases)
if self.options.cleanup_nailguns or self.config.get('nailgun', 'autokill', default = False):
if log:
log.debug('auto-killing nailguns')
if NailgunTask.killall:
NailgunTask.killall(log)
if self.options.time:
print('Timing report')
print('=============')
self.timer.print_timings()
return ret
def run(self, lock):
# Update the reporting settings, now that we have flags etc.
log_level = Report.log_level_from_string(self.options.log_level
or 'info')
color = not self.options.no_color
timing = self.options.time
cache_stats = self.options.time # TODO: Separate flag for this?
settings_updates_map = {
'console': {
'log_level': log_level,
'color': color,
'timing': timing,
'cache_stats': cache_stats
},
'html': {
'log_level': log_level
}
}
self.run_tracker.update_report_settings(settings_updates_map)
# TODO: Do something useful with --logdir.
if self.options.dry_run:
print '****** Dry Run ******'
context = Context(self.config,
self.options,
self.run_tracker,
self.targets,
requested_goals=self.requested_goals,
lock=lock)
# TODO: Time to get rid of this hack.
if self.options.recursive_directory:
context.log.warn(
'--all-recursive is deprecated, use a target spec with the form [dir]:: instead'
)
for dir in self.options.recursive_directory:
self.add_target_recursive(dir)
if self.options.target_directory:
context.log.warn(
'--all is deprecated, use a target spec with the form [dir]: instead'
)
for dir in self.options.target_directory:
self.add_target_directory(dir)
unknown = []
for phase in self.phases:
if not phase.goals():
unknown.append(phase)
if unknown:
print('Unknown goal(s): %s' % ' '.join(phase.name
for phase in unknown))
print('')
return Phase.execute(context, 'goals')
ret = Phase.attempt(context, self.phases)
if self.options.cleanup_nailguns or self.config.get(
'nailgun', 'autokill', default=False):
if log:
log.debug('auto-killing nailguns')
if NailgunTask.killall:
NailgunTask.killall(log)
return ret
def list_goals(self, message):
print(message)
print()
return Phase.execute(self.context, 'goals')
def list_goals(self, message): print(message) print() return Phase.execute(self.context, 'goals')
def as_phase(cls, phase_name):
"""Returns a ``Phase`` object of the given name"""
return Phase(cls._namespace(phase_name))
