def run(self, lock): if self.options.dry_run: print "****** Dry Run ******" logger = None if self.options.log or self.options.log_level: from twitter.common.log import init from twitter.common.log.options import LogOptions LogOptions.set_stderr_log_level((self.options.log_level or "info").upper()) logdir = self.options.logdir or self.config.get("goals", "logdir", default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) init("goals") else: init() logger = log if self.options.recursive_directory: log.warn("--all-recursive is deprecated, use a target spec with the form [dir]:: instead") for dir in self.options.recursive_directory: self.add_target_recursive(dir) if self.options.target_directory: log.warn("--all is deprecated, use a target spec with the form [dir]: instead") for dir in self.options.target_directory: self.add_target_directory(dir) context = Context( self.config, self.options, self.targets, requested_goals=self.requested_goals, lock=lock, log=logger, timer=self.timer if self.options.time else None, ) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: print ("Unknown goal(s): %s" % " ".join(phase.name for phase in unknown)) print ("") return Phase.execute(context, "goals") if logger: logger.debug("Operating on targets: %s", self.targets) ret = Phase.attempt(context, self.phases) if self.options.time: print ("Timing report") print ("=============") self.timer.print_timings() return ret
def execute(cls, context, *names): parser = OptionParser() cls.add_global_options(parser) phases = [Phase(name) for name in names] Phase.setup_parser(parser, [], phases) options, _ = parser.parse_args([]) context = Context(context.config, options, context.run_tracker, context.target_roots, requested_goals=list(names)) return cls._execute(context, phases, print_timing=False)
def run(self, lock): with self.check_errors("Target contains a dependency cycle") as error: for target in self.targets: try: InternalTarget.check_cycles(target) except InternalTarget.CycleException as e: error(target.id) timer = None if self.options.time: class Timer(object): def now(self): return time.time() def log(self, message): print(message) timer = Timer() logger = None if self.options.log or self.options.log_level: from twitter.common.log import init from twitter.common.log.options import LogOptions LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper()) logdir = self.options.logdir or self.config.get('goals', 'logdir', default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) init('goals') else: init() logger = log if self.options.recursive_directory: log.warn('--all-recursive is deprecated, use a target spec with the form [dir]:: instead') for dir in self.options.recursive_directory: self.add_target_recursive(dir) if self.options.target_directory: log.warn('--all is deprecated, use a target spec with the form [dir]: instead') for dir in self.options.target_directory: self.add_target_directory(dir) context = Context(self.config, self.options, self.targets, lock=lock, log=logger) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown)) print('') return Phase.execute(context, 'goals') if logger: logger.debug('Operating on targets: %s', self.targets) return Phase.attempt(context, self.phases, timer=timer)
def run(self, lock): # Update the reporting settings, now that we have flags etc. def is_console_task(): for phase in self.phases: for goal in phase.goals(): if issubclass(goal.task_type, ConsoleTask): return True return False update_reporting(self.options, is_console_task(), self.run_tracker) if self.options.dry_run: print '****** Dry Run ******' context = Context(self.config, self.options, self.run_tracker, self.targets, requested_goals=self.requested_goals, lock=lock) # TODO: Time to get rid of this hack. if self.options.recursive_directory: context.log.warn( '--all-recursive is deprecated, use a target spec with the form [dir]:: instead' ) for dir in self.options.recursive_directory: self.add_target_recursive(dir) if self.options.target_directory: context.log.warn( '--all is deprecated, use a target spec with the form [dir]: instead' ) for dir in self.options.target_directory: self.add_target_directory(dir) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown)) print('') return Phase.execute(context, 'goals') ret = Phase.attempt(context, self.phases) if self.options.cleanup_nailguns or self.config.get( 'nailgun', 'autokill', default=False): if log: log.debug('auto-killing nailguns') if NailgunTask.killall: NailgunTask.killall(log) return ret
def execute(self): def add_targets(dir, buildfile): try: self.targets.extend(Target.get(addr) for addr in Target.get_all_addresses(buildfile)) except (TypeError, ImportError): error(dir, include_traceback=True) except (IOError, SyntaxError): error(dir) if self.options.recursive_directory: with self.check_errors('There was a problem scanning the ' 'following directories for targets:') as error: for dir in self.options.recursive_directory: for buildfile in BuildFile.scan_buildfiles(self.root_dir, dir): add_targets(dir, buildfile) if self.options.target_directory: with self.check_errors("There was a problem loading targets " "from the following directory's BUILD files") as error: for dir in self.options.target_directory: add_targets(dir, BuildFile(self.root_dir, dir)) timer = None if self.options.time: class Timer(object): def now(self): return time.time() def log(self, message): print(message) timer = Timer() logger = None if self.options.log or self.options.log_level: from twitter.common.log import init from twitter.common.log.options import LogOptions LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper()) logdir = self.config.get('goals', 'logdir') if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) init('goals') logger = log context = Context(self.config, self.options, self.targets, log=logger) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown)) print() return Phase.execute(context, 'goals') return Phase.attempt(context, self.phases, timer=timer)
def run(self, lock): with self.check_errors("Target contains a dependency cycle") as error: with self.timer.timing("parse:check_cycles"): for target in self.targets: try: InternalTarget.check_cycles(target) except InternalTarget.CycleException as e: error(target.id) logger = None if self.options.log or self.options.log_level: from twitter.common.log import init from twitter.common.log.options import LogOptions LogOptions.set_stderr_log_level((self.options.log_level or "info").upper()) logdir = self.options.logdir or self.config.get("goals", "logdir", default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) init("goals") else: init() logger = log if self.options.recursive_directory: log.warn("--all-recursive is deprecated, use a target spec with the form [dir]:: instead") for dir in self.options.recursive_directory: self.add_target_recursive(dir) if self.options.target_directory: log.warn("--all is deprecated, use a target spec with the form [dir]: instead") for dir in self.options.target_directory: self.add_target_directory(dir) context = Context(self.config, self.options, self.targets, lock=lock, log=logger) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: print("Unknown goal(s): %s" % " ".join(phase.name for phase in unknown)) print("") return Phase.execute(context, "goals") if logger: logger.debug("Operating on targets: %s", self.targets) ret = Phase.attempt(context, self.phases, timer=self.timer if self.options.time else None) if self.options.time: print("Timing report") print("=============") self.timer.print_timings() return ret
def run(self, lock): if self.options.dry_run: print '****** Dry Run ******' logger = None if self.options.log or self.options.log_level: from twitter.common.log import init from twitter.common.log.options import LogOptions LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper()) logdir = self.options.logdir or self.config.get('goals', 'logdir', default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) init('goals') else: init() logger = log if self.options.recursive_directory: log.warn('--all-recursive is deprecated, use a target spec with the form [dir]:: instead') for dir in self.options.recursive_directory: self.add_target_recursive(dir) if self.options.target_directory: log.warn('--all is deprecated, use a target spec with the form [dir]: instead') for dir in self.options.target_directory: self.add_target_directory(dir) context = Context( self.config, self.options, self.targets, lock=lock, log=logger, timer=self.timer if self.options.time else None) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown)) print('') return Phase.execute(context, 'goals') if logger: logger.debug('Operating on targets: %s', self.targets) ret = Phase.attempt(context, self.phases) if self.options.time: print('Timing report') print('=============') self.timer.print_timings() return ret
def run(self, lock): with self.check_errors("Target contains a dependency cycle") as error: with self.timer.timing('parse:check_cycles'): for target in self.targets: try: InternalTarget.check_cycles(target) except InternalTarget.CycleException as e: error(target.id) logger = None if self.options.log or self.options.log_level: from twitter.common.log import init from twitter.common.log.options import LogOptions LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper()) logdir = self.options.logdir or self.config.get('goals', 'logdir', default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) init('goals') else: init() logger = log if self.options.recursive_directory: log.warn('--all-recursive is deprecated, use a target spec with the form [dir]:: instead') for dir in self.options.recursive_directory: self.add_target_recursive(dir) if self.options.target_directory: log.warn('--all is deprecated, use a target spec with the form [dir]: instead') for dir in self.options.target_directory: self.add_target_directory(dir) context = Context(self.config, self.options, self.targets, lock=lock, log=logger) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown)) print('') return Phase.execute(context, 'goals') if logger: logger.debug('Operating on targets: %s', self.targets) ret = Phase.attempt(context, self.phases, timer=self.timer if self.options.time else None) if self.options.time: print('Timing report') print('=============') self.timer.print_timings() return ret
def run(self, lock): # Update the reporting settings, now that we have flags etc. def is_console_task(): for phase in self.phases: for goal in phase.goals(): if issubclass(goal.task_type, ConsoleTask): return True return False update_reporting(self.options, is_console_task(), self.run_tracker) if self.options.dry_run: print '****** Dry Run ******' context = Context( self.config, self.options, self.run_tracker, self.targets, requested_goals=self.requested_goals, lock=lock) # TODO: Time to get rid of this hack. if self.options.recursive_directory: context.log.warn( '--all-recursive is deprecated, use a target spec with the form [dir]:: instead') for dir in self.options.recursive_directory: self.add_target_recursive(dir) if self.options.target_directory: context.log.warn('--all is deprecated, use a target spec with the form [dir]: instead') for dir in self.options.target_directory: self.add_target_directory(dir) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown)) print('') return Phase.execute(context, 'goals') ret = Phase.attempt(context, self.phases) if self.options.cleanup_nailguns or self.config.get('nailgun', 'autokill', default = False): if log: log.debug('auto-killing nailguns') if NailgunTask.killall: NailgunTask.killall(log) return ret
def execute(self, targets): goal = self.context.options.help_goal if goal is None: return self.list_goals('You must supply a goal name to provide help for.') phase = Phase(goal) if not phase.goals(): return self.list_goals('Goal %s is unknown.' % goal) parser = Help.parser() parser.set_usage('%s goal %s ([target]...)' % (sys.argv[0], goal)) parser.epilog = phase.description Phase.setup_parser(parser, [], [phase]) parser.parse_args(['--help'])
def execute(self, targets): goal = self.context.options.help_goal if goal is None: return self.list_goals('You must supply a goal name to provide help for.') phase = Phase(goal) if not phase.goals(): return self.list_goals('Goal %s is unknown.' % goal) parser = OptionParser() parser.set_usage('%s goal %s ([target]...)' % (sys.argv[0], goal)) parser.epilog = phase.description Goal.add_global_options(parser) Phase.setup_parser(parser, [], [phase]) parser.parse_args(['--help'])
def execute(self, targets): goal = self.context.options.help_goal if goal is None: return self.list_goals("You must supply a goal name to provide help for.") phase = Phase(goal) if not phase.goals(): return self.list_goals("Goal %s is unknown." % goal) parser = OptionParser() parser.set_usage("%s goal %s ([target]...)" % (sys.argv[0], goal)) parser.epilog = phase.description Goal.add_global_options(parser) Phase.setup_parser(parser, [], [phase]) parser.parse_args(["--help"])
def graph(): def get_cluster_name(phase): return 'cluster_%s' % phase.name.replace('-', '_') def get_goal_name(phase, goal): name = '%s_%s' % (phase.name, goal.name) return name.replace('-', '_') phase_by_phasename = {} for phase, goals in Phase.all(): phase_by_phasename[phase.name] = phase yield '\n'.join([ 'digraph G {', ' rankdir=LR;', ' graph [compound=true];', ]) for phase, installed_goals in Phase.all(): yield '\n'.join([ ' subgraph %s {' % get_cluster_name(phase), ' node [style=filled];', ' color = blue;', ' label = "%s";' % phase.name, ]) for installed_goal in installed_goals: yield ' %s [label="%s"];' % (get_goal_name(phase, installed_goal), installed_goal.name) yield ' }' edges = set() for phase, installed_goals in Phase.all(): for installed_goal in installed_goals: for dependency in installed_goal.dependencies: tail_goal = phase_by_phasename.get(dependency.name).goals()[-1] edge = 'ltail=%s lhead=%s' % (get_cluster_name(phase), get_cluster_name(Phase.of(tail_goal))) if edge not in edges: yield ' %s -> %s [%s];' % (get_goal_name(phase, installed_goal), get_goal_name(Phase.of(tail_goal), tail_goal), edge) edges.add(edge) yield '}'
def _record(self, goal, elapsed): phase = Phase.of(goal) phase_timings = self._timings.get(phase) if phase_timings is None: phase_timings = OrderedDict(()) self._timings[phase] = phase_timings goal_timings = phase_timings.get(goal) if goal_timings is None: goal_timings = [] phase_timings[goal] = goal_timings goal_timings.append(elapsed)
def execute(self, targets): print('Installed goals:') documented_rows = [] undocumented = [] max_width = 0 for phase, _ in Phase.all(): if phase.description: documented_rows.append((phase.name, phase.description)) max_width = max(max_width, len(phase.name)) elif self.context.options.goal_list_all: undocumented.append(phase.name) for name, description in documented_rows: print(' %s: %s' % (name.rjust(max_width), description)) if undocumented: print('\nUndocumented goals:\n %s' % ' '.join(undocumented))
def console_output(self, targets): yield "Installed goals:" documented_rows = [] undocumented = [] max_width = 0 for phase, _ in Phase.all(): if phase.description: documented_rows.append((phase.name, phase.description)) max_width = max(max_width, len(phase.name)) elif self.context.options.goal_list_all: undocumented.append(phase.name) for name, description in documented_rows: yield " %s: %s" % (name.rjust(max_width), description) if undocumented: yield "" yield "Undocumented goals:" yield " %s" % " ".join(undocumented)
def report(): yield 'Installed goals:' documented_rows = [] undocumented = [] max_width = 0 for phase, _ in Phase.all(): if phase.description: documented_rows.append((phase.name, phase.description)) max_width = max(max_width, len(phase.name)) elif self.context.options.goal_list_all: undocumented.append(phase.name) for name, description in documented_rows: yield ' %s: %s' % (name.rjust(max_width), description) if undocumented: yield '' yield 'Undocumented goals:' yield ' %s' % ' '.join(undocumented)
def console_output(self, targets): yield 'Installed goals:' documented_rows = [] undocumented = [] max_width = 0 for phase, _ in Phase.all(): if phase.description: documented_rows.append((phase.name, phase.description)) max_width = max(max_width, len(phase.name)) elif self.context.options.goal_list_all: undocumented.append(phase.name) for name, description in documented_rows: yield ' %s: %s' % (name.rjust(max_width), description) if undocumented: yield '' yield 'Undocumented goals:' yield ' %s' % ' '.join(undocumented)
def run(self, lock): # Update the reporting settings, now that we have flags etc. log_level = Report.log_level_from_string(self.options.log_level or 'info') color = not self.options.no_color timing = self.options.time cache_stats = self.options.time # TODO: Separate flag for this? settings_updates_map = { 'console': { 'log_level': log_level, 'color': color, 'timing': timing, 'cache_stats': cache_stats }, 'html': { 'log_level': log_level } } self.run_tracker.update_report_settings(settings_updates_map) # TODO: Do something useful with --logdir. if self.options.dry_run: print '****** Dry Run ******' context = Context( self.config, self.options, self.run_tracker, self.targets, requested_goals=self.requested_goals, lock=lock) # TODO: Time to get rid of this hack. if self.options.recursive_directory: context.log.warn( '--all-recursive is deprecated, use a target spec with the form [dir]:: instead') for dir in self.options.recursive_directory: self.add_target_recursive(dir) if self.options.target_directory: context.log.warn('--all is deprecated, use a target spec with the form [dir]: instead') for dir in self.options.target_directory: self.add_target_directory(dir) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown)) print('') return Phase.execute(context, 'goals') ret = Phase.attempt(context, self.phases) if self.options.cleanup_nailguns or self.config.get('nailgun', 'autokill', default = False): if log: log.debug('auto-killing nailguns') if NailgunTask.killall: NailgunTask.killall(log) return ret
def setup_parser(self, parser, args): self.config = Config.load() Goal.add_global_options(parser) # We support attempting zero or more goals. Multiple goals must be delimited from further # options and non goal args with a '--'. The key permutations we need to support: # ./pants goal => goals # ./pants goal goals => goals # ./pants goal compile src/java/... => compile # ./pants goal compile -x src/java/... => compile # ./pants goal compile src/java/... -x => compile # ./pants goal compile run -- src/java/... => compile, run # ./pants goal compile run -- src/java/... -x => compile, run # ./pants goal compile run -- -x src/java/... => compile, run if not args: args.append('goals') if len(args) == 1 and args[0] in set(['-h', '--help', 'help']): def format_usage(usages): left_colwidth = 0 for left, right in usages: left_colwidth = max(left_colwidth, len(left)) lines = [] for left, right in usages: lines.append(' %s%s%s' % (left, ' ' * (left_colwidth - len(left) + 1), right)) return '\n'.join(lines) usages = [ ("%prog goal goals ([spec]...)", Phase('goals').description), ("%prog goal help [goal] ([spec]...)", Phase('help').description), ("%prog goal [goal] [spec]...", "Attempt goal against one or more targets."), ("%prog goal [goal] ([goal]...) -- [spec]...", "Attempts all the specified goals."), ] parser.set_usage("\n%s" % format_usage(usages)) parser.epilog = ("Either lists all installed goals, provides extra help for a goal or else " "attempts to achieve the specified goal for the listed targets." """ Note that target specs accept two special forms: [dir]: to include all targets in the specified directory [dir]:: to include all targets found in all BUILD files recursively under the directory""") parser.print_help() sys.exit(0) else: goals, specs = Goal.parse_args(args) # TODO(John Sirois): kill PANTS_NEW and its usages when pants.new is rolled out ParseContext.enable_pantsnew() # Bootstrap goals by loading any configured bootstrap BUILD files with self.check_errors('The following bootstrap_buildfiles cannot be loaded:') as error: for path in self.config.getlist('goals', 'bootstrap_buildfiles', default = []): try: buildfile = BuildFile(get_buildroot(), os.path.relpath(path, get_buildroot())) ParseContext(buildfile).parse() except (TypeError, ImportError, TaskError, GoalError): error(path, include_traceback=True) except (IOError, SyntaxError): error(path) # Bootstrap user goals by loading any BUILD files implied by targets with self.check_errors('The following targets could not be loaded:') as error: for spec in specs: self.parse_spec(error, spec) self.phases = [Phase(goal) for goal in goals] rcfiles = self.config.getdefault('rcfiles', type=list, default=[]) if rcfiles: rcfile = RcFile(rcfiles, default_prepend=False, process_default=True) # Break down the goals specified on the command line to the full set that will be run so we # can apply default flags to inner goal nodes. Also break down goals by Task subclass and # register the task class hierarchy fully qualified names so we can apply defaults to # baseclasses. all_goals = Phase.execution_order(Phase(goal) for goal in goals) sections = OrderedSet() for goal in all_goals: sections.add(goal.name) for clazz in goal.task_type.mro(): if clazz == Task: break sections.add('%s.%s' % (clazz.__module__, clazz.__name__)) augmented_args = rcfile.apply_defaults(sections, args) if augmented_args != args: del args[:] args.extend(augmented_args) print("(using pantsrc expansion: pants goal %s)" % ' '.join(augmented_args)) Phase.setup_parser(parser, args, self.phases)
def setup_parser(self, parser, args): self.config = Config.load() Goal.add_global_options(parser) # We support attempting zero or more goals. Multiple goals must be delimited from further # options and non goal args with a '--'. The key permutations we need to support: # ./pants goal => goals # ./pants goal goals => goals # ./pants goal compile src/java/... => compile # ./pants goal compile -x src/java/... => compile # ./pants goal compile src/java/... -x => compile # ./pants goal compile run -- src/java/... => compile, run # ./pants goal compile run -- src/java/... -x => compile, run # ./pants goal compile run -- -x src/java/... => compile, run if not args: args.append('goals') if len(args) == 1 and args[0] in set(['-h', '--help', 'help']): def format_usage(usages): left_colwidth = 0 for left, right in usages: left_colwidth = max(left_colwidth, len(left)) lines = [] for left, right in usages: lines.append(' %s%s%s' % (left, ' ' * (left_colwidth - len(left) + 1), right)) return '\n'.join(lines) usages = [ ("%prog goal goals ([spec]...)", Phase('goals').description), ("%prog goal help [goal] ([spec]...)", Phase('help').description), ("%prog goal [goal] [spec]...", "Attempt goal against one or more targets."), ("%prog goal [goal] ([goal]...) -- [spec]...", "Attempts all the specified goals."), ] parser.set_usage("\n%s" % format_usage(usages)) parser.epilog = ("Either lists all installed goals, provides extra help for a goal or else " "attempts to achieve the specified goal for the listed targets." """ Note that target specs accept two special forms: [dir]: to include all targets in the specified directory [dir]:: to include all targets found in all BUILD files recursively under the directory""") parser.print_help() sys.exit(0) else: goals, specs = Goal.parse_args(args) self.requested_goals = goals with self.run_tracker.new_workunit(name='setup', labels=[WorkUnit.SETUP]): # Bootstrap goals by loading any configured bootstrap BUILD files with self.check_errors('The following bootstrap_buildfiles cannot be loaded:') as error: with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]): for path in self.config.getlist('goals', 'bootstrap_buildfiles', default = []): try: buildfile = BuildFile(get_buildroot(), os.path.relpath(path, get_buildroot())) ParseContext(buildfile).parse() except (TypeError, ImportError, TaskError, GoalError): error(path, include_traceback=True) except (IOError, SyntaxError): error(path) # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self.run_tracker.run_info.add_scm_info() # Bootstrap user goals by loading any BUILD files implied by targets. spec_parser = SpecParser(self.root_dir) with self.check_errors('The following targets could not be loaded:') as error: with self.run_tracker.new_workunit(name='parse', labels=[WorkUnit.SETUP]): for spec in specs: try: for target, address in spec_parser.parse(spec): if target: self.targets.append(target) # Force early BUILD file loading if this target is an alias that expands # to others. unused = list(target.resolve()) else: siblings = Target.get_all_addresses(address.buildfile) prompt = 'did you mean' if len(siblings) == 1 else 'maybe you meant one of these' error('%s => %s?:\n %s' % (address, prompt, '\n '.join(str(a) for a in siblings))) except (TypeError, ImportError, TaskError, GoalError): error(spec, include_traceback=True) except (IOError, SyntaxError, TargetDefinitionException): error(spec) self.phases = [Phase(goal) for goal in goals] rcfiles = self.config.getdefault('rcfiles', type=list, default=['/etc/pantsrc', '~/.pants.rc']) if rcfiles: rcfile = RcFile(rcfiles, default_prepend=False, process_default=True) # Break down the goals specified on the command line to the full set that will be run so we # can apply default flags to inner goal nodes. Also break down goals by Task subclass and # register the task class hierarchy fully qualified names so we can apply defaults to # baseclasses. sections = OrderedSet() for phase in Engine.execution_order(self.phases): for goal in phase.goals(): sections.add(goal.name) for clazz in goal.task_type.mro(): if clazz == Task: break sections.add('%s.%s' % (clazz.__module__, clazz.__name__)) augmented_args = rcfile.apply_defaults(sections, args) if augmented_args != args: del args[:] args.extend(augmented_args) sys.stderr.write("(using pantsrc expansion: pants goal %s)\n" % ' '.join(augmented_args)) Phase.setup_parser(parser, args, self.phases)
def run(self, lock): timer = None if self.options.time: class Timer(object): def now(self): return time.time() def log(self, message): print(message) timer = Timer() logger = None if self.options.log or self.options.log_level: from twitter.common.log import init from twitter.common.log.options import LogOptions LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper()) logdir = self.options.logdir or self.config.get( 'goals', 'logdir', default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) init('goals') else: init() logger = log if self.options.recursive_directory: log.warn( '--all-recursive is deprecated, use a target spec with the form [dir]:: instead' ) for dir in self.options.recursive_directory: self.add_target_recursive(dir) if self.options.target_directory: log.warn( '--all is deprecated, use a target spec with the form [dir]: instead' ) for dir in self.options.target_directory: self.add_target_directory(dir) context = Context(self.config, self.options, self.targets, lock=lock, log=logger) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown)) print('') return Phase.execute(context, 'goals') if logger: logger.debug('Operating on targets: %s', self.targets) return Phase.attempt(context, self.phases, timer=timer)
def _list_goals(context, message): """Show all installed goals.""" context.log.error(message) # Execute as if the user had run "./pants goals". return Phase.execute(context, 'goals')
def setup_parser(self, parser, args): self.config = Config.load() Goal.add_global_options(parser) # We support attempting zero or more goals. Multiple goals must be delimited from further # options and non goal args with a '--'. The key permutations we need to support: # ./pants goal => goals # ./pants goal goals => goals # ./pants goal compile src/java/... => compile # ./pants goal compile -x src/java/... => compile # ./pants goal compile src/java/... -x => compile # ./pants goal compile run -- src/java/... => compile, run # ./pants goal compile run -- src/java/... -x => compile, run # ./pants goal compile run -- -x src/java/... => compile, run if not args: args.append('goals') if len(args) == 1 and args[0] in set(['-h', '--help', 'help']): def format_usage(usages): left_colwidth = 0 for left, right in usages: left_colwidth = max(left_colwidth, len(left)) lines = [] for left, right in usages: lines.append(' %s%s%s' % (left, ' ' * (left_colwidth - len(left) + 1), right)) return '\n'.join(lines) usages = [ ("%prog goal goals ([spec]...)", Phase('goals').description), ("%prog goal help [goal] ([spec]...)", Phase('help').description), ("%prog goal [goal] [spec]...", "Attempt goal against one or more targets."), ("%prog goal [goal] ([goal]...) -- [spec]...", "Attempts all the specified goals."), ] parser.set_usage("\n%s" % format_usage(usages)) parser.epilog = ( "Either lists all installed goals, provides extra help for a goal or else " "attempts to achieve the specified goal for the listed targets." """ Note that target specs accept two special forms: [dir]: to include all targets in the specified directory [dir]:: to include all targets found in all BUILD files recursively under the directory""") parser.print_help() sys.exit(0) else: goals, specs = Goal.parse_args(args) self.requested_goals = goals with self.run_tracker.new_workunit(name='setup', labels=[WorkUnit.SETUP]): # Bootstrap goals by loading any configured bootstrap BUILD files with self.check_errors( 'The following bootstrap_buildfiles cannot be loaded:' ) as error: with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP ]): for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]): try: buildfile = BuildFile( get_buildroot(), os.path.relpath(path, get_buildroot())) ParseContext(buildfile).parse() except (TypeError, ImportError, TaskError, GoalError): error(path, include_traceback=True) except (IOError, SyntaxError): error(path) # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self.run_tracker.run_info.add_scm_info() # Bootstrap user goals by loading any BUILD files implied by targets. spec_parser = SpecParser(self.root_dir) with self.check_errors( 'The following targets could not be loaded:') as error: with self.run_tracker.new_workunit(name='parse', labels=[WorkUnit.SETUP ]): for spec in specs: try: for target, address in spec_parser.parse(spec): if target: self.targets.append(target) # Force early BUILD file loading if this target is an alias that expands # to others. unused = list(target.resolve()) else: siblings = Target.get_all_addresses( address.buildfile) prompt = 'did you mean' if len( siblings ) == 1 else 'maybe you meant one of these' error('%s => %s?:\n %s' % (address, prompt, '\n '.join( str(a) for a in siblings))) except (TypeError, ImportError, TaskError, GoalError): error(spec, include_traceback=True) except (IOError, SyntaxError, TargetDefinitionException): error(spec) self.phases = [Phase(goal) for goal in goals] rcfiles = self.config.getdefault('rcfiles', type=list, default=[]) if rcfiles: rcfile = RcFile(rcfiles, default_prepend=False, process_default=True) # Break down the goals specified on the command line to the full set that will be run so we # can apply default flags to inner goal nodes. Also break down goals by Task subclass and # register the task class hierarchy fully qualified names so we can apply defaults to # baseclasses. sections = OrderedSet() for phase in Engine.execution_order(self.phases): for goal in phase.goals(): sections.add(goal.name) for clazz in goal.task_type.mro(): if clazz == Task: break sections.add('%s.%s' % (clazz.__module__, clazz.__name__)) augmented_args = rcfile.apply_defaults(sections, args) if augmented_args != args: del args[:] args.extend(augmented_args) sys.stderr.write( "(using pantsrc expansion: pants goal %s)\n" % ' '.join(augmented_args)) Phase.setup_parser(parser, args, self.phases)
def setup_parser(self, parser, args): self.config = Config.load() parser.add_option("-x", "--time", action="store_true", dest = "time", default = False, help = "Times goal phases and outputs a report.") parser.add_option("-v", "--log", action="store_true", dest = "log", default = False, help = "[%default] Logs extra build output.") parser.add_option("-l", "--level", dest = "log_level", type="choice", choices=['debug', 'info', 'warn'], help = "[info] Sets the logging level to one of 'debug', 'info' or 'warn', " "implies -v if set.") parser.add_option("--all", dest="target_directory", action="append", help = "Adds all targets found in the given directory's BUILD file. Can " "be specified more than once.") parser.add_option("--all-recursive", dest="recursive_directory", action="append", help = "Adds all targets found recursively under the given directory. Can " "be specified more than once to add more than one root target " "directory to scan.") # We support attempting zero or more goals. Multiple goals must be delimited from further # options and non goal args with a '--'. The key permutations we need to support: # ./pants goal => goals # ./pants goal goals => goals # ./pants goal compile src/java/... => compile # ./pants goal compile -x src/java/... => compile # ./pants goal compile src/java/... -x => compile # ./pants goal compile run -- src/java/... => compile, run # ./pants goal compile run -- src/java/... -x => compile, run # ./pants goal compile run -- -x src/java/... => compile, run if not args: args.append('goals') if len(args) == 1 and args[0] in set(['-h', '--help', 'help']): def format_usage(usages): left_colwidth = 0 for left, right in usages: left_colwidth = max(left_colwidth, len(left)) lines = [] for left, right in usages: lines.append(' %s%s%s' % (left, ' ' * (left_colwidth - len(left) + 1), right)) return '\n'.join(lines) usages = [ ("%prog goal goals ([spec]...)", Phase('goals').description), ("%prog goal help [goal] ([spec]...)", Phase('help').description), ("%prog goal [goal] [spec]...", "Attempt goal against one or more targets."), ("%prog goal [goal] ([goal]...) -- [spec]...", "Attempts all the specified goals."), ] parser.set_usage("\n%s" % format_usage(usages)) parser.epilog = ("Either lists all installed goals, provides extra help for a goal or else " "attempts to achieve the specified goal for the listed targets.") parser.print_help() sys.exit(0) else: goals = [] help = False multi = False for i, arg in enumerate(args): help = help or 'help' == arg goals.append(arg) if '--' == arg: multi = True del args[i] goals.pop() break if arg.startswith('-'): break if not multi: goals = [goals[0]] spec_offset = len(goals) + 1 if help else len(goals) specs = [arg for arg in args[spec_offset:] if not arg.startswith('-')] def parse_build(buildfile): # TODO(John Sirois): kill PANTS_NEW and its usages when pants.new is rolled out ParseContext(buildfile).parse(PANTS_NEW=True) # Bootstrap goals by loading any configured bootstrap BUILD files with self.check_errors('The following bootstrap_buildfiles cannot be loaded:') as error: for path in self.config.getlist('goals', 'bootstrap_buildfiles', default = []): try: buildfile = BuildFile(get_buildroot(), os.path.relpath(path, get_buildroot())) parse_build(buildfile) except (TypeError, ImportError, TaskError, GoalError): error(path, include_traceback=True) except (IOError, SyntaxError): error(path) # Bootstrap user goals by loading any BUILD files implied by targets self.targets = [] with self.check_errors('The following targets could not be loaded:') as error: for spec in specs: try: address = Address.parse(get_buildroot(), spec) parse_build(address.buildfile) target = Target.get(address) if target: self.targets.append(target) else: siblings = Target.get_all_addresses(address.buildfile) prompt = 'did you mean' if len(siblings) == 1 else 'maybe you meant one of these' error('%s => %s?:\n %s' % (address, prompt, '\n '.join(str(a) for a in siblings))) except (TypeError, ImportError, TaskError, GoalError): error(spec, include_traceback=True) except (IOError, SyntaxError): error(spec) self.phases = [Phase(goal) for goal in goals] Phase.setup_parser(parser, args, self.phases)
def setup_parser(self, parser, args): self.config = Config.load() Goal.add_global_options(parser) # We support attempting zero or more goals. Multiple goals must be delimited from further # options and non goal args with a '--'. The key permutations we need to support: # ./pants goal => goals # ./pants goal goals => goals # ./pants goal compile src/java/... => compile # ./pants goal compile -x src/java/... => compile # ./pants goal compile src/java/... -x => compile # ./pants goal compile run -- src/java/... => compile, run # ./pants goal compile run -- src/java/... -x => compile, run # ./pants goal compile run -- -x src/java/... => compile, run if not args: args.append('goals') if len(args) == 1 and args[0] in set(['-h', '--help', 'help']): def format_usage(usages): left_colwidth = 0 for left, right in usages: left_colwidth = max(left_colwidth, len(left)) lines = [] for left, right in usages: lines.append(' %s%s%s' % (left, ' ' * (left_colwidth - len(left) + 1), right)) return '\n'.join(lines) usages = [ ("%prog goal goals ([spec]...)", Phase('goals').description), ("%prog goal help [goal] ([spec]...)", Phase('help').description), ("%prog goal [goal] [spec]...", "Attempt goal against one or more targets."), ("%prog goal [goal] ([goal]...) -- [spec]...", "Attempts all the specified goals."), ] parser.set_usage("\n%s" % format_usage(usages)) parser.epilog = ("Either lists all installed goals, provides extra help for a goal or else " "attempts to achieve the specified goal for the listed targets." """ Note that target specs accept two special forms: [dir]: to include all targets in the specified directory [dir]:: to include all targets found in all BUILD files recursively under the directory""") parser.print_help() sys.exit(0) else: goals, specs = Goal.parse_args(args) # TODO(John Sirois): kill PANTS_NEW and its usages when pants.new is rolled out ParseContext.enable_pantsnew() # Bootstrap goals by loading any configured bootstrap BUILD files with self.check_errors('The following bootstrap_buildfiles cannot be loaded:') as error: with self.timer.timing('parse:bootstrap'): for path in self.config.getlist('goals', 'bootstrap_buildfiles', default = []): try: buildfile = BuildFile(get_buildroot(), os.path.relpath(path, get_buildroot())) ParseContext(buildfile).parse() except (TypeError, ImportError, TaskError, GoalError): error(path, include_traceback=True) except (IOError, SyntaxError): error(path) # Bootstrap user goals by loading any BUILD files implied by targets with self.check_errors('The following targets could not be loaded:') as error: with self.timer.timing('parse:BUILD'): for spec in specs: self.parse_spec(error, spec) self.phases = [Phase(goal) for goal in goals] rcfiles = self.config.getdefault('rcfiles', type=list, default=[]) if rcfiles: rcfile = RcFile(rcfiles, default_prepend=False, process_default=True) # Break down the goals specified on the command line to the full set that will be run so we # can apply default flags to inner goal nodes. Also break down goals by Task subclass and # register the task class hierarchy fully qualified names so we can apply defaults to # baseclasses. all_goals = Phase.execution_order(Phase(goal) for goal in goals) sections = OrderedSet() for goal in all_goals: sections.add(goal.name) for clazz in goal.task_type.mro(): if clazz == Task: break sections.add('%s.%s' % (clazz.__module__, clazz.__name__)) augmented_args = rcfile.apply_defaults(sections, args) if augmented_args != args: del args[:] args.extend(augmented_args) print("(using pantsrc expansion: pants goal %s)" % ' '.join(augmented_args)) Phase.setup_parser(parser, args, self.phases)
def run(self, lock): if self.options.dry_run: print '****** Dry Run ******' logger = None if self.options.log or self.options.log_level: from twitter.common.log import init from twitter.common.log.options import LogOptions LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper()) logdir = self.options.logdir or self.config.get( 'goals', 'logdir', default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) init('goals') else: init() logger = log if self.options.recursive_directory: log.warn( '--all-recursive is deprecated, use a target spec with the form [dir]:: instead' ) for dir in self.options.recursive_directory: self.add_target_recursive(dir) if self.options.target_directory: log.warn( '--all is deprecated, use a target spec with the form [dir]: instead' ) for dir in self.options.target_directory: self.add_target_directory(dir) context = Context(self.config, self.options, self.targets, requested_goals=self.requested_goals, lock=lock, log=logger, timer=self.timer if self.options.time else None) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown)) print('') return Phase.execute(context, 'goals') if logger: logger.debug('Operating on targets: %s', self.targets) ret = Phase.attempt(context, self.phases) if self.options.cleanup_nailguns or self.config.get( 'nailgun', 'autokill', default=False): if log: log.debug('auto-killing nailguns') if NailgunTask.killall: NailgunTask.killall(log) if self.options.time: print('Timing report') print('=============') self.timer.print_timings() return ret
def run(self, lock): if self.options.dry_run: print('****** Dry Run ******') logger = None if self.options.log or self.options.log_level: log.LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper()) logdir = self.options.logdir or self.config.get('goals', 'logdir', default=None) if logdir: safe_mkdir(logdir) log.LogOptions.set_log_dir(logdir) log.init('goals') else: log.init() logger = log if not self.options.no_color: def colorwrapper(func, clrname): @wraps(func) def wrapper(msg, *args, **kwargs): return func(clrname(msg), *args, **kwargs) return wrapper log.info = colorwrapper(log.info, green) log.warn = colorwrapper(log.warn, yellow) log.debug = colorwrapper(log.debug, cyan) if self.options.recursive_directory: log.warn('--all-recursive is deprecated, use a target spec with the form [dir]:: instead') for dir in self.options.recursive_directory: self.add_target_recursive(dir) if self.options.target_directory: log.warn('--all is deprecated, use a target spec with the form [dir]: instead') for dir in self.options.target_directory: self.add_target_directory(dir) context = Context( self.config, self.options, self.targets, requested_goals=self.requested_goals, lock=lock, log=logger, timer=self.timer if self.options.time else None) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown)) print('') return Phase.execute(context, 'goals') if logger: logger.debug('Operating on targets: %s' % self.targets) ret = Phase.attempt(context, self.phases) if self.options.cleanup_nailguns or self.config.get('nailgun', 'autokill', default = False): if log: log.debug('auto-killing nailguns') if NailgunTask.killall: NailgunTask.killall(log) if self.options.time: print('Timing report') print('=============') self.timer.print_timings() return ret
def run(self, lock): # Update the reporting settings, now that we have flags etc. log_level = Report.log_level_from_string(self.options.log_level or 'info') color = not self.options.no_color timing = self.options.time cache_stats = self.options.time # TODO: Separate flag for this? settings_updates_map = { 'console': { 'log_level': log_level, 'color': color, 'timing': timing, 'cache_stats': cache_stats }, 'html': { 'log_level': log_level } } self.run_tracker.update_report_settings(settings_updates_map) # TODO: Do something useful with --logdir. if self.options.dry_run: print '****** Dry Run ******' context = Context(self.config, self.options, self.run_tracker, self.targets, requested_goals=self.requested_goals, lock=lock) # TODO: Time to get rid of this hack. if self.options.recursive_directory: context.log.warn( '--all-recursive is deprecated, use a target spec with the form [dir]:: instead' ) for dir in self.options.recursive_directory: self.add_target_recursive(dir) if self.options.target_directory: context.log.warn( '--all is deprecated, use a target spec with the form [dir]: instead' ) for dir in self.options.target_directory: self.add_target_directory(dir) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown)) print('') return Phase.execute(context, 'goals') ret = Phase.attempt(context, self.phases) if self.options.cleanup_nailguns or self.config.get( 'nailgun', 'autokill', default=False): if log: log.debug('auto-killing nailguns') if NailgunTask.killall: NailgunTask.killall(log) return ret
def list_goals(self, message): print(message) print() return Phase.execute(self.context, 'goals')
def as_phase(cls, phase_name): """Returns a ``Phase`` object of the given name""" return Phase(cls._namespace(phase_name))