def __init__(self, root_dir, parser, args): self.targets = [] # Note that we can't gate this on the self.options.time flag, because self.options is # only set up in Command.__init__, and only after it calls setup_parser(), which uses the timer. self.timer = Timer() Command.__init__(self, root_dir, parser, args)
def __init__(self, root_dir, parser, args): self.targets = [] # Note that we can't gate this on the self.options.time flag, because self.options is # only set up in Command.__init__, and only after it calls setup_parser(), which uses the timer. self.timer = Timer() Command.__init__(self, root_dir, parser, args)
class Goal(Command): """Lists installed goals or else executes a named goal.""" __command__ = 'goal' GLOBAL_OPTIONS = [ Option("-x", "--time", action="store_true", dest="time", default=False, help="Times goal phases and outputs a report."), Option("-v", "--log", action="store_true", dest="log", default=False, help="[%default] Logs extra build output."), Option("-d", "--logdir", dest="logdir", help="[%default] Forks logs to files under this directory."), Option("-l", "--level", dest="log_level", type="choice", choices=['debug', 'info', 'warn'], help="[info] Sets the logging level to one of 'debug', 'info' or 'warn', implies -v " "if set."), Option("-n", "--dry-run", action="store_true", dest="dry_run", default=False, help="Print the commands that would be run, without actually running them."), Option("--read-from-artifact-cache", "--no-read-from-artifact-cache", action="callback", callback=_set_bool, dest="read_from_artifact_cache", default=False, help="Whether to read artifacts from cache instead of building them, when possible."), Option("--write-to-artifact-cache", "--no-write-to-artifact-cache", action="callback", callback=_set_bool, dest="write_to_artifact_cache", default=False, help="Whether to write artifacts to cache ."), Option("--verify-artifact-cache", "--no-verify-artifact-cache", action="callback", callback=_set_bool, dest="verify_artifact_cache", default=False, help="Whether to verify that cached artifacts are identical after rebuilding them."), Option("--all", dest="target_directory", action="append", help="DEPRECATED: Use [dir]: with no flag in a normal target position on the command " "line. (Adds all targets found in the given directory's BUILD file. Can be " "specified more than once.)"), Option("--all-recursive", dest="recursive_directory", action="append", help="DEPRECATED: Use [dir]:: with no flag in a normal target position on the command " "line. (Adds all targets found recursively under the given directory. Can be " "specified more than once to add more than one root target directory to scan.)"), ] @staticmethod def add_global_options(parser): for option in Goal.GLOBAL_OPTIONS: parser.add_option(option) @staticmethod def parse_args(args): goals = OrderedSet() specs = OrderedSet() help = False explicit_multi = False def is_spec(spec): return os.sep in spec or ':' in spec for i, arg in enumerate(args): help = help or 'help' == arg if not arg.startswith('-'): specs.add(arg) if is_spec(arg) else goals.add(arg) elif '--' == arg: if specs: raise GoalError('Cannot intermix targets with goals when using --. Targets should ' 'appear on the right') explicit_multi = True del args[i] break if explicit_multi: spec_offset = len(goals) + 1 if help else len(goals) specs.update(arg for arg in args[spec_offset:] if not arg.startswith('-')) return goals, specs # TODO(John Sirois): revisit wholesale locking when we move py support into pants new @classmethod def serialized(cls): # Goal serialization is now handled in goal execution during group processing. # The goal command doesn't need to hold the serialization lock; individual goals will # acquire the lock if they need to be serialized. return False def __init__(self, root_dir, parser, args): self.targets = [] # Note that we can't gate this on the self.options.time flag, because self.options is # only set up in Command.__init__, and only after it calls setup_parser(), which uses the timer. self.timer = Timer() Command.__init__(self, root_dir, parser, args) @contextmanager def check_errors(self, banner): errors = {} def error(key, include_traceback=False): exc_type, exc_value, _ = sys.exc_info() msg = StringIO() if include_traceback: frame = inspect.trace()[-1] filename = frame[1] lineno = frame[2] funcname = frame[3] code = ''.join(frame[4]) traceback.print_list([(filename, lineno, funcname, code)], file=msg) if exc_type: msg.write(''.join(traceback.format_exception_only(exc_type, exc_value))) errors[key] = msg.getvalue() sys.exc_clear() yield error if errors: msg = StringIO() msg.write(banner) invalid_keys = [key for key, exc in errors.items() if not exc] if invalid_keys: msg.write('\n %s' % '\n '.join(invalid_keys)) for key, exc in errors.items(): if exc: msg.write('\n %s =>\n %s' % (key, '\n '.join(exc.splitlines()))) # The help message for goal is extremely verbose, and will obscure the # actual error message, so we don't show it in this case. self.error(msg.getvalue(), show_help = False) def add_targets(self, error, dir, buildfile): try: self.targets.extend(Target.get(addr) for addr in Target.get_all_addresses(buildfile)) except (TypeError, ImportError): error(dir, include_traceback=True) except (IOError, SyntaxError): error(dir) def get_dir(self, spec): path = spec.split(':', 1)[0] if os.path.isdir(path): return path else: if os.path.isfile(path): return os.path.dirname(path) else: return spec def add_target_recursive(self, *specs): with self.check_errors('There was a problem scanning the ' 'following directories for targets:') as error: for spec in specs: dir = self.get_dir(spec) for buildfile in BuildFile.scan_buildfiles(self.root_dir, dir): self.add_targets(error, dir, buildfile) def add_target_directory(self, *specs): with self.check_errors("There was a problem loading targets " "from the following directory's BUILD files") as error: for spec in specs: dir = self.get_dir(spec) try: self.add_targets(error, dir, BuildFile(self.root_dir, dir)) except IOError: error(dir) def parse_spec(self, error, spec): if spec.endswith('::'): self.add_target_recursive(spec[:-len('::')]) elif spec.endswith(':'): self.add_target_directory(spec[:-len(':')]) else: try: address = Address.parse(get_buildroot(), spec) ParseContext(address.buildfile).parse() target = Target.get(address) if target: self.targets.append(target) else: siblings = Target.get_all_addresses(address.buildfile) prompt = 'did you mean' if len(siblings) == 1 else 'maybe you meant one of these' error('%s => %s?:\n %s' % (address, prompt, '\n '.join(str(a) for a in siblings))) except (TypeError, ImportError, TaskError, GoalError): error(spec, include_traceback=True) except (IOError, SyntaxError): error(spec) def setup_parser(self, parser, args): self.config = Config.load() Goal.add_global_options(parser) # We support attempting zero or more goals. Multiple goals must be delimited from further # options and non goal args with a '--'. The key permutations we need to support: # ./pants goal => goals # ./pants goal goals => goals # ./pants goal compile src/java/... => compile # ./pants goal compile -x src/java/... => compile # ./pants goal compile src/java/... -x => compile # ./pants goal compile run -- src/java/... => compile, run # ./pants goal compile run -- src/java/... -x => compile, run # ./pants goal compile run -- -x src/java/... => compile, run if not args: args.append('goals') if len(args) == 1 and args[0] in set(['-h', '--help', 'help']): def format_usage(usages): left_colwidth = 0 for left, right in usages: left_colwidth = max(left_colwidth, len(left)) lines = [] for left, right in usages: lines.append(' %s%s%s' % (left, ' ' * (left_colwidth - len(left) + 1), right)) return '\n'.join(lines) usages = [ ("%prog goal goals ([spec]...)", Phase('goals').description), ("%prog goal help [goal] ([spec]...)", Phase('help').description), ("%prog goal [goal] [spec]...", "Attempt goal against one or more targets."), ("%prog goal [goal] ([goal]...) -- [spec]...", "Attempts all the specified goals."), ] parser.set_usage("\n%s" % format_usage(usages)) parser.epilog = ("Either lists all installed goals, provides extra help for a goal or else " "attempts to achieve the specified goal for the listed targets." """ Note that target specs accept two special forms: [dir]: to include all targets in the specified directory [dir]:: to include all targets found in all BUILD files recursively under the directory""") parser.print_help() sys.exit(0) else: goals, specs = Goal.parse_args(args) self.requested_goals = goals # TODO(John Sirois): kill PANTS_NEW and its usages when pants.new is rolled out ParseContext.enable_pantsnew() # Bootstrap goals by loading any configured bootstrap BUILD files with self.check_errors('The following bootstrap_buildfiles cannot be loaded:') as error: with self.timer.timing('parse:bootstrap'): for path in self.config.getlist('goals', 'bootstrap_buildfiles', default = []): try: buildfile = BuildFile(get_buildroot(), os.path.relpath(path, get_buildroot())) ParseContext(buildfile).parse() except (TypeError, ImportError, TaskError, GoalError): error(path, include_traceback=True) except (IOError, SyntaxError): error(path) # Bootstrap user goals by loading any BUILD files implied by targets with self.check_errors('The following targets could not be loaded:') as error: with self.timer.timing('parse:BUILD'): for spec in specs: self.parse_spec(error, spec) self.phases = [Phase(goal) for goal in goals] rcfiles = self.config.getdefault('rcfiles', type=list, default=[]) if rcfiles: rcfile = RcFile(rcfiles, default_prepend=False, process_default=True) # Break down the goals specified on the command line to the full set that will be run so we # can apply default flags to inner goal nodes. Also break down goals by Task subclass and # register the task class hierarchy fully qualified names so we can apply defaults to # baseclasses. all_goals = Phase.execution_order(Phase(goal) for goal in goals) sections = OrderedSet() for goal in all_goals: sections.add(goal.name) for clazz in goal.task_type.mro(): if clazz == Task: break sections.add('%s.%s' % (clazz.__module__, clazz.__name__)) augmented_args = rcfile.apply_defaults(sections, args) if augmented_args != args: del args[:] args.extend(augmented_args) print("(using pantsrc expansion: pants goal %s)" % ' '.join(augmented_args)) Phase.setup_parser(parser, args, self.phases) def run(self, lock): if self.options.dry_run: print '****** Dry Run ******' logger = None if self.options.log or self.options.log_level: from twitter.common.log import init from twitter.common.log.options import LogOptions LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper()) logdir = self.options.logdir or self.config.get('goals', 'logdir', default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) init('goals') else: init() logger = log if self.options.recursive_directory: log.warn('--all-recursive is deprecated, use a target spec with the form [dir]:: instead') for dir in self.options.recursive_directory: self.add_target_recursive(dir) if self.options.target_directory: log.warn('--all is deprecated, use a target spec with the form [dir]: instead') for dir in self.options.target_directory: self.add_target_directory(dir) context = Context( self.config, self.options, self.targets, requested_goals=self.requested_goals, lock=lock, log=logger, timer=self.timer if self.options.time else None) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown)) print('') return Phase.execute(context, 'goals') if logger: logger.debug('Operating on targets: %s', self.targets) ret = Phase.attempt(context, self.phases) if self.options.time: print('Timing report') print('=============') self.timer.print_timings() return ret def cleanup(self): # TODO: Make this more selective? Only kill nailguns that affect state? E.g., checkstyle # may not need to be killed. if NailgunTask.killall: NailgunTask.killall(log) sys.exit(1)
class Goal(Command): """Lists installed goals or else executes a named goal.""" __command__ = 'goal' GLOBAL_OPTIONS = [ Option("-x", "--time", action="store_true", dest="time", default=False, help="Times goal phases and outputs a report."), Option("-v", "--log", action="store_true", dest="log", default=False, help="[%default] Logs extra build output."), Option("-d", "--logdir", dest="logdir", help="[%default] Forks logs to files under this directory."), Option("-l", "--level", dest="log_level", type="choice", choices=['debug', 'info', 'warn'], help="[info] Sets the logging level to one of 'debug', 'info' or 'warn', implies -v " "if set."), Option("--read-from-artifact-cache", "--no-read-from-artifact-cache", action="callback", callback=_set_bool, dest="read_from_artifact_cache", default=False, help="Whether to read artifacts from cache instead of building them, when possible."), Option("--write-to-artifact-cache", "--no-write-to-artifact-cache", action="callback", callback=_set_bool, dest="write_to_artifact_cache", default=False, help="Whether to write artifacts to cache ."), Option("--verify-artifact-cache", "--no-verify-artifact-cache", action="callback", callback=_set_bool, dest="verify_artifact_cache", default=False, help="Whether to verify that cached artifacts are identical after rebuilding them."), Option("--all", dest="target_directory", action="append", help="DEPRECATED: Use [dir]: with no flag in a normal target position on the command " "line. (Adds all targets found in the given directory's BUILD file. Can be " "specified more than once.)"), Option("--all-recursive", dest="recursive_directory", action="append", help="DEPRECATED: Use [dir]:: with no flag in a normal target position on the command " "line. (Adds all targets found recursively under the given directory. Can be " "specified more than once to add more than one root target directory to scan.)"), ] @staticmethod def add_global_options(parser): for option in Goal.GLOBAL_OPTIONS: parser.add_option(option) @staticmethod def parse_args(args): goals = OrderedSet() specs = OrderedSet() help = False explicit_multi = False def is_spec(spec): return os.sep in spec or ':' in spec for i, arg in enumerate(args): help = help or 'help' == arg if not arg.startswith('-'): specs.add(arg) if is_spec(arg) else goals.add(arg) elif '--' == arg: if specs: raise GoalError('Cannot intermix targets with goals when using --. Targets should ' 'appear on the right') explicit_multi = True del args[i] break if explicit_multi: spec_offset = len(goals) + 1 if help else len(goals) specs.update(arg for arg in args[spec_offset:] if not arg.startswith('-')) return goals, specs # TODO(John Sirois): revisit wholesale locking when we move py support into pants new @classmethod def serialized(cls): return True def __init__(self, root_dir, parser, args): self.targets = [] # Note that we can't gate this on the self.options.time flag, because self.options is # only set up in Command.__init__, and only after it calls setup_parser(), which uses the timer. self.timer = Timer() Command.__init__(self, root_dir, parser, args) @contextmanager def check_errors(self, banner): errors = {} def error(key, include_traceback=False): exc_type, exc_value, _ = sys.exc_info() msg = StringIO() if include_traceback: frame = inspect.trace()[-1] filename = frame[1] lineno = frame[2] funcname = frame[3] code = ''.join(frame[4]) traceback.print_list([(filename, lineno, funcname, code)], file=msg) if exc_type: msg.write(''.join(traceback.format_exception_only(exc_type, exc_value))) errors[key] = msg.getvalue() sys.exc_clear() yield error if errors: msg = StringIO() msg.write(banner) invalid_keys = [key for key, exc in errors.items() if not exc] if invalid_keys: msg.write('\n %s' % '\n '.join(invalid_keys)) for key, exc in errors.items(): if exc: msg.write('\n %s =>\n %s' % (key, '\n '.join(exc.splitlines()))) # The help message for goal is extremely verbose, and will obscure the # actual error message, so we don't show it in this case. self.error(msg.getvalue(), show_help = False) def add_targets(self, error, dir, buildfile): try: self.targets.extend(Target.get(addr) for addr in Target.get_all_addresses(buildfile)) except (TypeError, ImportError): error(dir, include_traceback=True) except (IOError, SyntaxError): error(dir) def get_dir(self, spec): path = spec.split(':', 1)[0] if os.path.isdir(path): return path else: if os.path.isfile(path): return os.path.dirname(path) else: return spec def add_target_recursive(self, *specs): with self.check_errors('There was a problem scanning the ' 'following directories for targets:') as error: for spec in specs: dir = self.get_dir(spec) for buildfile in BuildFile.scan_buildfiles(self.root_dir, dir): self.add_targets(error, dir, buildfile) def add_target_directory(self, *specs): with self.check_errors("There was a problem loading targets " "from the following directory's BUILD files") as error: for spec in specs: dir = self.get_dir(spec) try: self.add_targets(error, dir, BuildFile(self.root_dir, dir)) except IOError: error(dir) def parse_spec(self, error, spec): if spec.endswith('::'): self.add_target_recursive(spec[:-len('::')]) elif spec.endswith(':'): self.add_target_directory(spec[:-len(':')]) else: try: address = Address.parse(get_buildroot(), spec) ParseContext(address.buildfile).parse() target = Target.get(address) if target: self.targets.append(target) else: siblings = Target.get_all_addresses(address.buildfile) prompt = 'did you mean' if len(siblings) == 1 else 'maybe you meant one of these' error('%s => %s?:\n %s' % (address, prompt, '\n '.join(str(a) for a in siblings))) except (TypeError, ImportError, TaskError, GoalError): error(spec, include_traceback=True) except (IOError, SyntaxError): error(spec) def setup_parser(self, parser, args): self.config = Config.load() Goal.add_global_options(parser) # We support attempting zero or more goals. Multiple goals must be delimited from further # options and non goal args with a '--'. The key permutations we need to support: # ./pants goal => goals # ./pants goal goals => goals # ./pants goal compile src/java/... => compile # ./pants goal compile -x src/java/... => compile # ./pants goal compile src/java/... -x => compile # ./pants goal compile run -- src/java/... => compile, run # ./pants goal compile run -- src/java/... -x => compile, run # ./pants goal compile run -- -x src/java/... => compile, run if not args: args.append('goals') if len(args) == 1 and args[0] in set(['-h', '--help', 'help']): def format_usage(usages): left_colwidth = 0 for left, right in usages: left_colwidth = max(left_colwidth, len(left)) lines = [] for left, right in usages: lines.append(' %s%s%s' % (left, ' ' * (left_colwidth - len(left) + 1), right)) return '\n'.join(lines) usages = [ ("%prog goal goals ([spec]...)", Phase('goals').description), ("%prog goal help [goal] ([spec]...)", Phase('help').description), ("%prog goal [goal] [spec]...", "Attempt goal against one or more targets."), ("%prog goal [goal] ([goal]...) -- [spec]...", "Attempts all the specified goals."), ] parser.set_usage("\n%s" % format_usage(usages)) parser.epilog = ("Either lists all installed goals, provides extra help for a goal or else " "attempts to achieve the specified goal for the listed targets." """ Note that target specs accept two special forms: [dir]: to include all targets in the specified directory [dir]:: to include all targets found in all BUILD files recursively under the directory""") parser.print_help() sys.exit(0) else: goals, specs = Goal.parse_args(args) # TODO(John Sirois): kill PANTS_NEW and its usages when pants.new is rolled out ParseContext.enable_pantsnew() # Bootstrap goals by loading any configured bootstrap BUILD files with self.check_errors('The following bootstrap_buildfiles cannot be loaded:') as error: with self.timer.timing('parse:bootstrap'): for path in self.config.getlist('goals', 'bootstrap_buildfiles', default = []): try: buildfile = BuildFile(get_buildroot(), os.path.relpath(path, get_buildroot())) ParseContext(buildfile).parse() except (TypeError, ImportError, TaskError, GoalError): error(path, include_traceback=True) except (IOError, SyntaxError): error(path) # Bootstrap user goals by loading any BUILD files implied by targets with self.check_errors('The following targets could not be loaded:') as error: with self.timer.timing('parse:BUILD'): for spec in specs: self.parse_spec(error, spec) self.phases = [Phase(goal) for goal in goals] rcfiles = self.config.getdefault('rcfiles', type=list, default=[]) if rcfiles: rcfile = RcFile(rcfiles, default_prepend=False, process_default=True) # Break down the goals specified on the command line to the full set that will be run so we # can apply default flags to inner goal nodes. Also break down goals by Task subclass and # register the task class hierarchy fully qualified names so we can apply defaults to # baseclasses. all_goals = Phase.execution_order(Phase(goal) for goal in goals) sections = OrderedSet() for goal in all_goals: sections.add(goal.name) for clazz in goal.task_type.mro(): if clazz == Task: break sections.add('%s.%s' % (clazz.__module__, clazz.__name__)) augmented_args = rcfile.apply_defaults(sections, args) if augmented_args != args: del args[:] args.extend(augmented_args) print("(using pantsrc expansion: pants goal %s)" % ' '.join(augmented_args)) Phase.setup_parser(parser, args, self.phases) def run(self, lock): with self.check_errors("Target contains a dependency cycle") as error: with self.timer.timing('parse:check_cycles'): for target in self.targets: try: InternalTarget.check_cycles(target) except InternalTarget.CycleException as e: error(target.id) logger = None if self.options.log or self.options.log_level: from twitter.common.log import init from twitter.common.log.options import LogOptions LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper()) logdir = self.options.logdir or self.config.get('goals', 'logdir', default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) init('goals') else: init() logger = log if self.options.recursive_directory: log.warn('--all-recursive is deprecated, use a target spec with the form [dir]:: instead') for dir in self.options.recursive_directory: self.add_target_recursive(dir) if self.options.target_directory: log.warn('--all is deprecated, use a target spec with the form [dir]: instead') for dir in self.options.target_directory: self.add_target_directory(dir) context = Context(self.config, self.options, self.targets, lock=lock, log=logger) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown)) print('') return Phase.execute(context, 'goals') if logger: logger.debug('Operating on targets: %s', self.targets) ret = Phase.attempt(context, self.phases, timer=self.timer if self.options.time else None) if self.options.time: print('Timing report') print('=============') self.timer.print_timings() return ret def cleanup(self): # TODO: Make this more selective? Only kill nailguns that affect state? E.g., checkstyle # may not need to be killed. if NailgunTask.killall: NailgunTask.killall(log) sys.exit(1)