def test_file_have_coding_utf8(self): """ Look through all .py files and ensure they start with the line '# coding=utf8' """ build_configuration = load_build_configuration_from_source() build_file_parser = BuildFileParser(root_dir=get_buildroot(), build_configuration=build_configuration) address_mapper = BuildFileAddressMapper(build_file_parser) build_graph = BuildGraph(address_mapper=address_mapper) for address in address_mapper.scan_addresses(get_buildroot()): build_graph.inject_address_closure(address) def has_hand_coded_python_files(tgt): return (not tgt.is_synthetic) and tgt.is_original and tgt.has_sources('.py') nonconforming_files = [] for target in build_graph.targets(has_hand_coded_python_files): for src in target.sources_relative_to_buildroot(): with open(os.path.join(get_buildroot(), src), 'r') as python_file: coding_line = python_file.readline() if '' == coding_line and os.path.basename(src) == '__init__.py': continue if coding_line[0:2] == '#!': # Executable file: look for the coding on the second line. coding_line = python_file.readline() if not coding_line.rstrip() == '# coding=utf-8': nonconforming_files.append(src) if len(nonconforming_files) > 0: self.fail('Expected these files to contain first line "# coding=utf8": ' + str(nonconforming_files))
def test_address_lookup_error_hierarcy(self): self.assertIsInstance(BuildFileAddressMapper.AddressNotInBuildFile(), AddressLookupError) self.assertIsInstance(BuildFileAddressMapper.EmptyBuildFileError(), AddressLookupError) self.assertIsInstance( BuildFileAddressMapper.InvalidBuildFileReference(), AddressLookupError) self.assertIsInstance(BuildFileAddressMapper.InvalidAddressError(), AddressLookupError) self.assertIsInstance(BuildFileAddressMapper.BuildFileScanError(), AddressLookupError)
def setUp(self): super(BaseTest, self).setUp() Goal.clear() Subsystem.reset() self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, '.pants.d') safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[''] = { 'pants_workdir': self.pants_workdir, 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'pants_configdir': os.path.join(self.build_root, 'config'), 'cache_key_gen_version': '0-test', } BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) # We need a pants.ini, even if empty. get_buildroot() uses its presence. self.create_file('pants.ini') self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit): """ :param str root_dir: The root directory of the pants workspace (aka the "build root"). :param Options options: The global, pre-initialized Options instance. :param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance. :param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance. :param Reporting reporting: The global, pre-initialized Reporting instance. :param func exiter: A function that accepts an exit code value and exits (for tests, Optional). """ self._root_dir = root_dir self._options = options self._build_config = build_config self._run_tracker = run_tracker self._reporting = reporting self._exiter = exiter self._goals = [] self._targets = [] self._requested_goals = self._options.goals self._target_specs = self._options.target_specs self._help_request = self._options.help_request self._global_options = options.for_global_scope() self._tag = self._global_options.tag self._fail_fast = self._global_options.fail_fast self._spec_excludes = self._global_options.spec_excludes self._explain = self._global_options.explain self._kill_nailguns = self._global_options.kill_nailguns self._build_file_type = self._get_buildfile_type(self._global_options.build_file_rev) self._build_file_parser = BuildFileParser(self._build_config, self._root_dir) self._address_mapper = BuildFileAddressMapper(self._build_file_parser, self._build_file_type) self._build_graph = BuildGraph(self._address_mapper) self._spec_parser = CmdLineSpecParser( self._root_dir, self._address_mapper, spec_excludes=self._spec_excludes, exclude_target_regexps=self._global_options.exclude_target_regexp )
def setUp(self): Goal.clear() self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) BuildRoot().path = self.build_root self.create_file('pants.ini') build_configuration = BuildConfiguration() build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(build_configuration, self.build_root) self.address_mapper = BuildFileAddressMapper(self.build_file_parser) self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit): """ :param str root_dir: The root directory of the pants workspace (aka the "build root"). :param Options options: The global, pre-initialized Options instance. :param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance. :param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance. :param Reporting reporting: The global, pre-initialized Reporting instance. :param func exiter: A function that accepts an exit code value and exits (for tests, Optional). """ self._root_dir = root_dir self._options = options self._build_config = build_config self._run_tracker = run_tracker self._reporting = reporting self._exiter = exiter self._goals = [] self._targets = [] self._requested_goals = self._options.goals self._target_specs = self._options.target_specs self._help_request = self._options.help_request self._global_options = options.for_global_scope() self._tag = self._global_options.tag self._fail_fast = self._global_options.fail_fast self._spec_excludes = self._global_options.spec_excludes self._explain = self._global_options.explain self._kill_nailguns = self._global_options.kill_nailguns self._build_file_type = self._get_buildfile_type(self._global_options.build_file_rev) self._build_file_parser = BuildFileParser(self._build_config, self._root_dir) self._address_mapper = BuildFileAddressMapper(self._build_file_parser, self._build_file_type) self._build_graph = BuildGraph(self._address_mapper) self._spec_parser = CmdLineSpecParser( self._root_dir, self._address_mapper, spec_excludes=self._spec_excludes, exclude_target_regexps=self._global_options.exclude_target_regexp, )
class GoalRunnerFactory(object): def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit): """ :param str root_dir: The root directory of the pants workspace (aka the "build root"). :param Options options: The global, pre-initialized Options instance. :param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance. :param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance. :param Reporting reporting: The global, pre-initialized Reporting instance. :param func exiter: A function that accepts an exit code value and exits (for tests, Optional). """ self._root_dir = root_dir self._options = options self._build_config = build_config self._run_tracker = run_tracker self._reporting = reporting self._exiter = exiter self._goals = [] self._targets = [] self._requested_goals = self._options.goals self._target_specs = self._options.target_specs self._help_request = self._options.help_request self._global_options = options.for_global_scope() self._tag = self._global_options.tag self._fail_fast = self._global_options.fail_fast self._spec_excludes = self._global_options.spec_excludes self._explain = self._global_options.explain self._kill_nailguns = self._global_options.kill_nailguns self._build_file_type = self._get_buildfile_type(self._global_options.build_file_rev) self._build_file_parser = BuildFileParser(self._build_config, self._root_dir) self._address_mapper = BuildFileAddressMapper(self._build_file_parser, self._build_file_type) self._build_graph = BuildGraph(self._address_mapper) self._spec_parser = CmdLineSpecParser( self._root_dir, self._address_mapper, spec_excludes=self._spec_excludes, exclude_target_regexps=self._global_options.exclude_target_regexp ) def _get_buildfile_type(self, build_file_rev): """Selects the BuildFile type for use in a given pants run.""" if build_file_rev: ScmBuildFile.set_rev(build_file_rev) ScmBuildFile.set_scm(get_scm()) return ScmBuildFile else: return FilesystemBuildFile def _expand_goals(self, goals): """Check and populate the requested goals for a given run.""" for goal in goals: if self._address_mapper.from_cache(self._root_dir, goal, must_exist=False).file_exists(): logger.warning("Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal)) if self._help_request: help_printer = HelpPrinter(self._options) help_printer.print_help() self._exiter(0) self._goals.extend([Goal.by_name(goal) for goal in goals]) def _expand_specs(self, specs, fail_fast): """Populate the BuildGraph and target list from a set of input specs.""" with self._run_tracker.new_workunit(name='parse', labels=[WorkUnitLabel.SETUP]): def filter_for_tag(tag): return lambda target: tag in map(str, target.tags) tag_filter = wrap_filters(create_filters(self._tag, filter_for_tag)) for spec in specs: for address in self._spec_parser.parse_addresses(spec, fail_fast): self._build_graph.inject_address_closure(address) target = self._build_graph.get_target(address) if tag_filter(target): self._targets.append(target) def _is_quiet(self): return any(goal.has_task_of_type(QuietTaskMixin) for goal in self._goals) or self._explain def _setup_context(self): # TODO(John Sirois): Kill when source root registration is lifted out of BUILD files. with self._run_tracker.new_workunit(name='bootstrap', labels=[WorkUnitLabel.SETUP]): source_root_bootstrapper = SourceRootBootstrapper.global_instance() source_root_bootstrapper.bootstrap(self._address_mapper, self._build_file_parser) with self._run_tracker.new_workunit(name='setup', labels=[WorkUnitLabel.SETUP]): self._expand_goals(self._requested_goals) self._expand_specs(self._target_specs, self._fail_fast) # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self._run_tracker.run_info.add_scm_info() # Update the Reporting settings now that we have options and goal info. invalidation_report = self._reporting.update_reporting(self._global_options, self._is_quiet(), self._run_tracker) context = Context(options=self._options, run_tracker=self._run_tracker, target_roots=self._targets, requested_goals=self._requested_goals, build_graph=self._build_graph, build_file_parser=self._build_file_parser, address_mapper=self._address_mapper, spec_excludes=self._spec_excludes, invalidation_report=invalidation_report) return context, invalidation_report def setup(self): context, invalidation_report = self._setup_context() return GoalRunner(context=context, goals=self._goals, kill_nailguns=self._kill_nailguns, run_tracker=self._run_tracker, invalidation_report=invalidation_report)
def _run(): # place the registration of the unhandled exception hook # as early as possible in the code sys.excepthook = _unhandled_exception_hook """ To add additional paths to sys.path, add a block to the config similar to the following: [main] roots: ['src/python/pants_internal/test/',] """ logging.basicConfig() version = pants_version() if len(sys.argv) == 2 and sys.argv[1] == _VERSION_OPTION: _do_exit(msg=version, out=sys.stdout) root_dir = get_buildroot() if not os.path.exists(root_dir): _exit_and_fail('PANTS_BUILD_ROOT does not point to a valid path: %s' % root_dir) if len(sys.argv) < 2: argv = ['goal'] else: argv = sys.argv[1:] # Hack to force ./pants -h etc. to redirect to goal. if argv[0] != 'goal' and set(['-h', '--help', 'help']).intersection(argv): argv = ['goal'] + argv parser = optparse.OptionParser(add_help_option=False, version=version) RcFile.install_disable_rc_option(parser) parser.add_option(_LOG_EXIT_OPTION, action='store_true', default=False, dest='log_exit', help='Log an exit message on success or failure.') config = Config.load() # XXX(wickman) This should be in the command goal, not in pants_exe.py! run_tracker = RunTracker.from_config(config) report = initial_reporting(config, run_tracker) run_tracker.start(report) url = run_tracker.run_info.get_info('report_url') if url: run_tracker.log(Report.INFO, 'See a report at: %s' % url) else: run_tracker.log(Report.INFO, '(To run a reporting server: ./pants goal server)') backend_packages = config.getlist('backends', 'packages') build_configuration = load_build_configuration_from_source( additional_backends=backend_packages) build_file_parser = BuildFileParser( build_configuration=build_configuration, root_dir=root_dir, run_tracker=run_tracker) address_mapper = BuildFileAddressMapper(build_file_parser) build_graph = BuildGraph(run_tracker=run_tracker, address_mapper=address_mapper) command_class, command_args = _parse_command(root_dir, argv) command = command_class(run_tracker, root_dir, parser, command_args, build_file_parser, address_mapper, build_graph) try: if command.serialized(): def onwait(pid): process = psutil.Process(pid) print('Waiting on pants process %d (%s) to complete' % (pid, ' '.join(process.cmdline)), file=sys.stderr) return True runfile = os.path.join(root_dir, '.pants.run') lock = Lock.acquire(runfile, onwait=onwait) else: lock = Lock.unlocked() try: result = command.run(lock) if result: run_tracker.set_root_outcome(WorkUnit.FAILURE) _do_exit(result) except KeyboardInterrupt: command.cleanup() raise finally: lock.release() finally: run_tracker.end() # Must kill nailguns only after run_tracker.end() is called, because there may still # be pending background work that needs a nailgun. if (hasattr(command.options, 'cleanup_nailguns') and command.options.cleanup_nailguns) \ or config.get('nailgun', 'autokill', default=False): NailgunTask.killall(None)
def setup(self): options_bootstrapper = OptionsBootstrapper() bootstrap_options = options_bootstrapper.get_bootstrap_options() # Get logging setup prior to loading backends so that they can log as needed. self._setup_logging(bootstrap_options.for_global_scope()) # Add any extra paths to python path (eg for loading extra source backends) for path in bootstrap_options.for_global_scope().pythonpath: sys.path.append(path) pkg_resources.fixup_namespace_packages(path) # Load plugins and backends. plugins = bootstrap_options.for_global_scope().plugins backend_packages = bootstrap_options.for_global_scope().backend_packages build_configuration = load_plugins_and_backends(plugins, backend_packages) # Now that plugins and backends are loaded, we can gather the known scopes. self.targets = [] known_scope_infos = [ScopeInfo.for_global_scope()] # Add scopes for all needed subsystems. subsystems = (set(self.subsystems) | Goal.subsystems() | build_configuration.subsystems()) for subsystem in subsystems: known_scope_infos.append(ScopeInfo(subsystem.options_scope, ScopeInfo.GLOBAL_SUBSYSTEM)) # Add scopes for all tasks in all goals. for goal in Goal.all(): known_scope_infos.extend(filter(None, goal.known_scope_infos())) # Now that we have the known scopes we can get the full options. self.options = options_bootstrapper.get_full_options(known_scope_infos) self.register_options(subsystems) # Make the options values available to all subsystems. Subsystem._options = self.options # Now that we have options we can instantiate subsystems. self.run_tracker = RunTracker.global_instance() self.reporting = Reporting.global_instance() report = self.reporting.initial_reporting(self.run_tracker) self.run_tracker.start(report) url = self.run_tracker.run_info.get_info('report_url') if url: self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url)) else: self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)') self.build_file_parser = BuildFileParser(build_configuration=build_configuration, root_dir=self.root_dir, run_tracker=self.run_tracker) rev = self.options.for_global_scope().build_file_rev if rev: ScmBuildFile.set_rev(rev) ScmBuildFile.set_scm(get_scm()) build_file_type = ScmBuildFile else: build_file_type = FilesystemBuildFile self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type) self.build_graph = BuildGraph(run_tracker=self.run_tracker, address_mapper=self.address_mapper) # TODO(John Sirois): Kill when source root registration is lifted out of BUILD files. with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]): source_root_bootstrapper = SourceRootBootstrapper.global_instance() source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser) self._expand_goals_and_specs() # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self.run_tracker.run_info.add_scm_info()
class GoalRunner(object): """Lists installed goals or else executes a named goal.""" def __init__(self, root_dir): """ :param root_dir: The root directory of the pants workspace. """ self.root_dir = root_dir @property def subsystems(self): # Subsystems used outside of any task. return SourceRootBootstrapper, Reporting, RunTracker def setup(self): options_bootstrapper = OptionsBootstrapper() bootstrap_options = options_bootstrapper.get_bootstrap_options() # Get logging setup prior to loading backends so that they can log as needed. self._setup_logging(bootstrap_options.for_global_scope()) # Add any extra paths to python path (eg for loading extra source backends) for path in bootstrap_options.for_global_scope().pythonpath: sys.path.append(path) pkg_resources.fixup_namespace_packages(path) # Load plugins and backends. plugins = bootstrap_options.for_global_scope().plugins backend_packages = bootstrap_options.for_global_scope().backend_packages build_configuration = load_plugins_and_backends(plugins, backend_packages) # Now that plugins and backends are loaded, we can gather the known scopes. self.targets = [] known_scope_infos = [ScopeInfo.for_global_scope()] # Add scopes for all needed subsystems. subsystems = (set(self.subsystems) | Goal.subsystems() | build_configuration.subsystems()) for subsystem in subsystems: known_scope_infos.append(ScopeInfo(subsystem.options_scope, ScopeInfo.GLOBAL_SUBSYSTEM)) # Add scopes for all tasks in all goals. for goal in Goal.all(): known_scope_infos.extend(filter(None, goal.known_scope_infos())) # Now that we have the known scopes we can get the full options. self.options = options_bootstrapper.get_full_options(known_scope_infos) self.register_options(subsystems) # Make the options values available to all subsystems. Subsystem._options = self.options # Now that we have options we can instantiate subsystems. self.run_tracker = RunTracker.global_instance() self.reporting = Reporting.global_instance() report = self.reporting.initial_reporting(self.run_tracker) self.run_tracker.start(report) url = self.run_tracker.run_info.get_info('report_url') if url: self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url)) else: self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)') self.build_file_parser = BuildFileParser(build_configuration=build_configuration, root_dir=self.root_dir, run_tracker=self.run_tracker) rev = self.options.for_global_scope().build_file_rev if rev: ScmBuildFile.set_rev(rev) ScmBuildFile.set_scm(get_scm()) build_file_type = ScmBuildFile else: build_file_type = FilesystemBuildFile self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type) self.build_graph = BuildGraph(run_tracker=self.run_tracker, address_mapper=self.address_mapper) # TODO(John Sirois): Kill when source root registration is lifted out of BUILD files. with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]): source_root_bootstrapper = SourceRootBootstrapper.global_instance() source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser) self._expand_goals_and_specs() # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self.run_tracker.run_info.add_scm_info() @property def spec_excludes(self): # Note: Only call after register_options() has been called. return self.options.for_global_scope().spec_excludes @property def global_options(self): return self.options.for_global_scope() def register_options(self, subsystems): # Standalone global options. GlobalOptionsRegistrar.register_options_on_scope(self.options) # Options for subsystems. for subsystem in subsystems: subsystem.register_options_on_scope(self.options) # TODO(benjy): Should Goals be subsystems? Or should the entire goal-running mechanism # be a subsystem? for goal in Goal.all(): # Register task options. goal.register_options(self.options) def _expand_goals_and_specs(self): goals = self.options.goals specs = self.options.target_specs fail_fast = self.options.for_global_scope().fail_fast for goal in goals: if self.address_mapper.from_cache(get_buildroot(), goal, must_exist=False).file_exists(): logger.warning(" Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal)) if self.options.print_help_if_requested(): sys.exit(0) self.requested_goals = goals with self.run_tracker.new_workunit(name='setup', labels=[WorkUnit.SETUP]): spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper, spec_excludes=self.spec_excludes, exclude_target_regexps=self.global_options.exclude_target_regexp) with self.run_tracker.new_workunit(name='parse', labels=[WorkUnit.SETUP]): def filter_for_tag(tag): return lambda target: tag in map(str, target.tags) tag_filter = wrap_filters(create_filters(self.global_options.tag, filter_for_tag)) for spec in specs: for address in spec_parser.parse_addresses(spec, fail_fast): self.build_graph.inject_address_closure(address) tgt = self.build_graph.get_target(address) if tag_filter(tgt): self.targets.append(tgt) self.goals = [Goal.by_name(goal) for goal in goals] def run(self): def fail(): self.run_tracker.set_root_outcome(WorkUnit.FAILURE) kill_nailguns = self.options.for_global_scope().kill_nailguns try: result = self._do_run() if result: fail() except KeyboardInterrupt: fail() # On ctrl-c we always kill nailguns, otherwise they might keep running # some heavyweight compilation and gum up the system during a subsequent run. kill_nailguns = True raise except Exception: fail() raise finally: self.run_tracker.end() # Must kill nailguns only after run_tracker.end() is called, otherwise there may still # be pending background work that needs a nailgun. if kill_nailguns: # TODO: This is JVM-specific and really doesn't belong here. # TODO: Make this more selective? Only kill nailguns that affect state? # E.g., checkstyle may not need to be killed. NailgunProcessGroup().killall() return result def _do_run(self): # Update the reporting settings, now that we have flags etc. def is_quiet_task(): for goal in self.goals: if goal.has_task_of_type(QuietTaskMixin): return True return False is_explain = self.global_options.explain if self.reporting.global_instance().get_options().invalidation_report: invalidation_report = InvalidationReport() else: invalidation_report = None self.reporting.update_reporting(self.global_options, is_quiet_task() or is_explain, self.run_tracker, invalidation_report=invalidation_report) context = Context( options=self.options, run_tracker=self.run_tracker, target_roots=self.targets, requested_goals=self.requested_goals, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, spec_excludes=self.spec_excludes, invalidation_report=invalidation_report ) unknown = [] for goal in self.goals: if not goal.ordered_task_names(): unknown.append(goal) if unknown: context.log.error('Unknown goal(s): {}\n'.format(' '.join(goal.name for goal in unknown))) return 1 engine = RoundEngine() result = engine.execute(context, self.goals) if invalidation_report: invalidation_report.report() return result def _setup_logging(self, global_options): # NB: quiet help says 'Squelches all console output apart from errors'. level = 'ERROR' if global_options.quiet else global_options.level.upper() setup_logging(level, log_dir=global_options.logdir)
def setup(self, options_bootstrapper, working_set): bootstrap_options = options_bootstrapper.get_bootstrap_options() global_bootstrap_options = bootstrap_options.for_global_scope() # The pants_version may be set in pants.ini for bootstrapping, so we make sure the user actually # requested the version on the command line before deciding to print the version and exit. if global_bootstrap_options.is_flagged('pants_version'): print(global_bootstrap_options.pants_version) self._exiter(0) # Get logging setup prior to loading backends so that they can log as needed. self._setup_logging(global_bootstrap_options) # Add any extra paths to python path (e.g., for loading extra source backends). for path in global_bootstrap_options.pythonpath: sys.path.append(path) pkg_resources.fixup_namespace_packages(path) # Load plugins and backends. plugins = global_bootstrap_options.plugins backend_packages = global_bootstrap_options.backend_packages build_configuration = load_plugins_and_backends(plugins, working_set, backend_packages) # Now that plugins and backends are loaded, we can gather the known scopes. self.targets = [] known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()] # Add scopes for all needed subsystems. subsystems = Subsystem.closure(set(self.subsystems) | Goal.subsystems() | build_configuration.subsystems()) for subsystem in subsystems: known_scope_infos.append(subsystem.get_scope_info()) # Add scopes for all tasks in all goals. for goal in Goal.all(): known_scope_infos.extend(filter(None, goal.known_scope_infos())) # Now that we have the known scopes we can get the full options. self.options = options_bootstrapper.get_full_options(known_scope_infos) self.register_options(subsystems) # Make the options values available to all subsystems. Subsystem._options = self.options # Now that we have options we can instantiate subsystems. self.run_tracker = RunTracker.global_instance() self.reporting = Reporting.global_instance() report = self.reporting.initial_reporting(self.run_tracker) self.run_tracker.start(report) url = self.run_tracker.run_info.get_info('report_url') if url: self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url)) else: self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)') self.build_file_parser = BuildFileParser(build_configuration=build_configuration, root_dir=self.root_dir, run_tracker=self.run_tracker) rev = self.options.for_global_scope().build_file_rev if rev: ScmBuildFile.set_rev(rev) ScmBuildFile.set_scm(get_scm()) build_file_type = ScmBuildFile else: build_file_type = FilesystemBuildFile self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type) self.build_graph = BuildGraph(run_tracker=self.run_tracker, address_mapper=self.address_mapper) # TODO(John Sirois): Kill when source root registration is lifted out of BUILD files. with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnitLabel.SETUP]): source_root_bootstrapper = SourceRootBootstrapper.global_instance() source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser) self._expand_goals_and_specs() # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self.run_tracker.run_info.add_scm_info()
def setup(self): options_bootstrapper = OptionsBootstrapper() # Force config into the cache so we (and plugin/backend loading code) can use it. # TODO: Plumb options in explicitly. bootstrap_options = options_bootstrapper.get_bootstrap_options() self.config = Config.from_cache() # Get logging setup prior to loading backends so that they can log as needed. self._setup_logging(bootstrap_options.for_global_scope()) # Add any extra paths to python path (eg for loading extra source backends) for path in bootstrap_options.for_global_scope().pythonpath: sys.path.append(path) pkg_resources.fixup_namespace_packages(path) # Load plugins and backends. backend_packages = self.config.getlist('backends', 'packages', []) plugins = self.config.getlist('backends', 'plugins', []) build_configuration = load_plugins_and_backends(plugins, backend_packages) # Now that plugins and backends are loaded, we can gather the known scopes. self.targets = [] known_scopes = [''] # Add scopes for global subsystem instances. for subsystem_type in set(self.subsystems) | Goal.global_subsystem_types(): known_scopes.append(subsystem_type.qualify_scope(Options.GLOBAL_SCOPE)) # Add scopes for all tasks in all goals. for goal in Goal.all(): # Note that enclosing scopes will appear before scopes they enclose. known_scopes.extend(filter(None, goal.known_scopes())) # Now that we have the known scopes we can get the full options. self.options = options_bootstrapper.get_full_options(known_scopes=known_scopes) self.register_options() # Make the options values available to all subsystems. Subsystem._options = self.options # Now that we have options we can instantiate subsystems. self.run_tracker = RunTracker.global_instance() report = initial_reporting(self.config, self.run_tracker) self.run_tracker.start(report) url = self.run_tracker.run_info.get_info('report_url') if url: self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url)) else: self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)') self.build_file_parser = BuildFileParser(build_configuration=build_configuration, root_dir=self.root_dir, run_tracker=self.run_tracker) rev = self.options.for_global_scope().build_file_rev if rev: ScmBuildFile.set_rev(rev) ScmBuildFile.set_scm(get_scm()) build_file_type = ScmBuildFile else: build_file_type = FilesystemBuildFile self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type) self.build_graph = BuildGraph(run_tracker=self.run_tracker, address_mapper=self.address_mapper) with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]): # construct base parameters to be filled in for BuildGraph for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]): build_file = self.address_mapper.from_cache(root_dir=self.root_dir, relpath=path) # TODO(pl): This is an unfortunate interface leak, but I don't think # in the long run that we should be relying on "bootstrap" BUILD files # that do nothing except modify global state. That type of behavior # (e.g. source roots, goal registration) should instead happen in # project plugins, or specialized configuration files. self.build_file_parser.parse_build_file_family(build_file) self._expand_goals_and_specs() # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self.run_tracker.run_info.add_scm_info()
class GoalRunner(object): """Lists installed goals or else executes a named goal.""" # Subsytems used outside of any task. subsystems = (RunTracker, ) def __init__(self, root_dir): """ :param root_dir: The root directory of the pants workspace. """ self.root_dir = root_dir def setup(self): options_bootstrapper = OptionsBootstrapper() # Force config into the cache so we (and plugin/backend loading code) can use it. # TODO: Plumb options in explicitly. bootstrap_options = options_bootstrapper.get_bootstrap_options() self.config = Config.from_cache() # Get logging setup prior to loading backends so that they can log as needed. self._setup_logging(bootstrap_options.for_global_scope()) # Add any extra paths to python path (eg for loading extra source backends) for path in bootstrap_options.for_global_scope().pythonpath: sys.path.append(path) pkg_resources.fixup_namespace_packages(path) # Load plugins and backends. backend_packages = self.config.getlist('backends', 'packages', []) plugins = self.config.getlist('backends', 'plugins', []) build_configuration = load_plugins_and_backends(plugins, backend_packages) # Now that plugins and backends are loaded, we can gather the known scopes. self.targets = [] known_scopes = [''] # Add scopes for global subsystem instances. for subsystem_type in set(self.subsystems) | Goal.global_subsystem_types(): known_scopes.append(subsystem_type.qualify_scope(Options.GLOBAL_SCOPE)) # Add scopes for all tasks in all goals. for goal in Goal.all(): # Note that enclosing scopes will appear before scopes they enclose. known_scopes.extend(filter(None, goal.known_scopes())) # Now that we have the known scopes we can get the full options. self.options = options_bootstrapper.get_full_options(known_scopes=known_scopes) self.register_options() # Make the options values available to all subsystems. Subsystem._options = self.options # Now that we have options we can instantiate subsystems. self.run_tracker = RunTracker.global_instance() report = initial_reporting(self.config, self.run_tracker) self.run_tracker.start(report) url = self.run_tracker.run_info.get_info('report_url') if url: self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url)) else: self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)') self.build_file_parser = BuildFileParser(build_configuration=build_configuration, root_dir=self.root_dir, run_tracker=self.run_tracker) rev = self.options.for_global_scope().build_file_rev if rev: ScmBuildFile.set_rev(rev) ScmBuildFile.set_scm(get_scm()) build_file_type = ScmBuildFile else: build_file_type = FilesystemBuildFile self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type) self.build_graph = BuildGraph(run_tracker=self.run_tracker, address_mapper=self.address_mapper) with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]): # construct base parameters to be filled in for BuildGraph for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]): build_file = self.address_mapper.from_cache(root_dir=self.root_dir, relpath=path) # TODO(pl): This is an unfortunate interface leak, but I don't think # in the long run that we should be relying on "bootstrap" BUILD files # that do nothing except modify global state. That type of behavior # (e.g. source roots, goal registration) should instead happen in # project plugins, or specialized configuration files. self.build_file_parser.parse_build_file_family(build_file) self._expand_goals_and_specs() # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self.run_tracker.run_info.add_scm_info() @property def spec_excludes(self): # Note: Only call after register_options() has been called. return self.options.for_global_scope().spec_excludes @property def global_options(self): return self.options.for_global_scope() def register_options(self): # Standalone global options. register_global_options(self.options.registration_function_for_global_scope()) # Options for global-level subsystems. for subsystem_type in set(self.subsystems) | Goal.global_subsystem_types(): subsystem_type.register_options_on_scope(self.options, Options.GLOBAL_SCOPE) # TODO(benjy): Should Goals be subsystems? Or should the entire goal-running mechanism # be a subsystem? for goal in Goal.all(): # Register task options (including per-task subsystem options). goal.register_options(self.options) def _expand_goals_and_specs(self): goals = self.options.goals specs = self.options.target_specs fail_fast = self.options.for_global_scope().fail_fast for goal in goals: if self.address_mapper.from_cache(get_buildroot(), goal, must_exist=False).file_exists(): logger.warning(" Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal)) if self.options.print_help_if_requested(): sys.exit(0) self.requested_goals = goals with self.run_tracker.new_workunit(name='setup', labels=[WorkUnit.SETUP]): spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper, spec_excludes=self.spec_excludes, exclude_target_regexps=self.global_options.exclude_target_regexp) with self.run_tracker.new_workunit(name='parse', labels=[WorkUnit.SETUP]): for spec in specs: for address in spec_parser.parse_addresses(spec, fail_fast): self.build_graph.inject_address_closure(address) self.targets.append(self.build_graph.get_target(address)) self.goals = [Goal.by_name(goal) for goal in goals] def run(self): def fail(): self.run_tracker.set_root_outcome(WorkUnit.FAILURE) kill_nailguns = self.options.for_global_scope().kill_nailguns try: result = self._do_run() if result: fail() except KeyboardInterrupt: fail() # On ctrl-c we always kill nailguns, otherwise they might keep running # some heavyweight compilation and gum up the system during a subsequent run. kill_nailguns = True raise except Exception: fail() raise finally: self.run_tracker.end() # Must kill nailguns only after run_tracker.end() is called, otherwise there may still # be pending background work that needs a nailgun. if kill_nailguns: # TODO: This is JVM-specific and really doesn't belong here. # TODO: Make this more selective? Only kill nailguns that affect state? # E.g., checkstyle may not need to be killed. NailgunTask.killall() return result def _do_run(self): # Update the reporting settings, now that we have flags etc. def is_quiet_task(): for goal in self.goals: if goal.has_task_of_type(QuietTaskMixin): return True return False is_explain = self.global_options.explain update_reporting(self.global_options, is_quiet_task() or is_explain, self.run_tracker) context = Context( config=self.config, options=self.options, run_tracker=self.run_tracker, target_roots=self.targets, requested_goals=self.requested_goals, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, spec_excludes=self.spec_excludes ) unknown = [] for goal in self.goals: if not goal.ordered_task_names(): unknown.append(goal) if unknown: context.log.error('Unknown goal(s): {}\n'.format(' '.join(goal.name for goal in unknown))) return 1 engine = RoundEngine() return engine.execute(context, self.goals) def _setup_logging(self, global_options): # NB: quiet help says 'Squelches all console output apart from errors'. level = 'ERROR' if global_options.quiet else global_options.level.upper() setup_logging(level, log_dir=global_options.logdir)
class GoalRunnerFactory(object): def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit): """ :param str root_dir: The root directory of the pants workspace (aka the "build root"). :param Options options: The global, pre-initialized Options instance. :param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance. :param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance. :param Reporting reporting: The global, pre-initialized Reporting instance. :param func exiter: A function that accepts an exit code value and exits (for tests, Optional). """ self._root_dir = root_dir self._options = options self._build_config = build_config self._run_tracker = run_tracker self._reporting = reporting self._exiter = exiter self._goals = [] self._targets = [] self._requested_goals = self._options.goals self._target_specs = self._options.target_specs self._help_request = self._options.help_request self._global_options = options.for_global_scope() self._tag = self._global_options.tag self._fail_fast = self._global_options.fail_fast self._spec_excludes = self._global_options.spec_excludes self._explain = self._global_options.explain self._kill_nailguns = self._global_options.kill_nailguns self._build_file_type = self._get_buildfile_type(self._global_options.build_file_rev) self._build_file_parser = BuildFileParser(self._build_config, self._root_dir) self._address_mapper = BuildFileAddressMapper(self._build_file_parser, self._build_file_type) self._build_graph = BuildGraph(self._address_mapper) self._spec_parser = CmdLineSpecParser( self._root_dir, self._address_mapper, spec_excludes=self._spec_excludes, exclude_target_regexps=self._global_options.exclude_target_regexp, ) def _get_buildfile_type(self, build_file_rev): """Selects the BuildFile type for use in a given pants run.""" if build_file_rev: ScmBuildFile.set_rev(build_file_rev) ScmBuildFile.set_scm(get_scm()) return ScmBuildFile else: return FilesystemBuildFile def _expand_goals(self, goals): """Check and populate the requested goals for a given run.""" for goal in goals: if self._address_mapper.from_cache(self._root_dir, goal, must_exist=False).file_exists(): logger.warning( "Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal) ) if self._help_request: help_printer = HelpPrinter(self._options) help_printer.print_help() self._exiter(0) self._goals.extend([Goal.by_name(goal) for goal in goals]) def _expand_specs(self, specs, fail_fast): """Populate the BuildGraph and target list from a set of input specs.""" with self._run_tracker.new_workunit(name="parse", labels=[WorkUnitLabel.SETUP]): def filter_for_tag(tag): return lambda target: tag in map(str, target.tags) tag_filter = wrap_filters(create_filters(self._tag, filter_for_tag)) for spec in specs: for address in self._spec_parser.parse_addresses(spec, fail_fast): self._build_graph.inject_address_closure(address) target = self._build_graph.get_target(address) if tag_filter(target): self._targets.append(target) def _is_quiet(self): return any(goal.has_task_of_type(QuietTaskMixin) for goal in self._goals) or self._explain def _setup_context(self): # TODO(John Sirois): Kill when source root registration is lifted out of BUILD files. with self._run_tracker.new_workunit(name="bootstrap", labels=[WorkUnitLabel.SETUP]): source_root_bootstrapper = SourceRootBootstrapper.global_instance() source_root_bootstrapper.bootstrap(self._address_mapper, self._build_file_parser) with self._run_tracker.new_workunit(name="setup", labels=[WorkUnitLabel.SETUP]): self._expand_goals(self._requested_goals) self._expand_specs(self._target_specs, self._fail_fast) # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self._run_tracker.run_info.add_scm_info() # Update the Reporting settings now that we have options and goal info. invalidation_report = self._reporting.update_reporting( self._global_options, self._is_quiet(), self._run_tracker ) context = Context( options=self._options, run_tracker=self._run_tracker, target_roots=self._targets, requested_goals=self._requested_goals, build_graph=self._build_graph, build_file_parser=self._build_file_parser, address_mapper=self._address_mapper, spec_excludes=self._spec_excludes, invalidation_report=invalidation_report, ) return context, invalidation_report def setup(self): context, invalidation_report = self._setup_context() return GoalRunner( context=context, goals=self._goals, kill_nailguns=self._kill_nailguns, run_tracker=self._run_tracker, invalidation_report=invalidation_report, )