예제 #1
0
 def test_complete_scopes(self):
     _global = GlobalOptionsRegistrar.get_scope_info()
     self.assertEquals(
         {_global, intermediate("foo"), intermediate("foo.bar"), task("foo.bar.baz")},
         Options.complete_scopes({task("foo.bar.baz")}),
     )
     self.assertEquals(
         {_global, intermediate("foo"), intermediate("foo.bar"), task("foo.bar.baz")},
         Options.complete_scopes({GlobalOptionsRegistrar.get_scope_info(), task("foo.bar.baz")}),
     )
     self.assertEquals(
         {_global, intermediate("foo"), intermediate("foo.bar"), task("foo.bar.baz")},
         Options.complete_scopes({intermediate("foo"), task("foo.bar.baz")}),
     )
     self.assertEquals(
         {
             _global,
             intermediate("foo"),
             intermediate("foo.bar"),
             task("foo.bar.baz"),
             intermediate("qux"),
             task("qux.quux"),
         },
         Options.complete_scopes({task("foo.bar.baz"), task("qux.quux")}),
     )
예제 #2
0
 def test_complete_scopes(self):
   self.assertEquals({'', 'foo', 'foo.bar', 'foo.bar.baz'},
                     Options.complete_scopes({'foo.bar.baz'}))
   self.assertEquals({'', 'foo', 'foo.bar', 'foo.bar.baz'},
                     Options.complete_scopes({'', 'foo.bar.baz'}))
   self.assertEquals({'', 'foo', 'foo.bar', 'foo.bar.baz'},
                     Options.complete_scopes({'foo', 'foo.bar.baz'}))
   self.assertEquals({'', 'foo', 'foo.bar', 'foo.bar.baz', 'qux', 'qux.quux'},
                     Options.complete_scopes({'foo.bar.baz', 'qux.quux'}))
예제 #3
0
 def test_complete_scopes(self):
   _global = ScopeInfo.for_global_scope()
   self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz')},
                     Options.complete_scopes({task('foo.bar.baz')}))
   self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz')},
                     Options.complete_scopes({ScopeInfo.for_global_scope(), task('foo.bar.baz')}))
   self.assertEquals({_global, goal('foo'), intermediate('foo.bar'), task('foo.bar.baz')},
                     Options.complete_scopes({goal('foo'), task('foo.bar.baz')}))
   self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz'),
                      intermediate('qux'), task('qux.quux')},
                     Options.complete_scopes({task('foo.bar.baz'), task('qux.quux')}))
예제 #4
0
 def test_complete_scopes(self):
   _global = GlobalOptionsRegistrar.get_scope_info()
   self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz')},
                     Options.complete_scopes({task('foo.bar.baz')}))
   self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz')},
                     Options.complete_scopes({GlobalOptionsRegistrar.get_scope_info(),
                                              task('foo.bar.baz')}))
   self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz')},
                     Options.complete_scopes({intermediate('foo'), task('foo.bar.baz')}))
   self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz'),
                      intermediate('qux'), task('qux.quux')},
                     Options.complete_scopes({task('foo.bar.baz'), task('qux.quux')}))
예제 #5
0
def get_bootstrap_option_values(env=None, config=None, args=None, buildroot=None):
  """Get the values of just the bootstrap options."""
  # Filter just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
  flags = set()
  def capture_the_flags(*args, **kwargs):
    flags.update(args)
  register_bootstrap_options(capture_the_flags, buildroot=buildroot)
  bargs = filter(lambda x: x.partition('=')[0] in flags, args)

  bootstrap_options = Options(env=env, config=config, known_scopes=[GLOBAL_SCOPE], args=bargs)
  register_bootstrap_options(bootstrap_options.register_global, buildroot=buildroot)
  return bootstrap_options.for_global_scope()
예제 #6
0
  def get_bootstrap_options(self):
    """Returns an Options instance that only knows about the bootstrap options."""
    if not self._bootstrap_options:
      flags = set()
      def capture_the_flags(*args, **kwargs):
        flags.update(args)
      register_bootstrap_options(capture_the_flags, buildroot=self._buildroot)
      # Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
      bargs = filter(lambda x: x.partition('=')[0] in flags, self._args or [])

      self._bootstrap_options = Options(env=self._env, config=self._pre_bootstrap_config,
                                        known_scopes=[GLOBAL_SCOPE], args=bargs)
      register_bootstrap_options(self._bootstrap_options.register_global, buildroot=self._buildroot)
      bootstrap_option_values = self._bootstrap_options.for_global_scope()
      Config.reset_default_bootstrap_option_values(values=bootstrap_option_values)

      # Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
      # from (typically pants.ini), then config override, then rcfiles.
      configpaths = list(self._pre_bootstrap_config.sources())
      if bootstrap_option_values.config_override:
        configpaths.append(bootstrap_option_values.config_override)
      if bootstrap_option_values.pantsrc:
        rcfiles = [os.path.expanduser(rcfile) for rcfile in bootstrap_option_values.pantsrc_files]
        existing_rcfiles = filter(os.path.exists, rcfiles)
        configpaths.extend(existing_rcfiles)

      self._post_bootstrap_config = Config.load(configpaths)
      Config.cache(self._post_bootstrap_config)

    return self._bootstrap_options
예제 #7
0
  def test_env_type_int(self):
    options = Options.create(env={'PANTS_FOO_BAR': "['123','456']"},
                             config=self._create_config({}),
                             known_scope_infos=OptionsTest._known_scope_infos,
                             args=shlex.split('./pants'),
                             option_tracker=OptionTracker())
    options.register(GLOBAL_SCOPE, '--foo-bar', type=list, member_type=int)
    self.assertEqual([123, 456], options.for_global_scope().foo_bar)

    options = Options.create(env={'PANTS_FOO_BAR': '123'},
                             config=self._create_config({}),
                             known_scope_infos=OptionsTest._known_scope_infos,
                             args=shlex.split('./pants'),
                             option_tracker=OptionTracker())
    options.register(GLOBAL_SCOPE, '--foo-bar', type=int)
    self.assertEqual(123, options.for_global_scope().foo_bar)
예제 #8
0
 def __init__(self, bootstrap_options: Options, daemon_entrypoint: str):
     super().__init__(
         name="pantsd",
         metadata_base_dir=bootstrap_options.for_global_scope().pants_subprocessdir,
     )
     self._bootstrap_options = bootstrap_options
     self._daemon_entrypoint = daemon_entrypoint
예제 #9
0
 def bootstrap_options_from_config(config):
   bootstrap_options = Options.create(env=self._env, config=config,
       known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()], args=bargs)
   def register_global(*args, **kwargs):
     bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
   GlobalOptionsRegistrar.register_bootstrap_options(register_global)
   return bootstrap_options
예제 #10
0
 def test_double_registration(self):
   options = Options.create(env={},
                            config=self._create_config({}),
                            known_scope_infos=OptionsTest._known_scope_infos,
                            args=shlex.split('./pants'),
                            option_tracker=OptionTracker())
   options.register(GLOBAL_SCOPE, '--foo-bar')
   self.assertRaises(OptionAlreadyRegistered, lambda: options.register(GLOBAL_SCOPE, '--foo-bar'))
예제 #11
0
 def test_double_registration(self):
   options = Options.create(env={},
                            config=self._create_config({}),
                            known_scope_infos=OptionsTest._known_scope_infos,
                            args=shlex.split('./pants'),
                            option_tracker=OptionTracker())
   options.register(GLOBAL_SCOPE, '--foo-bar')
   self.assertRaises(OptionAlreadyRegistered, lambda: options.register(GLOBAL_SCOPE, '--foo-bar'))
예제 #12
0
 def test_deprecated_option_past_removal(self):
   with self.assertRaises(PastRemovalVersionError):
     options = Options.create(env={},
                              config=self._create_config({}),
                              known_scope_infos=OptionsTest._known_scope_infos,
                              args="./pants")
     options.register(GLOBAL_SCOPE, '--too-old-option', deprecated_version='0.0.24',
                      deprecated_hint='The semver for this option has already passed.')
예제 #13
0
 def _full_options(self, known_scope_infos):
     bootstrap_option_values = self.get_bootstrap_options(
     ).for_global_scope()
     return Options.create(self.env,
                           self.config,
                           known_scope_infos,
                           args=self.args,
                           bootstrap_option_values=bootstrap_option_values)
예제 #14
0
    def create(
        cls,
        options_bootstrapper: OptionsBootstrapper,
        options: Options,
        session: SchedulerSession,
        build_root: Optional[str] = None,
    ) -> Specs:
        specs = cls.parse_specs(raw_specs=options.specs, build_root=build_root)
        changed_options = ChangedOptions.from_options(
            options.for_scope("changed"))

        logger.debug("specs are: %s", specs)
        logger.debug("changed_options are: %s", changed_options)

        if specs.provided and changed_options.provided:
            changed_name = "--changed-since" if changed_options.since else "--changed-diffspec"
            if specs.filesystem_specs and specs.address_specs:
                specs_description = "target and file arguments"
            elif specs.filesystem_specs:
                specs_description = "file arguments"
            else:
                specs_description = "target arguments"
            raise InvalidSpecConstraint(
                f"You used `{changed_name}` at the same time as using {specs_description}. Please "
                "use only one.")

        if not changed_options.provided:
            return specs

        scm = get_scm()
        if not scm:
            raise InvalidSpecConstraint(
                "The `--changed-*` options are not available without a recognized SCM (usually "
                "Git).")
        changed_request = ChangedRequest(
            sources=tuple(changed_options.changed_files(scm=scm)),
            dependees=changed_options.dependees,
        )
        (changed_addresses, ) = session.product_request(
            ChangedAddresses, [Params(changed_request, options_bootstrapper)])
        logger.debug("changed addresses: %s", changed_addresses)

        address_specs = []
        filesystem_specs = []
        for address in cast(ChangedAddresses, changed_addresses):
            if not address.is_base_target:
                # TODO: Should adjust Specs parsing to support parsing the disambiguated file
                # Address, which would bypass-rediscovering owners.
                filesystem_specs.append(FilesystemLiteralSpec(
                    address.filename))
            else:
                address_specs.append(
                    SingleAddress(address.spec_path, address.target_name))

        return Specs(
            AddressSpecs(address_specs, filter_by_global_options=True),
            FilesystemSpecs(filesystem_specs),
        )
예제 #15
0
    def __init__(
        self,
        root_dir: str,
        options_bootstrapper: OptionsBootstrapper,
        options: Options,
        build_config: BuildConfiguration,
        run_tracker: RunTracker,
        reporting: Reporting,
        graph_session: LegacyGraphSession,
        specs: Specs,
        exiter=sys.exit,
    ) -> None:
        """
        :param root_dir: The root directory of the pants workspace (aka the "build root").
        :param options: The global, pre-initialized Options instance.
        :param build_config: A pre-initialized BuildConfiguration instance.
        :param run_tracker: The global, pre-initialized/running RunTracker instance.
        :param reporting: The global, pre-initialized Reporting instance.
        :param graph_session: The graph session for this run.
        :param specs: The specs for this run, i.e. either the address or filesystem specs.
        :param func exiter: A function that accepts an exit code value and exits. (for tests, Optional)
        """
        self._root_dir = root_dir
        self._options_bootstrapper = options_bootstrapper
        self._options = options
        self._build_config = build_config
        self._run_tracker = run_tracker
        self._reporting = reporting
        self._graph_session = graph_session
        self._specs = specs
        self._exiter = exiter

        self._global_options = options.for_global_scope()
        self._fail_fast = self._global_options.fail_fast
        self._explain = self._global_options.explain
        self._kill_nailguns = self._global_options.kill_nailguns

        # V1 tasks do not understand FilesystemSpecs, so we eagerly convert them into AddressSpecs.
        if self._specs.filesystem_specs.dependencies:
            (owned_addresses,
             ) = self._graph_session.scheduler_session.product_request(
                 Addresses, [
                     Params(self._specs.filesystem_specs,
                            self._options_bootstrapper)
                 ])
            updated_address_specs = AddressSpecs(
                dependencies=tuple(
                    SingleAddress(a.spec_path, a.target_name)
                    for a in owned_addresses),
                tags=self._specs.address_specs.matcher.tags,
                exclude_patterns=self._specs.address_specs.matcher.
                exclude_patterns,
            )
            self._specs = Specs(
                address_specs=updated_address_specs,
                filesystem_specs=FilesystemSpecs([]),
            )
예제 #16
0
    def run_goal_rules(
        self,
        *,
        options_bootstrapper: OptionsBootstrapper,
        union_membership: UnionMembership,
        options: Options,
        goals: Iterable[str],
        specs: Specs,
    ):
        """Runs @goal_rules sequentially and interactively by requesting their implicit Goal
        products.

        For retryable failures, raises scheduler.ExecutionError.

        :returns: An exit code.
        """

        global_options = options.for_global_scope()

        console = Console(
            use_colors=global_options.colors,
            session=self.scheduler_session
            if global_options.get("v2_ui") else None,
        )
        workspace = Workspace(self.scheduler_session)
        interactive_runner = InteractiveRunner(self.scheduler_session)

        for goal in goals:
            goal_product = self.goal_map[goal]
            # NB: We no-op for goals that have no V2 implementation because no relevant backends are
            # registered. This allows us to safely set `--v1 --v2`, even if no V2 backends are registered.
            # Once V1 is removed, we might want to reconsider the behavior to instead warn or error when
            # trying to run something like `./pants run` without any backends registered.
            is_implemented = union_membership.has_members_for_all(
                goal_product.subsystem_cls.required_union_implementations)
            if not is_implemented:
                continue
            params = Params(
                specs.provided_specs,
                options_bootstrapper,
                console,
                workspace,
                interactive_runner,
            )
            logger.debug(
                f"requesting {goal_product} to satisfy execution of `{goal}` goal"
            )
            try:
                exit_code = self.scheduler_session.run_goal_rule(
                    goal_product, params)
            finally:
                console.flush()

            if exit_code != PANTS_SUCCEEDED_EXIT_CODE:
                return exit_code

        return PANTS_SUCCEEDED_EXIT_CODE
예제 #17
0
 def _parse(self, args_str, env=None, config=None, bootstrap_option_values=None):
   args = shlex.split(str(args_str))
   options = Options.create(env=env or {},
                            config=self._create_config(config or {}),
                            known_scope_infos=OptionsTest._known_scope_infos,
                            args=args,
                            bootstrap_option_values=bootstrap_option_values)
   self._register(options)
   return options
예제 #18
0
      def bootstrap_options_from_config(config):
        bootstrap_options = Options.create(env=self._env, config=config,
            known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()], args=bargs,
            option_tracker=self._option_tracker)

        def register_global(*args, **kwargs):
          bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
        GlobalOptionsRegistrar.register_bootstrap_options(register_global)
        return bootstrap_options
예제 #19
0
 def test_frozen_registration(self):
     options = Options.create(args=[],
                              env={},
                              config=self._create_config({}),
                              known_scope_infos=[task('foo')],
                              option_tracker=OptionTracker())
     options.register('foo', '--arg1')
     with self.assertRaises(FrozenRegistration):
         options.register(GLOBAL_SCOPE, '--arg2')
예제 #20
0
 def assertError(expected_error, *args, **kwargs):
     with self.assertRaises(expected_error):
         options = Options.create(args=[],
                                  env={},
                                  config=self._create_config({}),
                                  known_scope_infos=[],
                                  option_tracker=OptionTracker())
         options.register(GLOBAL_SCOPE, *args, **kwargs)
         options.for_global_scope()
예제 #21
0
    def get_all_help_info(
        cls,
        options: Options,
        union_membership: UnionMembership,
        consumed_scopes_mapper: ConsumedScopesMapper,
    ) -> AllHelpInfo:
        scope_to_help_info = {}
        name_to_goal_info = {}
        for scope_info in sorted(options.known_scope_to_info.values(), key=lambda x: x.scope):
            options.for_scope(scope_info.scope)  # Force parsing.
            optionable_cls = scope_info.optionable_cls
            if not scope_info.description:
                cls_name = (
                    f"{optionable_cls.__module__}.{optionable_cls.__qualname__}"
                    if optionable_cls
                    else ""
                )
                raise ValueError(
                    f"Subsystem {cls_name} with scope `{scope_info.scope}` has no description. "
                    f"Add a docstring or implement get_description()."
                )
            is_goal = optionable_cls is not None and issubclass(optionable_cls, GoalSubsystem)
            oshi: OptionScopeHelpInfo = HelpInfoExtracter(
                scope_info.scope
            ).get_option_scope_help_info(
                scope_info.description, options.get_parser(scope_info.scope), is_goal
            )
            scope_to_help_info[oshi.scope] = oshi

            if is_goal:
                goal_subsystem_cls = cast(Type[GoalSubsystem], optionable_cls)
                is_implemented = union_membership.has_members_for_all(
                    goal_subsystem_cls.required_union_implementations
                )
                name_to_goal_info[scope_info.scope] = GoalHelpInfo(
                    goal_subsystem_cls.name,
                    scope_info.description,
                    is_implemented,
                    consumed_scopes_mapper(scope_info.scope),
                )

        return AllHelpInfo(
            scope_to_help_info=scope_to_help_info, name_to_goal_info=name_to_goal_info
        )
예제 #22
0
def calculate_specs(
    options_bootstrapper: OptionsBootstrapper,
    options: Options,
    session: SchedulerSession,
    *,
    build_root: Optional[str] = None,
) -> Specs:
    """Determine the specs for a given Pants run."""
    build_root = build_root or get_buildroot()
    specs = SpecsParser(build_root).parse_specs(options.specs)
    changed_options = ChangedOptions.from_options(options.for_scope("changed"))

    logger.debug("specs are: %s", specs)
    logger.debug("changed_options are: %s", changed_options)

    if specs.provided and changed_options.provided:
        changed_name = "--changed-since" if changed_options.since else "--changed-diffspec"
        if specs.filesystem_specs and specs.address_specs:
            specs_description = "target and file arguments"
        elif specs.filesystem_specs:
            specs_description = "file arguments"
        else:
            specs_description = "target arguments"
        raise InvalidSpecConstraint(
            f"You used `{changed_name}` at the same time as using {specs_description}. Please "
            "use only one.")

    if not changed_options.provided:
        return specs

    git = get_git()
    if not git:
        raise InvalidSpecConstraint(
            "The `--changed-*` options are only available if Git is used for the repository."
        )
    changed_request = ChangedRequest(
        sources=tuple(changed_options.changed_files(git)),
        dependees=changed_options.dependees,
    )
    (changed_addresses, ) = session.product_request(
        ChangedAddresses, [Params(changed_request, options_bootstrapper)])
    logger.debug("changed addresses: %s", changed_addresses)

    address_specs = []
    for address in cast(ChangedAddresses, changed_addresses):
        address_input = AddressInput.parse(address.spec)
        address_specs.append(
            AddressLiteralSpec(
                path_component=address_input.path_component,
                # NB: AddressInput.target_component may be None, but AddressLiteralSpec expects a
                # string.
                target_component=address_input.target_component
                or address.target_name,
            ))
    return Specs(AddressSpecs(address_specs, filter_by_global_options=True),
                 FilesystemSpecs([]))
예제 #23
0
 def __init__(self, options: Options):
     super().__init__(options.for_global_scope().colors)
     self._all_help_info = HelpInfoExtracter.get_all_help_info(
         options,
         # We only care about the options-related help info, so we pass in
         # dummy values for the other arguments.
         UnionMembership({}),
         lambda x: tuple(),
         RegisteredTargetTypes({}),
     )
예제 #24
0
 def _parse_type_int(self, args_str, env=None, config=None, bootstrap_option_values=None,
                     action=None):
   args = shlex.split(str(args_str))
   options = Options.create(env=env or {},
                            config=self._create_config(config or {}),
                            known_scope_infos=OptionsTest._known_scope_infos,
                            args=args,
                            bootstrap_option_values=bootstrap_option_values)
   options.register(GLOBAL_SCOPE, '--config-override', action=action, type=int)
   return options
예제 #25
0
 def __init__(self, options: Options):
     super().__init__(options.for_global_scope().colors)
     self._bin_name = options.for_global_scope().pants_bin_name
     self._all_help_info = HelpInfoExtracter.get_all_help_info(
         options,
         # We only care about the options-related help info, so we pass in
         # dummy values for union_membership and consumed_scopes_mapper.
         UnionMembership({}),
         lambda x: tuple(),
     )
예제 #26
0
 def test_deprecated_option_past_removal(self):
     with self.assertRaises(PastRemovalVersionError):
         options = Options.create({}, FakeConfig({}),
                                  OptionsTest._known_scope_infos, "./pants")
         options.register(
             GLOBAL_SCOPE,
             '--too-old-option',
             deprecated_version='0.0.24',
             deprecated_hint='The semver for this option has already passed.'
         )
예제 #27
0
 def _parse(self, args_str, env=None, config=None, bootstrap_option_values=None):
   args = shlex.split(str(args_str))
   options = Options.create(env=env or {},
                            config=self._create_config(config or {}),
                            known_scope_infos=OptionsTest._known_scope_infos,
                            args=args,
                            bootstrap_option_values=bootstrap_option_values,
                            option_tracker=OptionTracker())
   self._register(options)
   return options
예제 #28
0
 def test_deprecated_option_past_removal(self):
   with self.assertRaises(PastRemovalVersionError):
     options = Options.create(env={},
                              config=self._create_config({}),
                              known_scope_infos=OptionsTest._known_scope_infos,
                              args='./pants',
                              option_tracker=OptionTracker())
     options.register(GLOBAL_SCOPE, '--too-old-option', deprecated_version='0.0.24',
                      deprecated_hint='The semver for this option has already passed.')
     options.for_global_scope()
예제 #29
0
 def _init_graph_session(
     cls,
     options_initializer: OptionsInitializer,
     options_bootstrapper: OptionsBootstrapper,
     build_config: BuildConfiguration,
     env: CompleteEnvironment,
     run_id: str,
     options: Options,
     scheduler: GraphScheduler | None = None,
     cancellation_latch: PySessionCancellationLatch | None = None,
 ) -> GraphSession:
     native_engine.maybe_set_panic_handler()
     if scheduler is None:
         dynamic_remote_options, _ = DynamicRemoteOptions.from_options(options, env)
         bootstrap_options = options.bootstrap_option_values()
         assert bootstrap_options is not None
         scheduler = EngineInitializer.setup_graph(
             bootstrap_options, build_config, dynamic_remote_options
         )
     with options_initializer.handle_unknown_flags(options_bootstrapper, env, raise_=True):
         global_options = options.for_global_scope()
     return scheduler.new_session(
         run_id,
         dynamic_ui=global_options.dynamic_ui,
         ui_use_prodash=global_options.dynamic_ui_renderer
         == DynamicUIRenderer.experimental_prodash,
         use_colors=global_options.get("colors", True),
         max_workunit_level=max(
             global_options.streaming_workunits_level,
             global_options.level,
             *(
                 LogLevel[level.upper()]
                 for level in global_options.log_levels_by_target.values()
             ),
         ),
         session_values=SessionValues(
             {
                 OptionsBootstrapper: options_bootstrapper,
                 CompleteEnvironment: env,
             }
         ),
         cancellation_latch=cancellation_latch,
     )
예제 #30
0
 def test_shadowing(self):
     options = Options.create(
         env={},
         config=self._create_config({}),
         known_scope_infos=[task("foo")],
         args="./pants",
         option_tracker=OptionTracker(),
     )
     options.register("", "--bar")
     with self.assertRaises(RegistrationError):
         options.register("foo", "--bar")
예제 #31
0
    def create(
        cls,
        options: Options,
        session: SchedulerSession,
        build_root: Optional[str] = None,
        exclude_patterns: Optional[Iterable[str]] = None,
        tags: Optional[Iterable[str]] = None,
    ) -> Specs:
        specs = cls.parse_specs(
            raw_specs=options.specs,
            build_root=build_root,
            exclude_patterns=exclude_patterns,
            tags=tags,
        )

        changed_options = ChangedOptions.from_options(
            options.for_scope("changed"))

        logger.debug("specs are: %s", specs)
        logger.debug("changed_options are: %s", changed_options)

        if changed_options.is_actionable(
        ) and specs.provided_specs.dependencies:
            # We've been provided both a change request and specs.
            raise InvalidSpecConstraint(
                "Multiple target selection methods provided. Please use only one of "
                "`--changed-*`, address specs, or filesystem specs.")

        if changed_options.is_actionable():
            scm = get_scm()
            if not scm:
                raise InvalidSpecConstraint(
                    "The `--changed-*` options are not available without a recognized SCM (usually Git)."
                )
            changed_request = ChangedRequest(
                sources=tuple(changed_options.changed_files(scm=scm)),
                include_dependees=changed_options.include_dependees,
            )
            (changed_addresses, ) = session.product_request(
                ChangedAddresses, [changed_request])
            logger.debug("changed addresses: %s", changed_addresses.addresses)
            dependencies = tuple(
                SingleAddress(a.spec_path, a.target_name)
                for a in changed_addresses.addresses)
            return Specs(
                address_specs=AddressSpecs(
                    dependencies=dependencies,
                    exclude_patterns=exclude_patterns,
                    tags=tags,
                ),
                filesystem_specs=FilesystemSpecs([]),
            )

        return specs
예제 #32
0
 def _parse_type_int(self, args_str, env=None, config=None, bootstrap_option_values=None,
                     action=None):
   args = shlex.split(str(args_str))
   options = Options.create(env=env or {},
                            config=self._create_config(config or {}),
                            known_scope_infos=OptionsTest._known_scope_infos,
                            args=args,
                            bootstrap_option_values=bootstrap_option_values,
                            option_tracker=OptionTracker())
   options.register(GLOBAL_SCOPE, '--config-override', action=action, type=int)
   return options
예제 #33
0
    def _init_graph_session(
        options_bootstrapper: OptionsBootstrapper,
        build_config: BuildConfiguration,
        options: Options,
        scheduler: Optional[LegacyGraphScheduler] = None,
    ) -> LegacyGraphSession:
        native = Native()
        native.set_panic_handler()
        graph_scheduler_helper = scheduler or EngineInitializer.setup_legacy_graph(
            options_bootstrapper, build_config)

        global_scope = options.for_global_scope()

        if global_scope.v2:
            dynamic_ui = resolve_conflicting_options(
                old_option="v2_ui",
                new_option="dynamic_ui",
                old_scope=GLOBAL_SCOPE,
                new_scope=GLOBAL_SCOPE,
                old_container=global_scope,
                new_container=global_scope,
            )
        else:
            dynamic_ui = False
        use_colors = global_scope.get("colors", True)

        zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2
        # TODO(#8658) This should_report_workunits flag must be set to True for
        # StreamingWorkunitHandler to receive WorkUnits. It should eventually
        # be merged with the zipkin_trace_v2 flag, since they both involve most
        # of the same engine functionality, but for now is separate to avoid
        # breaking functionality associated with zipkin tracing while iterating on streaming workunit reporting.
        stream_workunits = len(
            options.for_global_scope().streaming_workunits_handlers) != 0
        return graph_scheduler_helper.new_session(
            zipkin_trace_v2,
            RunTracker.global_instance().run_id,
            dynamic_ui=dynamic_ui,
            use_colors=use_colors,
            should_report_workunits=stream_workunits,
        )
예제 #34
0
def get_bootstrap_option_values(env=None,
                                config=None,
                                args=None,
                                buildroot=None):
    """Get the values of just the bootstrap options."""
    # Filter just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
    flags = set()

    def capture_the_flags(*args, **kwargs):
        flags.update(args)

    register_bootstrap_options(capture_the_flags, buildroot=buildroot)
    bargs = filter(lambda x: x.partition('=')[0] in flags, args)

    bootstrap_options = Options(env=env,
                                config=config,
                                known_scopes=[GLOBAL_SCOPE],
                                args=bargs)
    register_bootstrap_options(bootstrap_options.register_global,
                               buildroot=buildroot)
    return bootstrap_options.for_global_scope()
예제 #35
0
  def prepare_task(self,
                   config=None,
                   args=None,
                   targets=None,
                   build_graph=None,
                   build_file_parser=None,
                   address_mapper=None,
                   console_outstream=None,
                   workspace=None):
    """Prepares a Task for execution.

    task_type: The class of the Task to create.
    config: An optional string representing the contents of a pants.ini config.
    args: optional list of command line flags, these should be prefixed with '--test-'.
    targets: optional list of Target objects passed on the command line.

    Returns a new Task ready to execute.
    """

    task_type = self.task_type()
    assert issubclass(task_type, Task), 'task_type must be a Task subclass, got %s' % task_type

    config = create_config(config or '')
    workdir = os.path.join(config.getdefault('pants_workdir'), 'test', task_type.__name__)

    bootstrap_options = OptionsBootstrapper().get_bootstrap_options()

    options = Options(env={}, config=config, known_scopes=['', 'test'], args=args or [])
    # A lot of basic code uses these options, so always register them.
    register_bootstrap_options(options.register_global)

    # We need to wrap register_global (can't set .bootstrap attr on the bound instancemethod).
    def register_global_wrapper(*args, **kwargs):
      return options.register_global(*args, **kwargs)

    register_global_wrapper.bootstrap = bootstrap_options.for_global_scope()
    register_global_options(register_global_wrapper)

    task_type.options_scope = 'test'
    task_type.register_options_on_scope(options)

    run_tracker = create_run_tracker()

    context = Context(config,
                      options,
                      run_tracker,
                      targets or [],
                      build_graph=build_graph,
                      build_file_parser=build_file_parser,
                      address_mapper=address_mapper,
                      console_outstream=console_outstream,
                      workspace=workspace)
    return task_type(context, workdir)
예제 #36
0
  def get_bootstrap_options(self):
    """:returns: an Options instance that only knows about the bootstrap options.
    :rtype: Options
    """
    if not self._bootstrap_options:
      flags = set()
      short_flags = set()

      def capture_the_flags(*args, **kwargs):
        for flag in Parser.expand_flags(*args, **kwargs):
          flags.add(flag.name)
          if len(flag.name) == 2:
            short_flags.add(flag.name)
          if flag.inverse_name:
            flags.add(flag.inverse_name)

      register_bootstrap_options(capture_the_flags, buildroot=self._buildroot)

      def is_bootstrap_option(arg):
        components = arg.split('=', 1)
        if components[0] in flags:
          return True
        for flag in short_flags:
          if arg.startswith(flag):
            return True
        return False

      # Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
      # Stop before '--' since args after that are pass-through and may have duplicate names to our
      # bootstrap options.
      bargs = filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', self._args))

      self._bootstrap_options = Options(env=self._env, config=self._pre_bootstrap_config,
                                        known_scopes=[GLOBAL_SCOPE], args=bargs)
      register_bootstrap_options(self._bootstrap_options.register_global, buildroot=self._buildroot)
      bootstrap_option_values = self._bootstrap_options.for_global_scope()
      Config.reset_default_bootstrap_option_values(values=bootstrap_option_values)

      # Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
      # from (typically pants.ini), then config override, then rcfiles.
      configpaths = list(self._pre_bootstrap_config.sources())
      if bootstrap_option_values.config_override:
        configpaths.append(bootstrap_option_values.config_override)
      if bootstrap_option_values.pantsrc:
        rcfiles = [os.path.expanduser(rcfile) for rcfile in bootstrap_option_values.pantsrc_files]
        existing_rcfiles = filter(os.path.exists, rcfiles)
        configpaths.extend(existing_rcfiles)

      self._post_bootstrap_config = Config.load(configpaths)
      Config.cache(self._post_bootstrap_config)

    return self._bootstrap_options
예제 #37
0
            def bootstrap_options_from_config(config):
                bootstrap_options = Options(
                    env=self._env,
                    config=config,
                    known_scope_infos=[ScopeInfo.for_global_scope()],
                    args=bargs)

                def register_global(*args, **kwargs):
                    bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)

                GlobalOptionsRegistrar.register_bootstrap_options(
                    register_global)
                return bootstrap_options
예제 #38
0
    def parse_bootstrap_options(
        env: Mapping[str, str], args: Sequence[str], config: Config
    ) -> Options:
        bootstrap_options = Options.create(
            env=env, config=config, known_scope_infos=[GlobalOptions.get_scope_info()], args=args,
        )

        def register_global(*args, **kwargs):
            # Only use of Options.register?
            bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)

        GlobalOptions.register_bootstrap_options(register_global)
        return bootstrap_options
예제 #39
0
 def _parse(self,
            args_str,
            env=None,
            config=None,
            bootstrap_option_values=None):
     args = shlex.split(str(args_str))
     options = Options(env or {},
                       FakeConfig(config or {}),
                       OptionsTest._known_scope_infos,
                       args,
                       bootstrap_option_values=bootstrap_option_values)
     self._register(options)
     return options
예제 #40
0
 def test_complete_scopes(self):
     _global = ScopeInfo.for_global_scope()
     self.assertEquals(
         {
             _global,
             intermediate('foo'),
             intermediate('foo.bar'),
             task('foo.bar.baz')
         }, Options.complete_scopes({task('foo.bar.baz')}))
     self.assertEquals(
         {
             _global,
             intermediate('foo'),
             intermediate('foo.bar'),
             task('foo.bar.baz')
         },
         Options.complete_scopes(
             {ScopeInfo.for_global_scope(),
              task('foo.bar.baz')}))
     self.assertEquals(
         {
             _global,
             intermediate('foo'),
             intermediate('foo.bar'),
             task('foo.bar.baz')
         },
         Options.complete_scopes({intermediate('foo'),
                                  task('foo.bar.baz')}))
     self.assertEquals(
         {
             _global,
             intermediate('foo'),
             intermediate('foo.bar'),
             task('foo.bar.baz'),
             intermediate('qux'),
             task('qux.quux')
         }, Options.complete_scopes({task('foo.bar.baz'),
                                     task('qux.quux')}))
예제 #41
0
 def test_complete_scopes(self):
     _global = GlobalOptionsRegistrar.get_scope_info()
     self.assertEquals(
         {
             _global,
             intermediate('foo'),
             intermediate('foo.bar'),
             task('foo.bar.baz')
         }, Options.complete_scopes({task('foo.bar.baz')}))
     self.assertEquals(
         {
             _global,
             intermediate('foo'),
             intermediate('foo.bar'),
             task('foo.bar.baz')
         },
         Options.complete_scopes(
             {GlobalOptionsRegistrar.get_scope_info(),
              task('foo.bar.baz')}))
     self.assertEquals(
         {
             _global,
             intermediate('foo'),
             intermediate('foo.bar'),
             task('foo.bar.baz')
         },
         Options.complete_scopes({intermediate('foo'),
                                  task('foo.bar.baz')}))
     self.assertEquals(
         {
             _global,
             intermediate('foo'),
             intermediate('foo.bar'),
             task('foo.bar.baz'),
             intermediate('qux'),
             task('qux.quux')
         }, Options.complete_scopes({task('foo.bar.baz'),
                                     task('qux.quux')}))
예제 #42
0
  def parse_bootstrap_options(env, args, config):
    bootstrap_options = Options.create(
      env=env,
      config=config,
      known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()],
      args=args,
    )

    def register_global(*args, **kwargs):
      ## Only use of Options.register?
      bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)

    GlobalOptionsRegistrar.register_bootstrap_options(register_global)
    return bootstrap_options
예제 #43
0
    def parse_bootstrap_options(env, args, config):
        bootstrap_options = Options.create(
            env=env,
            config=config,
            known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()],
            args=args,
        )

        def register_global(*args, **kwargs):
            ## Only use of Options.register?
            bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)

        GlobalOptionsRegistrar.register_bootstrap_options(register_global)
        return bootstrap_options
예제 #44
0
    def _init_graph_session(
        options_bootstrapper: OptionsBootstrapper,
        build_config: BuildConfiguration,
        options: Options,
        scheduler: Optional[LegacyGraphScheduler] = None,
    ) -> LegacyGraphSession:
        native = Native()
        native.set_panic_handler()
        graph_scheduler_helper = scheduler or EngineInitializer.setup_legacy_graph(
            options_bootstrapper, build_config)

        global_scope = options.for_global_scope()
        dynamic_ui = global_scope.dynamic_ui if global_scope.v2 else False
        use_colors = global_scope.get("colors", True)

        stream_workunits = len(
            options.for_global_scope().streaming_workunits_handlers) != 0
        return graph_scheduler_helper.new_session(
            RunTracker.global_instance().run_id,
            dynamic_ui=dynamic_ui,
            use_colors=use_colors,
            should_report_workunits=stream_workunits,
        )
예제 #45
0
def test_register_options_blessed(caplog) -> None:
    class GoodToGo(Subsystem):
        options_scope = "good-to-go"

    options = Options.create(
        env={},
        config=Config.load([]),
        known_scope_infos=[GoodToGo.get_scope_info()],
        args=["./pants"],
        bootstrap_option_values=None,
    )
    GoodToGo.register_options_on_scope(options)

    assert not caplog.records, "The current blessed means of registering options should never warn."
예제 #46
0
 def test_no_recursive_subsystem_options(self):
   options = Options.create(env={},
                            config=self._create_config({}),
                            known_scope_infos=[subsystem('foo')],
                            args='./pants',
                            option_tracker=OptionTracker())
   # All subsystem options are implicitly recursive (a subscope of subsystem scope represents
   # a separate instance of the subsystem, so it needs all the options).
   # We disallow explicit specification of recursive (even if set to True), to avoid confusion.
   with self.assertRaises(RecursiveSubsystemOption):
     options.register('foo', '--bar', recursive=False)
     options.for_scope('foo')
   with self.assertRaises(RecursiveSubsystemOption):
     options.register('foo', '--baz', recursive=True)
     options.for_scope('foo')
예제 #47
0
  def _full_options(self, known_scope_infos):
    bootstrap_option_values = self.get_bootstrap_options().for_global_scope()
    options = Options.create(self.env,
                             self.config,
                             known_scope_infos,
                             args=self.args,
                             bootstrap_option_values=bootstrap_option_values)

    distinct_optionable_classes = set()
    for ksi in sorted(known_scope_infos, key=lambda si: si.scope):
      if not ksi.optionable_cls or ksi.optionable_cls in distinct_optionable_classes:
        continue
      distinct_optionable_classes.add(ksi.optionable_cls)
      ksi.optionable_cls.register_options_on_scope(options)

    return options
예제 #48
0
  def __init__(self, *args, **kwargs):
    self.targets = []
    self.config = Config.load()
    known_scopes = ['']
    for goal in Goal.all():
      # Note that enclosing scopes will appear before scopes they enclose.
      known_scopes.extend(filter(None, goal.known_scopes()))

    # Annoying but temporary hack to get the parser.  We can't use self.parser because
    # that only gets set up in the superclass ctor, and we can't call that until we have
    # self.new_options set up because the superclass ctor calls our register_options().
    # Fortunately this will all go away once we're fully off the old "Command" mechanism.
    legacy_parser = args[2] if len(args) > 2 else kwargs['parser']
    self.new_options = Options(os.environ.copy(), self.config, known_scopes, args=sys.argv,
                               legacy_parser=legacy_parser)
    super(GoalRunner, self).__init__(*args, **kwargs)
예제 #49
0
 def test_shadowing(self):
   options = Options.create(env={},
                            config=self._create_config({}),
                            known_scope_infos=[task('bar'), intermediate('foo'), task('foo.bar')],
                            args='./pants',
                            option_tracker=OptionTracker())
   options.register('', '--opt1')
   options.register('foo', '-o', '--opt2')
   with self.assertRaises(Shadowing):
     options.register('bar', '--opt1')
   with self.assertRaises(Shadowing):
     options.register('foo.bar', '--opt1')
   with self.assertRaises(Shadowing):
     options.register('foo.bar', '--opt2')
   with self.assertRaises(Shadowing):
     options.register('foo.bar', '--opt1', '--opt3')
   with self.assertRaises(Shadowing):
     options.register('foo.bar', '--opt3', '--opt2')
예제 #50
0
  def get_full_options(self, known_scope_infos):
    """Get the full Options instance bootstrapped by this object for the given known scopes.

    :param known_scope_infos: ScopeInfos for all scopes that may be encountered.
    :returns: A bootrapped Options instance that also carries options for all the supplied known
              scopes.
    :rtype: :class:`Options`
    """
    key = frozenset(sorted(known_scope_infos))
    if key not in self._full_options:
      # Note: Don't inline this into the Options() call, as this populates
      # self._post_bootstrap_config, which is another argument to that call.
      bootstrap_option_values = self.get_bootstrap_options().for_global_scope()
      self._full_options[key] = Options.create(self._env,
                                               self._post_bootstrap_config,
                                               known_scope_infos,
                                               args=self._args,
                                               bootstrap_option_values=bootstrap_option_values)
    return self._full_options[key]
예제 #51
0
 def test_option_tracker_required(self):
   with self.assertRaises(Options.OptionTrackerRequiredError):
     Options.create(None, None, [])
예제 #52
0
class GoalRunner(Command):
  """Lists installed goals or else executes a named goal."""

  class IntermixedArgumentsError(GoalError):
    pass

  __command__ = 'goal'
  output = None

  @staticmethod
  def parse_args(args):
    goals = OrderedSet()
    specs = OrderedSet()
    explicit_multi = False
    fail_fast = False
    logger = logging.getLogger(__name__)
    has_double_dash = u'--' in args
    goal_names = [goal.name for goal in Goal.all()]
    if not goal_names:
      raise GoalError(
        'Arguments cannot be parsed before the list of goals from Goal.all() is populated.')

    def is_spec(spec):
      if os.sep in spec or ':' in spec:
        return True # Definitely not a goal.
      if not (spec in goal_names):
        return True # Definitely not a (known) goal.
      if has_double_dash:
        # This means that we're parsing the half of the expression before a --, so assume it's a
        # goal without warning.
        return False
      # Here, it's possible we have a goal and target with the same name. For now, always give
      # priority to the goal, but give a warning if they might have meant the target (if the BUILD
      # file exists).
      if BuildFile.from_cache(get_buildroot(), spec, must_exist=False).exists():
        logger.warning(' Command-line argument "{spec}" is ambiguous, and was assumed to be a '
                       'goal.  If this is incorrect, disambiguate it with the "--" argument to '
                       'separate goals from targets.'.format(spec=spec))
      return False

    for i, arg in enumerate(args):
      if not arg.startswith('-'):
        specs.add(arg) if is_spec(arg) else goals.add(arg)
      elif '--' == arg:
        if specs:
          raise GoalRunner.IntermixedArgumentsError(
            'Cannot intermix targets with goals when using --. Targets should appear on the right')
        explicit_multi = True
        del args[i]
        break
      elif '--fail-fast' == arg.lower():
        fail_fast = True

    if explicit_multi:
      specs.update(arg for arg in args[len(goals):] if not arg.startswith('-'))

    return goals, specs, fail_fast

  # TODO(John Sirois): revisit wholesale locking when we move py support into pants new
  @classmethod
  def serialized(cls):
    # Goal serialization is now handled in goal execution during group processing.
    # The goal command doesn't need to hold the serialization lock; individual goals will
    # acquire the lock if they need to be serialized.
    return False

  def __init__(self, *args, **kwargs):
    self.targets = []
    self.config = Config.load()
    known_scopes = ['']
    for goal in Goal.all():
      # Note that enclosing scopes will appear before scopes they enclose.
      known_scopes.extend(filter(None, goal.known_scopes()))

    # Annoying but temporary hack to get the parser.  We can't use self.parser because
    # that only gets set up in the superclass ctor, and we can't call that until we have
    # self.new_options set up because the superclass ctor calls our register_options().
    # Fortunately this will all go away once we're fully off the old "Command" mechanism.
    legacy_parser = args[2] if len(args) > 2 else kwargs['parser']
    self.new_options = Options(os.environ.copy(), self.config, known_scopes, args=sys.argv,
                               legacy_parser=legacy_parser)
    super(GoalRunner, self).__init__(*args, **kwargs)

  def get_spec_excludes(self):
    spec_excludes = self.config.getlist(Config.DEFAULT_SECTION, 'spec_excludes',
                                        default=None)
    if spec_excludes is None:
       return [self.config.getdefault('pants_workdir')]
    return  [os.path.join(self.root_dir, spec_exclude) for spec_exclude in spec_excludes]


  @contextmanager
  def check_errors(self, banner):
    errors = {}
    def error(key, include_traceback=False):
      exc_type, exc_value, _ = sys.exc_info()
      msg = StringIO()
      if include_traceback:
        frame = inspect.trace()[-2]
        filename = frame[1]
        lineno = frame[2]
        funcname = frame[3]
        code = ''.join(frame[4]) if frame[4] else None
        traceback.print_list([(filename, lineno, funcname, code)], file=msg)
      if exc_type:
        msg.write(''.join(traceback.format_exception_only(exc_type, exc_value)))
      errors[key] = msg.getvalue()
      sys.exc_clear()

    yield error

    if errors:
      msg = StringIO()
      msg.write(banner)
      invalid_keys = [key for key, exc in errors.items() if not exc]
      if invalid_keys:
        msg.write('\n  %s' % '\n  '.join(invalid_keys))
      for key, exc in errors.items():
        if exc:
          msg.write('\n  %s =>\n    %s' % (key, '\n      '.join(exc.splitlines())))
      # The help message for goal is extremely verbose, and will obscure the
      # actual error message, so we don't show it in this case.
      self.error(msg.getvalue(), show_help=False)

  def register_options(self):
    register_global_options(self.new_options.register_global)
    for goal in Goal.all():
      goal.register_options(self.new_options)

  def setup_parser(self, parser, args):
    # We support attempting zero or more goals.  Multiple goals must be delimited from further
    # options and non goal args with a '--'.  The key permutations we need to support:
    # ./pants goal => goals
    # ./pants goal goals => goals
    # ./pants goal compile src/java/... => compile
    # ./pants goal compile -x src/java/... => compile
    # ./pants goal compile src/java/... -x => compile
    # ./pants goal compile run -- src/java/... => compile, run
    # ./pants goal compile run -- src/java/... -x => compile, run
    # ./pants goal compile run -- -x src/java/... => compile, run

    if not args:
      args.append('help')

    help_flags = set(['-h', '--help', 'help'])
    show_help = len(help_flags.intersection(args)) > 0
    non_help_args = filter(lambda f: f not in help_flags, args)

    goals, specs, fail_fast = GoalRunner.parse_args(non_help_args)
    if show_help:
      self.new_options.print_help(goals=goals, legacy=True)
      sys.exit(0)

    self.requested_goals = goals

    with self.run_tracker.new_workunit(name='setup', labels=[WorkUnit.SETUP]):

      spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper, spec_excludes=self.get_spec_excludes())
      with self.run_tracker.new_workunit(name='parse', labels=[WorkUnit.SETUP]):
        for spec in specs:
          for address in spec_parser.parse_addresses(spec, fail_fast):
            self.build_graph.inject_address_closure(address)
            self.targets.append(self.build_graph.get_target(address))
    self.goals = [Goal.by_name(goal) for goal in goals]

    rcfiles = self.config.getdefault('rcfiles', type=list,
                                     default=['/etc/pantsrc', '~/.pants.rc'])
    if rcfiles:
      rcfile = RcFile(rcfiles, default_prepend=False, process_default=True)

      # Break down the goals specified on the command line to the full set that will be run so we
      # can apply default flags to inner goal nodes.  Also break down goals by Task subclass and
      # register the task class hierarchy fully qualified names so we can apply defaults to
      # baseclasses.

      sections = OrderedSet()
      for goal in Engine.execution_order(self.goals):
        for task_name in goal.ordered_task_names():
          sections.add(task_name)
          task_type = goal.task_type_by_name(task_name)
          for clazz in task_type.mro():
            if clazz == Task:
              break
            sections.add('%s.%s' % (clazz.__module__, clazz.__name__))

      augmented_args = rcfile.apply_defaults(sections, args)
      if augmented_args != args:
        # TODO(John Sirois): Cleanup this currently important mutation of the passed in args
        # once the 2-layer of command -> goal is squashed into one.
        del args[:]
        args.extend(augmented_args)
        sys.stderr.write("(using pantsrc expansion: pants goal %s)\n" % ' '.join(augmented_args))

    Goal.setup_parser(parser, args, self.goals)

  def run(self, lock):
    # TODO(John Sirois): Consider moving to straight python logging.  The divide between the
    # context/work-unit logging and standard python logging doesn't buy us anything.

    # Enable standard python logging for code with no handle to a context/work-unit.
    if self.old_options.log_level:
      LogOptions.set_stderr_log_level((self.old_options.log_level or 'info').upper())
      logdir = self.old_options.logdir or self.config.get('goals', 'logdir', default=None)
      if logdir:
        safe_mkdir(logdir)
        LogOptions.set_log_dir(logdir)
        log.init('goals')
      else:
        log.init()

    # Update the reporting settings, now that we have flags etc.
    def is_quiet_task():
      for goal in self.goals:
        if goal.has_task_of_type(QuietTaskMixin):
          return True
      return False

    # Target specs are mapped to the patterns which match them, if any. This variable is a key for
    # specs which don't match any exclusion regexes. We know it won't already be in the list of
    # patterns, because the asterisks in its name make it an invalid regex.
    _UNMATCHED_KEY = '** unmatched **'

    def targets_by_pattern(targets, patterns):
      mapping = defaultdict(list)
      for target in targets:
        matched_pattern = None
        for pattern in patterns:
          if re.search(pattern, target.address.spec) is not None:
            matched_pattern = pattern
            break
        if matched_pattern is None:
          mapping[_UNMATCHED_KEY].append(target)
        else:
          mapping[matched_pattern].append(target)
      return mapping

    is_explain = self.old_options.explain
    update_reporting(self.old_options, is_quiet_task() or is_explain, self.run_tracker)

    if self.old_options.target_excludes:
      excludes = self.old_options.target_excludes
      log.debug('excludes:\n  {excludes}'.format(excludes='\n  '.join(excludes)))
      by_pattern = targets_by_pattern(self.targets, excludes)
      self.targets = by_pattern[_UNMATCHED_KEY]
      # The rest of this if-statement is just for debug logging.
      log.debug('Targets after excludes: {targets}'.format(
          targets=', '.join(t.address.spec for t in self.targets)))
      excluded_count = sum(len(by_pattern[p]) for p in excludes)
      log.debug('Excluded {count} target{plural}.'.format(count=excluded_count,
          plural=('s' if excluded_count != 1 else '')))
      for pattern in excludes:
        log.debug('Targets excluded by pattern {pattern}\n  {targets}'.format(pattern=pattern,
            targets='\n  '.join(t.address.spec for t in by_pattern[pattern])))

    context = Context(
      config=self.config,
      old_options=self.old_options,
      new_options=self.new_options,
      run_tracker=self.run_tracker,
      target_roots=self.targets,
      requested_goals=self.requested_goals,
      build_graph=self.build_graph,
      build_file_parser=self.build_file_parser,
      address_mapper=self.address_mapper,
      spec_excludes=self.get_spec_excludes(),
      lock=lock)

    unknown = []
    for goal in self.goals:
      if not goal.ordered_task_names():
        unknown.append(goal)

    if unknown:
      context.log.error('Unknown goal(s): %s\n' % ' '.join(goal.name for goal in unknown))
      return 1

    engine = RoundEngine()
    return engine.execute(context, self.goals)

  def cleanup(self):
    # TODO: This is JVM-specific and really doesn't belong here.
    # TODO: Make this more selective? Only kill nailguns that affect state? E.g., checkstyle
    # may not need to be killed.
    NailgunTask.killall(log.info)
    sys.exit(1)
예제 #53
0
 def test_frozen_registration(self):
   options = Options.create(args=[], env={}, config=self._create_config({}),
                            known_scope_infos=[task('foo')], option_tracker=OptionTracker())
   options.register('foo', '--arg1')
   with self.assertRaises(FrozenRegistration):
     options.register(GLOBAL_SCOPE, '--arg2')
예제 #54
0
 def assertError(expected_error, *args, **kwargs):
   with self.assertRaises(expected_error):
     options = Options.create(args=[], env={}, config=self._create_config({}),
                              known_scope_infos=[], option_tracker=OptionTracker())
     options.register(GLOBAL_SCOPE, *args, **kwargs)
     options.for_global_scope()
예제 #55
0
 def test_deprecated_option_past_removal(self):
   with self.assertRaises(PastRemovalVersionError):
     options = Options({}, FakeConfig({}), OptionsTest._known_scopes, "./pants")
     options.register_global('--too-old-option',
                             deprecated_version='0.0.24',
                             deprecated_hint='The semver for this option has already passed.')
예제 #56
0
class OptionsBootstrapper(object):
  """An object that knows how to create options in two stages: bootstrap, and then full options."""
  def __init__(self, env=None, configpath=None, args=None, buildroot=None):
    self._buildroot = buildroot or get_buildroot()
    self._env = env or os.environ.copy()
    Config.reset_default_bootstrap_option_values(buildroot=self._buildroot)
    self._pre_bootstrap_config = Config.load([configpath] if configpath else None)
    self._post_bootstrap_config = None  # Will be set later.
    self._args = args or sys.argv
    self._bootstrap_options = None  # We memoize the bootstrap options here.
    self._full_options = None  # We memoize the full options here.
    # So other startup code has config to work with. This will go away once we replace direct
    # config accesses with options, and plumb those through everywhere that needs them.
    Config.cache(self._pre_bootstrap_config)

  def get_bootstrap_options(self):
    """Returns an Options instance that only knows about the bootstrap options."""
    if not self._bootstrap_options:
      flags = set()
      short_flags = set()

      def capture_the_flags(*args, **kwargs):
        for flag in Parser.expand_flags(*args, **kwargs):
          flags.add(flag.name)
          if len(flag.name) == 2:
            short_flags.add(flag.name)
          if flag.inverse_name:
            flags.add(flag.inverse_name)

      register_bootstrap_options(capture_the_flags, buildroot=self._buildroot)

      def is_bootstrap_option(arg):
        components = arg.split('=', 1)
        if components[0] in flags:
          return True
        for flag in short_flags:
          if arg.startswith(flag):
            return True
        return False

      # Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
      # Stop before '--' since args after that are pass-through and may have duplicate names to our
      # bootstrap options.
      bargs = filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', self._args))

      self._bootstrap_options = Options(env=self._env, config=self._pre_bootstrap_config,
                                        known_scopes=[GLOBAL_SCOPE], args=bargs)
      register_bootstrap_options(self._bootstrap_options.register_global, buildroot=self._buildroot)
      bootstrap_option_values = self._bootstrap_options.for_global_scope()
      Config.reset_default_bootstrap_option_values(values=bootstrap_option_values)

      # Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
      # from (typically pants.ini), then config override, then rcfiles.
      configpaths = list(self._pre_bootstrap_config.sources())
      if bootstrap_option_values.config_override:
        configpaths.append(bootstrap_option_values.config_override)
      if bootstrap_option_values.pantsrc:
        rcfiles = [os.path.expanduser(rcfile) for rcfile in bootstrap_option_values.pantsrc_files]
        existing_rcfiles = filter(os.path.exists, rcfiles)
        configpaths.extend(existing_rcfiles)

      self._post_bootstrap_config = Config.load(configpaths)
      Config.cache(self._post_bootstrap_config)

    return self._bootstrap_options

  def get_full_options(self, known_scopes):
    if not self._full_options:
      # Note: Don't inline this into the Options() call, as this populates
      # self._post_bootstrap_config, which is another argument to that call.
      bootstrap_options = self.get_bootstrap_options()
      self._full_options = Options(self._env,
                                   self._post_bootstrap_config,
                                   known_scopes,
                                   args=self._args,
                                   bootstrap_option_values=bootstrap_options.for_global_scope())

      # The bootstrap options need to be registered on the post-bootstrap Options instance, so it
      # won't choke on them on the command line, and also so we can access their values as regular
      # global-scope options, for convenience.
      register_bootstrap_options(self._full_options.register_global, buildroot=self._buildroot)
    return self._full_options
예제 #57
0
  def test_scope_deprecation(self):
    # Note: This test demonstrates that two different new scopes can deprecate the same
    # old scope. I.e., it's possible to split an old scope's options among multiple new scopes.
    class DummyOptionable1(Optionable):
      options_scope = 'new-scope1'
      options_scope_category = ScopeInfo.SUBSYSTEM
      deprecated_options_scope = 'deprecated-scope'
      deprecated_options_scope_removal_version = '9999.9.9'

    class DummyOptionable2(Optionable):
      options_scope = 'new-scope2'
      options_scope_category = ScopeInfo.SUBSYSTEM
      deprecated_options_scope = 'deprecated-scope'
      deprecated_options_scope_removal_version = '9999.9.9'

    options = Options.create(env={},
                             config=self._create_config({
                               DummyOptionable1.options_scope: {
                                 'foo': 'xx'
                               },
                               DummyOptionable1.deprecated_options_scope: {
                                 'foo': 'yy',
                                 'bar': 'zz',
                                 'baz': 'ww',
                                 'qux': 'uu'
                               },
                             }),
                             known_scope_infos=[
                               DummyOptionable1.get_scope_info(),
                               DummyOptionable2.get_scope_info()
                             ],
                             args=shlex.split('./pants --new-scope1-baz=vv'),
                             option_tracker=OptionTracker())

    options.register(DummyOptionable1.options_scope, '--foo')
    options.register(DummyOptionable1.options_scope, '--bar')
    options.register(DummyOptionable1.options_scope, '--baz')
    options.register(DummyOptionable2.options_scope, '--qux')

    with self.warnings_catcher() as w:
      vals1 = options.for_scope(DummyOptionable1.options_scope)

    # Check that we got a warning.
    self.assertEquals(1, len(w))
    self.assertTrue(isinstance(w[0].message, DeprecationWarning))

    # Check values.
    # Deprecated scope takes precedence at equal rank.
    self.assertEquals('yy', vals1.foo)
    self.assertEquals('zz', vals1.bar)
    # New scope takes precedence at higher rank.
    self.assertEquals('vv', vals1.baz)

    with self.warnings_catcher() as w:
      vals2 = options.for_scope(DummyOptionable2.options_scope)

    # Check that we got a warning.
    self.assertEquals(1, len(w))
    self.assertTrue(isinstance(w[0].message, DeprecationWarning))

    # Check values.
    self.assertEquals('uu', vals2.qux)