示例#1
0
  def pre_fork(self):
    """Pre-fork() callback for ProcessManager.daemonize()."""
    for service in self._services:
      service.pre_fork()

    # Teardown the RunTracker's SubprocPool pre-fork.
    RunTracker.global_instance().shutdown_worker_pool()
示例#2
0
  def pre_fork(self):
    """Pre-fork() callback for ProcessManager.daemonize()."""
    for service in self._services:
      service.pre_fork()

    # Teardown the RunTracker's SubprocPool pre-fork.
    RunTracker.global_instance().shutdown_worker_pool()
示例#3
0
def clean_global_runtime_state():
    """Resets the global runtime state of a pants runtime for cleaner forking."""
    # Reset RunTracker state.
    RunTracker.global_instance().reset(reset_options=False)

    # Reset Goals and Tasks.
    Goal.clear()
示例#4
0
文件: util.py 项目: adamchainz/pants
def clean_global_runtime_state():
  """Resets the global runtime state of a pants runtime for cleaner forking."""
  # Reset RunTracker state.
  RunTracker.global_instance().reset(reset_options=False)

  # Reset Goals and Tasks.
  Goal.clear()
示例#5
0
    def _execute_engine(self):
        workdir = self._context.options.for_global_scope().pants_workdir
        if not workdir.endswith('.pants.d'):
            self._context.log.error(
                'Pants working directory should end with \'.pants.d\', currently it is {}\n'
                .format(workdir))
            return 1

        unknown_goals = [
            goal.name for goal in self._goals if not goal.ordered_task_names()
        ]
        if unknown_goals:
            self._context.log.error('Unknown goal(s): {}\n'.format(
                ' '.join(unknown_goals)))
            return 1

        engine = RoundEngine()

        sorted_goal_infos = engine.sort_goals(self._context, self._goals)
        RunTracker.global_instance().set_sorted_goal_infos(sorted_goal_infos)

        result = engine.execute(self._context, self._goals)

        if self._context.invalidation_report:
            self._context.invalidation_report.report()

        return result
示例#6
0
    def _finish_run(self, run_tracker: RunTracker, code: ExitCode) -> None:
        """Cleans up the run tracker."""

        metrics = self.graph_session.scheduler_session.metrics()
        run_tracker.set_pantsd_scheduler_metrics(metrics)
        outcome = WorkUnit.SUCCESS if code == PANTS_SUCCEEDED_EXIT_CODE else WorkUnit.FAILURE
        run_tracker.set_root_outcome(outcome)
示例#7
0
    def assert_upload_stats(self, *, response_code) -> None:
        stats = {'stats': {'foo': 'bar', 'baz': 42}}

        class Handler(http.server.BaseHTTPRequestHandler):
            def do_POST(handler):
                try:
                    if handler.path.startswith('/redirect'):
                        code = int(handler.path[-3:])
                        handler.send_response(code)
                        handler.send_header('location', mk_url('/upload'))
                        handler.end_headers()
                    else:
                        self.assertEqual('/upload', handler.path)
                        self.assertEqual('application/x-www-form-urlencoded',
                                         handler.headers['Content-type'])
                        length = int(handler.headers['Content-Length'])
                        post_data = parse_qs(
                            handler.rfile.read(length).decode())
                        decoded_post_data = {
                            k: json.loads(v[0])
                            for k, v in post_data.items()
                        }
                        self.assertEqual(stats, decoded_post_data)
                        self.assertEqual(handler.headers['User-Agent'],
                                         f"pants/v{VERSION}")
                        self.assertIn(handler.headers['X-Pants-Stats-Version'],
                                      {"1", "2"})
                        handler.send_response(response_code)
                        handler.end_headers()
                except Exception:
                    handler.send_response(
                        400)  # Ensure the main thread knows the test failed.
                    raise

        server_address = ('', 0)
        server = http.server.HTTPServer(server_address, Handler)
        host, port = server.server_address

        def mk_url(path):
            return f'http://{host}:{port}{path}'

        server_thread = threading.Thread(target=server.serve_forever)
        server_thread.daemon = True
        server_thread.start()

        self.context(for_subsystems=[Cookies])
        self.assertTrue(
            RunTracker.post_stats(mk_url('/upload'), stats, stats_version=1))
        self.assertTrue(
            RunTracker.post_stats(mk_url('/redirect307'),
                                  stats,
                                  stats_version=1))
        self.assertFalse(
            RunTracker.post_stats(mk_url('/redirect302'),
                                  stats,
                                  stats_version=2))

        server.shutdown()
        server.server_close()
示例#8
0
  def test_write_stats_to_json_file(self):
    # Set up
    stats = {'stats': {'foo': 'bar', 'baz': 42}}

    # Execute & verify
    with temporary_file_path() as file_name:
      RunTracker.write_stats_to_json(file_name, stats)
      with open(file_name, 'r') as f:
        result = json.load(f)
        self.assertEqual(stats, result)
示例#9
0
  def _clean_runtime_state(self):
    """Resets the runtime state from running ./pants -> running in the fork()'d daemon context."""
    # TODO(kwlzn): Make this logic available to PantsRunner et al for inline state reset before
    # pants runs to improve testability and avoid potential bitrot.

    # Reset RunTracker state.
    RunTracker.global_instance().reset(reset_options=False)

    # Reset Goals and Tasks.
    Goal.clear()
示例#10
0
  def _execute_engine(self):
    engine = RoundEngine()
    sorted_goal_infos = engine.sort_goals(self._context, self._goals)
    RunTracker.global_instance().set_sorted_goal_infos(sorted_goal_infos)
    result = engine.execute(self._context, self._goals)

    if self._context.invalidation_report:
      self._context.invalidation_report.report()

    return result
示例#11
0
    def test_write_stats_to_json_file(self):
        # Set up
        stats = {"stats": {"foo": "bar", "baz": 42}}

        # Execute & verify
        with temporary_file_path() as file_name:
            RunTracker.write_stats_to_json(file_name, stats)
            with open(file_name, "r") as f:
                result = json.load(f)
                self.assertEqual(stats, result)
示例#12
0
  def test_write_stats_to_json_file(self):
    # Set up
    stats = {'stats': {'foo': 'bar', 'baz': 42}}

    # Execute & verify
    with temporary_file_path() as file_name:
      RunTracker.write_stats_to_json(file_name, stats)
      with open(file_name, 'r') as f:
        result = json.load(f)
        self.assertEqual(stats, result)
示例#13
0
    def _execute_engine(self):
        engine = RoundEngine()
        sorted_goal_infos = engine.sort_goals(self._context, self._goals)
        RunTracker.global_instance().set_sorted_goal_infos(sorted_goal_infos)
        result = engine.execute(self._context, self._goals)

        if self._context.invalidation_report:
            self._context.invalidation_report.report()

        return result
示例#14
0
    def _start_run(self, run_tracker: RunTracker, start_time: float) -> None:
        run_tracker.start(self.options, run_start_time=start_time)

        spec_parser = SpecsParser(get_buildroot())
        specs = [
            str(spec_parser.parse_spec(spec)) for spec in self.options.specs
        ]
        # Note: This will not include values from `--changed-*` flags.
        run_tracker.run_info.add_info("specs_from_command_line",
                                      specs,
                                      stringify=False)
示例#15
0
def test_run_information(exit_code, expected, **kwargs) -> None:
    with temporary_dir() as buildroot:
        with environment_as(PANTS_BUILDROOT_OVERRIDE=buildroot):
            run_tracker = RunTracker(
                create_options_bootstrapper([]).bootstrap_options)

            specs = ["src/python/pants/goal/run_tracker_test.py"]
            run_tracker.start(run_start_time=time.time(), specs=specs)

            run_information = run_tracker.run_information()
            assert run_information["buildroot"] == get_buildroot()
            assert run_information["path"] == get_buildroot()
            # freezegun doesn't seem to accurately mock the time zone,
            # (i.e. the time zone used depends on that of the machine that
            # executes the test), so we can only safely assert that the
            # month and year appear in the human-readable string contained
            # in the "datetime" key
            assert "Jan" in run_information["datetime"]
            assert "2020" in run_information["datetime"]
            assert run_information["timestamp"] == 1578657601.0
            assert run_information["user"] == getpass.getuser()
            assert run_information["version"] == VERSION
            assert re.match("pants.*run_tracker_test.py",
                            run_information["cmd_line"])
            assert run_information["specs_from_command_line"] == [
                "src/python/pants/goal/run_tracker_test.py"
            ]

            frozen_time = kwargs["frozen_time"]
            frozen_time.tick(delta=datetime.timedelta(seconds=1))

            run_tracker.end_run(exit_code)
            run_information_after_ended = run_tracker.run_information()
            assert run_information_after_ended["outcome"] == expected
示例#16
0
def test_run_information(exit_code: ExitCode, expected: str, tmp_path: Path,
                         **kwargs) -> None:
    frozen_time = kwargs["frozen_time"]
    buildroot = tmp_path.as_posix()
    with environment_as(PANTS_BUILDROOT_OVERRIDE=buildroot):
        spec = "test/example.py"
        ob = create_options_bootstrapper(["list", spec])
        run_tracker = RunTracker(ob.args, ob.bootstrap_options)

        specs = [spec]
        run_tracker.start(run_start_time=time.time(), specs=specs)

        run_information = run_tracker.run_information()
        assert run_information["buildroot"] == get_buildroot()
        assert run_information["path"] == get_buildroot()
        # freezegun doesn't seem to accurately mock the time zone,
        # (i.e. the time zone used depends on that of the machine that
        # executes the test), so we can only safely assert that the
        # month and year appear in the human-readable string contained
        # in the "datetime" key
        assert "Jan" in run_information["datetime"]
        assert "2020" in run_information["datetime"]
        assert run_information["timestamp"] == 1578657601.0
        assert run_information["user"] == getpass.getuser()
        assert run_information["version"] == VERSION
        assert re.match(f"pants.*{spec}", run_information["cmd_line"])
        assert run_information["specs_from_command_line"] == [spec]

        frozen_time.tick(delta=datetime.timedelta(seconds=1))
        run_tracker.end_run(exit_code)
        run_information_after_ended = run_tracker.run_information()
        assert run_information_after_ended["outcome"] == expected
示例#17
0
  def setup(self):
    options_bootstrapper = OptionsBootstrapper()

    # Force config into the cache so we (and plugin/backend loading code) can use it.
    # TODO: Plumb options in explicitly.
    options_bootstrapper.get_bootstrap_options()
    self.config = Config.from_cache()

    # Add any extra paths to python path (eg for loading extra source backends)
    extra_paths = self.config.getlist('backends', 'python-path', [])
    if extra_paths:
      sys.path.extend(extra_paths)

    # Load plugins and backends.
    backend_packages = self.config.getlist('backends', 'packages', [])
    plugins = self.config.getlist('backends', 'plugins', [])
    build_configuration = load_plugins_and_backends(plugins, backend_packages)

    # Now that plugins and backends are loaded, we can gather the known scopes.
    self.targets = []
    known_scopes = ['']
    for goal in Goal.all():
      # Note that enclosing scopes will appear before scopes they enclose.
      known_scopes.extend(filter(None, goal.known_scopes()))

    # Now that we have the known scopes we can get the full options.
    self.options = options_bootstrapper.get_full_options(known_scopes=known_scopes)
    self.register_options()

    self.run_tracker = RunTracker.from_config(self.config)
    report = initial_reporting(self.config, self.run_tracker)
    self.run_tracker.start(report)
    url = self.run_tracker.run_info.get_info('report_url')
    if url:
      self.run_tracker.log(Report.INFO, 'See a report at: %s' % url)
    else:
      self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')

    self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
                                             root_dir=self.root_dir,
                                             run_tracker=self.run_tracker)
    self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
    self.build_graph = BuildGraph(run_tracker=self.run_tracker,
                                  address_mapper=self.address_mapper)

    with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
      # construct base parameters to be filled in for BuildGraph
      for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]):
        build_file = BuildFile.from_cache(root_dir=self.root_dir, relpath=path)
        # TODO(pl): This is an unfortunate interface leak, but I don't think
        # in the long run that we should be relying on "bootstrap" BUILD files
        # that do nothing except modify global state.  That type of behavior
        # (e.g. source roots, goal registration) should instead happen in
        # project plugins, or specialized configuration files.
        self.build_file_parser.parse_build_file_family(build_file)

    # Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
    self.run_tracker.run_info.add_scm_info()

    self._expand_goals_and_specs()
示例#18
0
  def test_upload_stats(self):
    stats = {'stats': {'foo': 'bar', 'baz': 42}}

    class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
      def do_POST(handler):
        try:
          self.assertEquals('/upload', handler.path)
          self.assertEquals('application/x-www-form-urlencoded', handler.headers['Content-type'])
          length = int(handler.headers['Content-Length'])
          post_data = urlparse.parse_qs(handler.rfile.read(length).decode('utf-8'))
          decoded_post_data = {k: json.loads(v[0]) for k, v in post_data.items()}
          self.assertEquals(stats, decoded_post_data)
          handler.send_response(200)
        except Exception:
          handler.send_response(400)  # Ensure the main thread knows the test failed.
          raise


    server_address = ('', 0)
    server = BaseHTTPServer.HTTPServer(server_address, Handler)
    host, port = server.server_address

    server_thread = threading.Thread(target=server.serve_forever)
    server_thread.daemon = True
    server_thread.start()

    self.assertTrue(RunTracker.post_stats('http://{}:{}/upload'.format(host, port), stats))

    server.shutdown()
    server.server_close()
示例#19
0
    def _init_graph_session(
        options_bootstrapper: OptionsBootstrapper,
        build_config: BuildConfiguration,
        options: Options,
    ) -> LegacyGraphSession:
        native = Native()
        native.set_panic_handler()
        graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
            native, options_bootstrapper, build_config)

        v2_ui = options.for_global_scope().get("v2_ui", False)
        zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2
        # TODO(#8658) This should_report_workunits flag must be set to True for
        # StreamingWorkunitHandler to receive WorkUnits. It should eventually
        # be merged with the zipkin_trace_v2 flag, since they both involve most
        # of the same engine functionality, but for now is separate to avoid
        # breaking functionality associated with zipkin tracing while iterating on streaming workunit reporting.
        stream_workunits = len(
            options.for_global_scope().streaming_workunits_handlers) != 0
        return graph_scheduler_helper.new_session(
            zipkin_trace_v2,
            RunTracker.global_instance().run_id,
            v2_ui,
            should_report_workunits=stream_workunits,
        )
示例#20
0
  def _run(self):
    # Launch RunTracker as early as possible (just after Subsystem options are initialized).
    run_tracker = RunTracker.global_instance()
    reporting = Reporting.global_instance()
    reporting.initialize(run_tracker, self._run_start_time)

    try:
      # Capture a repro of the 'before' state for this build, if needed.
      repro = Reproducer.global_instance().create_repro()
      if repro:
        repro.capture(run_tracker.run_info.get_as_dict())

      engine_result = self._maybe_run_v2()
      goal_runner_result = self._maybe_run_v1(run_tracker, reporting)

      if repro:
        # TODO: Have Repro capture the 'after' state (as a diff) as well?
        repro.log_location_of_repro_file()
    finally:
      run_tracker_result = run_tracker.end()

    final_exit_code = self._compute_final_exit_code(
      engine_result,
      goal_runner_result,
      run_tracker_result
    )
    self._exiter.exit(final_exit_code)
示例#21
0
  def test_upload_stats(self):
    stats = {'stats': {'foo': 'bar', 'baz': 42}}

    class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
      def do_POST(handler):
        try:
          self.assertEquals('/upload', handler.path)
          self.assertEquals('application/x-www-form-urlencoded', handler.headers['Content-type'])
          length = int(handler.headers['Content-Length'])
          post_data = urlparse.parse_qs(handler.rfile.read(length).decode('utf-8'))
          decoded_post_data = {k: json.loads(v[0]) for k, v in post_data.items()}
          self.assertEquals(stats, decoded_post_data)
          handler.send_response(200)
        except Exception:
          handler.send_response(400)  # Ensure the main thread knows the test failed.
          raise


    server_address = ('', 0)
    server = BaseHTTPServer.HTTPServer(server_address, Handler)
    host, port = server.server_address

    server_thread = threading.Thread(target=server.serve_forever)
    server_thread.daemon = True
    server_thread.start()

    self.assertTrue(RunTracker.post_stats('http://{}:{}/upload'.format(host, port), stats))

    server.shutdown()
    server.server_close()
示例#22
0
    def prepare_v1_graph_run_v2(
        self,
        options: Options,
        options_bootstrapper: OptionsBootstrapper,
    ) -> Tuple[LegacyGraphSession, Specs, int]:
        """For v1 (and v2): computing Specs for a later v1 run.

        For v2: running an entire v2 run The exit_code in the return indicates whether any issue was
        encountered
        """
        # If any nodes exist in the product graph, wait for the initial watchman event to avoid
        # racing watchman startup vs invalidation events.
        graph_len = self._scheduler.graph_len()
        if graph_len > 0:
            self._logger.debug(
                "graph len was {}, waiting for initial watchman event".format(
                    graph_len))
            self._watchman_is_running.wait()
        build_id = RunTracker.global_instance().run_id
        v2_ui = options.for_global_scope().get("v2_ui", False)
        zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2
        session = self._graph_helper.new_session(zipkin_trace_v2, build_id,
                                                 v2_ui)

        if options.for_global_scope().get("loop", False):
            fn = self._loop
        else:
            fn = self._body

        specs, exit_code = fn(session, options, options_bootstrapper)
        return session, specs, exit_code
示例#23
0
    def prepare_v1_graph_run_v2(self, options, options_bootstrapper):
        """For v1 (and v2): computing TargetRoots for a later v1 run

    For v2: running an entire v2 run
    The exit_code in the return indicates whether any issue was encountered

    :returns: `(LegacyGraphSession, TargetRoots, exit_code)`
    """
        # If any nodes exist in the product graph, wait for the initial watchman event to avoid
        # racing watchman startup vs invalidation events.
        graph_len = self._scheduler.graph_len()
        if graph_len > 0:
            self._logger.debug(
                'graph len was {}, waiting for initial watchman event'.format(
                    graph_len))
            self._watchman_is_running.wait()
        build_id = RunTracker.global_instance().run_id
        v2_ui = options.for_global_scope().v2_ui
        zipkin_trace_v2 = options.for_scope('reporting').zipkin_trace_v2
        session = self._graph_helper.new_session(zipkin_trace_v2, build_id,
                                                 v2_ui)

        if options.for_global_scope().loop:
            fn = self._loop
        else:
            fn = self._body

        target_roots, exit_code = fn(session, options, options_bootstrapper)
        return session, target_roots, exit_code
示例#24
0
    def create(
        cls,
        env: Mapping[str, str],
        options_bootstrapper: OptionsBootstrapper,
        scheduler: Optional[LegacyGraphScheduler] = None,
    ) -> "LocalPantsRunner":
        """Creates a new LocalPantsRunner instance by parsing options.

        By the time this method runs, logging will already have been initialized in either
        PantsRunner or DaemonPantsRunner.

        :param env: The environment (e.g. os.environ) for this run.
        :param options_bootstrapper: The OptionsBootstrapper instance to reuse.
        :param scheduler: If being called from the daemon, a warmed scheduler to use.
        """
        build_root = get_buildroot()
        global_bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope(
        )
        options, build_config = LocalPantsRunner.parse_options(
            options_bootstrapper)

        # Option values are usually computed lazily on demand,
        # but command line options are eagerly computed for validation.
        for scope in options.scope_to_flags.keys():
            options.for_scope(scope)

        # Verify configs.
        if global_bootstrap_options.verify_config:
            options.verify_configs(options_bootstrapper.config)

        union_membership = UnionMembership(build_config.union_rules())

        # If we're running with the daemon, we'll be handed a warmed Scheduler, which we use
        # to initialize a session here.
        graph_session = cls._init_graph_session(options_bootstrapper,
                                                build_config, options,
                                                scheduler)

        global_options = options.for_global_scope()
        specs = SpecsCalculator.create(
            options=options,
            build_root=build_root,
            session=graph_session.scheduler_session,
            exclude_patterns=tuple(global_options.exclude_target_regexp),
            tags=tuple(global_options.tag),
        )

        profile_path = env.get("PANTS_PROFILE")

        return cls(
            build_root=build_root,
            options=options,
            options_bootstrapper=options_bootstrapper,
            build_config=build_config,
            specs=specs,
            graph_session=graph_session,
            union_membership=union_membership,
            profile_path=profile_path,
            _run_tracker=RunTracker.global_instance(),
        )
示例#25
0
    def run(self, start_time: float) -> ExitCode:
        run_tracker = RunTracker.global_instance()
        self._start_run(run_tracker, start_time)

        with maybe_profiled(self.profile_path):
            global_options = self.options.for_global_scope()

            if self.options.help_request:
                return self._print_help(self.options.help_request)

            streaming_handlers = global_options.streaming_workunits_handlers
            callbacks = Subsystem.get_streaming_workunit_callbacks(
                streaming_handlers)
            streaming_reporter = StreamingWorkunitHandler(
                self.graph_session.scheduler_session,
                callbacks=callbacks,
                report_interval_seconds=global_options.
                streaming_workunits_report_interval,
            )

            goals = tuple(self.options.goals)
            with streaming_reporter.session():
                engine_result = PANTS_FAILED_EXIT_CODE
                try:
                    engine_result = self._run_v2(goals)
                except Exception as e:
                    ExceptionSink.log_exception(e)

                self._finish_run(run_tracker, engine_result)
            return engine_result
示例#26
0
    def _init_graph_session(
        cls,
        options_bootstrapper: OptionsBootstrapper,
        build_config: BuildConfiguration,
        options: Options,
        scheduler: Optional[GraphScheduler] = None,
    ) -> GraphSession:
        native = Native()
        native.set_panic_handler()
        graph_scheduler_helper = scheduler or EngineInitializer.setup_graph(
            options_bootstrapper, build_config)

        try:
            global_scope = options.for_global_scope()
        except UnknownFlagsError as err:
            cls._handle_unknown_flags(err, options_bootstrapper)
            raise

        stream_workunits = len(
            options.for_global_scope().streaming_workunits_handlers) != 0
        return graph_scheduler_helper.new_session(
            RunTracker.global_instance().run_id,
            dynamic_ui=global_scope.dynamic_ui,
            use_colors=global_scope.get("colors", True),
            should_report_workunits=stream_workunits,
            session_values=SessionValues({
                OptionsBootstrapper:
                options_bootstrapper,
                PantsEnvironment:
                PantsEnvironment(os.environ),
            }),
        )
示例#27
0
    def set_start_time(self, start_time: Optional[float]) -> None:
        # Launch RunTracker as early as possible (before .run() is called).
        self._run_tracker = RunTracker.global_instance()

        # Propagates parent_build_id to pants runs that may be called from this pants run.
        os.environ["PANTS_PARENT_BUILD_ID"] = self._run_tracker.run_id

        self._reporting = Reporting.global_instance()
        self._reporting.initialize(self._run_tracker,
                                   self.options,
                                   start_time=start_time)

        spec_parser = CmdLineSpecParser(get_buildroot())
        specs = [
            spec_parser.parse_spec(spec).to_spec_string()
            for spec in self.options.specs
        ]
        # Note: This will not include values from `--changed-*` flags.
        self._run_tracker.run_info.add_info("specs_from_command_line",
                                            specs,
                                            stringify=False)

        # Capture a repro of the 'before' state for this build, if needed.
        self._repro = Reproducer.global_instance().create_repro()
        if self._repro:
            self._repro.capture(self._run_tracker.run_info.get_as_dict())
示例#28
0
def test_anonymous_telemetry_with_no_repo_id() -> None:
    with temporary_dir() as buildroot:
        with environment_as(PANTS_BUILDROOT_OVERRIDE=buildroot):
            opts = create_options_bootstrapper([]).bootstrap_options
            run_tracker = RunTracker(opts)
            run_tracker.start(run_start_time=time.time(), specs=[])
            run_tracker.end_run(PANTS_SUCCEEDED_EXIT_CODE)
            repo_id = ""
            telemetry = run_tracker.get_anonymous_telemetry_data(repo_id)

            # Check that these keys have non-trivial values.
            for key in (
                    "run_id",
                    "timestamp",
                    "duration",
                    "outcome",
                    "platform",
                    "python_implementation",
                    "python_version",
                    "pants_version",
            ):
                assert bool(telemetry.get(key))

            for key in ("repo_id", "machine_id", "user_id"):
                assert telemetry.get(key) == ""
示例#29
0
  def register_options(self):
    # Add a 'bootstrap' attribute to the register function, so that register_global can
    # access the bootstrap option values.
    def register_global(*args, **kwargs):
      return self.options.register_global(*args, **kwargs)
    register_global.bootstrap = self.options.bootstrap_option_values()
    register_global_options(register_global)

    # This is the first case we have of non-task, non-global options.
    # The current implementation special-cases RunTracker, and is temporary.
    # In the near future it will be replaced with a 'Subsystem' abstraction.
    # But for now this is useful for kicking the tires.
    def register_run_tracker(*args, **kwargs):
      self.options.register('run-tracker', *args, **kwargs)
    RunTracker.register_options(register_run_tracker)

    for goal in Goal.all():
      goal.register_options(self.options)
示例#30
0
  def _run(self):
    # Bootstrap options and logging.
    options_bootstrapper = self._options_bootstrapper or OptionsBootstrapper(env=self._env,
                                                                             args=self._args)
    bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
    setup_logging_from_options(bootstrap_options)
    build_config = BuildConfigInitializer.get(options_bootstrapper)
    options = OptionsInitializer.create(options_bootstrapper, build_config)
    global_options = options.for_global_scope()

    # Apply exiter options.
    self._exiter.apply_options(options)

    # Option values are usually computed lazily on demand,
    # but command line options are eagerly computed for validation.
    for scope in options.scope_to_flags.keys():
      options.for_scope(scope)

    # Verify the configs here.
    if global_options.verify_config:
      options_bootstrapper.verify_configs_against_options(options)

    # Launch RunTracker as early as possible (just after Subsystem options are initialized).
    run_tracker = RunTracker.global_instance()
    reporting = Reporting.global_instance()
    reporting.initialize(run_tracker, self._run_start_time)

    try:
      # Determine the build root dir.
      root_dir = get_buildroot()

      # Capture a repro of the 'before' state for this build, if needed.
      repro = Reproducer.global_instance().create_repro()
      if repro:
        repro.capture(run_tracker.run_info.get_as_dict())

      # Setup and run GoalRunner.
      goal_runner = GoalRunner.Factory(root_dir,
                                       options,
                                       build_config,
                                       run_tracker,
                                       reporting,
                                       self._target_roots,
                                       self._daemon_build_graph,
                                       self._exiter).setup()

      goal_runner_result = goal_runner.run()

      if repro:
        # TODO: Have Repro capture the 'after' state (as a diff) as well?
        repro.log_location_of_repro_file()
    finally:
      run_tracker_result = run_tracker.end()

    # Take the exit code with higher abs value in case of negative values.
    final_exit_code = goal_runner_result if abs(goal_runner_result) > abs(run_tracker_result) else run_tracker_result
    self._exiter.exit(final_exit_code)
示例#31
0
    def _run(self):
        # Bootstrap options and logging.
        options_bootstrapper = self._options_bootstrapper or OptionsBootstrapper(
            env=self._env, args=self._args)
        options, build_config = OptionsInitializer(
            options_bootstrapper, exiter=self._exiter).setup()
        global_options = options.for_global_scope()

        # Apply exiter options.
        self._exiter.apply_options(options)

        # Option values are usually computed lazily on demand,
        # but command line options are eagerly computed for validation.
        for scope in options.scope_to_flags.keys():
            options.for_scope(scope)

        # Verify the configs here.
        if global_options.verify_config:
            options_bootstrapper.verify_configs_against_options(options)

        # Launch RunTracker as early as possible (just after Subsystem options are initialized).
        run_tracker = RunTracker.global_instance()
        reporting = Reporting.global_instance()
        reporting.initialize(run_tracker, self._run_start_time)

        try:
            # Determine the build root dir.
            root_dir = get_buildroot()

            # Capture a repro of the 'before' state for this build, if needed.
            repro = Reproducer.global_instance().create_repro()
            if repro:
                repro.capture(run_tracker.run_info.get_as_dict())

            # Record the preceding product graph size.
            run_tracker.pantsd_stats.set_preceding_graph_size(
                self._preceding_graph_size)

            # Setup and run GoalRunner.
            goal_runner = GoalRunner.Factory(root_dir, options, build_config,
                                             run_tracker, reporting,
                                             self._target_roots,
                                             self._daemon_build_graph,
                                             self._exiter).setup()

            goal_runner_result = goal_runner.run()

            if repro:
                # TODO: Have Repro capture the 'after' state (as a diff) as well?
                repro.log_location_of_repro_file()
        finally:
            run_tracker_result = run_tracker.end()

        # Take the exit code with higher abs value in case of negative values.
        final_exit_code = goal_runner_result if abs(goal_runner_result) > abs(
            run_tracker_result) else run_tracker_result
        self._exiter.exit(final_exit_code)
示例#32
0
def clean_global_runtime_state(reset_runtracker=True, reset_subsystem=False):
    """Resets the global runtime state of a pants runtime for cleaner forking.

  :param bool reset_runtracker: Whether or not to clean RunTracker global state.
  :param bool reset_subsystem: Whether or not to clean Subsystem global state.
  """
    if reset_runtracker:
        # Reset RunTracker state.
        RunTracker.global_instance().reset(reset_options=False)

    if reset_subsystem:
        # Reset subsystem state.
        Subsystem.reset()

    # Reset Goals and Tasks.
    Goal.clear()

    # Reset backend/plugins state.
    OptionsInitializer.reset()
示例#33
0
文件: util.py 项目: RobinTec/pants
def clean_global_runtime_state(reset_runtracker=True, reset_subsystem=False):
  """Resets the global runtime state of a pants runtime for cleaner forking.

  :param bool reset_runtracker: Whether or not to clean RunTracker global state.
  :param bool reset_subsystem: Whether or not to clean Subsystem global state.
  """
  if reset_runtracker:
    # Reset RunTracker state.
    RunTracker.global_instance().reset(reset_options=False)

  if reset_subsystem:
    # Reset subsystem state.
    Subsystem.reset()

  # Reset Goals and Tasks.
  Goal.clear()

  # Reset backend/plugins state.
  OptionsInitializer.reset()
示例#34
0
  def test_upload_stats(self):
    stats = {'stats': {'foo': 'bar', 'baz': 42}}

    class Handler(http.server.BaseHTTPRequestHandler):
      def do_POST(handler):
        try:
          if handler.path.startswith('/redirect'):
            code = int(handler.path[-3:])
            handler.send_response(code)
            handler.send_header('location', mk_url('/upload'))
            handler.end_headers()
          else:
            self.assertEqual('/upload', handler.path)
            self.assertEqual('application/x-www-form-urlencoded', handler.headers['Content-type'])
            length = int(handler.headers['Content-Length'])
            post_data = parse_qs(handler.rfile.read(length).decode('utf-8'))
            decoded_post_data = {k: json.loads(v[0]) for k, v in post_data.items()}
            self.assertEqual(stats, decoded_post_data)
            handler.send_response(200)
            handler.end_headers()
        except Exception:
          handler.send_response(400)  # Ensure the main thread knows the test failed.
          raise

    server_address = ('', 0)
    server = http.server.HTTPServer(server_address, Handler)
    host, port = server.server_address

    def mk_url(path):
      return 'http://{}:{}{}'.format(host, port, path)

    server_thread = threading.Thread(target=server.serve_forever)
    server_thread.daemon = True
    server_thread.start()

    self.context(for_subsystems=[Cookies])
    self.assertTrue(RunTracker.post_stats(mk_url('/upload'), stats))
    self.assertTrue(RunTracker.post_stats(mk_url('/redirect307'), stats))
    self.assertFalse(RunTracker.post_stats(mk_url('/redirect302'), stats))

    server.shutdown()
    server.server_close()
示例#35
0
def create_run_tracker(info_dir=None):
    """Creates a ``RunTracker`` and starts it.

  :param string info_dir: An optional director for the run tracker to store state; defaults to a
    new temp dir that will be be cleaned up on interpreter exit.
  """
    # TODO(John Sirois): Rework uses around a context manager for cleanup of the info_dir in a more
    # disciplined manner
    info_dir = info_dir or safe_mkdtemp()
    run_tracker = RunTracker(info_dir)
    report = Report()
    settings = PlainTextReporter.Settings(outfile=sys.stdout,
                                          log_level=Report.INFO,
                                          color=False,
                                          indent=True,
                                          timing=False,
                                          cache_stats=False)
    report.add_reporter('test_debug', PlainTextReporter(run_tracker, settings))
    run_tracker.start(report)
    return run_tracker
示例#36
0
def create_run_tracker(info_dir=None):
  """Creates a ``RunTracker`` and starts it.

  :param string info_dir: An optional director for the run tracker to store state; defaults to a
    new temp dir that will be be cleaned up on interpreter exit.
  """
  # TODO(John Sirois): Rework uses around a context manager for cleanup of the info_dir in a more
  # disciplined manner
  info_dir = info_dir or safe_mkdtemp()
  run_tracker = RunTracker(info_dir)
  report = Report()
  settings = PlainTextReporter.Settings(outfile=sys.stdout,
                                        log_level=Report.INFO,
                                        color=False,
                                        indent=True,
                                        timing=False,
                                        cache_stats=False)
  report.add_reporter('test_debug', PlainTextReporter(run_tracker, settings))
  run_tracker.start(report)
  return run_tracker
示例#37
0
    def register_options(self):
        # Add a 'bootstrap' attribute to the register function, so that register_global can
        # access the bootstrap option values.
        def register_global(*args, **kwargs):
            return self.options.register_global(*args, **kwargs)

        register_global.bootstrap = self.options.bootstrap_option_values()
        register_global_options(register_global)

        # This is the first case we have of non-task, non-global options.
        # The current implementation special-cases RunTracker, and is temporary.
        # In the near future it will be replaced with a 'Subsystem' abstraction.
        # But for now this is useful for kicking the tires.
        def register_run_tracker(*args, **kwargs):
            self.options.register('run-tracker', *args, **kwargs)

        RunTracker.register_options(register_run_tracker)

        for goal in Goal.all():
            goal.register_options(self.options)
示例#38
0
  def test_create_dict_with_nested_keys_and_val(self):
    keys = []

    with self.assertRaises(ValueError):
      RunTracker._create_dict_with_nested_keys_and_val(keys, 'something')

    keys += ['one']
    self.assertEquals(
      RunTracker._create_dict_with_nested_keys_and_val(keys, 'something'),
      {'one': 'something'}
    )

    keys += ['two']
    self.assertEquals(
      RunTracker._create_dict_with_nested_keys_and_val(keys, 'something'),
      {'one': {'two': 'something'}}
    )

    keys += ['three']
    self.assertEquals(
      RunTracker._create_dict_with_nested_keys_and_val(keys, 'something'),
      {'one': {'two': {'three': 'something'}}}
    )

    keys += ['four']
    self.assertEquals(
      RunTracker._create_dict_with_nested_keys_and_val(keys, 'something'),
      {'one': {'two': {'three': {'four': 'something'}}}}
    )
示例#39
0
    def test_invalid_stats_version(self):
        stats = {'stats': {'foo': 'bar', 'baz': 42}}
        url = 'http://example.com/upload/'
        with self.assertRaises(ValueError):
            RunTracker.post_stats(url, stats, stats_version=0)

        with self.assertRaises(ValueError):
            RunTracker.post_stats(url, stats, stats_version=None)

        with self.assertRaises(ValueError):
            RunTracker.post_stats(url, stats, stats_version=9)

        with self.assertRaises(ValueError):
            RunTracker.post_stats(url, stats, stats_version="not a number")
示例#40
0
    def test_invalid_stats_version(self):
        stats = {"stats": {"foo": "bar", "baz": 42}}
        url = "http://example.com/upload/"
        with self.assertRaises(ValueError):
            RunTracker.post_stats(url, stats, stats_version=0)

        with self.assertRaises(ValueError):
            RunTracker.post_stats(url, stats, stats_version=None)

        with self.assertRaises(ValueError):
            RunTracker.post_stats(url, stats, stats_version=9)

        with self.assertRaises(ValueError):
            RunTracker.post_stats(url, stats, stats_version="not a number")
示例#41
0
  def test_create_dict_with_nested_keys_and_val(self):
    keys = []

    with self.assertRaises(ValueError):
      RunTracker._create_dict_with_nested_keys_and_val(keys, 'something')

    keys += ['one']
    self.assertEqual(
      RunTracker._create_dict_with_nested_keys_and_val(keys, 'something'),
      {'one': 'something'}
    )

    keys += ['two']
    self.assertEqual(
      RunTracker._create_dict_with_nested_keys_and_val(keys, 'something'),
      {'one': {'two': 'something'}}
    )

    keys += ['three']
    self.assertEqual(
      RunTracker._create_dict_with_nested_keys_and_val(keys, 'something'),
      {'one': {'two': {'three': 'something'}}}
    )

    keys += ['four']
    self.assertEqual(
      RunTracker._create_dict_with_nested_keys_and_val(keys, 'something'),
      {'one': {'two': {'three': {'four': 'something'}}}}
    )
示例#42
0
文件: util.py 项目: lahosken/pants
def clean_global_runtime_state(reset_runtracker=True, reset_subsystem=False):
    """Resets the global runtime state of a pants runtime for cleaner forking.

  :param bool reset_runtracker: Whether or not to clean RunTracker global state.
  :param bool reset_subsystem: Whether or not to clean Subsystem global state.
  """
    if reset_runtracker:
        # Reset RunTracker state.
        RunTracker.global_instance().reset(reset_options=False)

    if reset_subsystem:
        # Reset subsystem state.
        Subsystem.reset()

    #TODO: Think of an alternative for IntermediateTargetFactoryBase._targets to avoid this call
    IntermediateTargetFactoryBase.reset()

    # Reset Goals and Tasks.
    Goal.clear()

    # Reset backend/plugins state.
    OptionsInitializer.reset()
示例#43
0
  def _execute_engine(self):
    workdir = self._context.options.for_global_scope().pants_workdir
    if not workdir.endswith('.pants.d'):
      self._context.log.error('Pants working directory should end with \'.pants.d\', currently it is {}\n'
                              .format(workdir))
      return 1

    unknown_goals = [goal.name for goal in self._goals if not goal.ordered_task_names()]
    if unknown_goals:
      self._context.log.error('Unknown goal(s): {}\n'.format(' '.join(unknown_goals)))
      return 1

    engine = RoundEngine()

    sorted_goal_infos = engine.sort_goals(self._context, self._goals)
    RunTracker.global_instance().set_sorted_goal_infos(sorted_goal_infos)

    result = engine.execute(self._context, self._goals)

    if self._context.invalidation_report:
      self._context.invalidation_report.report()

    return result
示例#44
0
  def test_raise_no_zipkin_endpoint_set(self):

    options = {'reporting': {'zipkin_trace_id': self.trace_id, 'zipkin_parent_id': self.parent_id}}
    context = self.context(for_subsystems=[RunTracker, Reporting], options=options)
    run_tracker = RunTracker.global_instance()
    reporting = Reporting.global_instance()

    with self.assertRaises(ValueError) as result:
      reporting.initialize(run_tracker, context.options)

    self.assertTrue(
      "The zipkin-endpoint flag must be set if zipkin-trace-id and zipkin-parent-id flags are given."
      in str(result.exception)
    )
示例#45
0
  def test_raise_if_no_parent_id_and_zipkin_endpoint_set(self):

    options = {'reporting': {'zipkin_trace_id': self.trace_id}}
    context = self.context(for_subsystems=[RunTracker, Reporting], options=options)

    run_tracker = RunTracker.global_instance()
    reporting = Reporting.global_instance()

    with self.assertRaises(ValueError) as result:
      reporting.initialize(run_tracker, context.options)

    self.assertTrue(
      "Flags zipkin-trace-id and zipkin-parent-id must both either be set or not set."
      in str(result.exception)
    )
示例#46
0
  def set_start_time(self, start_time):
    # Launch RunTracker as early as possible (before .run() is called).
    self._run_tracker = RunTracker.global_instance()
    self._reporting = Reporting.global_instance()

    self._run_start_time = start_time
    self._reporting.initialize(self._run_tracker, self._options, start_time=self._run_start_time)

    # Capture a repro of the 'before' state for this build, if needed.
    self._repro = Reproducer.global_instance().create_repro()
    if self._repro:
      self._repro.capture(self._run_tracker.run_info.get_as_dict())

    # The __call__ method of the Exiter allows for the prototype pattern.
    self._exiter = LocalExiter(self._run_tracker, self._repro, exiter=self._exiter)
    ExceptionSink.reset_exiter(self._exiter)
示例#47
0
  def test_raise_if_trace_id_is_of_wrong_ch_format(self):
    trace_id = 'gggggggggggggggg'
    options = {'reporting': {
      'zipkin_trace_id': trace_id,
      'zipkin_parent_id': self.parent_id,
      'zipkin_endpoint': self.zipkin_endpoint
    }}
    context = self.context(for_subsystems=[RunTracker, Reporting], options=options)

    run_tracker = RunTracker.global_instance()
    reporting = Reporting.global_instance()

    with self.assertRaises(ValueError) as result:
      reporting.initialize(run_tracker, context.options)

    self.assertTrue(
      "Value of the flag zipkin-trace-id must be a 16-character or 32-character hex string. "
      + "Got {}.".format(trace_id)
      in str(result.exception)
    )
示例#48
0
  def setup(self, options_bootstrapper, working_set):
    bootstrap_options = options_bootstrapper.get_bootstrap_options()
    global_bootstrap_options = bootstrap_options.for_global_scope()

    # The pants_version may be set in pants.ini for bootstrapping, so we make sure the user actually
    # requested the version on the command line before deciding to print the version and exit.
    if global_bootstrap_options.is_flagged('pants_version'):
      print(global_bootstrap_options.pants_version)
      self._exiter(0)

    # Get logging setup prior to loading backends so that they can log as needed.
    self._setup_logging(global_bootstrap_options)

    # Add any extra paths to python path (e.g., for loading extra source backends).
    for path in global_bootstrap_options.pythonpath:
      sys.path.append(path)
      pkg_resources.fixup_namespace_packages(path)

    # Load plugins and backends.
    plugins = global_bootstrap_options.plugins
    backend_packages = global_bootstrap_options.backend_packages
    build_configuration = load_plugins_and_backends(plugins, working_set, backend_packages)

    # Now that plugins and backends are loaded, we can gather the known scopes.
    self.targets = []

    known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]

    # Add scopes for all needed subsystems.
    subsystems = Subsystem.closure(set(self.subsystems) |
                                   Goal.subsystems() |
                                   build_configuration.subsystems())
    for subsystem in subsystems:
      known_scope_infos.append(subsystem.get_scope_info())

    # Add scopes for all tasks in all goals.
    for goal in Goal.all():
      known_scope_infos.extend(filter(None, goal.known_scope_infos()))

    # Now that we have the known scopes we can get the full options.
    self.options = options_bootstrapper.get_full_options(known_scope_infos)
    self.register_options(subsystems)

    # Make the options values available to all subsystems.
    Subsystem._options = self.options

    # Now that we have options we can instantiate subsystems.
    self.run_tracker = RunTracker.global_instance()
    self.reporting = Reporting.global_instance()
    report = self.reporting.initial_reporting(self.run_tracker)
    self.run_tracker.start(report)
    url = self.run_tracker.run_info.get_info('report_url')
    if url:
      self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
    else:
      self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')

    self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
                                             root_dir=self.root_dir,
                                             run_tracker=self.run_tracker)

    rev = self.options.for_global_scope().build_file_rev
    if rev:
      ScmBuildFile.set_rev(rev)
      ScmBuildFile.set_scm(get_scm())
      build_file_type = ScmBuildFile
    else:
      build_file_type = FilesystemBuildFile
    self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
    self.build_graph = BuildGraph(run_tracker=self.run_tracker,
                                  address_mapper=self.address_mapper)

    # TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
    with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnitLabel.SETUP]):
      source_root_bootstrapper = SourceRootBootstrapper.global_instance()
      source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser)

    self._expand_goals_and_specs()

    # Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
    self.run_tracker.run_info.add_scm_info()
示例#49
0
  def setup(self):
    options_bootstrapper = OptionsBootstrapper()
    bootstrap_options = options_bootstrapper.get_bootstrap_options()

    # Get logging setup prior to loading backends so that they can log as needed.
    self._setup_logging(bootstrap_options.for_global_scope())

    # Add any extra paths to python path (eg for loading extra source backends)
    for path in bootstrap_options.for_global_scope().pythonpath:
      sys.path.append(path)
      pkg_resources.fixup_namespace_packages(path)

    # Load plugins and backends.
    plugins = bootstrap_options.for_global_scope().plugins
    backend_packages = bootstrap_options.for_global_scope().backend_packages
    build_configuration = load_plugins_and_backends(plugins, backend_packages)

    # Now that plugins and backends are loaded, we can gather the known scopes.
    self.targets = []

    known_scope_infos = [ScopeInfo.for_global_scope()]

    # Add scopes for all needed subsystems.
    subsystems = (set(self.subsystems) | Goal.subsystems() | build_configuration.subsystems())
    for subsystem in subsystems:
      known_scope_infos.append(ScopeInfo(subsystem.options_scope, ScopeInfo.GLOBAL_SUBSYSTEM))

    # Add scopes for all tasks in all goals.
    for goal in Goal.all():
      known_scope_infos.extend(filter(None, goal.known_scope_infos()))

    # Now that we have the known scopes we can get the full options.
    self.options = options_bootstrapper.get_full_options(known_scope_infos)
    self.register_options(subsystems)

    # Make the options values available to all subsystems.
    Subsystem._options = self.options

    # Now that we have options we can instantiate subsystems.
    self.run_tracker = RunTracker.global_instance()
    self.reporting = Reporting.global_instance()
    report = self.reporting.initial_reporting(self.run_tracker)
    self.run_tracker.start(report)
    url = self.run_tracker.run_info.get_info('report_url')
    if url:
      self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
    else:
      self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')

    self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
                                             root_dir=self.root_dir,
                                             run_tracker=self.run_tracker)

    rev = self.options.for_global_scope().build_file_rev
    if rev:
      ScmBuildFile.set_rev(rev)
      ScmBuildFile.set_scm(get_scm())
      build_file_type = ScmBuildFile
    else:
      build_file_type = FilesystemBuildFile
    self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
    self.build_graph = BuildGraph(run_tracker=self.run_tracker,
                                  address_mapper=self.address_mapper)

    # TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
    with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
      source_root_bootstrapper = SourceRootBootstrapper.global_instance()
      source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser)

    self._expand_goals_and_specs()

    # Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
    self.run_tracker.run_info.add_scm_info()
示例#50
0
def _run():
  # Place the registration of the unhandled exception hook as early as possible in the code.
  sys.excepthook = _unhandled_exception_hook

  """
  To add additional paths to sys.path, add a block to the config similar to the following:
  [main]
  roots: ['src/python/pants_internal/test/',]
  """

  logging.basicConfig()
  version = pants_version()
  if len(sys.argv) == 2 and sys.argv[1] == _VERSION_OPTION:
    _do_exit(msg=version, out=sys.stdout)

  root_dir = get_buildroot()
  if not os.path.exists(root_dir):
    _exit_and_fail('PANTS_BUILD_ROOT does not point to a valid path: %s' % root_dir)

  if len(sys.argv) < 2:
    argv = ['goal']
  else:
    argv = sys.argv[1:]
  # Hack to force ./pants -h etc. to redirect to goal.
  if argv[0] != 'goal' and set(['-h', '--help', 'help']).intersection(argv):
    argv = ['goal'] + argv

  parser = optparse.OptionParser(add_help_option=False, version=version)
  RcFile.install_disable_rc_option(parser)
  parser.add_option(_LOG_EXIT_OPTION,
                    action='store_true',
                    default=False,
                    dest='log_exit',
                    help='Log an exit message on success or failure.')

  config = Config.load()

  # XXX(wickman) This should be in the command goal, not in pants_exe.py!
  run_tracker = RunTracker.from_config(config)
  report = initial_reporting(config, run_tracker)
  run_tracker.start(report)

  url = run_tracker.run_info.get_info('report_url')
  if url:
    run_tracker.log(Report.INFO, 'See a report at: %s' % url)
  else:
    run_tracker.log(Report.INFO, '(To run a reporting server: ./pants goal server)')

  backend_packages = config.getlist('backends', 'packages')
  build_configuration = load_build_configuration_from_source(additional_backends=backend_packages)
  build_file_parser = BuildFileParser(build_configuration=build_configuration,
                                      root_dir=root_dir,
                                      run_tracker=run_tracker)
  address_mapper = BuildFileAddressMapper(build_file_parser)
  build_graph = BuildGraph(run_tracker=run_tracker, address_mapper=address_mapper)

  command_class, command_args = _parse_command(root_dir, argv)
  command = command_class(run_tracker,
                          root_dir,
                          parser,
                          command_args,
                          build_file_parser,
                          address_mapper,
                          build_graph)
  try:
    if command.serialized():
      def onwait(pid):
        process = psutil.Process(pid)
        print('Waiting on pants process %d (%s) to complete' %
              (pid, ' '.join(process.cmdline)), file=sys.stderr)
        return True
      runfile = os.path.join(root_dir, '.pants.run')
      lock = Lock.acquire(runfile, onwait=onwait)
    else:
      lock = Lock.unlocked()
    try:
      result = command.run(lock)
      if result:
        run_tracker.set_root_outcome(WorkUnit.FAILURE)
      _do_exit(result)
    except KeyboardInterrupt:
      command.cleanup()
      raise
    except Exception:
      run_tracker.set_root_outcome(WorkUnit.FAILURE)
      raise
    finally:
      lock.release()
  finally:
    run_tracker.end()
    # Must kill nailguns only after run_tracker.end() is called, because there may still
    # be pending background work that needs a nailgun.
    if (hasattr(command.old_options, 'cleanup_nailguns') and command.old_options.cleanup_nailguns) \
        or config.get('nailgun', 'autokill', default=False):
      NailgunTask.killall(None)
示例#51
0
  def test_merge_list_of_keys_into_dict(self):
    data = {}
    keys = []

    with self.assertRaises(ValueError):
      RunTracker._merge_list_of_keys_into_dict(data, keys, 'something')

    with self.assertRaises(ValueError):
      RunTracker._merge_list_of_keys_into_dict(data, keys, 'something', -1)

    keys = ['key']
    with self.assertRaises(ValueError):
      RunTracker._merge_list_of_keys_into_dict(data, keys, 'something', 1)

    keys = ['a']
    RunTracker._merge_list_of_keys_into_dict(data, keys, 'O-N-E')
    self.assertEquals(data, {'a': 'O-N-E'})

    keys = ['one', 'two', 'three']
    RunTracker._merge_list_of_keys_into_dict(data, keys, 'T-H-R-E-E')
    self.assertEquals(data, {'one': {'two': {'three': 'T-H-R-E-E'}}, 'a': 'O-N-E'})

    keys = ['one', 'two', 'a']
    RunTracker._merge_list_of_keys_into_dict(data, keys, 'L-A')
    self.assertEquals(data, {'one': {'two': {'a': 'L-A', 'three': 'T-H-R-E-E'}}, 'a': 'O-N-E'})

    keys = ['c', 'd', 'e', 'f']
    RunTracker._merge_list_of_keys_into_dict(data, keys, 'F-O-U-R')
    self.assertEquals(data, {
      'one': {'two': {'a': 'L-A', 'three': 'T-H-R-E-E'}}, 'a': 'O-N-E',
      'c': {'d': {'e': {'f': 'F-O-U-R'}}}
    })

    keys = ['one', 'two', 'x', 'y']
    RunTracker._merge_list_of_keys_into_dict(data, keys, 'W-H-Y')
    self.assertEquals(data, {
      'one': {'two': {'a': 'L-A', 'three': 'T-H-R-E-E', 'x': {'y': 'W-H-Y'}}}, 'a': 'O-N-E',
      'c': {'d': {'e': {'f': 'F-O-U-R'}}}
    })

    keys = ['c', 'd', 'e', 'g', 'h']
    RunTracker._merge_list_of_keys_into_dict(data, keys, 'H-E-L-L-O')
    self.assertEquals(data, {
      'one': {'two': {'a': 'L-A', 'three': 'T-H-R-E-E', 'x': {'y': 'W-H-Y'}}}, 'a': 'O-N-E',
      'c': {'d': {'e': {'f': 'F-O-U-R', 'g': {'h': 'H-E-L-L-O'}}}}
    })

    keys = ['one', 'two', 'x', 'z']
    RunTracker._merge_list_of_keys_into_dict(data, keys, 'Z-E-D')
    self.assertEquals(data, {
      'one': {'two': {'a': 'L-A', 'three': 'T-H-R-E-E', 'x': {'y': 'W-H-Y', 'z': 'Z-E-D'}}},
      'a': 'O-N-E', 'c': {'d': {'e': {'f': 'F-O-U-R', 'g': {'h': 'H-E-L-L-O'}}}}
    })

    keys = ['c', 'd', 'e', 'g', 'i']
    RunTracker._merge_list_of_keys_into_dict(data, keys, 'E-Y-E')
    self.assertEquals(data, {
      'one': {'two': {'a': 'L-A', 'three': 'T-H-R-E-E', 'x': {'y': 'W-H-Y', 'z': 'Z-E-D'}}},
      'a': 'O-N-E', 'c': {'d': {'e': {'f': 'F-O-U-R', 'g': {'h': 'H-E-L-L-O', 'i': 'E-Y-E'}}}}
    })

    keys = ['a']
    RunTracker._merge_list_of_keys_into_dict(data, keys, 'new O-N-E')
    self.assertEquals(data, {
      'one': {'two': {'a': 'L-A', 'three': 'T-H-R-E-E', 'x': {'y': 'W-H-Y', 'z': 'Z-E-D'}}},
      'a': 'new O-N-E', 'c': {'d': {'e': {'f': 'F-O-U-R', 'g': {'h': 'H-E-L-L-O', 'i': 'E-Y-E'}}}}
    })

    keys = ['one', 'two', 'a']
    RunTracker._merge_list_of_keys_into_dict(data, keys, 'L-A-L-A')
    self.assertEquals(data, {
      'one': {'two': {'a': 'L-A-L-A', 'three': 'T-H-R-E-E', 'x': {'y': 'W-H-Y', 'z': 'Z-E-D'}}},
      'a': 'new O-N-E', 'c': {'d': {'e': {'f': 'F-O-U-R', 'g': {'h': 'H-E-L-L-O', 'i': 'E-Y-E'}}}}
    })

    keys = ['one', 'two', 'a', 'b', 'c']
    with self.assertRaises(ValueError):
      RunTracker._merge_list_of_keys_into_dict(data, keys, 'new A')
示例#52
0
def _run():
  """
  To add additional paths to sys.path, add a block to the config similar to the following:
  [main]
  roots: ['src/python/pants_internal/test/',]
  """
  version = get_version()
  if len(sys.argv) == 2 and sys.argv[1] == _VERSION_OPTION:
    _do_exit(version)

  root_dir = get_buildroot()
  if not os.path.exists(root_dir):
    _exit_and_fail('PANTS_BUILD_ROOT does not point to a valid path: %s' % root_dir)

  if len(sys.argv) < 2 or (len(sys.argv) == 2 and sys.argv[1] in _HELP_ALIASES):
    _help(version, root_dir)

  command_class, command_args = _parse_command(root_dir, sys.argv[1:])

  parser = optparse.OptionParser(version=version)
  RcFile.install_disable_rc_option(parser)
  parser.add_option(_LOG_EXIT_OPTION,
                    action='store_true',
                    default=False,
                    dest='log_exit',
                    help = 'Log an exit message on success or failure.')

  config = Config.load()

  # TODO: This can be replaced once extensions are enabled with
  # https://github.com/pantsbuild/pants/issues/5
  roots = config.getlist('parse', 'roots', default=[])
  sys.path.extend(map(lambda root: os.path.join(root_dir, root), roots))

  # XXX(wickman) This should be in the command goal, not un pants_exe.py!
  run_tracker = RunTracker.from_config(config)
  report = initial_reporting(config, run_tracker)
  run_tracker.start(report)

  url = run_tracker.run_info.get_info('report_url')
  if url:
    run_tracker.log(Report.INFO, 'See a report at: %s' % url)
  else:
    run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')

  command = command_class(run_tracker, root_dir, parser, command_args)
  try:
    if command.serialized():
      def onwait(pid):
        print('Waiting on pants process %s to complete' % _process_info(pid), file=sys.stderr)
        return True
      runfile = os.path.join(root_dir, '.pants.run')
      lock = Lock.acquire(runfile, onwait=onwait)
    else:
      lock = Lock.unlocked()
    try:
      result = command.run(lock)
      _do_exit(result)
    except KeyboardInterrupt:
      command.cleanup()
      raise
    finally:
      lock.release()
  finally:
    run_tracker.end()
    # Must kill nailguns only after run_tracker.end() is called, because there may still
    # be pending background work that needs a nailgun.
    if (hasattr(command.options, 'cleanup_nailguns') and command.options.cleanup_nailguns) \
        or config.get('nailgun', 'autokill', default=False):
      NailgunTask.killall(None)
示例#53
0
 def __init__(self, run_tracker=None, reporting=None):
     self._run_tracker = run_tracker or RunTracker.global_instance()
     self._reporting = reporting or Reporting.global_instance()
示例#54
0
  def setup(self):
    options_bootstrapper = OptionsBootstrapper()

    # Force config into the cache so we (and plugin/backend loading code) can use it.
    # TODO: Plumb options in explicitly.
    bootstrap_options = options_bootstrapper.get_bootstrap_options()
    self.config = Config.from_cache()

    # Get logging setup prior to loading backends so that they can log as needed.
    self._setup_logging(bootstrap_options.for_global_scope())

    # Add any extra paths to python path (eg for loading extra source backends)
    for path in bootstrap_options.for_global_scope().pythonpath:
      sys.path.append(path)
      pkg_resources.fixup_namespace_packages(path)

    # Load plugins and backends.
    backend_packages = self.config.getlist('backends', 'packages', [])
    plugins = self.config.getlist('backends', 'plugins', [])
    build_configuration = load_plugins_and_backends(plugins, backend_packages)

    # Now that plugins and backends are loaded, we can gather the known scopes.
    self.targets = []

    known_scopes = ['']

    # Add scopes for global subsystem instances.
    for subsystem_type in set(self.subsystems) | Goal.global_subsystem_types():
      known_scopes.append(subsystem_type.qualify_scope(Options.GLOBAL_SCOPE))

    # Add scopes for all tasks in all goals.
    for goal in Goal.all():
      # Note that enclosing scopes will appear before scopes they enclose.
      known_scopes.extend(filter(None, goal.known_scopes()))

    # Now that we have the known scopes we can get the full options.
    self.options = options_bootstrapper.get_full_options(known_scopes=known_scopes)
    self.register_options()

    # Make the options values available to all subsystems.
    Subsystem._options = self.options

    # Now that we have options we can instantiate subsystems.
    self.run_tracker = RunTracker.global_instance()
    report = initial_reporting(self.config, self.run_tracker)
    self.run_tracker.start(report)
    url = self.run_tracker.run_info.get_info('report_url')
    if url:
      self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
    else:
      self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')

    self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
                                             root_dir=self.root_dir,
                                             run_tracker=self.run_tracker)

    rev = self.options.for_global_scope().build_file_rev
    if rev:
      ScmBuildFile.set_rev(rev)
      ScmBuildFile.set_scm(get_scm())
      build_file_type = ScmBuildFile
    else:
      build_file_type = FilesystemBuildFile
    self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
    self.build_graph = BuildGraph(run_tracker=self.run_tracker,
                                  address_mapper=self.address_mapper)

    with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
      # construct base parameters to be filled in for BuildGraph
      for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]):
        build_file = self.address_mapper.from_cache(root_dir=self.root_dir, relpath=path)
        # TODO(pl): This is an unfortunate interface leak, but I don't think
        # in the long run that we should be relying on "bootstrap" BUILD files
        # that do nothing except modify global state.  That type of behavior
        # (e.g. source roots, goal registration) should instead happen in
        # project plugins, or specialized configuration files.
        self.build_file_parser.parse_build_file_family(build_file)

    self._expand_goals_and_specs()

    # Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
    self.run_tracker.run_info.add_scm_info()