示例#1
0
  def _run(self):
    # Launch RunTracker as early as possible (just after Subsystem options are initialized).
    run_tracker = RunTracker.global_instance()
    reporting = Reporting.global_instance()
    reporting.initialize(run_tracker, self._run_start_time)

    try:
      # Capture a repro of the 'before' state for this build, if needed.
      repro = Reproducer.global_instance().create_repro()
      if repro:
        repro.capture(run_tracker.run_info.get_as_dict())

      engine_result = self._maybe_run_v2()
      goal_runner_result = self._maybe_run_v1(run_tracker, reporting)

      if repro:
        # TODO: Have Repro capture the 'after' state (as a diff) as well?
        repro.log_location_of_repro_file()
    finally:
      run_tracker_result = run_tracker.end()

    final_exit_code = self._compute_final_exit_code(
      engine_result,
      goal_runner_result,
      run_tracker_result
    )
    self._exiter.exit(final_exit_code)
示例#2
0
    def set_start_time(self, start_time: Optional[float]) -> None:
        # Launch RunTracker as early as possible (before .run() is called).
        self._run_tracker = RunTracker.global_instance()

        # Propagates parent_build_id to pants runs that may be called from this pants run.
        os.environ["PANTS_PARENT_BUILD_ID"] = self._run_tracker.run_id

        self._reporting = Reporting.global_instance()
        self._reporting.initialize(self._run_tracker,
                                   self.options,
                                   start_time=start_time)

        spec_parser = CmdLineSpecParser(get_buildroot())
        specs = [
            spec_parser.parse_spec(spec).to_spec_string()
            for spec in self.options.specs
        ]
        # Note: This will not include values from `--changed-*` flags.
        self._run_tracker.run_info.add_info("specs_from_command_line",
                                            specs,
                                            stringify=False)

        # Capture a repro of the 'before' state for this build, if needed.
        self._repro = Reproducer.global_instance().create_repro()
        if self._repro:
            self._repro.capture(self._run_tracker.run_info.get_as_dict())
示例#3
0
    def _run(self):
        # Bootstrap options and logging.
        options_bootstrapper = self._options_bootstrapper or OptionsBootstrapper(
            env=self._env, args=self._args)
        options, build_config = OptionsInitializer(
            options_bootstrapper, exiter=self._exiter).setup()
        global_options = options.for_global_scope()

        # Apply exiter options.
        self._exiter.apply_options(options)

        # Option values are usually computed lazily on demand,
        # but command line options are eagerly computed for validation.
        for scope in options.scope_to_flags.keys():
            options.for_scope(scope)

        # Verify the configs here.
        if global_options.verify_config:
            options_bootstrapper.verify_configs_against_options(options)

        # Launch RunTracker as early as possible (just after Subsystem options are initialized).
        run_tracker = RunTracker.global_instance()
        reporting = Reporting.global_instance()
        reporting.initialize(run_tracker, self._run_start_time)

        try:
            # Determine the build root dir.
            root_dir = get_buildroot()

            # Capture a repro of the 'before' state for this build, if needed.
            repro = Reproducer.global_instance().create_repro()
            if repro:
                repro.capture(run_tracker.run_info.get_as_dict())

            # Record the preceding product graph size.
            run_tracker.pantsd_stats.set_preceding_graph_size(
                self._preceding_graph_size)

            # Setup and run GoalRunner.
            goal_runner = GoalRunner.Factory(root_dir, options, build_config,
                                             run_tracker, reporting,
                                             self._target_roots,
                                             self._daemon_build_graph,
                                             self._exiter).setup()

            goal_runner_result = goal_runner.run()

            if repro:
                # TODO: Have Repro capture the 'after' state (as a diff) as well?
                repro.log_location_of_repro_file()
        finally:
            run_tracker_result = run_tracker.end()

        # Take the exit code with higher abs value in case of negative values.
        final_exit_code = goal_runner_result if abs(goal_runner_result) > abs(
            run_tracker_result) else run_tracker_result
        self._exiter.exit(final_exit_code)
示例#4
0
  def _run(self):
    # Bootstrap options and logging.
    options_bootstrapper = self._options_bootstrapper or OptionsBootstrapper(env=self._env,
                                                                             args=self._args)
    bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
    setup_logging_from_options(bootstrap_options)
    build_config = BuildConfigInitializer.get(options_bootstrapper)
    options = OptionsInitializer.create(options_bootstrapper, build_config)
    global_options = options.for_global_scope()

    # Apply exiter options.
    self._exiter.apply_options(options)

    # Option values are usually computed lazily on demand,
    # but command line options are eagerly computed for validation.
    for scope in options.scope_to_flags.keys():
      options.for_scope(scope)

    # Verify the configs here.
    if global_options.verify_config:
      options_bootstrapper.verify_configs_against_options(options)

    # Launch RunTracker as early as possible (just after Subsystem options are initialized).
    run_tracker = RunTracker.global_instance()
    reporting = Reporting.global_instance()
    reporting.initialize(run_tracker, self._run_start_time)

    try:
      # Determine the build root dir.
      root_dir = get_buildroot()

      # Capture a repro of the 'before' state for this build, if needed.
      repro = Reproducer.global_instance().create_repro()
      if repro:
        repro.capture(run_tracker.run_info.get_as_dict())

      # Setup and run GoalRunner.
      goal_runner = GoalRunner.Factory(root_dir,
                                       options,
                                       build_config,
                                       run_tracker,
                                       reporting,
                                       self._target_roots,
                                       self._daemon_build_graph,
                                       self._exiter).setup()

      goal_runner_result = goal_runner.run()

      if repro:
        # TODO: Have Repro capture the 'after' state (as a diff) as well?
        repro.log_location_of_repro_file()
    finally:
      run_tracker_result = run_tracker.end()

    # Take the exit code with higher abs value in case of negative values.
    final_exit_code = goal_runner_result if abs(goal_runner_result) > abs(run_tracker_result) else run_tracker_result
    self._exiter.exit(final_exit_code)
示例#5
0
  def test_raise_no_zipkin_endpoint_set(self):

    options = {'reporting': {'zipkin_trace_id': self.trace_id, 'zipkin_parent_id': self.parent_id}}
    context = self.context(for_subsystems=[RunTracker, Reporting], options=options)
    run_tracker = RunTracker.global_instance()
    reporting = Reporting.global_instance()

    with self.assertRaises(ValueError) as result:
      reporting.initialize(run_tracker, context.options)

    self.assertTrue(
      "The zipkin-endpoint flag must be set if zipkin-trace-id and zipkin-parent-id flags are given."
      in str(result.exception)
    )
示例#6
0
    def set_start_time(self, start_time):
        # Launch RunTracker as early as possible (before .run() is called).
        self._run_tracker = RunTracker.global_instance()
        self._reporting = Reporting.global_instance()

        self._run_start_time = start_time
        self._reporting.initialize(self._run_tracker,
                                   self._options,
                                   start_time=self._run_start_time)

        # Capture a repro of the 'before' state for this build, if needed.
        self._repro = Reproducer.global_instance().create_repro()
        if self._repro:
            self._repro.capture(self._run_tracker.run_info.get_as_dict())
示例#7
0
  def test_raise_no_zipkin_endpoint_set(self):

    options = {'reporting': {'zipkin_trace_id': self.trace_id, 'zipkin_parent_id': self.parent_id}}
    context = self.context(for_subsystems=[RunTracker, Reporting], options=options)
    run_tracker = RunTracker.global_instance()
    reporting = Reporting.global_instance()

    with self.assertRaises(ValueError) as result:
      reporting.initialize(run_tracker, context.options)

    self.assertTrue(
      "The zipkin-endpoint flag must be set if zipkin-trace-id and zipkin-parent-id flags are given."
      in str(result.exception)
    )
示例#8
0
    def test_raise_if_no_trace_id_and_zipkin_endpoint_set(self):

        options = {'reporting': {'zipkin_parent_id': self.parent_id}}
        context = self.context(for_subsystems=[RunTracker, Reporting],
                               options=options)

        run_tracker = RunTracker.global_instance()
        reporting = Reporting.global_instance()

        with self.assertRaises(ValueError) as result:
            reporting.initialize(run_tracker, context.options)

        self.assertTrue(
            "Flags zipkin-trace-id and zipkin-parent-id must both either be set or not set."
            in str(result.exception))
示例#9
0
  def test_raise_if_no_parent_id_and_zipkin_endpoint_set(self):

    options = {'reporting': {'zipkin_trace_id': self.trace_id}}
    context = self.context(for_subsystems=[RunTracker, Reporting], options=options)

    run_tracker = RunTracker.global_instance()
    reporting = Reporting.global_instance()

    with self.assertRaises(ValueError) as result:
      reporting.initialize(run_tracker, context.options)

    self.assertTrue(
      "Flags zipkin-trace-id and zipkin-parent-id must both either be set or not set."
      in str(result.exception)
    )
示例#10
0
  def set_start_time(self, start_time):
    # Launch RunTracker as early as possible (before .run() is called).
    self._run_tracker = RunTracker.global_instance()
    self._reporting = Reporting.global_instance()

    self._run_start_time = start_time
    self._reporting.initialize(self._run_tracker, self._options, start_time=self._run_start_time)

    # Capture a repro of the 'before' state for this build, if needed.
    self._repro = Reproducer.global_instance().create_repro()
    if self._repro:
      self._repro.capture(self._run_tracker.run_info.get_as_dict())

    # The __call__ method of the Exiter allows for the prototype pattern.
    self._exiter = LocalExiter(self._run_tracker, self._repro, exiter=self._exiter)
    ExceptionSink.reset_exiter(self._exiter)
示例#11
0
  def set_start_time(self, start_time):
    # Launch RunTracker as early as possible (before .run() is called).
    self._run_tracker = RunTracker.global_instance()
    self._reporting = Reporting.global_instance()

    self._run_start_time = start_time
    self._reporting.initialize(self._run_tracker, self._options, start_time=self._run_start_time)

    # Capture a repro of the 'before' state for this build, if needed.
    self._repro = Reproducer.global_instance().create_repro()
    if self._repro:
      self._repro.capture(self._run_tracker.run_info.get_as_dict())

    # The __call__ method of the Exiter allows for the prototype pattern.
    self._exiter = LocalExiter(self._run_tracker, self._repro, exiter=self._exiter)
    ExceptionSink.reset_exiter(self._exiter)
示例#12
0
  def set_start_time(self, start_time):
    # Launch RunTracker as early as possible (before .run() is called).
    self._run_tracker = RunTracker.global_instance()
    self._reporting = Reporting.global_instance()

    self._run_start_time = start_time
    self._reporting.initialize(self._run_tracker, self._options, start_time=self._run_start_time)

    spec_parser = CmdLineSpecParser(get_buildroot())
    target_specs = [
      spec_parser.parse_spec(spec).to_spec_string()
      for spec in self._options.positional_args
    ]
    # Note: This will not include values from `--owner-of` or `--changed-*` flags.
    self._run_tracker.run_info.add_info("specs_from_command_line", target_specs, stringify=False)

    # Capture a repro of the 'before' state for this build, if needed.
    self._repro = Reproducer.global_instance().create_repro()
    if self._repro:
      self._repro.capture(self._run_tracker.run_info.get_as_dict())
示例#13
0
  def test_raise_if_trace_id_is_of_wrong_ch_format(self):
    trace_id = 'gggggggggggggggg'
    options = {'reporting': {
      'zipkin_trace_id': trace_id,
      'zipkin_parent_id': self.parent_id,
      'zipkin_endpoint': self.zipkin_endpoint
    }}
    context = self.context(for_subsystems=[RunTracker, Reporting], options=options)

    run_tracker = RunTracker.global_instance()
    reporting = Reporting.global_instance()

    with self.assertRaises(ValueError) as result:
      reporting.initialize(run_tracker, context.options)

    self.assertTrue(
      "Value of the flag zipkin-trace-id must be a 16-character or 32-character hex string. "
      + "Got {}.".format(trace_id)
      in str(result.exception)
    )
示例#14
0
  def test_raise_if_trace_id_is_of_wrong_ch_format(self):
    trace_id = 'gggggggggggggggg'
    options = {'reporting': {
      'zipkin_trace_id': trace_id,
      'zipkin_parent_id': self.parent_id,
      'zipkin_endpoint': self.zipkin_endpoint
    }}
    context = self.context(for_subsystems=[RunTracker, Reporting], options=options)

    run_tracker = RunTracker.global_instance()
    reporting = Reporting.global_instance()

    with self.assertRaises(ValueError) as result:
      reporting.initialize(run_tracker, context.options)

    self.assertTrue(
      "Value of the flag zipkin-trace-id must be a 16-character or 32-character hex string. "
      + f"Got {trace_id}."
      in str(result.exception)
    )
示例#15
0
    def test_raise_if_parent_id_is_of_wrong_ch_format(self):
        parent_id = "gggggggggggggggg"
        options = {
            "reporting": {
                "zipkin_trace_id": self.trace_id,
                "zipkin_parent_id": parent_id,
                "zipkin_endpoint": self.zipkin_endpoint,
            }
        }
        context = self.context(for_subsystems=[RunTracker, Reporting],
                               options=options)

        run_tracker = RunTracker.global_instance()
        reporting = Reporting.global_instance()

        with self.assertRaises(ValueError) as result:
            reporting.initialize(run_tracker, context.options)

        self.assertTrue(
            "Value of the flag zipkin-parent-id must be a 16-character hex string. "
            + f"Got {parent_id}." in str(result.exception))
示例#16
0
 def __init__(self, run_tracker=None, reporting=None):
     self._run_tracker = run_tracker or RunTracker.global_instance()
     self._reporting = reporting or Reporting.global_instance()
示例#17
0
  def setup(self, options_bootstrapper, working_set):
    bootstrap_options = options_bootstrapper.get_bootstrap_options()
    global_bootstrap_options = bootstrap_options.for_global_scope()

    # The pants_version may be set in pants.ini for bootstrapping, so we make sure the user actually
    # requested the version on the command line before deciding to print the version and exit.
    if global_bootstrap_options.is_flagged('pants_version'):
      print(global_bootstrap_options.pants_version)
      self._exiter(0)

    # Get logging setup prior to loading backends so that they can log as needed.
    self._setup_logging(global_bootstrap_options)

    # Add any extra paths to python path (e.g., for loading extra source backends).
    for path in global_bootstrap_options.pythonpath:
      sys.path.append(path)
      pkg_resources.fixup_namespace_packages(path)

    # Load plugins and backends.
    plugins = global_bootstrap_options.plugins
    backend_packages = global_bootstrap_options.backend_packages
    build_configuration = load_plugins_and_backends(plugins, working_set, backend_packages)

    # Now that plugins and backends are loaded, we can gather the known scopes.
    self.targets = []

    known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]

    # Add scopes for all needed subsystems.
    subsystems = Subsystem.closure(set(self.subsystems) |
                                   Goal.subsystems() |
                                   build_configuration.subsystems())
    for subsystem in subsystems:
      known_scope_infos.append(subsystem.get_scope_info())

    # Add scopes for all tasks in all goals.
    for goal in Goal.all():
      known_scope_infos.extend(filter(None, goal.known_scope_infos()))

    # Now that we have the known scopes we can get the full options.
    self.options = options_bootstrapper.get_full_options(known_scope_infos)
    self.register_options(subsystems)

    # Make the options values available to all subsystems.
    Subsystem._options = self.options

    # Now that we have options we can instantiate subsystems.
    self.run_tracker = RunTracker.global_instance()
    self.reporting = Reporting.global_instance()
    report = self.reporting.initial_reporting(self.run_tracker)
    self.run_tracker.start(report)
    url = self.run_tracker.run_info.get_info('report_url')
    if url:
      self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
    else:
      self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')

    self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
                                             root_dir=self.root_dir,
                                             run_tracker=self.run_tracker)

    rev = self.options.for_global_scope().build_file_rev
    if rev:
      ScmBuildFile.set_rev(rev)
      ScmBuildFile.set_scm(get_scm())
      build_file_type = ScmBuildFile
    else:
      build_file_type = FilesystemBuildFile
    self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
    self.build_graph = BuildGraph(run_tracker=self.run_tracker,
                                  address_mapper=self.address_mapper)

    # TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
    with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnitLabel.SETUP]):
      source_root_bootstrapper = SourceRootBootstrapper.global_instance()
      source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser)

    self._expand_goals_and_specs()

    # Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
    self.run_tracker.run_info.add_scm_info()
示例#18
0
 def __init__(self, run_tracker=None, reporting=None):
     self._run_tracker = run_tracker or RunTracker.global_instance()
     self._reporting = reporting or Reporting.global_instance()
示例#19
0
  def setup(self):
    options_bootstrapper = OptionsBootstrapper()
    bootstrap_options = options_bootstrapper.get_bootstrap_options()

    # Get logging setup prior to loading backends so that they can log as needed.
    self._setup_logging(bootstrap_options.for_global_scope())

    # Add any extra paths to python path (eg for loading extra source backends)
    for path in bootstrap_options.for_global_scope().pythonpath:
      sys.path.append(path)
      pkg_resources.fixup_namespace_packages(path)

    # Load plugins and backends.
    plugins = bootstrap_options.for_global_scope().plugins
    backend_packages = bootstrap_options.for_global_scope().backend_packages
    build_configuration = load_plugins_and_backends(plugins, backend_packages)

    # Now that plugins and backends are loaded, we can gather the known scopes.
    self.targets = []

    known_scope_infos = [ScopeInfo.for_global_scope()]

    # Add scopes for all needed subsystems.
    subsystems = (set(self.subsystems) | Goal.subsystems() | build_configuration.subsystems())
    for subsystem in subsystems:
      known_scope_infos.append(ScopeInfo(subsystem.options_scope, ScopeInfo.GLOBAL_SUBSYSTEM))

    # Add scopes for all tasks in all goals.
    for goal in Goal.all():
      known_scope_infos.extend(filter(None, goal.known_scope_infos()))

    # Now that we have the known scopes we can get the full options.
    self.options = options_bootstrapper.get_full_options(known_scope_infos)
    self.register_options(subsystems)

    # Make the options values available to all subsystems.
    Subsystem._options = self.options

    # Now that we have options we can instantiate subsystems.
    self.run_tracker = RunTracker.global_instance()
    self.reporting = Reporting.global_instance()
    report = self.reporting.initial_reporting(self.run_tracker)
    self.run_tracker.start(report)
    url = self.run_tracker.run_info.get_info('report_url')
    if url:
      self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
    else:
      self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')

    self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
                                             root_dir=self.root_dir,
                                             run_tracker=self.run_tracker)

    rev = self.options.for_global_scope().build_file_rev
    if rev:
      ScmBuildFile.set_rev(rev)
      ScmBuildFile.set_scm(get_scm())
      build_file_type = ScmBuildFile
    else:
      build_file_type = FilesystemBuildFile
    self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
    self.build_graph = BuildGraph(run_tracker=self.run_tracker,
                                  address_mapper=self.address_mapper)

    # TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
    with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
      source_root_bootstrapper = SourceRootBootstrapper.global_instance()
      source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser)

    self._expand_goals_and_specs()

    # Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
    self.run_tracker.run_info.add_scm_info()