Esempio n. 1
0
def assert_spec_parsed(build_root: Path, spec_str: str,
                       expected_spec: Spec) -> None:
    parser = SpecsParser(str(build_root))
    spec, is_ignore = parser.parse_spec(spec_str)
    assert isinstance(spec, type(expected_spec))
    assert spec == expected_spec
    assert is_ignore is False

    # Check ignores are also parsed correctly.
    spec, is_ignore = parser.parse_spec(f"-{spec_str}")
    assert isinstance(spec, type(expected_spec))
    assert spec == expected_spec
    assert is_ignore is True
Esempio n. 2
0
    def run(self, start_time: float) -> ExitCode:
        spec_parser = SpecsParser(get_buildroot())
        specs = [str(spec_parser.parse_spec(spec)) for spec in self.options.specs]
        self.run_tracker.start(run_start_time=start_time, specs=specs)

        with maybe_profiled(self.profile_path):
            global_options = self.options.for_global_scope()

            if self.options.help_request:
                return self._print_help(self.options.help_request)

            streaming_reporter = StreamingWorkunitHandler(
                self.graph_session.scheduler_session,
                run_tracker=self.run_tracker,
                callbacks=self._get_workunits_callbacks(),
                report_interval_seconds=global_options.streaming_workunits_report_interval,
            )

            goals = tuple(self.options.goals)
            with streaming_reporter.session():
                if not goals:
                    return PANTS_SUCCEEDED_EXIT_CODE
                engine_result = PANTS_FAILED_EXIT_CODE
                try:
                    engine_result = self._perform_run(goals)
                except Exception as e:
                    ExceptionSink.log_exception(e)

                metrics = self.graph_session.scheduler_session.metrics()
                self.run_tracker.set_pantsd_scheduler_metrics(metrics)
                self.run_tracker.end_run(engine_result)

            return engine_result
Esempio n. 3
0
    def run(self, start_time: float) -> ExitCode:
        with maybe_profiled(self.profile_path):
            spec_parser = SpecsParser(get_buildroot())
            specs = [
                str(spec_parser.parse_spec(spec))
                for spec in self.options.specs
            ]
            self.run_tracker.start(run_start_time=start_time, specs=specs)
            global_options = self.options.for_global_scope()

            streaming_reporter = StreamingWorkunitHandler(
                self.graph_session.scheduler_session,
                run_tracker=self.run_tracker,
                specs=self.specs,
                options_bootstrapper=self.options_bootstrapper,
                callbacks=self._get_workunits_callbacks(),
                report_interval_seconds=global_options.
                streaming_workunits_report_interval,
                pantsd=global_options.pantsd,
            )
            with streaming_reporter:
                engine_result = PANTS_FAILED_EXIT_CODE
                try:
                    engine_result = self._run_inner()
                finally:
                    metrics = self.graph_session.scheduler_session.metrics()
                    self.run_tracker.set_pantsd_scheduler_metrics(metrics)
                    self.run_tracker.end_run(engine_result)

                return engine_result
    def _set_start_time(self, start_time: float) -> None:
        self._run_tracker.start(self.options, run_start_time=start_time)

        spec_parser = SpecsParser(get_buildroot())
        specs = [str(spec_parser.parse_spec(spec)) for spec in self.options.specs]
        # Note: This will not include values from `--changed-*` flags.
        self._run_tracker.run_info.add_info("specs_from_command_line", specs, stringify=False)
Esempio n. 5
0
    def run(self, start_time: float) -> ExitCode:
        with maybe_profiled(self.profile_path):
            spec_parser = SpecsParser()
            specs = []
            for spec_str in self.options.specs:
                spec, is_ignore = spec_parser.parse_spec(spec_str)
                specs.append(f"-{spec}" if is_ignore else str(spec))

            self.run_tracker.start(run_start_time=start_time, specs=specs)
            global_options = self.options.for_global_scope()

            streaming_reporter = StreamingWorkunitHandler(
                self.graph_session.scheduler_session,
                run_tracker=self.run_tracker,
                specs=self.specs,
                options_bootstrapper=self.options_bootstrapper,
                callbacks=self._get_workunits_callbacks(),
                report_interval_seconds=global_options.
                streaming_workunits_report_interval,
                allow_async_completion=(
                    global_options.pantsd
                    and global_options.streaming_workunits_complete_async),
                max_workunit_verbosity=global_options.
                streaming_workunits_level,
            )
            with streaming_reporter:
                engine_result = PANTS_FAILED_EXIT_CODE
                try:
                    engine_result = self._run_inner()
                finally:
                    metrics = self.graph_session.scheduler_session.metrics()
                    self.run_tracker.set_pantsd_scheduler_metrics(metrics)
                    self.run_tracker.end_run(engine_result)

                return engine_result
Esempio n. 6
0
    def _set_start_time(self, start_time: float) -> None:
        # Propagates parent_build_id to pants runs that may be called from this pants run.
        os.environ["PANTS_PARENT_BUILD_ID"] = self._run_tracker.run_id

        self._run_tracker.start(self.options, run_start_time=start_time)

        spec_parser = SpecsParser(get_buildroot())
        specs = [
            str(spec_parser.parse_spec(spec)) for spec in self.options.specs
        ]
        # Note: This will not include values from `--changed-*` flags.
        self._run_tracker.run_info.add_info("specs_from_command_line",
                                            specs,
                                            stringify=False)
Esempio n. 7
0
def assert_filesystem_spec_parsed(build_root: Path, spec_str: str,
                                  expected_spec: FilesystemSpec) -> None:
    parser = SpecsParser(str(build_root))
    spec = parser.parse_spec(spec_str)
    assert isinstance(spec, FilesystemSpec)
    assert spec == expected_spec
Esempio n. 8
0
def assert_address_spec_parsed(build_root: Path, spec_str: str,
                               expected_spec: AddressSpec) -> None:
    parser = SpecsParser(str(build_root))
    spec = parser.parse_spec(spec_str)
    assert isinstance(spec, AddressSpec)
    assert spec == expected_spec
Esempio n. 9
0
class SpecsParserTest(TestBase):
    def setUp(self) -> None:
        super().setUp()
        self._spec_parser = SpecsParser(self.build_root)

    def test_address_literal_specs(self) -> None:
        self.assert_address_spec_parsed(":root", address_literal("", "root"))
        self.assert_address_spec_parsed("//:root", address_literal("", "root"))

        self.assert_address_spec_parsed("a", address_literal("a"))
        self.assert_address_spec_parsed("a:a", address_literal("a", "a"))

        self.assert_address_spec_parsed("a/b", address_literal("a/b"))
        self.assert_address_spec_parsed("a/b:b", address_literal("a/b", "b"))
        self.assert_address_spec_parsed("a/b:c", address_literal("a/b", "c"))

    def test_sibling(self) -> None:
        self.assert_address_spec_parsed(":", sib(""))
        self.assert_address_spec_parsed("//:", sib(""))

        self.assert_address_spec_parsed("a:", sib("a"))
        self.assert_address_spec_parsed("//a:", sib("a"))

        self.assert_address_spec_parsed("a/b:", sib("a/b"))
        self.assert_address_spec_parsed("//a/b:", sib("a/b"))

    def test_descendant(self) -> None:
        self.assert_address_spec_parsed("::", desc(""))
        self.assert_address_spec_parsed("//::", desc(""))

        self.assert_address_spec_parsed("a::", desc("a"))
        self.assert_address_spec_parsed("//a::", desc("a"))

        self.assert_address_spec_parsed("a/b::", desc("a/b"))
        self.assert_address_spec_parsed("//a/b::", desc("a/b"))

    def test_files(self) -> None:
        # We assume that specs with an extension are meant to be interpreted as filesystem specs.
        for f in ["a.txt", "a.tmp.cache.txt.bak", "a/b/c.txt", ".a.txt"]:
            self.assert_filesystem_spec_parsed(f, file_literal(f))
        self.assert_filesystem_spec_parsed("./a.txt", file_literal("a.txt"))
        self.assert_filesystem_spec_parsed("//./a.txt", file_literal("a.txt"))

    def test_globs(self) -> None:
        for glob_str in [
                "*", "**/*", "a/b/*", "a/b/test_*.py", "a/b/**/test_*"
        ]:
            self.assert_filesystem_spec_parsed(glob_str, file_glob(glob_str))

    def test_excludes(self) -> None:
        for glob_str in ["!", "!a/b/", "!/a/b/*"]:
            self.assert_filesystem_spec_parsed(glob_str, ignore(glob_str[1:]))

    def test_files_with_original_targets(self) -> None:
        self.assert_address_spec_parsed("a.txt:tgt",
                                        address_literal("a.txt", "tgt"))
        self.assert_address_spec_parsed("a/b/c.txt:tgt",
                                        address_literal("a/b/c.txt", "tgt"))
        self.assert_address_spec_parsed("./a.txt:tgt",
                                        address_literal("a.txt", "tgt"))
        self.assert_address_spec_parsed("//./a.txt:tgt",
                                        address_literal("a.txt", "tgt"))
        self.assert_address_spec_parsed("a/b/c.txt:../tgt",
                                        address_literal("a/b/c.txt", "../tgt"))

    def test_ambiguous_files(self) -> None:
        # These could either be files or the shorthand for address_literal addresses. We check if
        # they exist on the file system to disambiguate.
        for spec in ["a", "b/c"]:
            self.assert_address_spec_parsed(spec, address_literal(spec))
            self.create_file(spec)
            self.assert_filesystem_spec_parsed(spec, file_literal(spec))

    def test_absolute(self) -> None:
        self.assert_address_spec_parsed(os.path.join(self.build_root, "a"),
                                        address_literal("a"))
        self.assert_address_spec_parsed(os.path.join(self.build_root, "a:a"),
                                        address_literal("a", "a"))
        self.assert_address_spec_parsed(os.path.join(self.build_root, "a:"),
                                        sib("a"))
        self.assert_address_spec_parsed(os.path.join(self.build_root, "a::"),
                                        desc("a"))
        self.assert_filesystem_spec_parsed(
            os.path.join(self.build_root, "a.txt"), file_literal("a.txt"))

        with self.assertRaises(SpecsParser.BadSpecError):
            self.assert_address_spec_parsed("/not/the/buildroot/a", sib("a"))
            self.assert_filesystem_spec_parsed("/not/the/buildroot/a.txt",
                                               file_literal("a.txt"))

    def test_absolute_double_slashed(self) -> None:
        # By adding a double slash, we are insisting that this absolute path is actually
        # relative to the buildroot. Thus, it should parse correctly.
        double_absolute_address = "/" + os.path.join(self.build_root, "a")
        double_absolute_file = "/" + os.path.join(self.build_root, "a.txt")
        for spec in [double_absolute_address, double_absolute_file]:
            assert "//" == spec[:2]
        self.assert_address_spec_parsed(
            double_absolute_address,
            address_literal(double_absolute_address[2:]))
        self.assert_filesystem_spec_parsed(
            double_absolute_file, file_literal(double_absolute_file[2:]))

    def test_cmd_line_affordances(self) -> None:
        self.assert_address_spec_parsed("./:root", address_literal("", "root"))
        self.assert_address_spec_parsed("//./:root",
                                        address_literal("", "root"))
        self.assert_address_spec_parsed("//./a/../:root",
                                        address_literal("", "root"))
        self.assert_address_spec_parsed(
            os.path.join(self.build_root, "./a/../:root"),
            address_literal("", "root"))

        self.assert_address_spec_parsed("a/", address_literal("a"))
        self.assert_address_spec_parsed("./a/", address_literal("a"))
        self.assert_address_spec_parsed(os.path.join(self.build_root, "./a/"),
                                        address_literal("a"))

        self.assert_address_spec_parsed("a/b/:b", address_literal("a/b", "b"))
        self.assert_address_spec_parsed("./a/b/:b",
                                        address_literal("a/b", "b"))
        self.assert_address_spec_parsed(
            os.path.join(self.build_root, "./a/b/:b"),
            address_literal("a/b", "b"))

    def assert_address_spec_parsed(self, spec_str: str,
                                   expected_spec: AddressSpec) -> None:
        spec = self._spec_parser.parse_spec(spec_str)
        assert isinstance(spec, AddressSpec)
        assert spec == expected_spec

    def assert_filesystem_spec_parsed(self, spec_str: str,
                                      expected_spec: FilesystemSpec) -> None:
        spec = self._spec_parser.parse_spec(spec_str)
        assert isinstance(spec, FilesystemSpec)
        assert spec == expected_spec
Esempio n. 10
0
def assert_spec_parsed(build_root: Path, spec_str: str, expected_spec: Spec) -> None:
    parser = SpecsParser(str(build_root))
    spec = parser.parse_spec(spec_str)
    assert isinstance(spec, type(expected_spec))
    assert spec == expected_spec