示例#1
0
    def _get_result_from_coursier(self, jars_to_resolve, global_excludes,
                                  pinned_coords, pants_workdir,
                                  coursier_cache_path, sources, javadoc):
        """
    Calling coursier and return the result per invocation.

    If coursier was called once for classifier '' and once for classifier 'tests', then the return value
    would be: {'default': [<first coursier output>, <second coursier output>]}

    :param jars_to_resolve: List of `JarDependency`s to resolve
    :param global_excludes: List of `M2Coordinate`s to exclude globally
    :param pinned_coords: List of `M2Coordinate`s that need to be pinned.
    :param pants_workdir: Pants' workdir
    :param coursier_cache_path: path to where coursier cache is stored.

    :return: The aggregation of results by conf from coursier. Each coursier call could return
    the following:
        {
          "conflict_resolution": {
            "org:name:version" (requested): "org:name:version" (reconciled)
          },
          "dependencies": [
            {
              "coord": "orgA:nameA:versionA",
              "file": <path>,
              "dependencies": [ // coodinates for its transitive dependencies
                <orgX:nameX:versionX>,
                <orgY:nameY:versionY>,
              ]
            },
            {
              "coord": "orgB:nameB:jar:classifier:versionB",
              "file": <path>,
              "dependencies": [ // coodinates for its transitive dependencies
                <orgX:nameX:versionX>,
                <orgZ:nameZ:versionZ>,
              ]
            },
            ... // more about orgX:nameX:versionX, orgY:nameY:versionY, orgZ:nameZ:versionZ
          ]
        }
    Hence the aggregation of the results will be in the following format, for example when default classifier
    and sources are fetched:
    {
      'default': [<result from coursier call with default conf with classifier X>,
                  <result from coursier call with default conf with classifier Y>],
      'src_doc': [<result from coursier call with --sources and/or --javadoc>],
    }
    """
        # Prepare coursier args
        coursier_subsystem_instance = CoursierSubsystem.global_instance()
        coursier_jar = coursier_subsystem_instance.bootstrap_coursier(
            self.context.new_workunit)

        common_args = [
            'fetch',
            # Print the resolution tree
            '-t',
            '--cache',
            coursier_cache_path
        ] + coursier_subsystem_instance.get_options().fetch_options

        coursier_work_temp_dir = os.path.join(pants_workdir, 'tmp')
        safe_mkdir(coursier_work_temp_dir)

        results_by_conf = self._get_default_conf_results(
            common_args, coursier_jar, global_excludes, jars_to_resolve,
            coursier_work_temp_dir, pinned_coords)
        if sources or javadoc:
            non_default_conf_results = self._get_non_default_conf_results(
                common_args, coursier_jar, global_excludes, jars_to_resolve,
                coursier_work_temp_dir, pinned_coords, sources, javadoc)
            results_by_conf.update(non_default_conf_results)

        return results_by_conf
示例#2
0
    def resolve(self, targets, compile_classpath, sources, javadoc):
        """
    This is the core function for coursier resolve.

    Validation strategy:

    1. All targets are going through the `invalidated` to get fingerprinted in the target level.
       No cache is fetched at this stage because it is disabled.
    2. Once each target is fingerprinted, we combine them into a `VersionedTargetSet` where they
       are fingerprinted together, because each run of 3rdparty resolve is context sensitive.

    Artifacts are stored in `VersionedTargetSet`'s results_dir, the contents are the aggregation of
    each coursier run happened within that context.

    Caching: (TODO): https://github.com/pantsbuild/pants/issues/5187
    Currently it is disabled due to absolute paths in the coursier results.

    :param targets: a collection of targets to do 3rdparty resolve against
    :param compile_classpath: classpath product that holds the resolution result. IMPORTANT: this parameter will be changed.
    :param sources: if True, fetch sources for 3rdparty
    :param javadoc: if True, fetch javadoc for 3rdparty
    :return: n/a
    """
        manager = JarDependencyManagement.global_instance()

        jar_targets = manager.targets_by_artifact_set(targets)

        for artifact_set, target_subset in jar_targets.items():
            # TODO(wisechengyi): this is the only place we are using IvyUtil method, which isn't specific to ivy really.
            raw_jar_deps, global_excludes = IvyUtils.calculate_classpath(
                target_subset)

            # ['sources'] * False = [], ['sources'] * True = ['sources']
            confs_for_fingerprint = ['sources'] * sources + ['javadoc'
                                                             ] * javadoc
            fp_strategy = CoursierResolveFingerprintStrategy(
                confs_for_fingerprint)

            compile_classpath.add_excludes_for_targets(target_subset)

            with self.invalidated(
                    target_subset,
                    invalidate_dependents=False,
                    silent=False,
                    fingerprint_strategy=fp_strategy) as invalidation_check:

                if not invalidation_check.all_vts:
                    continue

                pants_workdir = self.get_options().pants_workdir
                resolve_vts = VersionedTargetSet.from_versioned_targets(
                    invalidation_check.all_vts)

                vt_set_results_dir = self._prepare_vts_results_dir(
                    pants_workdir, resolve_vts)
                pants_jar_base_dir = self._prepare_workdir(pants_workdir)
                coursier_cache_dir = CoursierSubsystem.global_instance(
                ).get_options().cache_dir

                # If a report is requested, do not proceed with loading validated result.
                if not self.get_options().report:
                    # Check each individual target without context first
                    # If the individuals are valid, check them as a VersionedTargetSet
                    if not invalidation_check.invalid_vts and resolve_vts.valid:
                        # Load up from the results dir
                        success = self._load_from_results_dir(
                            compile_classpath, vt_set_results_dir,
                            coursier_cache_dir, invalidation_check,
                            pants_jar_base_dir)
                        if success:
                            return

                jars_to_resolve, pinned_coords = self._compute_jars_to_resolve_and_pin(
                    raw_jar_deps, artifact_set, manager)

                results = self._get_result_from_coursier(
                    jars_to_resolve, global_excludes, pinned_coords,
                    pants_workdir, coursier_cache_dir, sources, javadoc)

                for conf, result_list in results.items():
                    for result in result_list:
                        self._load_json_result(
                            conf, compile_classpath, coursier_cache_dir,
                            invalidation_check, pants_jar_base_dir, result,
                            self._override_classifiers_for_conf(conf))

                self._populate_results_dir(vt_set_results_dir, results)
                resolve_vts.update()
示例#3
0
  def _get_result_from_coursier(self, jars_to_resolve, global_excludes, pinned_coords,
                                coursier_cache_path, sources, javadoc, executor):
    """
    Calling coursier and return the result per invocation.

    If coursier was called once for classifier '' and once for classifier 'tests', then the return value
    would be: {'default': [<first coursier output>, <second coursier output>]}

    :param jars_to_resolve: List of `JarDependency`s to resolve
    :param global_excludes: List of `M2Coordinate`s to exclude globally
    :param pinned_coords: List of `M2Coordinate`s that need to be pinned.
    :param coursier_cache_path: path to where coursier cache is stored.
    :param executor: An instance of `pants.java.executor.Executor`

    :return: The aggregation of results by conf from coursier. Each coursier call could return
    the following:
        {
          "conflict_resolution": {
            "org:name:version" (requested): "org:name:version" (reconciled)
          },
          "dependencies": [
            {
              "coord": "orgA:nameA:versionA",
              "file": <path>,
              "dependencies": [ // coodinates for its transitive dependencies
                <orgX:nameX:versionX>,
                <orgY:nameY:versionY>,
              ]
            },
            {
              "coord": "orgB:nameB:jar:classifier:versionB",
              "file": <path>,
              "dependencies": [ // coodinates for its transitive dependencies
                <orgX:nameX:versionX>,
                <orgZ:nameZ:versionZ>,
              ]
            },
            ... // more about orgX:nameX:versionX, orgY:nameY:versionY, orgZ:nameZ:versionZ
          ]
        }
    Hence the aggregation of the results will be in the following format, for example when default classifier
    and sources are fetched:
    {
      'default': [<result from coursier call with default conf with classifier X>,
                  <result from coursier call with default conf with classifier Y>],
      'src_doc': [<result from coursier call with --sources and/or --javadoc>],
    }
    """
    # Prepare coursier args
    coursier_subsystem_instance = CoursierSubsystem.global_instance()
    coursier_jar = coursier_subsystem_instance.bootstrap_coursier(self.context.new_workunit)

    repos = coursier_subsystem_instance.get_options().repos
    # make [repoX, repoY] -> ['-r', repoX, '-r', repoY]
    repo_args = list(itertools.chain(*list(zip(['-r'] * len(repos), repos))))
    artifact_types_arg = ['-A', ','.join(coursier_subsystem_instance.get_options().artifact_types)]
    advanced_options = coursier_subsystem_instance.get_options().fetch_options
    common_args = ['fetch',
                   # Print the resolution tree
                   '-t',
                   '--cache', coursier_cache_path
                   ] + repo_args + artifact_types_arg + advanced_options

    coursier_work_temp_dir = os.path.join(self.versioned_workdir, 'tmp')
    safe_mkdir(coursier_work_temp_dir)

    results_by_conf = self._get_default_conf_results(common_args, coursier_jar, global_excludes, jars_to_resolve,
                                                     coursier_work_temp_dir,
                                                     pinned_coords, executor)
    if sources or javadoc:
      non_default_conf_results = self._get_non_default_conf_results(common_args, coursier_jar, global_excludes,
                                                                    jars_to_resolve, coursier_work_temp_dir,
                                                                    pinned_coords, sources, javadoc, executor)
      results_by_conf.update(non_default_conf_results)

    return results_by_conf
示例#4
0
  def resolve(self, targets, compile_classpath, sources, javadoc, executor):
    """
    This is the core function for coursier resolve.

    Validation strategy:

    1. All targets are going through the `invalidated` to get fingerprinted in the target level.
       No cache is fetched at this stage because it is disabled.
    2. Once each target is fingerprinted, we combine them into a `VersionedTargetSet` where they
       are fingerprinted together, because each run of 3rdparty resolve is context sensitive.

    Artifacts are stored in `VersionedTargetSet`'s results_dir, the contents are the aggregation of
    each coursier run happened within that context.

    Caching: (TODO): https://github.com/pantsbuild/pants/issues/5187
    Currently it is disabled due to absolute paths in the coursier results.

    :param targets: a collection of targets to do 3rdparty resolve against
    :param compile_classpath: classpath product that holds the resolution result. IMPORTANT: this parameter will be changed.
    :param sources: if True, fetch sources for 3rdparty
    :param javadoc: if True, fetch javadoc for 3rdparty
    :param executor: An instance of `pants.java.executor.Executor`. If None, a subprocess executor will be assigned.
    :return: n/a
    """
    manager = JarDependencyManagement.global_instance()

    jar_targets = manager.targets_by_artifact_set(targets)

    executor = executor or SubprocessExecutor(DistributionLocator.cached())
    if not isinstance(executor, Executor):
      raise ValueError('The executor argument must be an Executor instance, given {} of type {}'.format(
        executor, type(executor)))

    for artifact_set, target_subset in jar_targets.items():
      # TODO(wisechengyi): this is the only place we are using IvyUtil method, which isn't specific to ivy really.
      raw_jar_deps, global_excludes = IvyUtils.calculate_classpath(target_subset)

      # ['sources'] * False = [], ['sources'] * True = ['sources']
      confs_for_fingerprint = ['sources'] * sources + ['javadoc'] * javadoc
      fp_strategy = CoursierResolveFingerprintStrategy(confs_for_fingerprint)

      compile_classpath.add_excludes_for_targets(target_subset)

      with self.invalidated(target_subset,
                            invalidate_dependents=False,
                            silent=False,
                            fingerprint_strategy=fp_strategy) as invalidation_check:

        if not invalidation_check.all_vts:
          continue

        resolve_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)

        vt_set_results_dir = self._prepare_vts_results_dir(resolve_vts)
        pants_jar_base_dir = self._prepare_workdir()
        coursier_cache_dir = CoursierSubsystem.global_instance().get_options().cache_dir

        # If a report is requested, do not proceed with loading validated result.
        if not self.get_options().report:
          # Check each individual target without context first
          # If the individuals are valid, check them as a VersionedTargetSet
          if not invalidation_check.invalid_vts and resolve_vts.valid:
            # Load up from the results dir
            success = self._load_from_results_dir(compile_classpath, vt_set_results_dir,
                                                  coursier_cache_dir, invalidation_check, pants_jar_base_dir)
            if success:
              return

        jars_to_resolve, pinned_coords = self._compute_jars_to_resolve_and_pin(raw_jar_deps,
                                                                               artifact_set,
                                                                               manager)

        results = self._get_result_from_coursier(jars_to_resolve, global_excludes, pinned_coords,
                                                 coursier_cache_dir, sources, javadoc, executor)

        for conf, result_list in results.items():
          for result in result_list:
            self._load_json_result(conf, compile_classpath, coursier_cache_dir, invalidation_check,
                                   pants_jar_base_dir, result, self._override_classifiers_for_conf(conf))

        self._populate_results_dir(vt_set_results_dir, results)
        resolve_vts.update()
示例#5
0
    def _get_result_from_coursier(
        self,
        jars_to_resolve,
        global_excludes,
        pinned_coords,
        coursier_cache_path,
        sources,
        javadoc,
        executor,
    ):
        """Calling coursier and return the result per invocation.

        If coursier was called once for classifier '' and once for classifier 'tests', then the return value
        would be: {'default': [<first coursier output>, <second coursier output>]}

        :param jars_to_resolve: List of `JarDependency`s to resolve
        :param global_excludes: List of `M2Coordinate`s to exclude globally
        :param pinned_coords: List of `M2Coordinate`s that need to be pinned.
        :param coursier_cache_path: path to where coursier cache is stored.
        :param executor: An instance of `pants.java.executor.Executor`

        :return: The aggregation of results by conf from coursier. Each coursier call could return
        the following:
            {
              "conflict_resolution": {
                "org:name:version" (requested): "org:name:version" (reconciled)
              },
              "dependencies": [
                {
                  "coord": "orgA:nameA:versionA",
                  "file": <path>,
                  "dependencies": [ // coodinates for its transitive dependencies
                    <orgX:nameX:versionX>,
                    <orgY:nameY:versionY>,
                  ]
                },
                {
                  "coord": "orgB:nameB:jar:classifier:versionB",
                  "file": <path>,
                  "dependencies": [ // coodinates for its transitive dependencies
                    <orgX:nameX:versionX>,
                    <orgZ:nameZ:versionZ>,
                  ]
                },
                ... // more about orgX:nameX:versionX, orgY:nameY:versionY, orgZ:nameZ:versionZ
              ]
            }
        Hence the aggregation of the results will be in the following format, for example when default classifier
        and sources are fetched:
        {
          'default': [<result from coursier call with default conf with classifier X>,
                      <result from coursier call with default conf with classifier Y>],
          'src_doc': [<result from coursier call with --sources and/or --javadoc>],
        }
        """
        # Prepare coursier args
        coursier_subsystem_instance = CoursierSubsystem.global_instance()
        coursier_jar = coursier_subsystem_instance.select()

        repos = coursier_subsystem_instance.get_options().repos
        # make [repoX, repoY] -> ['-r', repoX, '-r', repoY]
        repo_args = list(
            itertools.chain(*list(zip(["-r"] * len(repos), repos))))
        artifact_types_arg = [
            "-A",
            ",".join(coursier_subsystem_instance.get_options().artifact_types),
        ]
        advanced_options = coursier_subsystem_instance.get_options(
        ).fetch_options
        common_args = ([
            "fetch",
            # Print the resolution tree
            "-t",
            "--cache",
            coursier_cache_path,
        ] + repo_args + artifact_types_arg + advanced_options)

        coursier_work_temp_dir = os.path.join(self.versioned_workdir, "tmp")
        safe_mkdir(coursier_work_temp_dir)

        results_by_conf = self._get_default_conf_results(
            common_args,
            coursier_jar,
            global_excludes,
            jars_to_resolve,
            coursier_work_temp_dir,
            pinned_coords,
            executor,
        )
        if sources or javadoc:
            non_default_conf_results = self._get_non_default_conf_results(
                common_args,
                coursier_jar,
                global_excludes,
                jars_to_resolve,
                coursier_work_temp_dir,
                pinned_coords,
                sources,
                javadoc,
                executor,
            )
            results_by_conf.update(non_default_conf_results)

        return results_by_conf
示例#6
0
    def resolve(self, targets, compile_classpath, sources, javadoc, executor):
        """This is the core function for coursier resolve.

        Validation strategy:

        1. All targets are going through the `invalidated` to get fingerprinted in the target level.
           No cache is fetched at this stage because it is disabled.
        2. Once each target is fingerprinted, we combine them into a `VersionedTargetSet` where they
           are fingerprinted together, because each run of 3rdparty resolve is context sensitive.

        Artifacts are stored in `VersionedTargetSet`'s results_dir, the contents are the aggregation of
        each coursier run happened within that context.

        Caching: (TODO): https://github.com/pantsbuild/pants/issues/5187
        Currently it is disabled due to absolute paths in the coursier results.

        :param targets: a collection of targets to do 3rdparty resolve against
        :param compile_classpath: classpath product that holds the resolution result. IMPORTANT: this parameter will be changed.
        :param sources: if True, fetch sources for 3rdparty
        :param javadoc: if True, fetch javadoc for 3rdparty
        :param executor: An instance of `pants.java.executor.Executor`. If None, a subprocess executor will be assigned.
        :return: n/a
        """
        manager = JarDependencyManagement.global_instance()

        jar_targets = manager.targets_by_artifact_set(targets)

        executor = executor or SubprocessExecutor(DistributionLocator.cached())
        if not isinstance(executor, Executor):
            raise ValueError(
                "The executor argument must be an Executor instance, given {} of type {}".format(
                    executor, type(executor)
                )
            )

        for artifact_set, target_subset in jar_targets.items():
            raw_jar_deps, global_excludes = calculate_classpath(target_subset)
            confs_for_fingerprint = ["sources"] * sources + ["javadoc"] * javadoc
            fp_strategy = CoursierResolveFingerprintStrategy(confs_for_fingerprint)

            compile_classpath.add_excludes_for_targets(target_subset)

            with self.invalidated(
                target_subset,
                invalidate_dependents=False,
                silent=False,
                fingerprint_strategy=fp_strategy,
            ) as invalidation_check:

                if not invalidation_check.all_vts:
                    continue

                resolve_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)

                vt_set_results_dir = self._prepare_vts_results_dir(resolve_vts)
                pants_jar_base_dir = self._prepare_workdir()
                coursier_cache_dir = CoursierSubsystem.global_instance().get_options().cache_dir

                # If a report is requested, do not proceed with loading validated result.
                if not self.get_options().report:
                    # Check each individual target without context first
                    # If the individuals are valid, check them as a VersionedTargetSet
                    # The order of 'or' statement matters, because checking for cache is more expensive.
                    if resolve_vts.valid or (
                        self.artifact_cache_reads_enabled()
                        and len(self.check_artifact_cache([resolve_vts])[0])
                        == len(resolve_vts.targets)
                    ):
                        # Load up from the results dir
                        success = self._load_from_results_dir(
                            compile_classpath,
                            vt_set_results_dir,
                            coursier_cache_dir,
                            invalidation_check,
                            pants_jar_base_dir,
                        )
                        if success:
                            resolve_vts.update()
                            return

                jars_to_resolve, pinned_coords = self._compute_jars_to_resolve_and_pin(
                    raw_jar_deps, artifact_set, manager
                )

                results = self._get_result_from_coursier(
                    jars_to_resolve,
                    global_excludes,
                    pinned_coords,
                    coursier_cache_dir,
                    sources,
                    javadoc,
                    executor,
                )

                for conf, result_list in results.items():
                    for result in result_list:
                        self._load_json_result(
                            conf,
                            compile_classpath,
                            coursier_cache_dir,
                            invalidation_check,
                            pants_jar_base_dir,
                            result,
                            self._override_classifiers_for_conf(conf),
                        )

                self._populate_results_dir(vt_set_results_dir, results)
                resolve_vts.update()

                if self.artifact_cache_writes_enabled():
                    self.update_artifact_cache([(resolve_vts, [vt_set_results_dir])])