def _get_result_from_coursier(self, jars_to_resolve, global_excludes,
pinned_coords, pants_workdir,
coursier_cache_path, sources, javadoc):
"""
Calling coursier and return the result per invocation.
If coursier was called once for classifier '' and once for classifier 'tests', then the return value
would be: {'default': [<first coursier output>, <second coursier output>]}
:param jars_to_resolve: List of `JarDependency`s to resolve
:param global_excludes: List of `M2Coordinate`s to exclude globally
:param pinned_coords: List of `M2Coordinate`s that need to be pinned.
:param pants_workdir: Pants' workdir
:param coursier_cache_path: path to where coursier cache is stored.
:return: The aggregation of results by conf from coursier. Each coursier call could return
the following:
{
"conflict_resolution": {
"org:name:version" (requested): "org:name:version" (reconciled)
},
"dependencies": [
{
"coord": "orgA:nameA:versionA",
"file": <path>,
"dependencies": [ // coodinates for its transitive dependencies
<orgX:nameX:versionX>,
<orgY:nameY:versionY>,
]
},
{
"coord": "orgB:nameB:jar:classifier:versionB",
"file": <path>,
"dependencies": [ // coodinates for its transitive dependencies
<orgX:nameX:versionX>,
<orgZ:nameZ:versionZ>,
]
},
... // more about orgX:nameX:versionX, orgY:nameY:versionY, orgZ:nameZ:versionZ
]
}
Hence the aggregation of the results will be in the following format, for example when default classifier
and sources are fetched:
{
'default': [<result from coursier call with default conf with classifier X>,
<result from coursier call with default conf with classifier Y>],
'src_doc': [<result from coursier call with --sources and/or --javadoc>],
}
"""
# Prepare coursier args
coursier_subsystem_instance = CoursierSubsystem.global_instance()
coursier_jar = coursier_subsystem_instance.bootstrap_coursier(
self.context.new_workunit)
common_args = [
'fetch',
# Print the resolution tree
'-t',
'--cache',
coursier_cache_path
] + coursier_subsystem_instance.get_options().fetch_options
coursier_work_temp_dir = os.path.join(pants_workdir, 'tmp')
safe_mkdir(coursier_work_temp_dir)
results_by_conf = self._get_default_conf_results(
common_args, coursier_jar, global_excludes, jars_to_resolve,
coursier_work_temp_dir, pinned_coords)
if sources or javadoc:
non_default_conf_results = self._get_non_default_conf_results(
common_args, coursier_jar, global_excludes, jars_to_resolve,
coursier_work_temp_dir, pinned_coords, sources, javadoc)
results_by_conf.update(non_default_conf_results)
return results_by_conf
def resolve(self, targets, compile_classpath, sources, javadoc):
"""
This is the core function for coursier resolve.
Validation strategy:
1. All targets are going through the `invalidated` to get fingerprinted in the target level.
No cache is fetched at this stage because it is disabled.
2. Once each target is fingerprinted, we combine them into a `VersionedTargetSet` where they
are fingerprinted together, because each run of 3rdparty resolve is context sensitive.
Artifacts are stored in `VersionedTargetSet`'s results_dir, the contents are the aggregation of
each coursier run happened within that context.
Caching: (TODO): https://github.com/pantsbuild/pants/issues/5187
Currently it is disabled due to absolute paths in the coursier results.
:param targets: a collection of targets to do 3rdparty resolve against
:param compile_classpath: classpath product that holds the resolution result. IMPORTANT: this parameter will be changed.
:param sources: if True, fetch sources for 3rdparty
:param javadoc: if True, fetch javadoc for 3rdparty
:return: n/a
"""
manager = JarDependencyManagement.global_instance()
jar_targets = manager.targets_by_artifact_set(targets)
for artifact_set, target_subset in jar_targets.items():
# TODO(wisechengyi): this is the only place we are using IvyUtil method, which isn't specific to ivy really.
raw_jar_deps, global_excludes = IvyUtils.calculate_classpath(
target_subset)
# ['sources'] * False = [], ['sources'] * True = ['sources']
confs_for_fingerprint = ['sources'] * sources + ['javadoc'
] * javadoc
fp_strategy = CoursierResolveFingerprintStrategy(
confs_for_fingerprint)
compile_classpath.add_excludes_for_targets(target_subset)
with self.invalidated(
target_subset,
invalidate_dependents=False,
silent=False,
fingerprint_strategy=fp_strategy) as invalidation_check:
if not invalidation_check.all_vts:
continue
pants_workdir = self.get_options().pants_workdir
resolve_vts = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts)
vt_set_results_dir = self._prepare_vts_results_dir(
pants_workdir, resolve_vts)
pants_jar_base_dir = self._prepare_workdir(pants_workdir)
coursier_cache_dir = CoursierSubsystem.global_instance(
).get_options().cache_dir
# If a report is requested, do not proceed with loading validated result.
if not self.get_options().report:
# Check each individual target without context first
# If the individuals are valid, check them as a VersionedTargetSet
if not invalidation_check.invalid_vts and resolve_vts.valid:
# Load up from the results dir
success = self._load_from_results_dir(
compile_classpath, vt_set_results_dir,
coursier_cache_dir, invalidation_check,
pants_jar_base_dir)
if success:
return
jars_to_resolve, pinned_coords = self._compute_jars_to_resolve_and_pin(
raw_jar_deps, artifact_set, manager)
results = self._get_result_from_coursier(
jars_to_resolve, global_excludes, pinned_coords,
pants_workdir, coursier_cache_dir, sources, javadoc)
for conf, result_list in results.items():
for result in result_list:
self._load_json_result(
conf, compile_classpath, coursier_cache_dir,
invalidation_check, pants_jar_base_dir, result,
self._override_classifiers_for_conf(conf))
self._populate_results_dir(vt_set_results_dir, results)
resolve_vts.update()
def _get_result_from_coursier(self, jars_to_resolve, global_excludes, pinned_coords,
coursier_cache_path, sources, javadoc, executor):
"""
Calling coursier and return the result per invocation.
If coursier was called once for classifier '' and once for classifier 'tests', then the return value
would be: {'default': [<first coursier output>, <second coursier output>]}
:param jars_to_resolve: List of `JarDependency`s to resolve
:param global_excludes: List of `M2Coordinate`s to exclude globally
:param pinned_coords: List of `M2Coordinate`s that need to be pinned.
:param coursier_cache_path: path to where coursier cache is stored.
:param executor: An instance of `pants.java.executor.Executor`
:return: The aggregation of results by conf from coursier. Each coursier call could return
the following:
{
"conflict_resolution": {
"org:name:version" (requested): "org:name:version" (reconciled)
},
"dependencies": [
{
"coord": "orgA:nameA:versionA",
"file": <path>,
"dependencies": [ // coodinates for its transitive dependencies
<orgX:nameX:versionX>,
<orgY:nameY:versionY>,
]
},
{
"coord": "orgB:nameB:jar:classifier:versionB",
"file": <path>,
"dependencies": [ // coodinates for its transitive dependencies
<orgX:nameX:versionX>,
<orgZ:nameZ:versionZ>,
]
},
... // more about orgX:nameX:versionX, orgY:nameY:versionY, orgZ:nameZ:versionZ
]
}
Hence the aggregation of the results will be in the following format, for example when default classifier
and sources are fetched:
{
'default': [<result from coursier call with default conf with classifier X>,
<result from coursier call with default conf with classifier Y>],
'src_doc': [<result from coursier call with --sources and/or --javadoc>],
}
"""
# Prepare coursier args
coursier_subsystem_instance = CoursierSubsystem.global_instance()
coursier_jar = coursier_subsystem_instance.bootstrap_coursier(self.context.new_workunit)
repos = coursier_subsystem_instance.get_options().repos
# make [repoX, repoY] -> ['-r', repoX, '-r', repoY]
repo_args = list(itertools.chain(*list(zip(['-r'] * len(repos), repos))))
artifact_types_arg = ['-A', ','.join(coursier_subsystem_instance.get_options().artifact_types)]
advanced_options = coursier_subsystem_instance.get_options().fetch_options
common_args = ['fetch',
# Print the resolution tree
'-t',
'--cache', coursier_cache_path
] + repo_args + artifact_types_arg + advanced_options
coursier_work_temp_dir = os.path.join(self.versioned_workdir, 'tmp')
safe_mkdir(coursier_work_temp_dir)
results_by_conf = self._get_default_conf_results(common_args, coursier_jar, global_excludes, jars_to_resolve,
coursier_work_temp_dir,
pinned_coords, executor)
if sources or javadoc:
non_default_conf_results = self._get_non_default_conf_results(common_args, coursier_jar, global_excludes,
jars_to_resolve, coursier_work_temp_dir,
pinned_coords, sources, javadoc, executor)
results_by_conf.update(non_default_conf_results)
return results_by_conf
def resolve(self, targets, compile_classpath, sources, javadoc, executor):
"""
This is the core function for coursier resolve.
Validation strategy:
1. All targets are going through the `invalidated` to get fingerprinted in the target level.
No cache is fetched at this stage because it is disabled.
2. Once each target is fingerprinted, we combine them into a `VersionedTargetSet` where they
are fingerprinted together, because each run of 3rdparty resolve is context sensitive.
Artifacts are stored in `VersionedTargetSet`'s results_dir, the contents are the aggregation of
each coursier run happened within that context.
Caching: (TODO): https://github.com/pantsbuild/pants/issues/5187
Currently it is disabled due to absolute paths in the coursier results.
:param targets: a collection of targets to do 3rdparty resolve against
:param compile_classpath: classpath product that holds the resolution result. IMPORTANT: this parameter will be changed.
:param sources: if True, fetch sources for 3rdparty
:param javadoc: if True, fetch javadoc for 3rdparty
:param executor: An instance of `pants.java.executor.Executor`. If None, a subprocess executor will be assigned.
:return: n/a
"""
manager = JarDependencyManagement.global_instance()
jar_targets = manager.targets_by_artifact_set(targets)
executor = executor or SubprocessExecutor(DistributionLocator.cached())
if not isinstance(executor, Executor):
raise ValueError('The executor argument must be an Executor instance, given {} of type {}'.format(
executor, type(executor)))
for artifact_set, target_subset in jar_targets.items():
# TODO(wisechengyi): this is the only place we are using IvyUtil method, which isn't specific to ivy really.
raw_jar_deps, global_excludes = IvyUtils.calculate_classpath(target_subset)
# ['sources'] * False = [], ['sources'] * True = ['sources']
confs_for_fingerprint = ['sources'] * sources + ['javadoc'] * javadoc
fp_strategy = CoursierResolveFingerprintStrategy(confs_for_fingerprint)
compile_classpath.add_excludes_for_targets(target_subset)
with self.invalidated(target_subset,
invalidate_dependents=False,
silent=False,
fingerprint_strategy=fp_strategy) as invalidation_check:
if not invalidation_check.all_vts:
continue
resolve_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
vt_set_results_dir = self._prepare_vts_results_dir(resolve_vts)
pants_jar_base_dir = self._prepare_workdir()
coursier_cache_dir = CoursierSubsystem.global_instance().get_options().cache_dir
# If a report is requested, do not proceed with loading validated result.
if not self.get_options().report:
# Check each individual target without context first
# If the individuals are valid, check them as a VersionedTargetSet
if not invalidation_check.invalid_vts and resolve_vts.valid:
# Load up from the results dir
success = self._load_from_results_dir(compile_classpath, vt_set_results_dir,
coursier_cache_dir, invalidation_check, pants_jar_base_dir)
if success:
return
jars_to_resolve, pinned_coords = self._compute_jars_to_resolve_and_pin(raw_jar_deps,
artifact_set,
manager)
results = self._get_result_from_coursier(jars_to_resolve, global_excludes, pinned_coords,
coursier_cache_dir, sources, javadoc, executor)
for conf, result_list in results.items():
for result in result_list:
self._load_json_result(conf, compile_classpath, coursier_cache_dir, invalidation_check,
pants_jar_base_dir, result, self._override_classifiers_for_conf(conf))
self._populate_results_dir(vt_set_results_dir, results)
resolve_vts.update()
def _get_result_from_coursier(
self,
jars_to_resolve,
global_excludes,
pinned_coords,
coursier_cache_path,
sources,
javadoc,
executor,
):
"""Calling coursier and return the result per invocation.
If coursier was called once for classifier '' and once for classifier 'tests', then the return value
would be: {'default': [<first coursier output>, <second coursier output>]}
:param jars_to_resolve: List of `JarDependency`s to resolve
:param global_excludes: List of `M2Coordinate`s to exclude globally
:param pinned_coords: List of `M2Coordinate`s that need to be pinned.
:param coursier_cache_path: path to where coursier cache is stored.
:param executor: An instance of `pants.java.executor.Executor`
:return: The aggregation of results by conf from coursier. Each coursier call could return
the following:
{
"conflict_resolution": {
"org:name:version" (requested): "org:name:version" (reconciled)
},
"dependencies": [
{
"coord": "orgA:nameA:versionA",
"file": <path>,
"dependencies": [ // coodinates for its transitive dependencies
<orgX:nameX:versionX>,
<orgY:nameY:versionY>,
]
},
{
"coord": "orgB:nameB:jar:classifier:versionB",
"file": <path>,
"dependencies": [ // coodinates for its transitive dependencies
<orgX:nameX:versionX>,
<orgZ:nameZ:versionZ>,
]
},
... // more about orgX:nameX:versionX, orgY:nameY:versionY, orgZ:nameZ:versionZ
]
}
Hence the aggregation of the results will be in the following format, for example when default classifier
and sources are fetched:
{
'default': [<result from coursier call with default conf with classifier X>,
<result from coursier call with default conf with classifier Y>],
'src_doc': [<result from coursier call with --sources and/or --javadoc>],
}
"""
# Prepare coursier args
coursier_subsystem_instance = CoursierSubsystem.global_instance()
coursier_jar = coursier_subsystem_instance.select()
repos = coursier_subsystem_instance.get_options().repos
# make [repoX, repoY] -> ['-r', repoX, '-r', repoY]
repo_args = list(
itertools.chain(*list(zip(["-r"] * len(repos), repos))))
artifact_types_arg = [
"-A",
",".join(coursier_subsystem_instance.get_options().artifact_types),
]
advanced_options = coursier_subsystem_instance.get_options(
).fetch_options
common_args = ([
"fetch",
# Print the resolution tree
"-t",
"--cache",
coursier_cache_path,
] + repo_args + artifact_types_arg + advanced_options)
coursier_work_temp_dir = os.path.join(self.versioned_workdir, "tmp")
safe_mkdir(coursier_work_temp_dir)
results_by_conf = self._get_default_conf_results(
common_args,
coursier_jar,
global_excludes,
jars_to_resolve,
coursier_work_temp_dir,
pinned_coords,
executor,
)
if sources or javadoc:
non_default_conf_results = self._get_non_default_conf_results(
common_args,
coursier_jar,
global_excludes,
jars_to_resolve,
coursier_work_temp_dir,
pinned_coords,
sources,
javadoc,
executor,
)
results_by_conf.update(non_default_conf_results)
return results_by_conf
def resolve(self, targets, compile_classpath, sources, javadoc, executor):
"""This is the core function for coursier resolve.
Validation strategy:
1. All targets are going through the `invalidated` to get fingerprinted in the target level.
No cache is fetched at this stage because it is disabled.
2. Once each target is fingerprinted, we combine them into a `VersionedTargetSet` where they
are fingerprinted together, because each run of 3rdparty resolve is context sensitive.
Artifacts are stored in `VersionedTargetSet`'s results_dir, the contents are the aggregation of
each coursier run happened within that context.
Caching: (TODO): https://github.com/pantsbuild/pants/issues/5187
Currently it is disabled due to absolute paths in the coursier results.
:param targets: a collection of targets to do 3rdparty resolve against
:param compile_classpath: classpath product that holds the resolution result. IMPORTANT: this parameter will be changed.
:param sources: if True, fetch sources for 3rdparty
:param javadoc: if True, fetch javadoc for 3rdparty
:param executor: An instance of `pants.java.executor.Executor`. If None, a subprocess executor will be assigned.
:return: n/a
"""
manager = JarDependencyManagement.global_instance()
jar_targets = manager.targets_by_artifact_set(targets)
executor = executor or SubprocessExecutor(DistributionLocator.cached())
if not isinstance(executor, Executor):
raise ValueError(
"The executor argument must be an Executor instance, given {} of type {}".format(
executor, type(executor)
)
)
for artifact_set, target_subset in jar_targets.items():
raw_jar_deps, global_excludes = calculate_classpath(target_subset)
confs_for_fingerprint = ["sources"] * sources + ["javadoc"] * javadoc
fp_strategy = CoursierResolveFingerprintStrategy(confs_for_fingerprint)
compile_classpath.add_excludes_for_targets(target_subset)
with self.invalidated(
target_subset,
invalidate_dependents=False,
silent=False,
fingerprint_strategy=fp_strategy,
) as invalidation_check:
if not invalidation_check.all_vts:
continue
resolve_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
vt_set_results_dir = self._prepare_vts_results_dir(resolve_vts)
pants_jar_base_dir = self._prepare_workdir()
coursier_cache_dir = CoursierSubsystem.global_instance().get_options().cache_dir
# If a report is requested, do not proceed with loading validated result.
if not self.get_options().report:
# Check each individual target without context first
# If the individuals are valid, check them as a VersionedTargetSet
# The order of 'or' statement matters, because checking for cache is more expensive.
if resolve_vts.valid or (
self.artifact_cache_reads_enabled()
and len(self.check_artifact_cache([resolve_vts])[0])
== len(resolve_vts.targets)
):
# Load up from the results dir
success = self._load_from_results_dir(
compile_classpath,
vt_set_results_dir,
coursier_cache_dir,
invalidation_check,
pants_jar_base_dir,
)
if success:
resolve_vts.update()
return
jars_to_resolve, pinned_coords = self._compute_jars_to_resolve_and_pin(
raw_jar_deps, artifact_set, manager
)
results = self._get_result_from_coursier(
jars_to_resolve,
global_excludes,
pinned_coords,
coursier_cache_dir,
sources,
javadoc,
executor,
)
for conf, result_list in results.items():
for result in result_list:
self._load_json_result(
conf,
compile_classpath,
coursier_cache_dir,
invalidation_check,
pants_jar_base_dir,
result,
self._override_classifiers_for_conf(conf),
)
self._populate_results_dir(vt_set_results_dir, results)
resolve_vts.update()
if self.artifact_cache_writes_enabled():
self.update_artifact_cache([(resolve_vts, [vt_set_results_dir])])