def resolve(self, executor, targets, classpath_products, confs=None, extra_args=None,
invalidate_dependents=False):
"""Resolves external classpath products (typically jars) for the given targets.
:API: public
:param executor: A java executor to run ivy with.
:type executor: :class:`pants.java.executor.Executor`
:param targets: The targets to resolve jvm dependencies for.
:type targets: :class:`collections.Iterable` of :class:`pants.build_graph.target.Target`
:param classpath_products: The classpath products to populate with the results of the resolve.
:type classpath_products: :class:`pants.backend.jvm.tasks.classpath_products.ClasspathProducts`
:param confs: The ivy configurations to resolve; ('default',) by default.
:type confs: :class:`collections.Iterable` of string
:param extra_args: Any extra command line arguments to pass to ivy.
:type extra_args: list of string
:param bool invalidate_dependents: `True` to invalidate dependents of targets that needed to be
resolved.
:returns: The results of each of the resolves run by this call.
:rtype: list of IvyResolveResult
"""
confs = confs or ('default',)
targets_by_sets = JarDependencyManagement.global_instance().targets_by_artifact_set(targets)
results = []
for artifact_set, target_subset in targets_by_sets.items():
results.append(self._resolve_subset(executor,
target_subset,
classpath_products,
confs=confs,
extra_args=extra_args,
invalidate_dependents=invalidate_dependents,
pinned_artifacts=artifact_set))
return results
def compute_fingerprint(self, target):
hash_elements_for_target = []
if isinstance(target, JarLibrary):
managed_jar_artifact_set = JarDependencyManagement.global_instance().for_target(target)
if managed_jar_artifact_set:
hash_elements_for_target.append(str(managed_jar_artifact_set.id))
hash_elements_for_target.append(target.payload.fingerprint())
elif isinstance(target, JvmTarget) and target.payload.excludes:
hash_elements_for_target.append(target.payload.fingerprint(field_keys=('excludes',)))
else:
pass
if not hash_elements_for_target:
return None
hasher = hashlib.sha1()
fingerprint = target.payload.fingerprint().encode('utf-8')
hasher.update(fingerprint)
for conf in self._confs:
hasher.update(conf.encode('utf-8'))
for element in hash_elements_for_target:
hasher.update(element.encode('utf-8'))
return hasher.hexdigest() if PY3 else hasher.hexdigest().decode('utf-8')
def compute_fingerprint(self, target):
hash_elements_for_target = []
if isinstance(target, JarLibrary):
managed_jar_artifact_set = JarDependencyManagement.global_instance(
).for_target(target)
if managed_jar_artifact_set:
hash_elements_for_target.append(
str(managed_jar_artifact_set.id))
hash_elements_for_target.append(target.payload.fingerprint())
elif isinstance(target, JvmTarget) and target.payload.excludes:
hash_elements_for_target.append(
target.payload.fingerprint(field_keys=('excludes', )))
else:
pass
if not hash_elements_for_target:
return None
hasher = hashlib.sha1()
hasher.update(target.payload.fingerprint())
for conf in self._confs:
hasher.update(conf)
for element in hash_elements_for_target:
hasher.update(element)
# Just in case so we do not collide with ivy cache
hasher.update('coursier')
return hasher.hexdigest()
def compute_fingerprint(self, target):
hash_elements_for_target = []
if isinstance(target, JarLibrary):
managed_jar_artifact_set = JarDependencyManagement.global_instance().for_target(target)
if managed_jar_artifact_set:
hash_elements_for_target.append(str(managed_jar_artifact_set.id))
hash_elements_for_target.append(target.payload.fingerprint())
elif isinstance(target, JvmTarget) and target.payload.excludes:
hash_elements_for_target.append(target.payload.fingerprint(field_keys=('excludes',)))
else:
pass
if not hash_elements_for_target:
return None
hasher = hashlib.sha1()
hasher.update(target.payload.fingerprint())
for conf in self._confs:
hasher.update(conf)
for element in hash_elements_for_target:
hasher.update(element)
# Just in case so we do not collide with ivy cache
hasher.update('coursier')
return hasher.hexdigest()
def compute_fingerprint(self, target):
hash_elements_for_target = []
if isinstance(target, JarLibrary):
managed_jar_artifact_set = JarDependencyManagement.global_instance().for_target(target)
if managed_jar_artifact_set:
hash_elements_for_target.append(str(managed_jar_artifact_set.id))
hash_elements_for_target.append(target.payload.fingerprint())
elif isinstance(target, JvmTarget) and target.payload.excludes:
hash_elements_for_target.append(target.payload.fingerprint(field_keys=("excludes",)))
else:
pass
if not hash_elements_for_target:
return None
hasher = self._build_hasher(target)
for conf in self._confs:
hasher.update(conf)
for element in hash_elements_for_target:
hasher.update(element)
return hasher.hexdigest()
def _generate_resolve_ivy(cls, jars, excludes, ivyxml, confs, resolve_hash_name, pinned_artifacts=None,
jar_dep_manager=None):
org = IvyUtils.INTERNAL_ORG_NAME
name = resolve_hash_name
extra_configurations = [conf for conf in confs if conf and conf != 'default']
jars_by_key = OrderedDict()
for jar in jars:
jars = jars_by_key.setdefault((jar.org, jar.name), [])
jars.append(jar)
manager = jar_dep_manager or JarDependencyManagement.global_instance()
artifact_set = PinnedJarArtifactSet(pinned_artifacts) # Copy, because we're modifying it.
for jars in jars_by_key.values():
for i, dep in enumerate(jars):
direct_coord = M2Coordinate.create(dep)
managed_coord = artifact_set[direct_coord]
if direct_coord.rev != managed_coord.rev:
# It may be necessary to actually change the version number of the jar we want to resolve
# here, because overrides do not apply directly (they are exclusively transitive). This is
# actually a good thing, because it gives us more control over what happens.
coord = manager.resolve_version_conflict(managed_coord, direct_coord, force=dep.force)
jars[i] = dep.copy(rev=coord.rev)
elif dep.force:
# If this dependency is marked as 'force' and there is no version conflict, use the normal
# pants behavior for 'force'.
artifact_set.put(direct_coord)
dependencies = [cls._generate_jar_template(jars) for jars in jars_by_key.values()]
# As it turns out force is not transitive - it only works for dependencies pants knows about
# directly (declared in BUILD files - present in generated ivy.xml). The user-level ivy docs
# don't make this clear [1], but the source code docs do (see isForce docs) [2]. I was able to
# edit the generated ivy.xml and use the override feature [3] though and that does work
# transitively as you'd hope.
#
# [1] http://ant.apache.org/ivy/history/2.3.0/settings/conflict-managers.html
# [2] https://svn.apache.org/repos/asf/ant/ivy/core/branches/2.3.0/
# src/java/org/apache/ivy/core/module/descriptor/DependencyDescriptor.java
# [3] http://ant.apache.org/ivy/history/2.3.0/ivyfile/override.html
overrides = [cls._generate_override_template(_coord) for _coord in artifact_set]
excludes = [cls._generate_exclude_template(exclude) for exclude in excludes]
template_data = TemplateData(
org=org,
module=name,
extra_configurations=extra_configurations,
dependencies=dependencies,
excludes=excludes,
overrides=overrides)
template_relpath = os.path.join('templates', 'ivy_utils', 'ivy.mustache')
cls._write_ivy_xml_file(ivyxml, template_data, template_relpath)
def compute_fingerprint(self, target):
hasher = sha1()
for conf in self._confs:
hasher.update(conf)
managed_jar_dependencies_artifacts = JarDependencyManagement.global_instance().for_target(target)
if managed_jar_dependencies_artifacts:
hasher.update(str(managed_jar_dependencies_artifacts.id))
if isinstance(target, JarLibrary):
hasher.update(target.payload.fingerprint())
return hasher.hexdigest()
if isinstance(target, JvmTarget):
if target.payload.excludes:
hasher.update(target.payload.fingerprint(field_keys=('excludes',)))
return hasher.hexdigest()
return None
def compute_fingerprint(self, target):
hasher = sha1()
for conf in self._confs:
hasher.update(conf)
managed_jar_dependencies_artifacts = JarDependencyManagement.global_instance(
).for_target(target)
if managed_jar_dependencies_artifacts:
hasher.update(str(managed_jar_dependencies_artifacts.id))
if isinstance(target, JarLibrary):
hasher.update(target.payload.fingerprint())
return hasher.hexdigest()
if isinstance(target, JvmTarget):
if target.payload.excludes:
hasher.update(
target.payload.fingerprint(field_keys=('excludes', )))
return hasher.hexdigest()
return None
def set_artifact_set_for(self, managed_jar_target, artifact_set):
JarDependencyManagement.global_instance()._artifact_set_map[
managed_jar_target.id] = artifact_set
def set_artifact_set_for(self, managed_jar_target, artifact_set):
JarDependencyManagement.global_instance()._artifact_set_map[
managed_jar_target.id] = artifact_set
def _ivy_resolve(self,
targets,
executor=None,
silent=False,
workunit_name=None,
confs=None,
extra_args=None,
invalidate_dependents=False,
pinned_artifacts=None):
"""Resolves external dependencies for the given targets."""
# If there are no targets, we don't need to do a resolve.
if not targets:
return NO_RESOLVE_RUN_RESULT
confs = confs or ('default', )
fingerprint_strategy = IvyResolveFingerprintStrategy(confs)
with self.invalidated(targets,
invalidate_dependents=invalidate_dependents,
silent=silent,
fingerprint_strategy=fingerprint_strategy
) as invalidation_check:
# In case all the targets were filtered out because they didn't participate in fingerprinting.
if not invalidation_check.all_vts:
return NO_RESOLVE_RUN_RESULT
resolve_vts = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts)
resolve_hash_name = resolve_vts.cache_key.hash
global_ivy_workdir = os.path.join(
self.context.options.for_global_scope().pants_workdir, 'ivy')
fetch = self._create_ivy_fetch_step(
confs, resolve_hash_name, pinned_artifacts,
self.get_options().soft_excludes, self.ivy_cache_dir,
global_ivy_workdir)
resolve = self._create_ivy_resolve_step(
confs, resolve_hash_name, pinned_artifacts,
self.get_options().soft_excludes, self.ivy_cache_dir,
global_ivy_workdir, self.global_excludes)
result = self._perform_resolution(
fetch,
resolve,
executor,
extra_args,
invalidation_check,
resolve_vts,
resolve_vts.targets,
workunit_name,
)
# NOTE(mateo): Wiring up our own reports, the full ivy report is too heavy weight for our purposes.
if result.resolved_artifact_paths and self.resolution_report_outdir and not self.get_options(
).disable_reports:
# This is added to get a reasonable handle for managed_dependencies target sets.
# If there is more than one VT it defaults to the VTS.id, which is a non-human-readable cache key.
# If we wanted to be more performant than rigorous, we could bail after the first query.
managed_dependencies = set(
j.target.managed_dependencies
for j in invalidation_check.all_vts
if isinstance(j.target, JarLibrary)
and j.target.managed_dependencies is not None)
if managed_dependencies and len(managed_dependencies) > 1:
raise TaskError(
'Each partition should be mapped to a single managed_dependencies target: (was: {})\n Targets: {}'
.format(managed_dependencies, resolve_vts.targets))
default_target_name = JarDependencyManagement.global_instance(
)._default_target.name
partition_name = list(
managed_dependencies
)[0].name if managed_dependencies else default_target_name
self.write_resolve_report(resolve.frozen_resolve_file,
partition_name)
return result
def resolve(self, targets, compile_classpath, sources, javadoc):
"""
This is the core function for coursier resolve.
Validation strategy:
1. All targets are going through the `invalidated` to get fingerprinted in the target level.
No cache is fetched at this stage because it is disabled.
2. Once each target is fingerprinted, we combine them into a `VersionedTargetSet` where they
are fingerprinted together, because each run of 3rdparty resolve is context sensitive.
Artifacts are stored in `VersionedTargetSet`'s results_dir, the contents are the aggregation of
each coursier run happened within that context.
Caching: (TODO): https://github.com/pantsbuild/pants/issues/5187
Currently it is disabled due to absolute paths in the coursier results.
:param targets: a collection of targets to do 3rdparty resolve against
:param compile_classpath: classpath product that holds the resolution result. IMPORTANT: this parameter will be changed.
:param sources: if True, fetch sources for 3rdparty
:param javadoc: if True, fetch javadoc for 3rdparty
:return: n/a
"""
manager = JarDependencyManagement.global_instance()
jar_targets = manager.targets_by_artifact_set(targets)
for artifact_set, target_subset in jar_targets.items():
# TODO(wisechengyi): this is the only place we are using IvyUtil method, which isn't specific to ivy really.
raw_jar_deps, global_excludes = IvyUtils.calculate_classpath(
target_subset)
# ['sources'] * False = [], ['sources'] * True = ['sources']
confs_for_fingerprint = ['sources'] * sources + ['javadoc'
] * javadoc
fp_strategy = CoursierResolveFingerprintStrategy(
confs_for_fingerprint)
compile_classpath.add_excludes_for_targets(target_subset)
with self.invalidated(
target_subset,
invalidate_dependents=False,
silent=False,
fingerprint_strategy=fp_strategy) as invalidation_check:
if not invalidation_check.all_vts:
continue
pants_workdir = self.get_options().pants_workdir
resolve_vts = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts)
vt_set_results_dir = self._prepare_vts_results_dir(
pants_workdir, resolve_vts)
coursier_cache_dir, pants_jar_base_dir = self._prepare_workdir(
pants_workdir)
# Check each individual target without context first
if not invalidation_check.invalid_vts:
# If the individuals are valid, check them as a VersionedTargetSet
if resolve_vts.valid:
# Load up from the results dir
success = self._load_from_results_dir(
compile_classpath, vt_set_results_dir,
coursier_cache_dir, invalidation_check,
pants_jar_base_dir)
if success:
return
jars_to_resolve, pinned_coords = self._compute_jars_to_resolve_and_pin(
raw_jar_deps, artifact_set, manager)
results = self._get_result_from_coursier(
jars_to_resolve, global_excludes, pinned_coords,
pants_workdir, coursier_cache_dir, sources, javadoc)
for conf, result_list in results.items():
for result in result_list:
self._load_json_result(
conf, compile_classpath, coursier_cache_dir,
invalidation_check, pants_jar_base_dir, result,
self._override_classifiers_for_conf(conf))
self._populate_results_dir(vt_set_results_dir, results)
resolve_vts.update()
def resolve(self, targets, compile_classpath, sources, javadoc, executor):
"""
This is the core function for coursier resolve.
Validation strategy:
1. All targets are going through the `invalidated` to get fingerprinted in the target level.
No cache is fetched at this stage because it is disabled.
2. Once each target is fingerprinted, we combine them into a `VersionedTargetSet` where they
are fingerprinted together, because each run of 3rdparty resolve is context sensitive.
Artifacts are stored in `VersionedTargetSet`'s results_dir, the contents are the aggregation of
each coursier run happened within that context.
Caching: (TODO): https://github.com/pantsbuild/pants/issues/5187
Currently it is disabled due to absolute paths in the coursier results.
:param targets: a collection of targets to do 3rdparty resolve against
:param compile_classpath: classpath product that holds the resolution result. IMPORTANT: this parameter will be changed.
:param sources: if True, fetch sources for 3rdparty
:param javadoc: if True, fetch javadoc for 3rdparty
:param executor: An instance of `pants.java.executor.Executor`. If None, a subprocess executor will be assigned.
:return: n/a
"""
manager = JarDependencyManagement.global_instance()
jar_targets = manager.targets_by_artifact_set(targets)
executor = executor or SubprocessExecutor(DistributionLocator.cached())
if not isinstance(executor, Executor):
raise ValueError('The executor argument must be an Executor instance, given {} of type {}'.format(
executor, type(executor)))
for artifact_set, target_subset in jar_targets.items():
# TODO(wisechengyi): this is the only place we are using IvyUtil method, which isn't specific to ivy really.
raw_jar_deps, global_excludes = IvyUtils.calculate_classpath(target_subset)
# ['sources'] * False = [], ['sources'] * True = ['sources']
confs_for_fingerprint = ['sources'] * sources + ['javadoc'] * javadoc
fp_strategy = CoursierResolveFingerprintStrategy(confs_for_fingerprint)
compile_classpath.add_excludes_for_targets(target_subset)
with self.invalidated(target_subset,
invalidate_dependents=False,
silent=False,
fingerprint_strategy=fp_strategy) as invalidation_check:
if not invalidation_check.all_vts:
continue
resolve_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
vt_set_results_dir = self._prepare_vts_results_dir(resolve_vts)
pants_jar_base_dir = self._prepare_workdir()
coursier_cache_dir = CoursierSubsystem.global_instance().get_options().cache_dir
# If a report is requested, do not proceed with loading validated result.
if not self.get_options().report:
# Check each individual target without context first
# If the individuals are valid, check them as a VersionedTargetSet
if not invalidation_check.invalid_vts and resolve_vts.valid:
# Load up from the results dir
success = self._load_from_results_dir(compile_classpath, vt_set_results_dir,
coursier_cache_dir, invalidation_check, pants_jar_base_dir)
if success:
return
jars_to_resolve, pinned_coords = self._compute_jars_to_resolve_and_pin(raw_jar_deps,
artifact_set,
manager)
results = self._get_result_from_coursier(jars_to_resolve, global_excludes, pinned_coords,
coursier_cache_dir, sources, javadoc, executor)
for conf, result_list in results.items():
for result in result_list:
self._load_json_result(conf, compile_classpath, coursier_cache_dir, invalidation_check,
pants_jar_base_dir, result, self._override_classifiers_for_conf(conf))
self._populate_results_dir(vt_set_results_dir, results)
resolve_vts.update()
def _ivy_resolve(self,
targets,
executor=None,
silent=False,
workunit_name=None,
confs=None,
extra_args=None,
invalidate_dependents=False,
pinned_artifacts=None):
"""Resolves external dependencies for the given targets."""
# If there are no targets, we don't need to do a resolve.
if not targets:
return NO_RESOLVE_RUN_RESULT
confs = confs or ('default',)
fingerprint_strategy = IvyResolveFingerprintStrategy(confs)
with self.invalidated(targets,
invalidate_dependents=invalidate_dependents,
silent=silent,
fingerprint_strategy=fingerprint_strategy) as invalidation_check:
# In case all the targets were filtered out because they didn't participate in fingerprinting.
if not invalidation_check.all_vts:
return NO_RESOLVE_RUN_RESULT
resolve_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
resolve_hash_name = resolve_vts.cache_key.hash
global_ivy_workdir = os.path.join(self.context.options.for_global_scope().pants_workdir,
'ivy')
fetch = self._create_ivy_fetch_step(confs,
resolve_hash_name,
pinned_artifacts,
self.get_options().soft_excludes,
self.ivy_cache_dir,
global_ivy_workdir)
resolve = self._create_ivy_resolve_step(confs,
resolve_hash_name,
pinned_artifacts,
self.get_options().soft_excludes,
self.ivy_cache_dir,
global_ivy_workdir,
self.global_excludes)
result = self._perform_resolution(
fetch, resolve, executor, extra_args, invalidation_check, resolve_vts, resolve_vts.targets, workunit_name,
)
# NOTE(mateo): Wiring up our own reports, the full ivy report is too heavy weight for our purposes.
if result.resolved_artifact_paths and self.resolution_report_outdir and not self.get_options().disable_reports:
# This is added to get a reasonable handle for managed_dependencies target sets.
# If there is more than one VT it defaults to the VTS.id, which is a non-human-readable cache key.
# If we wanted to be more performant than rigorous, we could bail after the first query.
managed_dependencies = set(
j.target.managed_dependencies
for j in invalidation_check.all_vts
if isinstance(j.target, JarLibrary) and
j.target.managed_dependencies is not None
)
if managed_dependencies and len(managed_dependencies) > 1:
raise TaskError(
'Each partition should be mapped to a single managed_dependencies target: (was: {})\n Targets: {}'
.format(managed_dependencies, resolve_vts.targets)
)
default_target_name = JarDependencyManagement.global_instance()._default_target.name
partition_name = list(managed_dependencies)[0].name if managed_dependencies else default_target_name
self.write_resolve_report(resolve.frozen_resolve_file, partition_name)
return result
def resolve(self, targets, compile_classpath, sources, javadoc, executor):
"""This is the core function for coursier resolve.
Validation strategy:
1. All targets are going through the `invalidated` to get fingerprinted in the target level.
No cache is fetched at this stage because it is disabled.
2. Once each target is fingerprinted, we combine them into a `VersionedTargetSet` where they
are fingerprinted together, because each run of 3rdparty resolve is context sensitive.
Artifacts are stored in `VersionedTargetSet`'s results_dir, the contents are the aggregation of
each coursier run happened within that context.
Caching: (TODO): https://github.com/pantsbuild/pants/issues/5187
Currently it is disabled due to absolute paths in the coursier results.
:param targets: a collection of targets to do 3rdparty resolve against
:param compile_classpath: classpath product that holds the resolution result. IMPORTANT: this parameter will be changed.
:param sources: if True, fetch sources for 3rdparty
:param javadoc: if True, fetch javadoc for 3rdparty
:param executor: An instance of `pants.java.executor.Executor`. If None, a subprocess executor will be assigned.
:return: n/a
"""
manager = JarDependencyManagement.global_instance()
jar_targets = manager.targets_by_artifact_set(targets)
executor = executor or SubprocessExecutor(DistributionLocator.cached())
if not isinstance(executor, Executor):
raise ValueError(
"The executor argument must be an Executor instance, given {} of type {}"
.format(executor, type(executor)))
for artifact_set, target_subset in jar_targets.items():
# TODO(wisechengyi): this is the only place we are using IvyUtil method, which isn't specific to ivy really.
raw_jar_deps, global_excludes = IvyUtils.calculate_classpath(
target_subset)
# ['sources'] * False = [], ['sources'] * True = ['sources']
confs_for_fingerprint = ["sources"] * sources + ["javadoc"
] * javadoc
fp_strategy = CoursierResolveFingerprintStrategy(
confs_for_fingerprint)
compile_classpath.add_excludes_for_targets(target_subset)
with self.invalidated(
target_subset,
invalidate_dependents=False,
silent=False,
fingerprint_strategy=fp_strategy,
) as invalidation_check:
if not invalidation_check.all_vts:
continue
resolve_vts = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts)
vt_set_results_dir = self._prepare_vts_results_dir(resolve_vts)
pants_jar_base_dir = self._prepare_workdir()
coursier_cache_dir = CoursierSubsystem.global_instance(
).get_options().cache_dir
# If a report is requested, do not proceed with loading validated result.
if not self.get_options().report:
# Check each individual target without context first
# If the individuals are valid, check them as a VersionedTargetSet
# The order of 'or' statement matters, because checking for cache is more expensive.
if resolve_vts.valid or (
self.artifact_cache_reads_enabled() and len(
self.check_artifact_cache([resolve_vts])[0])
== len(resolve_vts.targets)):
# Load up from the results dir
success = self._load_from_results_dir(
compile_classpath,
vt_set_results_dir,
coursier_cache_dir,
invalidation_check,
pants_jar_base_dir,
)
if success:
resolve_vts.update()
return
jars_to_resolve, pinned_coords = self._compute_jars_to_resolve_and_pin(
raw_jar_deps, artifact_set, manager)
results = self._get_result_from_coursier(
jars_to_resolve,
global_excludes,
pinned_coords,
coursier_cache_dir,
sources,
javadoc,
executor,
)
for conf, result_list in results.items():
for result in result_list:
self._load_json_result(
conf,
compile_classpath,
coursier_cache_dir,
invalidation_check,
pants_jar_base_dir,
result,
self._override_classifiers_for_conf(conf),
)
self._populate_results_dir(vt_set_results_dir, results)
resolve_vts.update()
if self.artifact_cache_writes_enabled():
self.update_artifact_cache([(resolve_vts,
[vt_set_results_dir])])