def test_resolved_jars_with_differing_paths_not_equal(self):
jar1 = ResolvedJar(M2Coordinate("org.example", "lib"), "ivy2/path",
"path1")
jar2 = ResolvedJar(M2Coordinate("org.example", "lib"), "ivy2/path",
"path2")
self.assertNotEqual(jar1, jar2)
def test_resolved_jars_with_differing_paths_not_equal(self):
jar1 = ResolvedJar(M2Coordinate('org.example', 'lib'), 'ivy2/path',
'path1')
jar2 = ResolvedJar(M2Coordinate('org.example', 'lib'), 'ivy2/path',
'path2')
self.assertNotEqual(jar1, jar2)
def test_resolved_jars_with_same_paths_equal(self):
jar1 = ResolvedJar(M2Coordinate("org.example", "lib"), "ivy2/path",
"path")
jar2 = ResolvedJar(M2Coordinate("org.example", "lib"), "ivy2/path",
"path")
self.assertEqual(jar1, jar2)
def test_resolved_jars_with_same_paths_equal(self):
jar1 = ResolvedJar(M2Coordinate('org.example', 'lib'), 'ivy2/path',
'path')
jar2 = ResolvedJar(M2Coordinate('org.example', 'lib'), 'ivy2/path',
'path')
self.assertEqual(jar1, jar2)
def test_classpath_by_targets(self):
b = self.make_target("b", JvmTarget)
a = self.make_target("a",
JvmTarget,
dependencies=[b],
excludes=[Exclude("com.example", "lib")])
classpath_products = ClasspathProducts(self.pants_workdir)
path1 = self._path("jar/path1")
path2 = self._path("jar/path2")
path3 = os.path.join(self.pants_workdir, "jar/path3")
resolved_jar = ResolvedJar(
M2Coordinate(org="com.example", name="lib", rev="1.0"),
cache_path="somewhere",
pants_path=path3,
)
classpath_products.add_for_target(a, [("default", path1)])
classpath_products.add_for_target(a, [("non-default", path2)])
classpath_products.add_for_target(b, [("default", path2)])
classpath_products.add_jars_for_targets([b], "default", [resolved_jar])
classpath_products.add_excludes_for_targets([a])
# (a, path2) filtered because of conf
# (b, path3) filtered because of excludes
self.assertEqual(
OrderedDict([(a, [ClasspathEntry(path1)]),
(b, [ClasspathEntry(path2)])]),
ClasspathUtil.classpath_by_targets(a.closure(bfs=True),
classpath_products),
)
def to_resolved_jar(jar_ref, jar_path):
return ResolvedJar(coordinate=M2Coordinate(org=jar_ref.org,
name=jar_ref.name,
rev=jar_ref.rev,
classifier=jar_ref.classifier,
ext=jar_ref.ext),
cache_path=jar_path)
def test_create_canonical_classpath_no_duplicate_entry(self):
"""Test no more than one symlink are created for the same classpath entry."""
jar_path = "ivy/jars/org.x/lib/x-1.0.jar"
resolved_jar = ResolvedJar(
M2Coordinate(org="org", name="x", rev="1.0"),
cache_path="somewhere",
pants_path=self._create_file(jar_path),
)
target_a = self.make_target("a", JvmTarget)
target_b = self.make_target("b", JvmTarget)
classpath_products = ClasspathProducts(self.pants_workdir)
# Both target a and target b depend on the same jar library
classpath_products.add_jars_for_targets([target_a], "default",
[resolved_jar])
classpath_products.add_jars_for_targets([target_b], "default",
[resolved_jar])
with temporary_dir() as base_dir:
# Only target a generates symlink to jar library, target b skips creating the
# symlink for the same jar library. Both targets' classpath.txt files should
# still contain the jar library.
self._test_canonical_classpath_helper(
classpath_products,
[target_a, target_b],
base_dir,
["a.a-0.jar"],
{
"a.a-classpath.txt": f"{self.pants_workdir}/{jar_path}\n",
"b.b-classpath.txt": f"{self.pants_workdir}/{jar_path}\n",
},
)
def test_classpath_by_targets(self):
b = self.make_target('b', JvmTarget)
a = self.make_target('a',
JvmTarget,
dependencies=[b],
excludes=[Exclude('com.example', 'lib')])
classpath_products = ClasspathProducts(self.pants_workdir)
path1 = self._path('jar/path1')
path2 = self._path('jar/path2')
path3 = os.path.join(self.pants_workdir, 'jar/path3')
resolved_jar = ResolvedJar(M2Coordinate(org='com.example',
name='lib',
rev='1.0'),
cache_path='somewhere',
pants_path=path3)
classpath_products.add_for_target(a, [('default', path1)])
classpath_products.add_for_target(a, [('non-default', path2)])
classpath_products.add_for_target(b, [('default', path2)])
classpath_products.add_jars_for_targets([b], 'default', [resolved_jar])
classpath_products.add_excludes_for_targets([a])
# (a, path2) filtered because of conf
# (b, path3) filtered because of excludes
self.assertEqual(
OrderedDict([(a, [ClasspathEntry(path1)]),
(b, [ClasspathEntry(path2)])]),
ClasspathUtil.classpath_by_targets(a.closure(bfs=True),
classpath_products))
def create_artifact(self,
org,
name,
rev,
classifier=None,
ext=None,
materialize=True):
"""
:API: public
:param string org: The maven dependency `groupId`.
:param string name: The maven dependency `artifactId`.
:param string rev: The maven dependency `version`.
:param string classifier: The maven dependency `classifier`.
:param string ext: There is no direct maven parallel, but the maven `packaging` value of the
depended-on artifact for simple cases, and in more complex cases the
extension of the artifact. For example, 'bundle' packaging implies an
extension of 'jar'. Defaults to 'jar'.
:param bool materialize: `False` to populate the returned resolved_jar with a `pants_path` that
does not exist; defaults to `True` and `touch`es the `pants_path`.
:returns: A resolved jar describing the artifact.
:rtype: :class:`pants.java.jar.ResolvedJar`
"""
coordinate = M2Coordinate(org=org,
name=name,
rev=rev,
classifier=classifier,
ext=ext)
cache_path = "not/a/real/cache/path"
jar_name = coordinate.artifact_filename
pants_path = (self.create_workdir_file(jar_name) if materialize else
os.path.join(self.pants_workdir, jar_name))
return ResolvedJar(coordinate=coordinate,
cache_path=cache_path,
pants_path=pants_path)
def _populate_compile_classpath(self):
products = self.context.products
compile_classpath = products.get_data('compile_classpath',
init_func=ClasspathProducts.init_func(self.get_options().pants_workdir))
sorted_targets = sorted(
self.context.targets(predicate=lambda t: t in self.all_jar_libs),
key=lambda t: t.address.spec,
)
for target in sorted_targets:
resolved_jars = []
for coord in sorted(self.target_to_maven_coordinate_closure[target.id]):
m2_coord = M2Coordinate(
org=coord.groupId,
name=coord.artifactId,
rev=coord.version,
classifier=coord.classifier,
ext=coord.packaging,
)
sorted_artifact_paths = sorted(
artifact.artifact_path
for artifact in self.maven_coordinate_to_provided_artifacts[coord]
)
for artifact_path in sorted_artifact_paths:
resolved_jar = ResolvedJar(
coordinate=m2_coord,
pants_path=os.path.join(self.artifact_symlink_dir, artifact_path),
cache_path=os.path.join(self.pom_cache_dir, artifact_path),
)
resolved_jars.append(resolved_jar)
compile_classpath.add_jars_for_targets([target], 'default', resolved_jars)
def test_create_canonical_classpath_no_duplicate_entry(self):
"""Test no more than one symlink are created for the same classpath entry."""
jar_path = 'ivy/jars/org.x/lib/x-1.0.jar'
resolved_jar = ResolvedJar(M2Coordinate(org='org', name='x',
rev='1.0'),
cache_path='somewhere',
pants_path=self._create_file(jar_path))
target_a = self.make_target('a', JvmTarget)
target_b = self.make_target('b', JvmTarget)
classpath_products = ClasspathProducts(self.pants_workdir)
# Both target a and target b depend on the same jar library
classpath_products.add_jars_for_targets([target_a], 'default',
[resolved_jar])
classpath_products.add_jars_for_targets([target_b], 'default',
[resolved_jar])
with temporary_dir() as base_dir:
# Only target a generates symlink to jar library, target b skips creating the
# symlink for the same jar library. Both targets' classpath.txt files should
# still contain the jar library.
self._test_canonical_classpath_helper(
classpath_products, [target_a, target_b], base_dir,
['a.a-0.jar'], {
'a.a-classpath.txt':
'{}/{}\n'.format(self.pants_workdir, jar_path),
'b.b-classpath.txt':
'{}/{}\n'.format(self.pants_workdir, jar_path),
})
def resolved_jarlib(name, jar_path):
resolved_jar = ResolvedJar(M2Coordinate(org='org.example', name=name, rev='0.0.1'),
cache_path=jar_path,
pants_path=jar_path)
jar_dep = JarDependency(org='org.example', name=name, rev='0.0.1')
jar_library = self.make_target(spec='3rdparty:{}'.format(name),
target_type=JarLibrary,
jars=[jar_dep])
return jar_library, resolved_jar
def resolved_jarlib(name, jar_path):
resolved_jar = ResolvedJar(
M2Coordinate(org="org.example", name=name, rev="0.0.1"),
cache_path=jar_path,
pants_path=jar_path,
)
jar_dep = JarDependency(org="org.example", name=name, rev="0.0.1")
jar_library = self.make_target(spec=f"3rdparty:{name}",
target_type=JarLibrary,
jars=[jar_dep])
return jar_library, resolved_jar
def test_jar_missing_pants_path_fails_adding(self):
b = self.make_target('b', JvmTarget)
classpath_products = ClasspathProducts(self.pants_workdir)
with self.assertRaises(TaskError) as cm:
classpath_products.add_jars_for_targets([b], 'default', [
ResolvedJar(M2Coordinate(org='org', name='name'),
cache_path='somewhere',
pants_path=None)
])
self.assertEqual('Jar: org:name: has no specified path.',
str(cm.exception))
def test_create_canonical_classpath(self):
a = self.make_target('a/b', JvmTarget)
jar_path = 'ivy/jars/org.x/lib/x-1.0.jar'
classpath_products = ClasspathProducts(self.pants_workdir)
resolved_jar = ResolvedJar(M2Coordinate(org='org', name='x',
rev='1.0'),
cache_path='somewhere',
pants_path=self._create_file(jar_path))
classpath_products.add_for_target(
a, [('default', self._create_file('a.jar')),
('default', self._create_file('resources'))])
classpath_products.add_jars_for_targets([a], 'default', [resolved_jar])
with temporary_dir() as base_dir:
self._test_canonical_classpath_helper(
classpath_products, [a],
base_dir, [
'a.b.b-0.jar',
'a.b.b-1',
'a.b.b-2.jar',
], {
'a.b.b-classpath.txt':
'{}/a.jar:{}/resources:{}/{}\n'.format(
self.pants_workdir, self.pants_workdir,
self.pants_workdir, jar_path)
},
excludes={Exclude(org='org', name='y')})
# incrementally delete the resource dendendency
classpath_products = ClasspathProducts(self.pants_workdir)
classpath_products.add_for_target(
a, [('default', self._create_file('a.jar'))])
self._test_canonical_classpath_helper(
classpath_products, [a], base_dir, [
'a.b.b-0.jar',
],
{'a.b.b-classpath.txt': '{}/a.jar\n'.format(self.pants_workdir)})
# incrementally add another jar dependency
classpath_products = ClasspathProducts(self.pants_workdir)
classpath_products.add_for_target(
a, [('default', self._create_file('a.jar')),
('default', self._create_file('b.jar'))])
self._test_canonical_classpath_helper(
classpath_products, [a], base_dir, ['a.b.b-0.jar', 'a.b.b-1.jar'],
{
'a.b.b-classpath.txt':
'{}/a.jar:{}/b.jar\n'.format(self.pants_workdir,
self.pants_workdir)
})
def add_directory_digests_for_jars(self, targets_and_jars):
"""For each target, get DirectoryDigests for its jars and return them zipped with the jars.
:param targets_and_jars: List of tuples of the form (Target, [pants.java.jar.jar_dependency_utils.ResolveJar])
:return: list[tuple[(Target, list[pants.java.jar.jar_dependency_utils.ResolveJar])]
"""
targets_and_jars = list(targets_and_jars)
if not targets_and_jars:
return targets_and_jars
jar_paths = []
for target, jars_to_snapshot in targets_and_jars:
for jar in jars_to_snapshot:
jar_paths.append(fast_relpath(jar.pants_path, get_buildroot()))
# Capture Snapshots for jars, using an optional adjacent digest. Create the digest afterward
# if it does not exist.
snapshots = self.context._scheduler.capture_snapshots(
tuple(
PathGlobsAndRoot(
PathGlobs([jar]),
get_buildroot(),
Digest.load(jar),
) for jar in jar_paths))
for snapshot, jar_path in zip(snapshots, jar_paths):
snapshot.digest.dump(jar_path)
# We want to map back the list[Snapshot] to targets_and_jars
# We assume that (1) jars_to_snapshot has the same number of ResolveJars as snapshots does Snapshots,
# and that (2) capture_snapshots preserves ordering.
digests = [snapshot.digest for snapshot in snapshots]
digest_iterator = iter(digests)
snapshotted_targets_and_jars = []
for target, jars_to_snapshot in targets_and_jars:
snapshotted_jars = [
ResolvedJar(
coordinate=jar.coordinate,
cache_path=jar.cache_path,
pants_path=jar.pants_path,
directory_digest=next(digest_iterator),
) for jar in jars_to_snapshot
]
snapshotted_targets_and_jars.append((target, snapshotted_jars))
return snapshotted_targets_and_jars
def test_jar_missing_pants_path_fails_adding(self):
b = self.make_target("b", JvmTarget)
classpath_products = ClasspathProducts(self.pants_workdir)
with self.assertRaises(TaskError) as cm:
classpath_products.add_jars_for_targets(
[b],
"default",
[
ResolvedJar(
M2Coordinate(org="org", name="name"),
cache_path="somewhere",
pants_path=None,
)
],
)
self.assertEqual("Jar: org:name: has no specified path.",
str(cm.exception))
def test_create_canonical_classpath_with_broken_classpath(self):
"""Test exception is thrown when the jar file is missing."""
a = self.make_target('a/b', JvmTarget)
classpath_products = ClasspathProducts(self.pants_workdir)
jar_path = 'ivy/jars/org.x/lib/x-1.0.jar'
# this sets the path for the artifact without actually materializing it.
resolved_jar = ResolvedJar(M2Coordinate(org='org', name='x',
rev='1.0'),
cache_path='somewhere',
pants_path=os.path.join(
self.pants_workdir, jar_path))
classpath_products.add_jars_for_targets([a], 'default', [resolved_jar])
with temporary_dir() as base_dir:
with self.assertRaises(MissingClasspathEntryError):
self._test_canonical_classpath_helper(classpath_products, [a],
base_dir, [], {})
def add_directory_digests_for_jars(self, jars):
"""Get DirectoryDigests for jars and return them zipped with the jars.
:param jars: List of pants.java.jar.jar_dependency_utils.ResolveJar
:return: List of ResolveJars.
"""
snapshots = self.context._scheduler.capture_snapshots(
tuple(
PathGlobsAndRoot(
PathGlobs([fast_relpath(jar.pants_path, get_buildroot())]),
get_buildroot()) for jar in jars))
return [
ResolvedJar(coordinate=jar.coordinate,
cache_path=jar.cache_path,
pants_path=jar.pants_path,
directory_digest=directory_digest)
for jar, directory_digest in list(
zip(jars,
[snapshot.directory_digest for snapshot in snapshots]))
]
def _new_resolved_jar_with_symlink_path(self, conf, target, resolved_jar_without_symlink):
def candidate_cache_paths():
# There is a focus on being lazy here to avoid `os.path.realpath` when we can.
yield resolved_jar_without_symlink.cache_path
yield os.path.realpath(resolved_jar_without_symlink.cache_path)
for cache_path in candidate_cache_paths():
pants_path = self._symlink_map.get(cache_path)
if pants_path:
break
else:
raise IvyResolveMappingError(
'Jar {resolved_jar} in {spec} not resolved to the ivy '
'symlink map in conf {conf}.'
.format(spec=target.address.spec,
resolved_jar=resolved_jar_without_symlink.cache_path,
conf=conf))
return ResolvedJar(coordinate=resolved_jar_without_symlink.coordinate,
pants_path=pants_path,
cache_path=resolved_jar_without_symlink.cache_path)
def test_resolved_jars_with_same_properties(self):
jar1 = ResolvedJar(M2Coordinate("org.example", "lib"), "path")
jar2 = ResolvedJar(M2Coordinate("org.example", "lib"), "path")
self.assertEqual(jar1, jar2)
self.assertEqual(hash(jar1), hash(jar2))
def resolved_example_jar_at(path, org="com.example", name="lib"):
return ResolvedJar(
M2Coordinate(org=org, name=name),
cache_path=os.path.join("resolver-cache-dir", path),
pants_path=path,
)
def resolved_example_jar_at(path, org='com.example', name='lib'):
return ResolvedJar(M2Coordinate(org=org, name=name),
cache_path=os.path.join('resolver-cache-dir', path),
pants_path=path)
def _map_coord_to_resolved_jars(cls, result, coursier_cache_path,
pants_jar_path_base):
"""
Map resolved files to each org:name:version
Example:
{
"conflict_resolution": {},
"dependencies": [
{
"coord": "a",
"dependencies": ["b", "c"],
"file": "a.jar"
},
{
"coord": "b",
"dependencies": [],
"file": "b.jar"
},
{
"coord": "c",
"dependencies": [],
"file": "c.jar"
},
{
"coord": "a:sources",
"dependencies": ["b", "c"],
"file": "a-sources.jar"
},
]
}
Should return:
{
M2Coordinate("a", ...): ResolvedJar(classifier='', path/cache_path="a.jar"),
M2Coordinate("a", ..., classifier="sources"): ResolvedJar(classifier='sources', path/cache_path="a-sources.jar"),
M2Coordinate("b", ...): ResolvedJar(classifier='', path/cache_path="b.jar"),
M2Coordinate("c", ...): ResolvedJar(classifier='', path/cache_path="c.jar"),
}
:param result: coursier json output
:param coursier_cache_path: coursier cache location
:param pants_jar_path_base: location under pants workdir to store the symlink to the coursier cache
:return: a map from maven coordinate to a resolved jar.
"""
coord_to_resolved_jars = dict()
for dep in result['dependencies']:
coord = dep['coord']
jar_path = dep.get('file', None)
if not jar_path:
# NB: Not all coordinates will have associated files.
# This is fine. Some coordinates will just have dependencies.
continue
if not os.path.exists(jar_path):
raise CoursierResultNotFound(
"Jar path not found: {}".format(jar_path))
pants_path = cls._get_path_to_jar(coursier_cache_path,
pants_jar_path_base, jar_path)
if not os.path.exists(pants_path):
safe_mkdir(os.path.dirname(pants_path))
os.symlink(jar_path, pants_path)
coord = cls.to_m2_coord(coord)
resolved_jar = ResolvedJar(coord,
cache_path=jar_path,
pants_path=pants_path)
coord_to_resolved_jars[coord] = resolved_jar
return coord_to_resolved_jars
def test_create_canonical_classpath(self):
a = self.make_target("a/b", JvmTarget)
jar_path = "ivy/jars/org.x/lib/x-1.0.jar"
classpath_products = ClasspathProducts(self.pants_workdir)
resolved_jar = ResolvedJar(
M2Coordinate(org="org", name="x", rev="1.0"),
cache_path="somewhere",
pants_path=self._create_file(jar_path),
)
classpath_products.add_for_target(
a,
[("default", self._create_file("a.jar")),
("default", self._create_file("resources"))],
)
classpath_products.add_jars_for_targets([a], "default", [resolved_jar])
with temporary_dir() as base_dir:
self._test_canonical_classpath_helper(
classpath_products,
[a],
base_dir,
["a.b.b-0.jar", "a.b.b-1", "a.b.b-2.jar"],
{
"a.b.b-classpath.txt":
"{}/a.jar:{}/resources:{}/{}\n".format(
self.pants_workdir, self.pants_workdir,
self.pants_workdir, jar_path)
},
excludes={Exclude(org="org", name="y")},
)
# incrementally delete the resource dependency
classpath_products = ClasspathProducts(self.pants_workdir)
classpath_products.add_for_target(
a, [("default", self._create_file("a.jar"))])
self._test_canonical_classpath_helper(
classpath_products,
[a],
base_dir,
["a.b.b-0.jar"],
{"a.b.b-classpath.txt": f"{self.pants_workdir}/a.jar\n"},
)
# incrementally add another jar dependency
classpath_products = ClasspathProducts(self.pants_workdir)
classpath_products.add_for_target(
a, [("default", self._create_file("a.jar")),
("default", self._create_file("b.jar"))])
self._test_canonical_classpath_helper(
classpath_products,
[a],
base_dir,
["a.b.b-0.jar", "a.b.b-1.jar"],
{
"a.b.b-classpath.txt":
"{}/a.jar:{}/b.jar\n".format(self.pants_workdir,
self.pants_workdir)
},
)
def test_resolved_jars_with_same_properties(self):
jar1 = ResolvedJar(M2Coordinate('org.example', 'lib'), 'path')
jar2 = ResolvedJar(M2Coordinate('org.example', 'lib'), 'path')
self.assertEqual(jar1, jar2)
self.assertEqual(hash(jar1), hash(jar2))
def _map_coord_to_resolved_jars(cls, result, coursier_cache_path, pants_jar_path_base):
"""
Map resolved files to each org:name:version
Example:
{
"conflict_resolution": {},
"dependencies": [
{
"coord": "a",
"dependencies": ["b", "c"],
"files": [ ["", "a.jar"], ["sources", "a-sources.jar"] ]
},
{
"coord": "b",
"dependencies": [],
"files": [ ["", "b.jar"] ]
},
{
"coord": "c",
"dependencies": [],
"files": [ ["", "c.jar"] ]
}
]
}
Should return:
{
"a": { ResolvedJar(classifier='', path/cache_path="a.jar"),
ResolvedJar(classifier='sources', path/cache_path="a-sources.jar") },
"b": { ResolvedJar(classifier='', path/cache_path="b.jar") },
"c": { ResolvedJar(classifier='', path/cache_path="c.jar") },
}
:param result: coursier json output
:param coursier_cache_path: coursier cache location
:param pants_jar_path_base: location under pants workdir to store the symlink to the coursier cache
:return: a map from org:name:version to a set of resolved jars.
"""
coord_to_resolved_jars = defaultdict(set)
for dep in result['dependencies']:
for classifier, jar_path in dep['files']:
simple_coord = dep['coord']
coord = cls.to_m2_coord(simple_coord, classifier)
pants_path = os.path.join(pants_jar_path_base, os.path.relpath(jar_path, coursier_cache_path))
if not os.path.exists(jar_path):
raise CoursierResultNotFound("Jar path not found: {}".format(jar_path))
if not os.path.exists(pants_path):
safe_mkdir(os.path.dirname(pants_path))
os.symlink(jar_path, pants_path)
resolved_jar = ResolvedJar(coord,
cache_path=jar_path,
pants_path=pants_path)
coord_to_resolved_jars[simple_coord].add(resolved_jar)
return coord_to_resolved_jars