def test_resolved_jars_with_differing_paths_not_equal(self): jar1 = ResolvedJar(M2Coordinate("org.example", "lib"), "ivy2/path", "path1") jar2 = ResolvedJar(M2Coordinate("org.example", "lib"), "ivy2/path", "path2") self.assertNotEqual(jar1, jar2)
def test_resolved_jars_with_differing_paths_not_equal(self): jar1 = ResolvedJar(M2Coordinate('org.example', 'lib'), 'ivy2/path', 'path1') jar2 = ResolvedJar(M2Coordinate('org.example', 'lib'), 'ivy2/path', 'path2') self.assertNotEqual(jar1, jar2)
def test_resolved_jars_with_same_paths_equal(self): jar1 = ResolvedJar(M2Coordinate("org.example", "lib"), "ivy2/path", "path") jar2 = ResolvedJar(M2Coordinate("org.example", "lib"), "ivy2/path", "path") self.assertEqual(jar1, jar2)
def test_resolved_jars_with_same_paths_equal(self): jar1 = ResolvedJar(M2Coordinate('org.example', 'lib'), 'ivy2/path', 'path') jar2 = ResolvedJar(M2Coordinate('org.example', 'lib'), 'ivy2/path', 'path') self.assertEqual(jar1, jar2)
def test_classpath_by_targets(self): b = self.make_target("b", JvmTarget) a = self.make_target("a", JvmTarget, dependencies=[b], excludes=[Exclude("com.example", "lib")]) classpath_products = ClasspathProducts(self.pants_workdir) path1 = self._path("jar/path1") path2 = self._path("jar/path2") path3 = os.path.join(self.pants_workdir, "jar/path3") resolved_jar = ResolvedJar( M2Coordinate(org="com.example", name="lib", rev="1.0"), cache_path="somewhere", pants_path=path3, ) classpath_products.add_for_target(a, [("default", path1)]) classpath_products.add_for_target(a, [("non-default", path2)]) classpath_products.add_for_target(b, [("default", path2)]) classpath_products.add_jars_for_targets([b], "default", [resolved_jar]) classpath_products.add_excludes_for_targets([a]) # (a, path2) filtered because of conf # (b, path3) filtered because of excludes self.assertEqual( OrderedDict([(a, [ClasspathEntry(path1)]), (b, [ClasspathEntry(path2)])]), ClasspathUtil.classpath_by_targets(a.closure(bfs=True), classpath_products), )
def to_resolved_jar(jar_ref, jar_path): return ResolvedJar(coordinate=M2Coordinate(org=jar_ref.org, name=jar_ref.name, rev=jar_ref.rev, classifier=jar_ref.classifier, ext=jar_ref.ext), cache_path=jar_path)
def test_create_canonical_classpath_no_duplicate_entry(self): """Test no more than one symlink are created for the same classpath entry.""" jar_path = "ivy/jars/org.x/lib/x-1.0.jar" resolved_jar = ResolvedJar( M2Coordinate(org="org", name="x", rev="1.0"), cache_path="somewhere", pants_path=self._create_file(jar_path), ) target_a = self.make_target("a", JvmTarget) target_b = self.make_target("b", JvmTarget) classpath_products = ClasspathProducts(self.pants_workdir) # Both target a and target b depend on the same jar library classpath_products.add_jars_for_targets([target_a], "default", [resolved_jar]) classpath_products.add_jars_for_targets([target_b], "default", [resolved_jar]) with temporary_dir() as base_dir: # Only target a generates symlink to jar library, target b skips creating the # symlink for the same jar library. Both targets' classpath.txt files should # still contain the jar library. self._test_canonical_classpath_helper( classpath_products, [target_a, target_b], base_dir, ["a.a-0.jar"], { "a.a-classpath.txt": f"{self.pants_workdir}/{jar_path}\n", "b.b-classpath.txt": f"{self.pants_workdir}/{jar_path}\n", }, )
def test_classpath_by_targets(self): b = self.make_target('b', JvmTarget) a = self.make_target('a', JvmTarget, dependencies=[b], excludes=[Exclude('com.example', 'lib')]) classpath_products = ClasspathProducts(self.pants_workdir) path1 = self._path('jar/path1') path2 = self._path('jar/path2') path3 = os.path.join(self.pants_workdir, 'jar/path3') resolved_jar = ResolvedJar(M2Coordinate(org='com.example', name='lib', rev='1.0'), cache_path='somewhere', pants_path=path3) classpath_products.add_for_target(a, [('default', path1)]) classpath_products.add_for_target(a, [('non-default', path2)]) classpath_products.add_for_target(b, [('default', path2)]) classpath_products.add_jars_for_targets([b], 'default', [resolved_jar]) classpath_products.add_excludes_for_targets([a]) # (a, path2) filtered because of conf # (b, path3) filtered because of excludes self.assertEqual( OrderedDict([(a, [ClasspathEntry(path1)]), (b, [ClasspathEntry(path2)])]), ClasspathUtil.classpath_by_targets(a.closure(bfs=True), classpath_products))
def create_artifact(self, org, name, rev, classifier=None, ext=None, materialize=True): """ :API: public :param string org: The maven dependency `groupId`. :param string name: The maven dependency `artifactId`. :param string rev: The maven dependency `version`. :param string classifier: The maven dependency `classifier`. :param string ext: There is no direct maven parallel, but the maven `packaging` value of the depended-on artifact for simple cases, and in more complex cases the extension of the artifact. For example, 'bundle' packaging implies an extension of 'jar'. Defaults to 'jar'. :param bool materialize: `False` to populate the returned resolved_jar with a `pants_path` that does not exist; defaults to `True` and `touch`es the `pants_path`. :returns: A resolved jar describing the artifact. :rtype: :class:`pants.java.jar.ResolvedJar` """ coordinate = M2Coordinate(org=org, name=name, rev=rev, classifier=classifier, ext=ext) cache_path = "not/a/real/cache/path" jar_name = coordinate.artifact_filename pants_path = (self.create_workdir_file(jar_name) if materialize else os.path.join(self.pants_workdir, jar_name)) return ResolvedJar(coordinate=coordinate, cache_path=cache_path, pants_path=pants_path)
def _populate_compile_classpath(self): products = self.context.products compile_classpath = products.get_data('compile_classpath', init_func=ClasspathProducts.init_func(self.get_options().pants_workdir)) sorted_targets = sorted( self.context.targets(predicate=lambda t: t in self.all_jar_libs), key=lambda t: t.address.spec, ) for target in sorted_targets: resolved_jars = [] for coord in sorted(self.target_to_maven_coordinate_closure[target.id]): m2_coord = M2Coordinate( org=coord.groupId, name=coord.artifactId, rev=coord.version, classifier=coord.classifier, ext=coord.packaging, ) sorted_artifact_paths = sorted( artifact.artifact_path for artifact in self.maven_coordinate_to_provided_artifacts[coord] ) for artifact_path in sorted_artifact_paths: resolved_jar = ResolvedJar( coordinate=m2_coord, pants_path=os.path.join(self.artifact_symlink_dir, artifact_path), cache_path=os.path.join(self.pom_cache_dir, artifact_path), ) resolved_jars.append(resolved_jar) compile_classpath.add_jars_for_targets([target], 'default', resolved_jars)
def test_create_canonical_classpath_no_duplicate_entry(self): """Test no more than one symlink are created for the same classpath entry.""" jar_path = 'ivy/jars/org.x/lib/x-1.0.jar' resolved_jar = ResolvedJar(M2Coordinate(org='org', name='x', rev='1.0'), cache_path='somewhere', pants_path=self._create_file(jar_path)) target_a = self.make_target('a', JvmTarget) target_b = self.make_target('b', JvmTarget) classpath_products = ClasspathProducts(self.pants_workdir) # Both target a and target b depend on the same jar library classpath_products.add_jars_for_targets([target_a], 'default', [resolved_jar]) classpath_products.add_jars_for_targets([target_b], 'default', [resolved_jar]) with temporary_dir() as base_dir: # Only target a generates symlink to jar library, target b skips creating the # symlink for the same jar library. Both targets' classpath.txt files should # still contain the jar library. self._test_canonical_classpath_helper( classpath_products, [target_a, target_b], base_dir, ['a.a-0.jar'], { 'a.a-classpath.txt': '{}/{}\n'.format(self.pants_workdir, jar_path), 'b.b-classpath.txt': '{}/{}\n'.format(self.pants_workdir, jar_path), })
def resolved_jarlib(name, jar_path): resolved_jar = ResolvedJar(M2Coordinate(org='org.example', name=name, rev='0.0.1'), cache_path=jar_path, pants_path=jar_path) jar_dep = JarDependency(org='org.example', name=name, rev='0.0.1') jar_library = self.make_target(spec='3rdparty:{}'.format(name), target_type=JarLibrary, jars=[jar_dep]) return jar_library, resolved_jar
def resolved_jarlib(name, jar_path): resolved_jar = ResolvedJar( M2Coordinate(org="org.example", name=name, rev="0.0.1"), cache_path=jar_path, pants_path=jar_path, ) jar_dep = JarDependency(org="org.example", name=name, rev="0.0.1") jar_library = self.make_target(spec=f"3rdparty:{name}", target_type=JarLibrary, jars=[jar_dep]) return jar_library, resolved_jar
def test_jar_missing_pants_path_fails_adding(self): b = self.make_target('b', JvmTarget) classpath_products = ClasspathProducts(self.pants_workdir) with self.assertRaises(TaskError) as cm: classpath_products.add_jars_for_targets([b], 'default', [ ResolvedJar(M2Coordinate(org='org', name='name'), cache_path='somewhere', pants_path=None) ]) self.assertEqual('Jar: org:name: has no specified path.', str(cm.exception))
def test_create_canonical_classpath(self): a = self.make_target('a/b', JvmTarget) jar_path = 'ivy/jars/org.x/lib/x-1.0.jar' classpath_products = ClasspathProducts(self.pants_workdir) resolved_jar = ResolvedJar(M2Coordinate(org='org', name='x', rev='1.0'), cache_path='somewhere', pants_path=self._create_file(jar_path)) classpath_products.add_for_target( a, [('default', self._create_file('a.jar')), ('default', self._create_file('resources'))]) classpath_products.add_jars_for_targets([a], 'default', [resolved_jar]) with temporary_dir() as base_dir: self._test_canonical_classpath_helper( classpath_products, [a], base_dir, [ 'a.b.b-0.jar', 'a.b.b-1', 'a.b.b-2.jar', ], { 'a.b.b-classpath.txt': '{}/a.jar:{}/resources:{}/{}\n'.format( self.pants_workdir, self.pants_workdir, self.pants_workdir, jar_path) }, excludes={Exclude(org='org', name='y')}) # incrementally delete the resource dendendency classpath_products = ClasspathProducts(self.pants_workdir) classpath_products.add_for_target( a, [('default', self._create_file('a.jar'))]) self._test_canonical_classpath_helper( classpath_products, [a], base_dir, [ 'a.b.b-0.jar', ], {'a.b.b-classpath.txt': '{}/a.jar\n'.format(self.pants_workdir)}) # incrementally add another jar dependency classpath_products = ClasspathProducts(self.pants_workdir) classpath_products.add_for_target( a, [('default', self._create_file('a.jar')), ('default', self._create_file('b.jar'))]) self._test_canonical_classpath_helper( classpath_products, [a], base_dir, ['a.b.b-0.jar', 'a.b.b-1.jar'], { 'a.b.b-classpath.txt': '{}/a.jar:{}/b.jar\n'.format(self.pants_workdir, self.pants_workdir) })
def add_directory_digests_for_jars(self, targets_and_jars): """For each target, get DirectoryDigests for its jars and return them zipped with the jars. :param targets_and_jars: List of tuples of the form (Target, [pants.java.jar.jar_dependency_utils.ResolveJar]) :return: list[tuple[(Target, list[pants.java.jar.jar_dependency_utils.ResolveJar])] """ targets_and_jars = list(targets_and_jars) if not targets_and_jars: return targets_and_jars jar_paths = [] for target, jars_to_snapshot in targets_and_jars: for jar in jars_to_snapshot: jar_paths.append(fast_relpath(jar.pants_path, get_buildroot())) # Capture Snapshots for jars, using an optional adjacent digest. Create the digest afterward # if it does not exist. snapshots = self.context._scheduler.capture_snapshots( tuple( PathGlobsAndRoot( PathGlobs([jar]), get_buildroot(), Digest.load(jar), ) for jar in jar_paths)) for snapshot, jar_path in zip(snapshots, jar_paths): snapshot.digest.dump(jar_path) # We want to map back the list[Snapshot] to targets_and_jars # We assume that (1) jars_to_snapshot has the same number of ResolveJars as snapshots does Snapshots, # and that (2) capture_snapshots preserves ordering. digests = [snapshot.digest for snapshot in snapshots] digest_iterator = iter(digests) snapshotted_targets_and_jars = [] for target, jars_to_snapshot in targets_and_jars: snapshotted_jars = [ ResolvedJar( coordinate=jar.coordinate, cache_path=jar.cache_path, pants_path=jar.pants_path, directory_digest=next(digest_iterator), ) for jar in jars_to_snapshot ] snapshotted_targets_and_jars.append((target, snapshotted_jars)) return snapshotted_targets_and_jars
def test_jar_missing_pants_path_fails_adding(self): b = self.make_target("b", JvmTarget) classpath_products = ClasspathProducts(self.pants_workdir) with self.assertRaises(TaskError) as cm: classpath_products.add_jars_for_targets( [b], "default", [ ResolvedJar( M2Coordinate(org="org", name="name"), cache_path="somewhere", pants_path=None, ) ], ) self.assertEqual("Jar: org:name: has no specified path.", str(cm.exception))
def test_create_canonical_classpath_with_broken_classpath(self): """Test exception is thrown when the jar file is missing.""" a = self.make_target('a/b', JvmTarget) classpath_products = ClasspathProducts(self.pants_workdir) jar_path = 'ivy/jars/org.x/lib/x-1.0.jar' # this sets the path for the artifact without actually materializing it. resolved_jar = ResolvedJar(M2Coordinate(org='org', name='x', rev='1.0'), cache_path='somewhere', pants_path=os.path.join( self.pants_workdir, jar_path)) classpath_products.add_jars_for_targets([a], 'default', [resolved_jar]) with temporary_dir() as base_dir: with self.assertRaises(MissingClasspathEntryError): self._test_canonical_classpath_helper(classpath_products, [a], base_dir, [], {})
def add_directory_digests_for_jars(self, jars): """Get DirectoryDigests for jars and return them zipped with the jars. :param jars: List of pants.java.jar.jar_dependency_utils.ResolveJar :return: List of ResolveJars. """ snapshots = self.context._scheduler.capture_snapshots( tuple( PathGlobsAndRoot( PathGlobs([fast_relpath(jar.pants_path, get_buildroot())]), get_buildroot()) for jar in jars)) return [ ResolvedJar(coordinate=jar.coordinate, cache_path=jar.cache_path, pants_path=jar.pants_path, directory_digest=directory_digest) for jar, directory_digest in list( zip(jars, [snapshot.directory_digest for snapshot in snapshots])) ]
def _new_resolved_jar_with_symlink_path(self, conf, target, resolved_jar_without_symlink): def candidate_cache_paths(): # There is a focus on being lazy here to avoid `os.path.realpath` when we can. yield resolved_jar_without_symlink.cache_path yield os.path.realpath(resolved_jar_without_symlink.cache_path) for cache_path in candidate_cache_paths(): pants_path = self._symlink_map.get(cache_path) if pants_path: break else: raise IvyResolveMappingError( 'Jar {resolved_jar} in {spec} not resolved to the ivy ' 'symlink map in conf {conf}.' .format(spec=target.address.spec, resolved_jar=resolved_jar_without_symlink.cache_path, conf=conf)) return ResolvedJar(coordinate=resolved_jar_without_symlink.coordinate, pants_path=pants_path, cache_path=resolved_jar_without_symlink.cache_path)
def test_resolved_jars_with_same_properties(self): jar1 = ResolvedJar(M2Coordinate("org.example", "lib"), "path") jar2 = ResolvedJar(M2Coordinate("org.example", "lib"), "path") self.assertEqual(jar1, jar2) self.assertEqual(hash(jar1), hash(jar2))
def resolved_example_jar_at(path, org="com.example", name="lib"): return ResolvedJar( M2Coordinate(org=org, name=name), cache_path=os.path.join("resolver-cache-dir", path), pants_path=path, )
def resolved_example_jar_at(path, org='com.example', name='lib'): return ResolvedJar(M2Coordinate(org=org, name=name), cache_path=os.path.join('resolver-cache-dir', path), pants_path=path)
def _map_coord_to_resolved_jars(cls, result, coursier_cache_path, pants_jar_path_base): """ Map resolved files to each org:name:version Example: { "conflict_resolution": {}, "dependencies": [ { "coord": "a", "dependencies": ["b", "c"], "file": "a.jar" }, { "coord": "b", "dependencies": [], "file": "b.jar" }, { "coord": "c", "dependencies": [], "file": "c.jar" }, { "coord": "a:sources", "dependencies": ["b", "c"], "file": "a-sources.jar" }, ] } Should return: { M2Coordinate("a", ...): ResolvedJar(classifier='', path/cache_path="a.jar"), M2Coordinate("a", ..., classifier="sources"): ResolvedJar(classifier='sources', path/cache_path="a-sources.jar"), M2Coordinate("b", ...): ResolvedJar(classifier='', path/cache_path="b.jar"), M2Coordinate("c", ...): ResolvedJar(classifier='', path/cache_path="c.jar"), } :param result: coursier json output :param coursier_cache_path: coursier cache location :param pants_jar_path_base: location under pants workdir to store the symlink to the coursier cache :return: a map from maven coordinate to a resolved jar. """ coord_to_resolved_jars = dict() for dep in result['dependencies']: coord = dep['coord'] jar_path = dep.get('file', None) if not jar_path: # NB: Not all coordinates will have associated files. # This is fine. Some coordinates will just have dependencies. continue if not os.path.exists(jar_path): raise CoursierResultNotFound( "Jar path not found: {}".format(jar_path)) pants_path = cls._get_path_to_jar(coursier_cache_path, pants_jar_path_base, jar_path) if not os.path.exists(pants_path): safe_mkdir(os.path.dirname(pants_path)) os.symlink(jar_path, pants_path) coord = cls.to_m2_coord(coord) resolved_jar = ResolvedJar(coord, cache_path=jar_path, pants_path=pants_path) coord_to_resolved_jars[coord] = resolved_jar return coord_to_resolved_jars
def test_create_canonical_classpath(self): a = self.make_target("a/b", JvmTarget) jar_path = "ivy/jars/org.x/lib/x-1.0.jar" classpath_products = ClasspathProducts(self.pants_workdir) resolved_jar = ResolvedJar( M2Coordinate(org="org", name="x", rev="1.0"), cache_path="somewhere", pants_path=self._create_file(jar_path), ) classpath_products.add_for_target( a, [("default", self._create_file("a.jar")), ("default", self._create_file("resources"))], ) classpath_products.add_jars_for_targets([a], "default", [resolved_jar]) with temporary_dir() as base_dir: self._test_canonical_classpath_helper( classpath_products, [a], base_dir, ["a.b.b-0.jar", "a.b.b-1", "a.b.b-2.jar"], { "a.b.b-classpath.txt": "{}/a.jar:{}/resources:{}/{}\n".format( self.pants_workdir, self.pants_workdir, self.pants_workdir, jar_path) }, excludes={Exclude(org="org", name="y")}, ) # incrementally delete the resource dependency classpath_products = ClasspathProducts(self.pants_workdir) classpath_products.add_for_target( a, [("default", self._create_file("a.jar"))]) self._test_canonical_classpath_helper( classpath_products, [a], base_dir, ["a.b.b-0.jar"], {"a.b.b-classpath.txt": f"{self.pants_workdir}/a.jar\n"}, ) # incrementally add another jar dependency classpath_products = ClasspathProducts(self.pants_workdir) classpath_products.add_for_target( a, [("default", self._create_file("a.jar")), ("default", self._create_file("b.jar"))]) self._test_canonical_classpath_helper( classpath_products, [a], base_dir, ["a.b.b-0.jar", "a.b.b-1.jar"], { "a.b.b-classpath.txt": "{}/a.jar:{}/b.jar\n".format(self.pants_workdir, self.pants_workdir) }, )
def test_resolved_jars_with_same_properties(self): jar1 = ResolvedJar(M2Coordinate('org.example', 'lib'), 'path') jar2 = ResolvedJar(M2Coordinate('org.example', 'lib'), 'path') self.assertEqual(jar1, jar2) self.assertEqual(hash(jar1), hash(jar2))
def _map_coord_to_resolved_jars(cls, result, coursier_cache_path, pants_jar_path_base): """ Map resolved files to each org:name:version Example: { "conflict_resolution": {}, "dependencies": [ { "coord": "a", "dependencies": ["b", "c"], "files": [ ["", "a.jar"], ["sources", "a-sources.jar"] ] }, { "coord": "b", "dependencies": [], "files": [ ["", "b.jar"] ] }, { "coord": "c", "dependencies": [], "files": [ ["", "c.jar"] ] } ] } Should return: { "a": { ResolvedJar(classifier='', path/cache_path="a.jar"), ResolvedJar(classifier='sources', path/cache_path="a-sources.jar") }, "b": { ResolvedJar(classifier='', path/cache_path="b.jar") }, "c": { ResolvedJar(classifier='', path/cache_path="c.jar") }, } :param result: coursier json output :param coursier_cache_path: coursier cache location :param pants_jar_path_base: location under pants workdir to store the symlink to the coursier cache :return: a map from org:name:version to a set of resolved jars. """ coord_to_resolved_jars = defaultdict(set) for dep in result['dependencies']: for classifier, jar_path in dep['files']: simple_coord = dep['coord'] coord = cls.to_m2_coord(simple_coord, classifier) pants_path = os.path.join(pants_jar_path_base, os.path.relpath(jar_path, coursier_cache_path)) if not os.path.exists(jar_path): raise CoursierResultNotFound("Jar path not found: {}".format(jar_path)) if not os.path.exists(pants_path): safe_mkdir(os.path.dirname(pants_path)) os.symlink(jar_path, pants_path) resolved_jar = ResolvedJar(coord, cache_path=jar_path, pants_path=pants_path) coord_to_resolved_jars[simple_coord].add(resolved_jar) return coord_to_resolved_jars