def _check_bundle_products(self, bundle_name_prefix, check_symlink=False, symlink_name_prefix=''):
products = self.task_context.products.get('jvm_bundles')
bundle_fullname = '{}-bundle'.format(bundle_name_prefix)
bundle_root = self._check_products(products, bundle_fullname)
self.assertTrue(os.path.isdir(bundle_root))
self.assertEqual(sorted(['foo-binary.jar',
'libs/foo.foo-binary-0.jar',
'libs/3rdparty.jvm.org.example.foo-0.jar',
'libs/3rdparty.jvm.org.example.foo-1.zip',
'libs/3rdparty.jvm.org.example.foo-2.jar',
'libs/3rdparty.jvm.org.example.foo-3.gz']),
sorted(self.iter_files(bundle_root)))
with open_zip(os.path.join(bundle_root, 'libs/foo.foo-binary-0.jar')) as zf:
self.assertEqual(sorted(['META-INF/',
'META-INF/MANIFEST.MF',
'Foo.class',
'foo.txt',
'foo/',
'foo/file']),
sorted(zf.namelist()))
# TODO verify Manifest's Class-Path
with open_zip(os.path.join(bundle_root, 'foo-binary.jar')) as jar:
self.assertEqual(sorted(['META-INF/', 'META-INF/MANIFEST.MF']),
sorted(jar.namelist()))
# Check symlink.
if check_symlink:
symlink_fullname = '{}-bundle'.format(symlink_name_prefix) if symlink_name_prefix else bundle_fullname
symlink_path = os.path.join(self.dist_root, symlink_fullname)
self.assertTrue(os.path.islink(symlink_path))
self.assertEqual(os.readlink(symlink_path), bundle_root)
def _check_bundle_products(self, bundle_basename):
products = self.task_context.products.get('jvm_bundles')
self.assertIsNotNone(products)
product_data = products.get(self.app_target)
self.assertEquals({self.dist_root: ['{basename}-bundle'.format(basename=bundle_basename)]},
product_data)
self.assertTrue(os.path.exists(self.dist_root))
bundle_root = os.path.join(self.dist_root,
'{basename}-bundle'.format(basename=bundle_basename))
self.assertEqual(sorted(['foo-binary.jar',
'libs/foo.foo-binary-0.jar',
'libs/3rdparty.jvm.org.example.foo-0.jar',
'libs/3rdparty.jvm.org.example.foo-1.zip',
'libs/3rdparty.jvm.org.example.foo-2.jar',
'libs/3rdparty.jvm.org.example.foo-3.gz']),
sorted(self.iter_files(bundle_root)))
with open_zip(os.path.join(bundle_root, 'libs/foo.foo-binary-0.jar')) as zf:
self.assertEqual(sorted(['META-INF/',
'META-INF/MANIFEST.MF',
'Foo.class',
'foo.txt',
'foo/',
'foo/file']),
sorted(zf.namelist()))
# TODO verify Manifest's Class-Path
with open_zip(os.path.join(bundle_root, 'foo-binary.jar')) as jar:
self.assertEqual(sorted(['META-INF/', 'META-INF/MANIFEST.MF']),
sorted(jar.namelist()))
def _check_bundle_products(self, bundle_basename):
products = self.task_context.products.get("jvm_bundles")
self.assertIsNotNone(products)
product_data = products.get(self.app_target)
self.assertEquals({self.dist_root: ["{basename}-bundle".format(basename=bundle_basename)]}, product_data)
self.assertTrue(os.path.exists(self.dist_root))
bundle_root = os.path.join(self.dist_root, "{basename}-bundle".format(basename=bundle_basename))
self.assertEqual(
sorted(
[
"foo-binary.jar",
"libs/foo.foo-binary-0.jar",
"libs/3rdparty.jvm.org.example.foo-0.jar",
"libs/3rdparty.jvm.org.example.foo-1.zip",
"libs/3rdparty.jvm.org.example.foo-2.jar",
"libs/3rdparty.jvm.org.example.foo-3.gz",
]
),
sorted(self.iter_files(bundle_root)),
)
with open_zip(os.path.join(bundle_root, "libs/foo.foo-binary-0.jar")) as zf:
self.assertEqual(
sorted(["META-INF/", "META-INF/MANIFEST.MF", "Foo.class", "foo.txt", "foo/", "foo/file"]),
sorted(zf.namelist()),
)
# TODO verify Manifest's Class-Path
with open_zip(os.path.join(bundle_root, "foo-binary.jar")) as jar:
self.assertEqual(sorted(["META-INF/", "META-INF/MANIFEST.MF"]), sorted(jar.namelist()))
def test_jvm_binaries_products(self):
binary_target = self.make_target(spec='//bar:bar-binary',
target_type=JvmBinary,
source='Bar.java')
context = self.context(target_roots=[binary_target])
classpath_products = self.ensure_classpath_products(context)
jar_artifact = self.create_artifact(org='org.example', name='foo', rev='1.0.0')
with open_zip(jar_artifact.pants_path, 'w') as jar:
jar.writestr('foo/Foo.class', '')
classpath_products.add_jars_for_targets(targets=[binary_target],
conf='default',
resolved_jars=[jar_artifact])
with self.add_data(context.products, 'classes_by_target', binary_target, 'Bar.class'):
with self.add_data(context.products, 'resources_by_target', binary_target, 'bar.txt'):
self.execute(context)
jvm_binary_products = context.products.get('jvm_binaries')
self.assertIsNotNone(jvm_binary_products)
product_data = jvm_binary_products.get(binary_target)
dist_root = os.path.join(self.build_root, 'dist')
self.assertEquals({dist_root: ['bar-binary.jar']}, product_data)
with open_zip(os.path.join(dist_root, 'bar-binary.jar')) as jar:
self.assertEqual(sorted(['META-INF/',
'META-INF/MANIFEST.MF',
'foo/',
'foo/Foo.class',
'Bar.class',
'bar.txt']),
sorted(jar.namelist()))
def test_jvm_binaries_products(self):
self.add_to_build_file('bar', 'jvm_binary(name = "bar-binary", source = "Bar.java")')
binary_target = self.target('//bar:bar-binary')
context = self.context(target_roots=[binary_target])
classpath_products = self.ensure_classpath_products(context)
jar_artifact = self.create_artifact(org='org.example', name='foo', rev='1.0.0')
with open_zip(jar_artifact.pants_path, 'w') as jar:
jar.writestr('foo/Foo.class', '')
classpath_products.add_jars_for_targets(targets=[binary_target],
conf='default',
resolved_jars=[jar_artifact])
self.add_to_runtime_classpath(context, binary_target, {'Bar.class': '', 'bar.txt': ''})
self.execute(context)
jvm_binary_products = context.products.get('jvm_binaries')
self.assertIsNotNone(jvm_binary_products)
product_data = jvm_binary_products.get(binary_target)
dist_root = os.path.join(self.build_root, 'dist')
self.assertEquals({dist_root: ['bar-binary.jar']}, product_data)
with open_zip(os.path.join(dist_root, 'bar-binary.jar')) as jar:
self.assertEqual(sorted(['META-INF/',
'META-INF/MANIFEST.MF',
'foo/',
'foo/Foo.class',
'Bar.class',
'bar.txt']),
sorted(jar.namelist()))
def test_open_zip_returns_realpath_on_badzipfile(self):
# In case of file corruption, deleting a Pants-constructed symlink would not resolve the error.
with temporary_file() as not_zip:
with temporary_dir() as tempdir:
file_symlink = os.path.join(tempdir, 'foo')
os.symlink(not_zip.name, file_symlink)
self.assertEquals(os.path.realpath(file_symlink), os.path.realpath(not_zip.name))
with self.assertRaisesRegexp(zipfile.BadZipfile, r'{}'.format(not_zip.name)):
open_zip(file_symlink).gen.next()
def test_jvm_binaries_deploy_excludes(self):
self.add_to_build_file(
'3rdparty/jvm/org/example',
'jar_library(name = "foo", jars = [jar(org = "org.example", name = "foo", rev = "1.0.0")])',
)
foo_jar_lib = self.target('3rdparty/jvm/org/example:foo')
self.add_to_build_file(
'bar',
'''jvm_binary(
name = "bar-binary",
source = "Bar.java",
dependencies = ["3rdparty/jvm/org/example:foo"],
deploy_excludes = [exclude(org = "org.pantsbuild")],
)'''
)
binary_target = self.target('//bar:bar-binary')
context = self.context(target_roots=[binary_target])
classpath_products = self.ensure_classpath_products(context)
foo_artifact = self.create_artifact(org='org.example', name='foo', rev='1.0.0')
with open_zip(foo_artifact.pants_path, 'w') as jar:
jar.writestr('foo/Foo.class', '')
baz_artifact = self.create_artifact(org='org.pantsbuild', name='baz', rev='2.0.0')
with open_zip(baz_artifact.pants_path, 'w') as jar:
# This file should not be included in the binary jar since org.pantsbuild is deploy excluded.
jar.writestr('baz/Baz.class', '')
classpath_products.add_jars_for_targets(targets=[foo_jar_lib],
conf='default',
resolved_jars=[foo_artifact, baz_artifact])
self.add_to_runtime_classpath(context, binary_target, {'Bar.class': '', 'bar.txt': ''})
self.execute(context)
jvm_binary_products = context.products.get('jvm_binaries')
self.assertIsNotNone(jvm_binary_products)
product_data = jvm_binary_products.get(binary_target)
dist_root = os.path.join(self.build_root, 'dist')
self.assertEquals({dist_root: ['bar-binary.jar']}, product_data)
with open_zip(os.path.join(dist_root, 'bar-binary.jar')) as jar:
self.assertEqual(sorted(['META-INF/',
'META-INF/MANIFEST.MF',
'foo/',
'foo/Foo.class',
'Bar.class',
'bar.txt']),
sorted(jar.namelist()))
def reversion(args):
with temporary_dir() as workspace:
# Extract the input.
with open_zip(args.whl_file, 'r') as whl:
src_filenames = whl.namelist()
whl.extractall(workspace)
# Determine the location of the `dist-info` directory.
dist_info_dir = locate_dist_info_dir(workspace)
record_file = os.path.join(dist_info_dir, 'RECORD')
# Get version from the input whl's metadata.
input_version = None
metadata_file = os.path.join(workspace, dist_info_dir, 'METADATA')
with open(metadata_file, 'r') as info:
for line in info:
mo = _version_re.match(line)
if mo:
input_version = mo.group('version')
break
if not input_version:
raise Exception('Could not find `Version:` line in {}'.format(metadata_file))
# Rewrite and move all files (including the RECORD file), recording which files need to be
# re-fingerprinted due to content changes.
dst_filenames = []
refingerprint = []
for src_filename in src_filenames:
if os.path.isdir(os.path.join(workspace, src_filename)):
continue
dst_filename = src_filename
if any_match(args.glob, src_filename):
rewritten = replace_in_file(workspace, src_filename, input_version, args.target_version)
if rewritten is not None:
dst_filename = rewritten
refingerprint.append((src_filename, dst_filename))
dst_filenames.append(dst_filename)
# Refingerprint relevant entries in the RECORD file under their new names.
rewrite_record_file(workspace, record_file, refingerprint)
# Create a new output whl in the destination.
dst_whl_filename = os.path.basename(args.whl_file).replace(input_version, args.target_version)
dst_whl_file = os.path.join(args.dest_dir, dst_whl_filename)
with open_zip(dst_whl_file, 'w', zipfile.ZIP_DEFLATED) as whl:
for dst_filename in dst_filenames:
whl.write(os.path.join(workspace, dst_filename), dst_filename)
print('Wrote whl with version {} to {}.\n'.format(args.target_version, dst_whl_file))
def test_deploy_excludes(self):
with temporary_dir() as distdir:
def build(name):
jar_filename = os.path.join(distdir, '{}.jar'.format(name))
command = [
'--pants-distdir={}'.format(distdir),
'--no-compile-zinc-capture-classpath',
'binary',
'testprojects/src/java/org/pantsbuild/testproject/deployexcludes:{}'.format(name),
]
self.assert_success(self.run_pants(command))
return jar_filename
# The excluded binary should not contain any guava classes, and should fail to run.
jar_filename = build('deployexcludes')
with open_zip(jar_filename) as jar_file:
self.assertEqual({'META-INF/',
'META-INF/MANIFEST.MF',
'org/',
'org/pantsbuild/',
'org/pantsbuild/testproject/',
'org/pantsbuild/testproject/deployexcludes/',
'org/pantsbuild/testproject/deployexcludes/DeployExcludesMain.class'},
set(jar_file.namelist()))
self.run_java(java_args=['-jar', jar_filename],
expected_returncode=1,
expected_output='java.lang.NoClassDefFoundError: '
'com/google/common/collect/ImmutableSortedSet')
# And the non excluded binary should succeed.
jar_filename = build('nodeployexcludes')
self.run_java(java_args=['-jar', jar_filename],
expected_output='DeployExcludes Hello World')
def classpath_entries_contents(cls, classpath_entries):
"""Provide a generator over the contents (classes/resources) of a classpath.
Subdirectories are included and differentiated via a trailing forward slash (for symmetry
across ZipFile.namelist and directory walks).
:param classpath_entries: A sequence of classpath_entries. Non-jars/dirs are ignored.
:returns: An iterator over all classpath contents, one directory, class or resource relative
path per iteration step.
:rtype: :class:`collections.Iterator` of string
"""
for entry in classpath_entries:
if cls.is_jar(entry):
# Walk the jar namelist.
with open_zip(entry, mode="r") as jar:
for name in jar.namelist():
yield name
elif os.path.isdir(entry):
# Walk the directory, including subdirs.
def rel_walk_name(abs_sub_dir, name):
return fast_relpath(os.path.join(abs_sub_dir, name), entry)
for abs_sub_dir, dirnames, filenames in safe_walk(entry):
for name in dirnames:
yield "{}/".format(rel_walk_name(abs_sub_dir, name))
for name in filenames:
yield rel_walk_name(abs_sub_dir, name)
else:
# non-jar and non-directory classpath entries should be ignored
pass
def test_deploy_excludes(self):
jar_filename = os.path.join('dist', 'deployexcludes.jar')
safe_delete(jar_filename)
command = [
'--no-compile-zinc-capture-classpath',
'binary',
'testprojects/src/java/org/pantsbuild/testproject/deployexcludes',
]
with self.pants_results(command) as pants_run:
self.assert_success(pants_run)
# The resulting binary should not contain any guava classes
with open_zip(jar_filename) as jar_file:
self.assertEquals({'META-INF/',
'META-INF/MANIFEST.MF',
'org/',
'org/pantsbuild/',
'org/pantsbuild/testproject/',
'org/pantsbuild/testproject/deployexcludes/',
'org/pantsbuild/testproject/deployexcludes/DeployExcludesMain.class'},
set(jar_file.namelist()))
# This jar should not run by itself, missing symbols
self.run_java(java_args=['-jar', jar_filename],
expected_returncode=1,
expected_output='java.lang.NoClassDefFoundError: '
'com/google/common/collect/ImmutableSortedSet')
# But adding back the deploy_excluded symbols should result in a clean run.
classpath = [jar_filename,
os.path.join(pants_run.workdir,
'ivy/jars/com.google.guava/guava/jars/guava-18.0.jar')]
self.run_java(java_args=['-cp', os.pathsep.join(classpath),
'org.pantsbuild.testproject.deployexcludes.DeployExcludesMain'],
expected_output='DeployExcludes Hello World')
def assert_prep_compile(self):
with temporary_dir() as tempdir:
with open_zip('/tmp/running-in-goal-compile.jar') as jar:
self.assertEquals(sorted(['BUILD',
'ExampleJvmPrepCommand.java',
'META-INF/', 'META-INF/MANIFEST.MF']),
sorted(jar.namelist()))
def _iter_jar_packages(cls, path):
with open_zip(path) as jar:
paths = set()
for pathname in jar.namelist():
if cls._potential_package_path(pathname):
paths.add(os.path.dirname(pathname))
return cls._iter_packages(paths)
def generate_jar(path, *class_name):
jar_path = os.path.join(self.test_workdir, 'jars', path)
safe_mkdir_for(jar_path)
with open_zip(jar_path, 'w') as zipfile:
for clazz in class_name:
zipfile.write(clazz, os.path.relpath(clazz, self.classes_dir))
return jar_path
def _compute_classpath_elements_by_class(self, classpath):
"""Computes a mapping of a .class file to its corresponding element on the given classpath."""
# Don't consider loose classes dirs in our classes dir. Those will be considered
# separately, by looking at products.
def non_product(path):
return path != self._classes_dir
classpath_entries = filter(non_product, classpath)
if self._upstream_class_to_path is None:
self._upstream_class_to_path = {}
for cp_entry in self._find_all_bootstrap_jars() + classpath_entries:
# Per the classloading spec, a 'jar' in this context can also be a .zip file.
if os.path.isfile(cp_entry) and (cp_entry.endswith('.jar') or cp_entry.endswith('.zip')):
with open_zip(cp_entry, 'r') as jar:
for cls in jar.namelist():
# First jar with a given class wins, just like when classloading.
if cls.endswith(b'.class') and not cls in self._upstream_class_to_path:
self._upstream_class_to_path[cls] = cp_entry
elif os.path.isdir(cp_entry):
for dirpath, _, filenames in safe_walk(cp_entry, followlinks=True):
for f in filter(lambda x: x.endswith('.class'), filenames):
cls = os.path.relpath(os.path.join(dirpath, f), cp_entry)
if not cls in self._upstream_class_to_path:
self._upstream_class_to_path[cls] = os.path.join(dirpath, f)
return self._upstream_class_to_path
def test_scala_compile_jar(self):
jar_suffix = 'z.jar'
with self.do_test_compile(SHAPELESS_TARGET,
expected_files=[jar_suffix]) as found:
with open_zip(self.get_only(found, jar_suffix), 'r') as jar:
self.assertTrue(jar.getinfo(SHAPELESS_CLSFILE),
'Expected a jar containing the expected class.')
def test_manifest_items(self):
self.add_to_build_file('src/java/hello', dedent("""
jvm_binary(
name='hello',
main='hello.Hello',
manifest_entries = {
'Foo': 'foo-value',
'Implementation-Version': '1.2.3',
},
)""").strip())
binary_target = self.target('src/java/hello:hello')
context = self.context(target_roots=[binary_target])
classfile = '.pants.d/javac/classes/hello/Hello.class'
self.create_file(classfile, '0xDEADBEEF')
self._add_to_classes_by_target(context, binary_target, classfile)
context.products.safe_create_data('resources_by_target',
lambda: defaultdict(MultipleRootedProducts))
jar_task = self.prepare_jar_task(context)
with self.jarfile() as existing_jarfile:
with jar_task.open_jar(existing_jarfile) as jar:
with jar_task.create_jar_builder(jar) as jar_builder:
jar_builder.add_target(binary_target)
with open_zip(existing_jarfile) as jar:
manifest = jar.read('META-INF/MANIFEST.MF').strip()
all_entries = dict(tuple(re.split(r'\s*:\s*', line, 1)) for line in manifest.splitlines())
expected_entries = {
'Foo': 'foo-value',
'Implementation-Version': '1.2.3',
}
self.assertEquals(set(expected_entries.items()),
set(expected_entries.items()).intersection(set(all_entries.items())))
def setUp(self):
super(ClassmapTaskTest, self).setUp()
init_subsystem(Target.Arguments)
self.add_to_build_file(
'a',
'java_library(sources=["a1.java", "a2.java"])',
)
self.jar_artifact = self.create_artifact(org='org.example', name='foo', rev='1.0.0')
with open_zip(self.jar_artifact.pants_path, 'w') as jar:
jar.writestr('foo/Foo.class', '')
self.add_to_build_file(
'b',
'jar_library(jars=[jar(org="org.example", name="foo", rev="1.0.0")])',
)
self.add_to_build_file(
'c',
'java_library(dependencies=["a", "b"])',
)
self.target_a = self.target('a')
self.target_b = self.target('b')
self.target_c = self.target('c')
def safe_classpath(classpath, synthetic_jar_dir):
"""Bundles classpath into one synthetic jar that includes original classpath in its manifest.
This is to ensure classpath length never exceeds platform ARG_MAX.
:param list classpath: Classpath to be bundled.
:param string synthetic_jar_dir: directory to store the synthetic jar, if `None`
a temp directory will be provided and cleaned up upon process exit. Otherwise synthetic
jar will remain in the supplied directory, only for debugging purpose.
:returns: A classpath (singleton list with just the synthetic jar).
:rtype: list of strings
"""
if synthetic_jar_dir:
safe_mkdir(synthetic_jar_dir)
else:
synthetic_jar_dir = safe_mkdtemp()
bundled_classpath = relativize_classpath(classpath, synthetic_jar_dir)
manifest = Manifest()
manifest.addentry(Manifest.CLASS_PATH, ' '.join(bundled_classpath))
with temporary_file(root_dir=synthetic_jar_dir, cleanup=False, suffix='.jar') as jar_file:
with open_zip(jar_file, mode='w', compression=ZIP_STORED) as jar:
jar.writestr(Manifest.PATH, manifest.contents())
return [jar_file.name]
def safe_classpath(classpath, synthetic_jar_dir, custom_name=None):
"""Bundles classpath into one synthetic jar that includes original classpath in its manifest.
This is to ensure classpath length never exceeds platform ARG_MAX.
:param list classpath: Classpath to be bundled.
:param string synthetic_jar_dir: directory to store the synthetic jar, if `None`
a temp directory will be provided and cleaned up upon process exit. Otherwise synthetic
jar will remain in the supplied directory, only for debugging purpose.
:param custom_name: filename of the synthetic jar to be created.
:returns: A classpath (singleton list with just the synthetic jar).
:rtype: list of strings
"""
if synthetic_jar_dir:
safe_mkdir(synthetic_jar_dir)
else:
synthetic_jar_dir = safe_mkdtemp()
# Quote the paths so that if they contain reserved characters can be safely passed to JVM classloader.
bundled_classpath = map(urllib.quote, relativize_classpath(classpath, synthetic_jar_dir))
manifest = Manifest()
manifest.addentry(Manifest.CLASS_PATH, ' '.join(bundled_classpath))
with temporary_file(root_dir=synthetic_jar_dir, cleanup=False, suffix='.jar') as jar_file:
with open_zip(jar_file, mode='w', compression=ZIP_STORED) as jar:
jar.writestr(Manifest.PATH, manifest.contents())
if custom_name:
custom_path = os.path.join(synthetic_jar_dir, custom_name)
safe_concurrent_rename(jar_file.name, custom_path)
return [custom_path]
else:
return [jar_file.name]
def test_manifest_items(self):
self.add_to_build_file('src/java/hello', dedent("""
jvm_binary(
name='hello',
main='hello.Hello',
manifest_entries = {
'Foo': 'foo-value',
'Implementation-Version': '1.2.3',
},
)""").strip())
binary_target = self.target('src/java/hello:hello')
context = self.context(target_roots=[binary_target])
self.add_to_runtime_classpath(context, binary_target, {'Hello.class': '0xDEADBEEF'})
jar_builder_task = self.prepare_execute(context)
with self.jarfile() as existing_jarfile:
with jar_builder_task.open_jar(existing_jarfile) as jar:
with jar_builder_task.create_jar_builder(jar) as jar_builder:
jar_builder.add_target(binary_target)
with open_zip(existing_jarfile) as jar:
manifest = jar.read('META-INF/MANIFEST.MF').strip()
all_entries = dict(tuple(re.split(r'\s*:\s*', line, 1)) for line in manifest.splitlines())
expected_entries = {
'Foo': 'foo-value',
'Implementation-Version': '1.2.3',
}
self.assertEquals(set(expected_entries.items()),
set(expected_entries.items()).intersection(set(all_entries.items())))
def sample_jarfile(self, name):
with temporary_dir() as temp_dir:
jar_name = os.path.join(temp_dir, '{}.jar'.format(name))
with open_zip(jar_name, 'w') as proto_jarfile:
proto_jarfile.writestr('a/b/c/{}.txt'.format(name), 'Some text')
proto_jarfile.writestr('a/b/c/{}.proto'.format(name), 'message Msg {}')
yield jar_name
def bundled_classpath(classpath):
"""Bundles classpath into one synthetic jar that includes original classpath in its manifest.
See https://docs.oracle.com/javase/7/docs/technotes/guides/extensions/spec.html#bundled
:param list classpath: Classpath to be bundled.
:returns: A classpath (singleton list with just the synthetic jar).
:rtype: list of strings
"""
def prepare_url(url):
url_in_bundle = os.path.realpath(url)
# append '/' for directories, those not ending with '/' are assumed to be jars
if os.path.isdir(url):
url_in_bundle += '/'
return url_in_bundle
bundled_classpath = [prepare_url(url) for url in classpath]
manifest = Manifest()
manifest.addentry(Manifest.CLASS_PATH, ' '.join(bundled_classpath))
with temporary_file(cleanup=False, suffix='.jar') as jar_file:
with open_zip(jar_file, mode='w', compression=ZIP_STORED) as jar:
jar.writestr(Manifest.PATH, manifest.contents())
yield [jar_file.name]
def _download_zip(self, zip_url, dest_dir):
"""Downloads a zip file at the given URL into the given directory.
:param str zip_url: Full URL pointing to zip file.
:param str dest_dir: Absolute path of directory into which the unzipped contents
will be placed into, not including the zip directory itself.
"""
# TODO(jsirois): Wrap with workunits, progress meters, checksums.
self.context.log.info('Downloading {}...'.format(zip_url))
sess = requests.session()
sess.mount('file://', self.LocalFileAdapter())
res = sess.get(zip_url)
if not res.status_code == requests.codes.ok:
raise TaskError('Failed to download {} ({} error)'.format(zip_url, res.status_code))
with open_zip(BytesIO(res.content)) as zfile:
safe_mkdir(dest_dir)
for info in zfile.infolist():
if info.filename.endswith('/'):
# Skip directories.
continue
# Strip zip directory name from files.
filename = os.path.relpath(info.filename, get_basedir(info.filename))
f = safe_open(os.path.join(dest_dir, filename), 'w')
f.write(zfile.read(info))
f.close()
def test_bundled_classpath(self):
"""This creates the following classpath
basedir/libs/A.jar:basedir/resources
"""
RESOURCES = 'resources'
LIB_DIR = 'libs'
JAR_FILE = 'A.jar'
basedir = safe_mkdtemp()
lib_dir = os.path.join(basedir, LIB_DIR)
resource_dir = os.path.join(lib_dir, RESOURCES)
jar_file = os.path.join(lib_dir, JAR_FILE)
for dir in (lib_dir, resource_dir):
safe_mkdir(dir)
touch(jar_file)
classpath = [jar_file, resource_dir]
with bundled_classpath(classpath) as bundled_cp:
self.assertEquals(1, len(bundled_cp))
bundled_jar = bundled_cp[0]
self.assertTrue(os.path.exists(bundled_jar))
with open_zip(bundled_jar) as synthetic_jar:
self.assertListEqual([Manifest.PATH], synthetic_jar.namelist())
# manifest should contain the absolute path of both jar and resource directory
self.assertEquals('{}: {} {}/\n'.format(Manifest.CLASS_PATH, os.path.realpath(jar_file),
os.path.realpath(resource_dir)),
synthetic_jar.read(Manifest.PATH).replace('\n ', ''))
safe_rmtree(resource_dir)
def _maybe_get_plugin_name(cls, classpath_element):
"""If classpath_element is a scalac plugin, returns its name.
Returns None otherwise.
"""
def process_info_file(cp_elem, info_file):
plugin_info = ElementTree.parse(info_file).getroot()
if plugin_info.tag != 'plugin':
raise TaskError('File {} in {} is not a valid scalac plugin descriptor'.format(
_SCALAC_PLUGIN_INFO_FILE, cp_elem))
return plugin_info.find('name').text
if os.path.isdir(classpath_element):
try:
with open(os.path.join(classpath_element, _SCALAC_PLUGIN_INFO_FILE)) as plugin_info_file:
return process_info_file(classpath_element, plugin_info_file)
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
with open_zip(classpath_element, 'r') as jarfile:
try:
with closing(jarfile.open(_SCALAC_PLUGIN_INFO_FILE, 'r')) as plugin_info_file:
return process_info_file(classpath_element, plugin_info_file)
except KeyError:
pass
return None
def test_scala_compile_jar(self):
# NB: generated with:
# hashlib.sha1('testprojects.src.scala.org.pantsbuild.testproject.unicode.shapeless.shapeless').hexdigest()[:12]
jar_suffix = "fd9f49e1153b.jar"
with self.do_test_compile(SHAPELESS_TARGET, expected_files=[jar_suffix]) as found:
with open_zip(self.get_only(found, jar_suffix), "r") as jar:
self.assertTrue(jar.getinfo(SHAPELESS_CLSFILE), "Expected a jar containing the expected class.")
def _find_plugins(self):
"""Returns a map from plugin name to plugin jar."""
# Allow multiple flags and also comma-separated values in a single flag.
plugin_names = set([p for val in self.get_options().scalac_plugins for p in val.split(',')])
plugins = {}
buildroot = get_buildroot()
for jar in self.plugin_jars:
with open_zip(jar, 'r') as jarfile:
try:
with closing(jarfile.open(_PLUGIN_INFO_FILE, 'r')) as plugin_info_file:
plugin_info = ElementTree.parse(plugin_info_file).getroot()
if plugin_info.tag != 'plugin':
raise TaskError(
'File {} in {} is not a valid scalac plugin descriptor'.format(_PLUGIN_INFO_FILE,
jar))
name = plugin_info.find('name').text
if name in plugin_names:
if name in plugins:
raise TaskError('Plugin {} defined in {} and in {}'.format(name, plugins[name], jar))
# It's important to use relative paths, as the compiler flags get embedded in the zinc
# analysis file, and we port those between systems via the artifact cache.
plugins[name] = os.path.relpath(jar, buildroot)
except KeyError:
pass
unresolved_plugins = plugin_names - set(plugins.keys())
if unresolved_plugins:
raise TaskError('Could not find requested plugins: {}'.format(list(unresolved_plugins)))
return plugins
def populate_input_jar(self, *entries):
fd, input_jar_path = tempfile.mkstemp()
os.close(fd)
self.addCleanup(safe_delete, input_jar_path)
with open_zip(input_jar_path, 'w') as jar:
for entry in entries:
jar.writestr(entry, '0xCAFEBABE')
return input_jar_path
def sample_jarfile(self):
"""Create a jar file with a/b/c/data.txt and a/b/c/foo.proto"""
with temporary_dir() as temp_dir:
jar_name = os.path.join(temp_dir, 'foo.jar')
with open_zip(jar_name, 'w') as proto_jarfile:
proto_jarfile.writestr('a/b/c/data.txt', 'Foo text')
proto_jarfile.writestr('a/b/c/foo.proto', 'message Foo {}')
yield jar_name
def test_overwrite_jars(self):
with self.jarfile() as main_jar:
with self.jarfile() as included_jar:
with self.jar_task.open_jar(main_jar) as jar:
jar.writestr('a/b', b'c')
with self.jar_task.open_jar(included_jar) as jar:
jar.writestr('e/f', b'g')
# Create lots of included jars (even though they're all the same)
# so the -jars argument to jar-tool will exceed max_args limit thus
# switch to @argfile calling style.
with self.jar_task.open_jar(main_jar, overwrite=True) as jar:
for i in range(self.MAX_SUBPROC_ARGS + 1):
jar.writejar(included_jar)
with open_zip(main_jar) as jar:
self.assert_listing(jar, 'e/', 'e/f')
def create(self, basedir, outdir, name, prefix=None):
"""
:API: public
"""
zippath = os.path.join(outdir, '{}.{}'.format(name, self.extension))
with open_zip(zippath, 'w', compression=self.compression) as zip:
# For symlinks, we want to archive the actual content of linked files but
# under the relpath derived from symlink.
for root, _, files in safe_walk(basedir, followlinks=True):
root = ensure_text(root)
for file in files:
file = ensure_text(file)
full_path = os.path.join(root, file)
relpath = os.path.relpath(full_path, basedir)
if prefix:
relpath = os.path.join(ensure_text(prefix), relpath)
zip.write(full_path, relpath)
return zippath
def assert_jar_contents(self, context, product_type, target, *contents):
"""Contents is a list of lists representing contents from particular classpath entries.
Ordering across classpath entries is guaranteed, but not within classpath entries.
"""
jar_mapping = context.products.get(product_type).get(target)
self.assertEqual(1, len(jar_mapping))
for basedir, jars in jar_mapping.items():
self.assertEqual(1, len(jars))
with open_zip(os.path.join(basedir, jars[0])) as jar:
actual_iter = iter(jar.namelist())
self.assertPrefixEqual(['META-INF/', 'META-INF/MANIFEST.MF'], actual_iter)
for content_set in list(contents):
self.assertUnorderedPrefixEqual(content_set, actual_iter)
for content in content_set:
if not content.endswith('/'):
with closing(jar.open(content)) as fp:
self.assertEqual(os.path.basename(content), fp.read())
def extract(cls, path, outdir, filter_func=None):
"""Extract from a zip file, with an optional filter
:API: public
:param string path: path to the zipfile to extract from
:param string outdir: directory to extract files into
:param function filter_func: optional filter with the filename as the parameter. Returns True if
the file should be extracted.
"""
with open_zip(path) as archive_file:
for name in archive_file.namelist():
# While we're at it, we also perform this safety test.
if name.startswith(b'/') or name.startswith(b'..'):
raise ValueError(
'Zip file contains unsafe path: {}'.format(name))
if (not filter_func or filter_func(name)):
archive_file.extract(name, outdir)
def test_safe_classpath(self):
"""For directory structure like:
./
./libs/A.jar
./libs/resources/
./synthetic_jar_dir
Verify a synthetic jar with the following classpath in manifest is created:
Class-Path: ../libs/A.jar:../libs/resources/
"""
RESOURCES = 'resources'
LIB_DIR = 'libs'
JAR_FILE = 'A.jar'
SYNTHENTIC_JAR_DIR = 'synthetic_jar_dir'
basedir = safe_mkdtemp()
lib_dir = os.path.join(basedir, LIB_DIR)
synthetic_jar_dir = os.path.join(basedir, SYNTHENTIC_JAR_DIR)
resource_dir = os.path.join(lib_dir, RESOURCES)
jar_file = os.path.join(lib_dir, JAR_FILE)
for dir in (lib_dir, resource_dir, synthetic_jar_dir):
safe_mkdir(dir)
touch(jar_file)
classpath = [jar_file, resource_dir]
safe_cp = safe_classpath(classpath, synthetic_jar_dir)
self.assertEqual(1, len(safe_cp))
safe_jar = safe_cp[0]
self.assertTrue(os.path.exists(safe_jar))
self.assertEqual(synthetic_jar_dir, os.path.dirname(safe_jar))
with open_zip(safe_jar) as synthetic_jar:
self.assertEqual([Manifest.PATH], synthetic_jar.namelist())
# manifest should contain the relative path of both jar and resource directory
expected = ('{}: ../{}/{} ../{}/{}/\n'.format(
Manifest.CLASS_PATH, LIB_DIR, JAR_FILE, LIB_DIR,
RESOURCES).encode('utf-8'))
self.assertEqual(
expected,
synthetic_jar.read(Manifest.PATH).replace(b'\n ', b''))
def test_agent_manifest(self):
self.add_to_build_file(
'src/java/pants/agents',
dedent("""
java_agent(
name='fake_agent',
premain='bob',
agent_class='fred',
can_redefine=True,
can_retransform=True,
can_set_native_method_prefix=True
)""").strip())
java_agent = self.target('src/java/pants/agents:fake_agent')
context = self.context(target_roots=[java_agent])
jar_builder_task = self.prepare_execute(context)
self.add_to_runtime_classpath(context, java_agent,
{'FakeAgent.class': '0xCAFEBABE'})
with self.jarfile() as existing_jarfile:
with jar_builder_task.open_jar(existing_jarfile) as jar:
with jar_builder_task.create_jar_builder(jar) as jar_builder:
jar_builder.add_target(java_agent)
with open_zip(existing_jarfile) as jar:
self.assert_listing(jar, 'FakeAgent.class')
self.assertEqual(b'0xCAFEBABE', jar.read('FakeAgent.class'))
manifest = jar.read('META-INF/MANIFEST.MF').decode(
'utf-8').strip()
all_entries = dict(
tuple(re.split(r'\s*:\s*', line, 1))
for line in manifest.splitlines())
expected_entries = {
'Agent-Class': 'fred',
'Premain-Class': 'bob',
'Can-Redefine-Classes': 'true',
'Can-Retransform-Classes': 'true',
'Can-Set-Native-Method-Prefix': 'true',
}
self.assertEqual(
set(expected_entries.items()),
set(expected_entries.items()).intersection(
set(all_entries.items())))
def test_compile_wire_roots(self):
pants_run = self.run_pants(['bundle.jvm', '--deployjar',
'examples/src/java/org/pantsbuild/example/wire/roots'])
self.assert_success(pants_run)
out_path = os.path.join(get_buildroot(), 'dist', 'wire-roots-example.jar')
with open_zip(out_path) as zipfile:
jar_entries = zipfile.namelist()
def is_relevant(entry):
return (entry.startswith('org/pantsbuild/example/roots/') and entry.endswith('.class')
and '$' not in entry)
expected_classes = {
'org/pantsbuild/example/roots/Bar.class',
'org/pantsbuild/example/roots/Foobar.class',
'org/pantsbuild/example/roots/Fooboo.class',
}
received_classes = {entry for entry in jar_entries if is_relevant(entry)}
self.assertEqual(expected_classes, received_classes)
def assert_classpath(classpath):
with self.jarfile() as existing_jarfile:
# Note for -classpath, there is no update, it's already overwriting.
# To verify this, first add a random classpath, and verify it's overwritten by
# the supplied classpath value.
with self.jar_task.open_jar(existing_jarfile) as jar:
# prefix with workdir since Class-Path is relative to jarfile.path
jar.append_classpath(
os.path.join(self.workdir,
"something_should_be_overwritten.jar"))
with self.jar_task.open_jar(existing_jarfile) as jar:
jar.append_classpath([
os.path.join(self.workdir, jar_path)
for jar_path in classpath
])
with open_zip(existing_jarfile) as jar:
self.assertEqual(manifest_content(classpath),
jar.read("META-INF/MANIFEST.MF"))
def extract(cls, path, outdir, filter_func=None):
"""Extract from a zip file, with an optional filter
:param string path: path to the zipfile to extract from
:param string outdir: directory to extract files into
:param function filter_func: optional filter with the filename as the parameter. Returns True if
the file should be extracted.
"""
with open_zip(path) as archive_file:
for name in archive_file.namelist():
# While we're at it, we also perform this safety test.
if name.startswith(b'/') or name.startswith(b'..'):
raise ValueError('Zip file contains unsafe path: {}'.format(name))
# Ignore directories. extract() will create parent dirs as needed.
# OS X's python 2.6.1 has a bug in zipfile that makes it unzip directories as regular files.
# This method should work on for python 2.6-3.x.
# TODO(Eric Ayers) Pants no longer builds with python 2.6. Can this be removed?
if not name.endswith(b'/'):
if (not filter_func or filter_func(name)):
archive_file.extract(name, outdir)
def publish(resource_content):
with temporary_dir() as publish_dir:
with self.temporary_file_content(resource, resource_content):
# Validate that the target depends on the relevant resource.
self.assertIn(resource, self.run_pants(['filedeps', target]).stdout_data)
pants_run = self.run_pants_with_workdir(['publish.jar',
'--local={}'.format(publish_dir),
'--named-snapshot=X',
'--no-dryrun',
target
],
workdir=workdir)
self.assert_success(pants_run)
# Validate that the content in the resulting jar matches.
jar = os.path.join(publish_dir,
'org/pantsbuild/testproject/publish/hello-greet/X/hello-greet-X.jar')
with open_zip(jar, mode='r') as j:
with j.open(resource_relative_to_sourceroot) as jar_entry:
self.assertEqual(resource_content, jar_entry.read())
def test_agent_dependency(self):
directory = "testprojects/src/java/org/pantsbuild/testproject/manifest"
target = "{}:manifest-with-agent".format(directory)
with self.temporary_workdir() as workdir:
pants_run = self.run_pants_with_workdir(["binary", target],
workdir=workdir)
self.assert_success(pants_run)
jar = "dist/manifest-with-agent.jar"
with open_zip(jar, mode='r') as j:
with j.open("META-INF/MANIFEST.MF") as jar_entry:
normalized_lines = (line.decode('utf-8').strip()
for line in jar_entry.readlines()
if line.strip())
entries = {
tuple(line.split(": ", 2))
for line in normalized_lines
}
self.assertIn(
('Agent-Class',
'org.pantsbuild.testproject.manifest.Agent'), entries)
def test_deploy_excludes(self):
with temporary_dir() as distdir:
def build(name):
jar_filename = os.path.join(distdir, f"{name}.jar")
command = [
f"--pants-distdir={distdir}",
"--no-compile-rsc-capture-classpath",
"binary",
f"testprojects/src/java/org/pantsbuild/testproject/deployexcludes:{name}",
]
self.assert_success(self.run_pants(command))
return jar_filename
# The excluded binary should not contain any guava classes, and should fail to run.
jar_filename = build("deployexcludes")
with open_zip(jar_filename) as jar_file:
self.assertEqual(
{
"META-INF/",
"META-INF/MANIFEST.MF",
"org/",
"org/pantsbuild/",
"org/pantsbuild/testproject/",
"org/pantsbuild/testproject/deployexcludes/",
"org/pantsbuild/testproject/deployexcludes/DeployExcludesMain.class",
},
set(jar_file.namelist()),
)
self.run_java(
java_args=["-jar", jar_filename],
expected_returncode=1,
expected_output="java.lang.NoClassDefFoundError: "
"com/google/common/collect/ImmutableSortedSet",
)
# And the non excluded binary should succeed.
jar_filename = build("nodeployexcludes")
self.run_java(
java_args=["-jar", jar_filename], expected_output="DeployExcludes Hello World"
)
def test_manifest_items(self):
self.add_to_build_file(
'src/java/hello',
dedent("""
jvm_binary(
name='hello',
main='hello.Hello',
manifest_entries = {
'Foo': 'foo-value',
'Implementation-Version': '1.2.3',
},
)""").strip())
binary_target = self.target('src/java/hello:hello')
context = self.context(target_roots=[binary_target])
classfile = '.pants.d/javac/classes/hello/Hello.class'
self.create_file(classfile, '0xDEADBEEF')
self._add_to_classes_by_target(context, binary_target, classfile)
context.products.safe_create_data(
'resources_by_target', lambda: defaultdict(MultipleRootedProducts))
jar_task = self.prepare_jar_task(context)
with self.jarfile() as existing_jarfile:
with jar_task.open_jar(existing_jarfile) as jar:
with jar_task.create_jar_builder(jar) as jar_builder:
jar_builder.add_target(binary_target)
with open_zip(existing_jarfile) as jar:
manifest = jar.read('META-INF/MANIFEST.MF').strip()
all_entries = dict(
tuple(re.split(r'\s*:\s*', line, 1))
for line in manifest.splitlines())
expected_entries = {
'Foo': 'foo-value',
'Implementation-Version': '1.2.3',
}
self.assertEquals(
set(expected_entries.items()),
set(expected_entries.items()).intersection(
set(all_entries.items())))
def test_deploy_excludes(self):
jar_filename = os.path.join('dist', 'deployexcludes.jar')
safe_delete(jar_filename)
command = [
'--no-compile-zinc-capture-classpath',
'binary',
'testprojects/src/java/org/pantsbuild/testproject/deployexcludes',
]
with self.pants_results(command) as pants_run:
self.assert_success(pants_run)
# The resulting binary should not contain any guava classes
with open_zip(jar_filename) as jar_file:
self.assertEquals(
{
'META-INF/', 'META-INF/MANIFEST.MF', 'org/',
'org/pantsbuild/', 'org/pantsbuild/testproject/',
'org/pantsbuild/testproject/deployexcludes/',
'org/pantsbuild/testproject/deployexcludes/DeployExcludesMain.class'
}, set(jar_file.namelist()))
# This jar should not run by itself, missing symbols
self.run_java(java_args=['-jar', jar_filename],
expected_returncode=1,
expected_output='java.lang.NoClassDefFoundError: '
'com/google/common/collect/ImmutableSortedSet')
# But adding back the deploy_excluded symbols should result in a clean run.
classpath = [
jar_filename,
os.path.join(
pants_run.workdir,
'ivy/jars/com.google.guava/guava/jars/guava-18.0.jar')
]
self.run_java(java_args=[
'-cp',
os.pathsep.join(classpath),
'org.pantsbuild.testproject.deployexcludes.DeployExcludesMain'
],
expected_output='DeployExcludes Hello World')
def safe_classpath(classpath, synthetic_jar_dir, custom_name=None):
"""Bundles classpath into one synthetic jar that includes original classpath in its manifest.
This is to ensure classpath length never exceeds platform ARG_MAX.
:param list classpath: Classpath to be bundled.
:param string synthetic_jar_dir: directory to store the synthetic jar, if `None`
a temp directory will be provided and cleaned up upon process exit. Otherwise synthetic
jar will remain in the supplied directory, only for debugging purpose.
:param custom_name: filename of the synthetic jar to be created.
:returns: A classpath (singleton list with just the synthetic jar).
:rtype: list of strings
"""
if synthetic_jar_dir:
safe_mkdir(synthetic_jar_dir)
else:
synthetic_jar_dir = safe_mkdtemp()
# Quote the paths so that if they contain reserved characters can be safely passed to JVM classloader.
bundled_classpath = [
parse.quote(cp)
for cp in relativize_classpath(classpath, synthetic_jar_dir)
]
manifest = Manifest()
manifest.addentry(Manifest.CLASS_PATH, ' '.join(bundled_classpath))
with temporary_file(root_dir=synthetic_jar_dir,
cleanup=False,
suffix='.jar') as jar_file:
with open_zip(jar_file, mode='w', compression=ZIP_STORED) as jar:
jar.writestr(Manifest.PATH, manifest.contents())
if custom_name:
custom_path = os.path.join(synthetic_jar_dir, custom_name)
safe_concurrent_rename(jar_file.name, custom_path)
return [custom_path]
else:
return [jar_file.name]
def test_manifest_items(self):
self.add_to_build_file(
"src/java/hello",
dedent("""
jvm_binary(
name='hello',
main='hello.Hello',
manifest_entries = {
'Foo': 'foo-value',
'Implementation-Version': '1.2.3',
},
)""").strip(),
)
binary_target = self.target("src/java/hello:hello")
context = self.context(target_roots=[binary_target])
self.add_to_runtime_classpath(context, binary_target,
{"Hello.class": "0xDEADBEEF"})
jar_builder_task = self.prepare_execute(context)
with self.jarfile() as existing_jarfile:
with jar_builder_task.open_jar(existing_jarfile) as jar:
with jar_builder_task.create_jar_builder(jar) as jar_builder:
jar_builder.add_target(binary_target)
with open_zip(existing_jarfile) as jar:
manifest = jar.read("META-INF/MANIFEST.MF").decode().strip()
all_entries = dict(
tuple(re.split(r"\s*:\s*", line, 1))
for line in manifest.splitlines())
expected_entries = {
"Foo": "foo-value",
"Implementation-Version": "1.2.3",
}
self.assertEqual(
set(expected_entries.items()),
set(expected_entries.items()).intersection(
set(all_entries.items())),
)
def test_ipex_gets_imprecise_constraint(self) -> None:
cur_interpreter_id = PythonInterpreter.get().identity
interpreter_name = cur_interpreter_id.requirement.name
major, minor, patch = cur_interpreter_id.version
# Pin the selected interpreter to the one used by pants to execute this test.
cur_interpreter_constraint = f"{interpreter_name}=={major}.{minor}.{patch}"
# Validate the the .ipex file specifically matches the major and minor versions, but allows
# any patch version.
imprecise_constraint = f"{interpreter_name}=={major}.{minor}.*"
with temporary_dir() as tmp_dir:
self.do_command(
"--binary-py-generate-ipex",
"binary",
self.binary_target_address,
config={
"GLOBAL": {
"pants_distdir": tmp_dir
},
"python-setup": {
"interpreter_constraints":
[cur_interpreter_constraint]
},
},
)
pex_path = os.path.join(tmp_dir, "test.ipex")
assert os.path.isfile(pex_path)
pex_execution_result = subprocess.run([pex_path],
stdout=subprocess.PIPE,
check=True)
assert pex_execution_result.stdout.decode() == "test!\n"
with open_zip(pex_path) as zf:
info = json.loads(zf.read("PEX-INFO"))
constraint = assert_single_element(
info["interpreter_constraints"])
assert constraint == imprecise_constraint
def _get_external_dependencies(self, binary_target):
artifacts_by_file_name = defaultdict(set)
for basedir, externaljar in self.list_external_jar_dependencies(
binary_target):
external_dep = os.path.join(basedir, externaljar)
self.context.log.debug(' scanning {}'.format(external_dep))
with open_zip(external_dep) as dep_zip:
for qualified_file_name in dep_zip.namelist():
# Zip entry names can come in any encoding and in practice we find some jars that have
# utf-8 encoded entry names, some not. As a result we cannot simply decode in all cases
# and need to do this to_bytes(...).decode('utf-8') dance to stay safe across all entry
# name flavors and under all supported pythons.
decoded_file_name = to_bytes(qualified_file_name).decode(
'utf-8')
if os.path.basename(
decoded_file_name).lower() in self._excludes:
continue
jar_name = os.path.basename(external_dep)
if (not self._isdir(decoded_file_name)
) and Manifest.PATH != decoded_file_name:
artifacts_by_file_name[decoded_file_name].add(jar_name)
return artifacts_by_file_name
def setUp(self):
super(ClassmapTaskTest, self).setUp()
init_subsystem(Target.Arguments)
self.target_a = self.make_target('a',
target_type=JavaLibrary,
sources=['a1.java', 'a2.java'])
self.jar_artifact = self.create_artifact(org='org.example',
name='foo',
rev='1.0.0')
with open_zip(self.jar_artifact.pants_path, 'w') as jar:
jar.writestr('foo/Foo.class', '')
self.target_b = self.make_target(
'b',
target_type=JarLibrary,
jars=[JarDependency(org='org.example', name='foo', rev='1.0.0')])
self.target_c = self.make_target(
'c',
dependencies=[self.target_a, self.target_b],
target_type=JavaLibrary)
def test_compile_wire_roots(self):
pants_run = self.run_pants(
["binary.jvm", "examples/src/java/org/pantsbuild/example/wire/roots"]
)
self.assert_success(pants_run)
out_path = os.path.join(get_buildroot(), "dist", "wire-roots-example.jar")
with open_zip(out_path) as zipfile:
jar_entries = zipfile.namelist()
def is_relevant(entry):
return (
entry.startswith("org/pantsbuild/example/roots/")
and entry.endswith(".class")
and "$" not in entry
)
expected_classes = {
"org/pantsbuild/example/roots/Bar.class",
"org/pantsbuild/example/roots/Foobar.class",
"org/pantsbuild/example/roots/Fooboo.class",
}
received_classes = {entry for entry in jar_entries if is_relevant(entry)}
self.assertEqual(expected_classes, received_classes)
def _find_plugins(self):
"""Returns a map from plugin name to plugin jar."""
# Allow multiple flags and also comma-separated values in a single flag.
plugin_names = set([
p for val in self.get_options().scalac_plugins
for p in val.split(',')
])
plugins = {}
buildroot = get_buildroot()
for jar in self.plugin_jars:
with open_zip(jar, 'r') as jarfile:
try:
with closing(jarfile.open(_PLUGIN_INFO_FILE,
'r')) as plugin_info_file:
plugin_info = ElementTree.parse(
plugin_info_file).getroot()
if plugin_info.tag != 'plugin':
raise TaskError(
'File {} in {} is not a valid scalac plugin descriptor'
.format(_PLUGIN_INFO_FILE, jar))
name = plugin_info.find('name').text
if name in plugin_names:
if name in plugins:
raise TaskError(
'Plugin {} defined in {} and in {}'.format(
name, plugins[name], jar))
# It's important to use relative paths, as the compiler flags get embedded in the zinc
# analysis file, and we port those between systems via the artifact cache.
plugins[name] = os.path.relpath(jar, buildroot)
except KeyError:
pass
unresolved_plugins = plugin_names - set(plugins.keys())
if unresolved_plugins:
raise TaskError('Could not find requested plugins: {}'.format(
list(unresolved_plugins)))
return plugins
def safe_classpath(classpath, synthetic_jar_dir):
"""Bundles classpath into one synthetic jar that includes original classpath in its manifest.
This is to ensure classpath length never exceeds platform ARG_MAX. Original classpath are
converted to URLs relative to synthetic jar path and saved in its manifest as attribute
`Class-Path`. See
https://docs.oracle.com/javase/7/docs/technotes/guides/extensions/spec.html#bundled
:param list classpath: Classpath to be bundled.
:param string synthetic_jar_dir: directory to store the synthetic jar, if `None`
a temp directory will be provided and cleaned up upon process exit. Otherwise synthetic
jar will remain in the supplied directory, only for debugging purpose.
:returns: A classpath (singleton list with just the synthetic jar).
:rtype: list of strings
"""
def prepare_url(url, root_dir):
url_in_bundle = os.path.relpath(os.path.realpath(url), os.path.realpath(root_dir))
# append '/' for directories, those not ending with '/' are assumed to be jars
if os.path.isdir(url):
url_in_bundle += '/'
return url_in_bundle
if synthetic_jar_dir:
safe_mkdir(synthetic_jar_dir)
else:
synthetic_jar_dir = safe_mkdtemp()
bundled_classpath = [prepare_url(url, synthetic_jar_dir) for url in classpath]
manifest = Manifest()
manifest.addentry(Manifest.CLASS_PATH, ' '.join(bundled_classpath))
with temporary_file(root_dir=synthetic_jar_dir, cleanup=False, suffix='.jar') as jar_file:
with open_zip(jar_file, mode='w', compression=ZIP_STORED) as jar:
jar.writestr(Manifest.PATH, manifest.contents())
return [jar_file.name]
def _check_bundle_products(self, bundle_basename):
products = self.task_context.products.get('jvm_bundles')
self.assertIsNotNone(products)
product_data = products.get(self.app_target)
self.assertEquals({self.dist_root: ['{basename}-bundle'.format(basename=bundle_basename)]},
product_data)
self.assertTrue(os.path.exists(self.dist_root))
bundle_root = os.path.join(self.dist_root,
'{basename}-bundle'.format(basename=bundle_basename))
self.assertEqual(sorted(['foo-binary.jar',
'libs/foo.foo-binary-0.jar',
'libs/3rdparty.jvm.org.example.foo-0.jar',
'libs/3rdparty.jvm.org.example.foo-1.zip',
'libs/3rdparty.jvm.org.example.foo-2.jar',
'libs/3rdparty.jvm.org.example.foo-3.gz']),
sorted(self.iter_files(bundle_root)))
check_zip_file_content(os.path.join(bundle_root, 'libs/foo.foo-binary-0.jar'),
self.FOO_JAR)
# TODO verify Manifest's Class-Path
with open_zip(os.path.join(bundle_root, 'foo-binary.jar')) as jar:
self.assertEqual(sorted(['META-INF/', 'META-INF/MANIFEST.MF']),
sorted(jar.namelist()))
def _compute_classpath_elements_by_class(self, classpath):
# Don't consider loose classes dirs in our classes dir. Those will be considered
# separately, by looking at products.
def non_product(path):
return path != self._classes_dir
if self._upstream_class_to_path is None:
self._upstream_class_to_path = {}
classpath_entries = filter(non_product, classpath)
for cp_entry in self._find_all_bootstrap_jars() + classpath_entries:
# Per the classloading spec, a 'jar' in this context can also be a .zip file.
if os.path.isfile(cp_entry) and (cp_entry.endswith('.jar') or cp_entry.endswith('.zip')):
with open_zip(cp_entry, 'r') as jar:
for cls in jar.namelist():
# First jar with a given class wins, just like when classloading.
if cls.endswith(b'.class') and not cls in self._upstream_class_to_path:
self._upstream_class_to_path[cls] = cp_entry
elif os.path.isdir(cp_entry):
for dirpath, _, filenames in safe_walk(cp_entry, followlinks=True):
for f in filter(lambda x: x.endswith('.class'), filenames):
cls = os.path.relpath(os.path.join(dirpath, f), cp_entry)
if not cls in self._upstream_class_to_path:
self._upstream_class_to_path[cls] = os.path.join(dirpath, f)
return self._upstream_class_to_path
def setUp(self):
super().setUp()
init_subsystem(Target.Arguments)
self.create_files("a", files=["a1.java", "a2.java"])
self.add_to_build_file(
"a", 'java_library(sources=["a1.java", "a2.java"])',
)
self.jar_artifact = self.create_artifact(org="org.example", name="foo", rev="1.0.0")
with open_zip(self.jar_artifact.pants_path, "w") as jar:
jar.writestr("foo/Foo.class", "")
self.add_to_build_file(
"b", 'jar_library(jars=[jar(org="org.example", name="foo", rev="1.0.0")])',
)
self.add_to_build_file(
"c", 'java_library(dependencies=["a", "b"], sources=[])',
)
self.target_a = self.target("a")
self.target_b = self.target("b")
self.target_c = self.target("c")
def _jar_classfiles(self, jar_file):
"""Returns an iterator over the classfiles inside jar_file."""
with open_zip(jar_file, 'r') as jar:
for cls in jar.namelist():
if cls.endswith(b'.class'):
yield cls
def test_open_zipFalse(self):
with temporary_dir() as tempdir:
with open_zip(os.path.join(tempdir, 'test'), 'w',
allowZip64=False) as zf:
self.assertFalse(zf._allowZip64)
def test_open_zip_raises_exception_on_falsey_paths(self):
falsey = (None, '', False)
for invalid in falsey:
with self.assertRaises(InvalidZipPath):
open_zip(invalid).gen.next()
def test_jvm_binaries_deploy_excludes(self):
self.add_to_build_file(
"3rdparty/jvm/org/example",
'jar_library(name = "foo", jars = [jar(org = "org.example", name = "foo", rev = "1.0.0")])',
)
foo_jar_lib = self.target("3rdparty/jvm/org/example:foo")
self.create_file("bar/Bar.java")
self.add_to_build_file(
"bar",
"""jvm_binary(
name = "bar-binary",
sources = ["Bar.java"],
dependencies = ["3rdparty/jvm/org/example:foo"],
deploy_excludes = [exclude(org = "org.pantsbuild")],
)""",
)
binary_target = self.target("//bar:bar-binary")
context = self.context(target_roots=[binary_target])
classpath_products = self.ensure_classpath_products(context)
foo_artifact = self.create_artifact(org="org.example",
name="foo",
rev="1.0.0")
with open_zip(foo_artifact.pants_path, "w") as jar:
jar.writestr("foo/Foo.class", "")
baz_artifact = self.create_artifact(org="org.pantsbuild",
name="baz",
rev="2.0.0")
with open_zip(baz_artifact.pants_path, "w") as jar:
# This file should not be included in the binary jar since org.pantsbuild is deploy excluded.
jar.writestr("baz/Baz.class", "")
classpath_products.add_jars_for_targets(
targets=[foo_jar_lib],
conf="default",
resolved_jars=[foo_artifact, baz_artifact])
self.add_to_runtime_classpath(context, binary_target, {
"Bar.class": "",
"bar.txt": ""
})
self.execute(context)
jvm_binary_products = context.products.get("jvm_binaries")
self.assertIsNotNone(jvm_binary_products)
product_data = jvm_binary_products.get(binary_target)
dist_root = os.path.join(self.build_root, "dist")
self.assertEqual({dist_root: ["bar-binary.jar"]}, product_data)
with open_zip(os.path.join(dist_root, "bar-binary.jar")) as jar:
self.assertEqual(
sorted([
"META-INF/",
"META-INF/MANIFEST.MF",
"foo/",
"foo/Foo.class",
"Bar.class",
"bar.txt",
]),
sorted(jar.namelist()),
)
def open_jar(self, mode):
with open_zip(self.jar_file, mode=mode,
compression=zipfile.ZIP_STORED) as jar:
yield jar