def safe_classpath(classpath, synthetic_jar_dir, custom_name=None):
"""Bundles classpath into one synthetic jar that includes original classpath in its manifest.
This is to ensure classpath length never exceeds platform ARG_MAX.
:param list classpath: Classpath to be bundled.
:param string synthetic_jar_dir: directory to store the synthetic jar, if `None`
a temp directory will be provided and cleaned up upon process exit. Otherwise synthetic
jar will remain in the supplied directory, only for debugging purpose.
:param custom_name: filename of the synthetic jar to be created.
:returns: A classpath (singleton list with just the synthetic jar).
:rtype: list of strings
"""
if synthetic_jar_dir:
safe_mkdir(synthetic_jar_dir)
else:
synthetic_jar_dir = safe_mkdtemp()
# Quote the paths so that if they contain reserved characters can be safely passed to JVM classloader.
bundled_classpath = map(urllib.quote, relativize_classpath(classpath, synthetic_jar_dir))
manifest = Manifest()
manifest.addentry(Manifest.CLASS_PATH, ' '.join(bundled_classpath))
with temporary_file(root_dir=synthetic_jar_dir, cleanup=False, suffix='.jar') as jar_file:
with open_zip(jar_file, mode='w', compression=ZIP_STORED) as jar:
jar.writestr(Manifest.PATH, manifest.contents())
if custom_name:
custom_path = os.path.join(synthetic_jar_dir, custom_name)
safe_concurrent_rename(jar_file.name, custom_path)
return [custom_path]
else:
return [jar_file.name]
def safe_classpath(classpath, synthetic_jar_dir):
"""Bundles classpath into one synthetic jar that includes original classpath in its manifest.
This is to ensure classpath length never exceeds platform ARG_MAX.
:param list classpath: Classpath to be bundled.
:param string synthetic_jar_dir: directory to store the synthetic jar, if `None`
a temp directory will be provided and cleaned up upon process exit. Otherwise synthetic
jar will remain in the supplied directory, only for debugging purpose.
:returns: A classpath (singleton list with just the synthetic jar).
:rtype: list of strings
"""
if synthetic_jar_dir:
safe_mkdir(synthetic_jar_dir)
else:
synthetic_jar_dir = safe_mkdtemp()
bundled_classpath = relativize_classpath(classpath, synthetic_jar_dir)
manifest = Manifest()
manifest.addentry(Manifest.CLASS_PATH, ' '.join(bundled_classpath))
with temporary_file(root_dir=synthetic_jar_dir, cleanup=False, suffix='.jar') as jar_file:
with open_zip(jar_file, mode='w', compression=ZIP_STORED) as jar:
jar.writestr(Manifest.PATH, manifest.contents())
return [jar_file.name]
def bundled_classpath(classpath):
"""Bundles classpath into one synthetic jar that includes original classpath in its manifest.
See https://docs.oracle.com/javase/7/docs/technotes/guides/extensions/spec.html#bundled
:param list classpath: Classpath to be bundled.
:returns: A classpath (singleton list with just the synthetic jar).
:rtype: list of strings
"""
def prepare_url(url):
url_in_bundle = os.path.realpath(url)
# append '/' for directories, those not ending with '/' are assumed to be jars
if os.path.isdir(url):
url_in_bundle += '/'
return url_in_bundle
bundled_classpath = [prepare_url(url) for url in classpath]
manifest = Manifest()
manifest.addentry(Manifest.CLASS_PATH, ' '.join(bundled_classpath))
with temporary_file(cleanup=False, suffix='.jar') as jar_file:
with open_zip(jar_file, mode='w', compression=ZIP_STORED) as jar:
jar.writestr(Manifest.PATH, manifest.contents())
yield [jar_file.name]
def safe_classpath(classpath, synthetic_jar_dir):
"""Bundles classpath into one synthetic jar that includes original classpath in its manifest.
This is to ensure classpath length never exceeds platform ARG_MAX. Original classpath are
converted to URLs relative to synthetic jar path and saved in its manifest as attribute
`Class-Path`. See
https://docs.oracle.com/javase/7/docs/technotes/guides/extensions/spec.html#bundled
:param list classpath: Classpath to be bundled.
:param string synthetic_jar_dir: directory to store the synthetic jar, if `None`
a temp directory will be provided and cleaned up upon process exit. Otherwise synthetic
jar will remain in the supplied directory, only for debugging purpose.
:returns: A classpath (singleton list with just the synthetic jar).
:rtype: list of strings
"""
def prepare_url(url, root_dir):
url_in_bundle = os.path.relpath(os.path.realpath(url), os.path.realpath(root_dir))
# append '/' for directories, those not ending with '/' are assumed to be jars
if os.path.isdir(url):
url_in_bundle += '/'
return url_in_bundle
if synthetic_jar_dir:
safe_mkdir(synthetic_jar_dir)
else:
synthetic_jar_dir = safe_mkdtemp()
bundled_classpath = [prepare_url(url, synthetic_jar_dir) for url in classpath]
manifest = Manifest()
manifest.addentry(Manifest.CLASS_PATH, ' '.join(bundled_classpath))
with temporary_file(root_dir=synthetic_jar_dir, cleanup=False, suffix='.jar') as jar_file:
with open_zip(jar_file, mode='w', compression=ZIP_STORED) as jar:
jar.writestr(Manifest.PATH, manifest.contents())
return [jar_file.name]
def test_too_long_entry(self):
manifest = Manifest()
with self.assertRaises(ValueError):
manifest.addentry(
'1234567890123456789012345678901234567890'
'12345678901234567890123456789', 'value')
def test_addentry(self):
manifest = Manifest()
manifest.addentry('Header', 'value')
self.assertEquals(
'Header: value\n', manifest.contents())
def test_isempty(self):
manifest = Manifest()
self.assertTrue(manifest.is_empty())
manifest.addentry('Header', 'value')
self.assertFalse(manifest.is_empty())
class JarBuilder(AbstractClass):
"""A utility to aid in adding the classes and resources associated with targets to a jar."""
@staticmethod
def _add_agent_manifest(agent, manifest):
# TODO(John Sirois): refactor an agent model to support 'Boot-Class-Path' properly.
manifest.addentry(Manifest.MANIFEST_VERSION, '1.0')
if agent.premain:
manifest.addentry('Premain-Class', agent.premain)
if agent.agent_class:
manifest.addentry('Agent-Class', agent.agent_class)
if agent.can_redefine:
manifest.addentry('Can-Redefine-Classes', 'true')
if agent.can_retransform:
manifest.addentry('Can-Retransform-Classes', 'true')
if agent.can_set_native_method_prefix:
manifest.addentry('Can-Set-Native-Method-Prefix', 'true')
@staticmethod
def _add_manifest_entries(jvm_binary_target, manifest):
"""Add additional fields to MANIFEST.MF as declared in the ManifestEntries structure.
:param JvmBinary jvm_binary_target:
:param Manifest manifest:
"""
for header, value in jvm_binary_target.manifest_entries.entries.iteritems():
manifest.addentry(header, value)
@staticmethod
def prepare(round_manager):
"""Prepares the products needed to use `create_jar_builder`.
This method should be called during task preparation to ensure the classes and resources
needed for jarring targets are mapped by upstream tasks that generate these.
Later, in execute context, the `create_jar_builder` method can be called to get back a
prepared ``JarTask.JarBuilder`` ready for use.
"""
round_manager.require_data('resources_by_target')
round_manager.require_data('classes_by_target')
def __init__(self, context, jar):
self._context = context
self._jar = jar
self._manifest = Manifest()
def add_target(self, target, recursive=False):
"""Adds the classes and resources for a target to an open jar.
:param target: The target to add generated classes and resources for.
:param bool recursive: `True` to add classes and resources for the target's transitive
internal dependency closure.
:returns: The list of targets that actually contributed classes or resources or both to the
jar.
"""
classes_by_target = self._context.products.get_data('classes_by_target')
resources_by_target = self._context.products.get_data('resources_by_target')
targets_added = []
def add_to_jar(tgt):
target_classes = classes_by_target.get(tgt)
target_resources = []
# TODO(pl): https://github.com/pantsbuild/pants/issues/206
resource_products_on_target = resources_by_target.get(tgt)
if resource_products_on_target:
target_resources.append(resource_products_on_target)
if tgt.has_resources:
target_resources.extend(resources_by_target.get(r) for r in tgt.resources)
if target_classes or target_resources:
targets_added.append(tgt)
def add_products(target_products):
if target_products:
for root, products in target_products.rel_paths():
for prod in products:
self._jar.write(os.path.join(root, prod), prod)
add_products(target_classes)
for resources_target in target_resources:
add_products(resources_target)
if isinstance(tgt, JavaAgent):
self._add_agent_manifest(tgt, self._manifest)
if isinstance(target, JvmBinary):
self._add_manifest_entries(target, self._manifest)
if recursive:
target.walk(add_to_jar)
else:
add_to_jar(target)
return targets_added
def commit_manifest(self, jar):
"""Updates the manifest in the jar being written to.
Typically done right before closing the .jar. This gives a chance for all targets to bundle
in their contributions to the manifest.
"""
if not self._manifest.is_empty():
jar.writestr(Manifest.PATH, self._manifest.contents())
def test_addentry(self):
manifest = Manifest()
manifest.addentry('Header', 'value')
self.assertEqual(b'Header: value\n', manifest.contents())
def test_too_long_entry(self):
manifest = Manifest()
with self.assertRaises(ValueError):
manifest.addentry(
'1234567890123456789012345678901234567890'
'12345678901234567890123456789', 'value')
def __init__(self, context, jar):
self._context = context
self._jar = jar
self._manifest = Manifest()
class JarBuilder(ABC):
"""A utility to aid in adding the classes and resources associated with targets to a jar.
:API: public
"""
@staticmethod
def _add_agent_manifest(agent, manifest):
# TODO(John Sirois): refactor an agent model to support 'Boot-Class-Path' properly.
manifest.addentry(Manifest.MANIFEST_VERSION, '1.0')
if agent.premain:
manifest.addentry('Premain-Class', agent.premain)
if agent.agent_class:
manifest.addentry('Agent-Class', agent.agent_class)
if agent.can_redefine:
manifest.addentry('Can-Redefine-Classes', 'true')
if agent.can_retransform:
manifest.addentry('Can-Retransform-Classes', 'true')
if agent.can_set_native_method_prefix:
manifest.addentry('Can-Set-Native-Method-Prefix', 'true')
@staticmethod
def _add_manifest_entries(jvm_binary_target, manifest):
"""Add additional fields to MANIFEST.MF as declared in the ManifestEntries structure.
:param JvmBinary jvm_binary_target:
:param Manifest manifest:
"""
for header, value in jvm_binary_target.manifest_entries.entries.items(
):
manifest.addentry(header, value)
@staticmethod
def prepare(round_manager):
"""Prepares the products needed to use `create_jar_builder`.
This method should be called during task preparation to ensure the classes and resources
needed for jarring targets are mapped by upstream tasks that generate these.
Later, in execute context, the `create_jar_builder` method can be called to get back a
prepared ``JarTask.JarBuilder`` ready for use.
"""
round_manager.require_data('runtime_classpath')
def __init__(self, context, jar):
self._context = context
self._jar = jar
self._manifest = Manifest()
def add_target(self, target, recursive=False):
"""Adds the classes and resources for a target to an open jar.
:param target: The target to add generated classes and resources for.
:param bool recursive: `True` to add classes and resources for the target's transitive
internal dependency closure.
:returns: `True` if the target contributed any files - manifest entries, classfiles or
resource files - to this jar.
:rtype: bool
"""
products_added = False
classpath_products = self._context.products.get_data(
'runtime_classpath')
# TODO(John Sirois): Manifest handling is broken. We should be tracking state and failing
# fast if any duplicate entries are added; ie: if we get a second binary or a second agent.
if isinstance(target, JvmBinary):
self._add_manifest_entries(target, self._manifest)
products_added = True
# Ensure that JavaAgent entries are added to the manifest. Either by adding all of the
# transitive JavaAgent deps, if recursive, or by adding the root target, if the root target
# is itself a JavaAgent.
if recursive:
agents = [
t for t in target.closure() if isinstance(t, JavaAgent)
]
if len(agents) > 1:
raise TaskError(
'Only 1 agent can be added to a jar, found {} for {}:\n\t{}'
.format(
len(agents), target.address.reference(),
'\n\t'.join(agent.address.reference()
for agent in agents)))
elif agents:
self._add_agent_manifest(agents[0], self._manifest)
products_added = True
elif isinstance(target, JavaAgent):
self._add_agent_manifest(target, self._manifest)
products_added = True
# In the transitive case we'll gather internal resources naturally as dependencies, but in the
# non-transitive case we need to manually add these special (in the context of jarring)
# dependencies.
targets = target.closure(bfs=True) if recursive else [target]
if not recursive and target.has_resources:
targets += target.resources
# We only gather internal classpath elements per our contract.
target_classpath = ClasspathUtil.internal_classpath(
targets, classpath_products)
for entry in target_classpath:
if ClasspathUtil.is_jar(entry):
self._jar.writejar(entry)
products_added = True
elif ClasspathUtil.is_dir(entry):
for rel_file in ClasspathUtil.classpath_entries_contents(
[entry]):
self._jar.write(os.path.join(entry, rel_file),
rel_file)
products_added = True
else:
# non-jar and non-directory classpath entries should be ignored
pass
return products_added
def commit_manifest(self, jar):
"""Updates the manifest in the jar being written to.
Typically done right before closing the .jar. This gives a chance for all targets to bundle
in their contributions to the manifest.
"""
if not self._manifest.is_empty():
jar.writestr(Manifest.PATH, self._manifest.contents())
def write_agent_manifest(self, agent, jarfile):
# TODO(John Sirois): refactor an agent model to suport 'Boot-Class-Path' properly.
manifest = Manifest()
manifest.addentry(Manifest.MANIFEST_VERSION, '1.0')
if agent.premain:
manifest.addentry('Premain-Class', agent.premain)
if agent.agent_class:
manifest.addentry('Agent-Class', agent.agent_class)
if agent.can_redefine:
manifest.addentry('Can-Redefine-Classes', 'true')
if agent.can_retransform:
manifest.addentry('Can-Retransform-Classes', 'true')
if agent.can_set_native_method_prefix:
manifest.addentry('Can-Set-Native-Method-Prefix', 'true')
jarfile.writestr(Manifest.PATH, manifest.contents())
class JarBuilder(AbstractClass):
"""A utility to aid in adding the classes and resources associated with targets to a jar."""
@staticmethod
def _add_agent_manifest(agent, manifest):
# TODO(John Sirois): refactor an agent model to support 'Boot-Class-Path' properly.
manifest.addentry(Manifest.MANIFEST_VERSION, '1.0')
if agent.premain:
manifest.addentry('Premain-Class', agent.premain)
if agent.agent_class:
manifest.addentry('Agent-Class', agent.agent_class)
if agent.can_redefine:
manifest.addentry('Can-Redefine-Classes', 'true')
if agent.can_retransform:
manifest.addentry('Can-Retransform-Classes', 'true')
if agent.can_set_native_method_prefix:
manifest.addentry('Can-Set-Native-Method-Prefix', 'true')
@staticmethod
def _add_manifest_entries(jvm_binary_target, manifest):
"""Add additional fields to MANIFEST.MF as declared in the ManifestEntries structure.
:param JvmBinary jvm_binary_target:
:param Manifest manifest:
"""
for header, value in jvm_binary_target.manifest_entries.entries.iteritems(
):
manifest.addentry(header, value)
@staticmethod
def prepare(round_manager):
"""Prepares the products needed to use `create_jar_builder`.
This method should be called during task preparation to ensure the classes and resources
needed for jarring targets are mapped by upstream tasks that generate these.
Later, in execute context, the `create_jar_builder` method can be called to get back a
prepared ``JarTask.JarBuilder`` ready for use.
"""
round_manager.require_data('resources_by_target')
round_manager.require_data('classes_by_target')
def __init__(self, context, jar):
self._context = context
self._jar = jar
self._manifest = Manifest()
def add_target(self, target, recursive=False):
"""Adds the classes and resources for a target to an open jar.
:param target: The target to add generated classes and resources for.
:param bool recursive: `True` to add classes and resources for the target's transitive
internal dependency closure.
:returns: The list of targets that actually contributed classes or resources or both to the
jar.
"""
classes_by_target = self._context.products.get_data(
'classes_by_target')
resources_by_target = self._context.products.get_data(
'resources_by_target')
targets_added = []
def add_to_jar(tgt):
target_classes = classes_by_target.get(tgt)
target_resources = []
# TODO(pl): https://github.com/pantsbuild/pants/issues/206
resource_products_on_target = resources_by_target.get(tgt)
if resource_products_on_target:
target_resources.append(resource_products_on_target)
if tgt.has_resources:
target_resources.extend(
resources_by_target.get(r) for r in tgt.resources)
if target_classes or target_resources:
targets_added.append(tgt)
def add_products(target_products):
if target_products:
for root, products in target_products.rel_paths():
for prod in products:
self._jar.write(os.path.join(root, prod),
prod)
add_products(target_classes)
for resources_target in target_resources:
add_products(resources_target)
if isinstance(tgt, JavaAgent):
self._add_agent_manifest(tgt, self._manifest)
if isinstance(target, JvmBinary):
self._add_manifest_entries(target, self._manifest)
if recursive:
target.walk(add_to_jar)
else:
add_to_jar(target)
return targets_added
def commit_manifest(self, jar):
"""Updates the manifest in the jar being written to.
Typically done right before closing the .jar. This gives a chance for all targets to bundle
in their contributions to the manifest.
"""
if not self._manifest.is_empty():
jar.writestr(Manifest.PATH, self._manifest.contents())
def test_nonascii_char(self):
manifest = Manifest()
with self.assertRaises(UnicodeEncodeError):
manifest.addentry('X-Copyright', '© 2015')
def test_nonascii_char(self):
manifest = Manifest()
with self.assertRaises(UnicodeEncodeError):
manifest.addentry('X-Copyright', '© 2015')
class JarBuilder(AbstractClass):
"""A utility to aid in adding the classes and resources associated with targets to a jar."""
@staticmethod
def _add_agent_manifest(agent, manifest):
# TODO(John Sirois): refactor an agent model to support 'Boot-Class-Path' properly.
manifest.addentry(Manifest.MANIFEST_VERSION, "1.0")
if agent.premain:
manifest.addentry("Premain-Class", agent.premain)
if agent.agent_class:
manifest.addentry("Agent-Class", agent.agent_class)
if agent.can_redefine:
manifest.addentry("Can-Redefine-Classes", "true")
if agent.can_retransform:
manifest.addentry("Can-Retransform-Classes", "true")
if agent.can_set_native_method_prefix:
manifest.addentry("Can-Set-Native-Method-Prefix", "true")
@staticmethod
def _add_manifest_entries(jvm_binary_target, manifest):
"""Add additional fields to MANIFEST.MF as declared in the ManifestEntries structure.
:param JvmBinary jvm_binary_target:
:param Manifest manifest:
"""
for header, value in six.iteritems(jvm_binary_target.manifest_entries.entries):
manifest.addentry(header, value)
@staticmethod
def prepare(round_manager):
"""Prepares the products needed to use `create_jar_builder`.
This method should be called during task preparation to ensure the classes and resources
needed for jarring targets are mapped by upstream tasks that generate these.
Later, in execute context, the `create_jar_builder` method can be called to get back a
prepared ``JarTask.JarBuilder`` ready for use.
"""
round_manager.require_data("runtime_classpath")
def __init__(self, context, jar):
self._context = context
self._jar = jar
self._manifest = Manifest()
def add_target(self, target, recursive=False, canonical_classpath_base_dir=None):
"""Adds the classes and resources for a target to an open jar.
:param target: The target to add generated classes and resources for.
:param bool recursive: `True` to add classes and resources for the target's transitive
internal dependency closure.
:param string canonical_classpath_base_dir: If set, instead of adding targets to the jar
bundle, create canonical symlinks to the original classpath and save canonical symlinks
to Manifest's Class-Path.
:returns: `True` if the target contributed any files - manifest entries, classfiles or
resource files - to this jar.
:rtype: bool
"""
products_added = False
classpath_products = self._context.products.get_data("runtime_classpath")
# TODO(John Sirois): Manifest handling is broken. We should be tracking state and failing
# fast if any duplicate entries are added; ie: if we get a second binary or a second agent.
if isinstance(target, JvmBinary):
self._add_manifest_entries(target, self._manifest)
products_added = True
elif isinstance(target, JavaAgent):
self._add_agent_manifest(target, self._manifest)
products_added = True
elif recursive:
agents = [t for t in target.closure() if isinstance(t, JavaAgent)]
if len(agents) > 1:
raise TaskError(
"Only 1 agent can be added to a jar, found {} for {}:\n\t{}".format(
len(agents),
target.address.reference(),
"\n\t".join(agent.address.reference() for agent in agents),
)
)
elif agents:
self._add_agent_manifest(agents[0], self._manifest)
products_added = True
# In the transitive case we'll gather internal resources naturally as dependencies, but in the
# non-transitive case we need to manually add these special (in the context of jarring)
# dependencies.
targets = target.closure(bfs=True) if recursive else [target]
if not recursive and target.has_resources:
targets += target.resources
# We only gather internal classpath elements per our contract.
if canonical_classpath_base_dir:
canonical_classpath = ClasspathUtil.create_canonical_classpath(
classpath_products, targets, canonical_classpath_base_dir
)
self._jar.append_classpath(canonical_classpath)
products_added = True
else:
target_classpath = ClasspathUtil.internal_classpath(targets, classpath_products)
for entry in target_classpath:
if ClasspathUtil.is_jar(entry):
self._jar.writejar(entry)
products_added = True
elif ClasspathUtil.is_dir(entry):
for rel_file in ClasspathUtil.classpath_entries_contents([entry]):
self._jar.write(os.path.join(entry, rel_file), rel_file)
products_added = True
else:
# non-jar and non-directory classpath entries should be ignored
pass
return products_added
def commit_manifest(self, jar):
"""Updates the manifest in the jar being written to.
Typically done right before closing the .jar. This gives a chance for all targets to bundle
in their contributions to the manifest.
"""
if not self._manifest.is_empty():
jar.writestr(Manifest.PATH, self._manifest.contents())
def __init__(self, context, jar):
self._context = context
self._jar = jar
self._manifest = Manifest()
def _write_agent_manifest(agent, jar):
# TODO(John Sirois): refactor an agent model to support 'Boot-Class-Path' properly.
manifest = Manifest()
manifest.addentry(Manifest.MANIFEST_VERSION, '1.0')
if agent.premain:
manifest.addentry('Premain-Class', agent.premain)
if agent.agent_class:
manifest.addentry('Agent-Class', agent.agent_class)
if agent.can_redefine:
manifest.addentry('Can-Redefine-Classes', 'true')
if agent.can_retransform:
manifest.addentry('Can-Retransform-Classes', 'true')
if agent.can_set_native_method_prefix:
manifest.addentry('Can-Set-Native-Method-Prefix', 'true')
jar.writestr(Manifest.PATH, manifest.contents())
def test_isempty(self):
manifest = Manifest()
self.assertTrue(manifest.is_empty())
manifest.addentry('Header', 'value')
self.assertFalse(manifest.is_empty())
