def bundle(self, app):
"""Create a self-contained application bundle containing the target
classes, dependencies and resources.
"""
assert(isinstance(app, BundleCreate.App))
bundledir = os.path.join(self.outdir, '%s-bundle' % app.basename)
self.context.log.info('creating %s' % os.path.relpath(bundledir, get_buildroot()))
safe_mkdir(bundledir, clean=True)
classpath = OrderedSet()
if not self.deployjar:
libdir = os.path.join(bundledir, 'libs')
os.mkdir(libdir)
# Add external dependencies to the bundle.
for basedir, externaljar in self.list_jar_dependencies(app.binary):
path = os.path.join(basedir, externaljar)
os.symlink(path, os.path.join(libdir, externaljar))
classpath.add(externaljar)
# TODO: There should probably be a separate 'binary_jars' product type,
# so we can more easily distinguish binary jars (that contain all the classes of their
# transitive deps) and per-target jars.
for basedir, jars in self.context.products.get('jars').get(app.binary).items():
if len(jars) != 1:
raise TaskError('Expected 1 mapped binary for %s but found: %s' % (app.binary, jars))
binary = jars[0]
binary_jar = os.path.join(basedir, binary)
bundle_jar = os.path.join(bundledir, binary)
# Add the internal classes into the bundle_jar.
if not classpath:
os.symlink(binary_jar, bundle_jar)
else:
# TODO: Can we copy the existing jar and inject the manifest in, instead of
# laboriously copying the contents one by one? Would that be more efficient?
with open_zip(binary_jar, 'r') as src:
with open_zip(bundle_jar, 'w', compression=ZIP_DEFLATED) as dest:
for item in src.infolist():
buf = src.read(item.filename)
if Manifest.PATH == item.filename:
manifest = Manifest(buf)
manifest.addentry(Manifest.CLASS_PATH,
' '.join(os.path.join('libs', jar) for jar in classpath))
buf = manifest.contents()
dest.writestr(item, buf)
for bundle in app.bundles:
for path, relpath in bundle.filemap.items():
bundlepath = os.path.join(bundledir, relpath)
safe_mkdir(os.path.dirname(bundlepath))
os.symlink(path, bundlepath)
return bundledir
def bundle(self, app):
bundledir = os.path.join(self.outdir, '%s-bundle' % app.basename)
self.context.log.info('creating %s' % os.path.relpath(bundledir, get_buildroot()))
safe_mkdir(bundledir, clean=True)
classpath = OrderedSet()
if not self.deployjar:
libdir = os.path.join(bundledir, 'libs')
os.mkdir(libdir)
for basedir, externaljar in self.list_jar_dependencies(app.binary):
src = os.path.join(basedir, externaljar)
link_name = os.path.join(libdir, externaljar)
try:
os.symlink(src, link_name)
except OSError as e:
if e.errno == errno.EEXIST:
raise TaskError('Trying to symlink %s to %s, but it is already symlinked to %s. ' %
(link_name, src, os.readlink(link_name)) +
'Does the bundled target depend on multiple jvm_binary targets?')
else:
raise
classpath.add(externaljar)
for basedir, jars in self.context.products.get('jars').get(app.binary).items():
if len(jars) != 1:
raise TaskError('Expected 1 mapped binary but found: %s' % jars)
binary = jars.pop()
binary_jar = os.path.join(basedir, binary)
bundle_jar = os.path.join(bundledir, binary)
if not classpath:
os.symlink(binary_jar, bundle_jar)
else:
with open_zip(binary_jar, 'r') as src:
with open_zip(bundle_jar, 'w', compression=ZIP_DEFLATED) as dest:
for item in src.infolist():
buffer = src.read(item.filename)
if Manifest.PATH == item.filename:
manifest = Manifest(buffer)
manifest.addentry(Manifest.CLASS_PATH,
' '.join(os.path.join('libs', jar) for jar in classpath))
buffer = manifest.contents()
dest.writestr(item, buffer)
for bundle in app.bundles:
for path, relpath in bundle.filemap.items():
bundlepath = os.path.join(bundledir, relpath)
safe_mkdir(os.path.dirname(bundlepath))
os.symlink(path, bundlepath)
return bundledir
def bundle(self, app):
bundledir = os.path.join(self.outdir, '%s-bundle' % app.basename)
self.context.log.info('creating %s' %
os.path.relpath(bundledir, get_buildroot()))
safe_mkdir(bundledir, clean=True)
classpath = OrderedSet()
if not self.deployjar:
libdir = os.path.join(bundledir, 'libs')
os.mkdir(libdir)
for basedir, externaljar in self.list_jar_dependencies(app.binary):
path = os.path.join(basedir, externaljar)
os.symlink(path, os.path.join(libdir, externaljar))
classpath.add(externaljar)
for basedir, jars in self.context.products.get('jars').get(
app.binary).items():
if len(jars) != 1:
raise TaskError('Expected 1 mapped binary but found: %s' %
jars)
binary = jars.pop()
binary_jar = os.path.join(basedir, binary)
bundle_jar = os.path.join(bundledir, binary)
if not classpath:
os.symlink(binary_jar, bundle_jar)
else:
with open_zip(binary_jar, 'r') as src:
with open_zip(bundle_jar, 'w',
compression=ZIP_DEFLATED) as dest:
for item in src.infolist():
buffer = src.read(item.filename)
if Manifest.PATH == item.filename:
manifest = Manifest(buffer)
manifest.addentry(
Manifest.CLASS_PATH, ' '.join(
os.path.join('libs', jar)
for jar in classpath))
buffer = manifest.contents()
dest.writestr(item, buffer)
for bundle in app.bundles:
for path, relpath in bundle.filemap.items():
bundlepath = os.path.join(bundledir, relpath)
safe_mkdir(os.path.dirname(bundlepath))
os.symlink(path, bundlepath)
return bundledir
def _extract(self, jarpath):
self.context.log.debug('Extracting idl jar to: %s' % self._EXTRACT_BASE)
ZIP.extract(jarpath, self._EXTRACT_BASE)
with open_zip(jarpath) as jar:
sources = filter(lambda path: path.endswith('.thrift'), jar.namelist())
self.context.log.debug('Found thrift IDL sources: %s' % sources)
return sources
def _compute_classpath_elements_by_class(self, classpath):
# Don't consider loose classes dirs in our classpath. Those will be considered
# separately, by looking at products.
def non_product(path):
return not (path.startswith(self._pants_workdir)
and os.path.isdir(path))
classpath_jars = filter(non_product, classpath)
if self._class_to_jarfile is None:
self._class_to_jarfile = {}
for jarpath in self.find_all_bootstrap_jars() + classpath_jars:
# Per the classloading spec, a 'jar' in this context can also be a .zip file.
if os.path.isfile(jarpath) and ((jarpath.endswith('.jar')
or jarpath.endswith('.zip'))):
with open_zip(jarpath, 'r') as jar:
for cls in jar.namelist():
# First jar with a given class wins, just like when classloading.
if cls.endswith(
b'.class'
) and not cls in self._class_to_jarfile:
self._class_to_jarfile[cls] = jarpath
elif os.path.isdir(jarpath):
for dirpath, _, filenames in os.walk(jarpath,
followlinks=True):
for f in filter(lambda x: x.endswith('.class'),
filenames):
cls = os.path.relpath(os.path.join(dirpath, f),
jarpath)
if not cls in self._class_to_jarfile:
self._class_to_jarfile[cls] = jarpath
return self._class_to_jarfile
def assert_writestr(path, contents, *entries):
with self.jarfile() as existing_jarfile:
with self.jar_task.open_jar(existing_jarfile) as jar:
jar.writestr(path, contents)
with open_zip(existing_jarfile) as jar:
self.assert_listing(jar, *entries)
self.assertEquals(contents, jar.read(path))
def assert_writestr(path, contents, *entries):
with self.jarfile() as existing_jarfile:
with self.jar_task.open_jar(existing_jarfile) as jar:
jar.writestr(path, contents)
with open_zip(existing_jarfile) as jar:
self.assert_listing(jar, *entries)
self.assertEquals(contents, jar.read(path))
def test_overwrite_writestr(self):
with self.jarfile() as existing_jarfile:
with self.jar_task.open_jar(existing_jarfile, overwrite=True) as jar:
jar.writestr('README', b'42')
with open_zip(existing_jarfile) as jar:
self.assert_listing(jar, 'README')
self.assertEquals('42', jar.read('README'))
def test_overwrite_writestr(self):
with self.jarfile() as existing_jarfile:
with self.jar_task.open_jar(existing_jarfile, overwrite=True) as jar:
jar.writestr('README', b'42')
with open_zip(existing_jarfile) as jar:
self.assert_listing(jar, 'README')
self.assertEquals('42', jar.read('README'))
def bundle(self, app):
bundledir = os.path.join(self.outdir, "%s-bundle" % app.basename)
self.context.log.info("creating %s" % os.path.relpath(bundledir, get_buildroot()))
safe_mkdir(bundledir, clean=True)
classpath = OrderedSet()
if not self.deployjar:
libdir = os.path.join(bundledir, "libs")
os.mkdir(libdir)
for basedir, externaljar in self.list_jar_dependencies(app.binary):
path = os.path.join(basedir, externaljar)
os.symlink(path, os.path.join(libdir, externaljar))
classpath.add(externaljar)
for basedir, jars in self.context.products.get("jars").get(app.binary).items():
if len(jars) != 1:
raise TaskError("Expected 1 mapped binary but found: %s" % jars)
binary = jars.pop()
binary_jar = os.path.join(basedir, binary)
bundle_jar = os.path.join(bundledir, binary)
if not classpath:
os.symlink(binary_jar, bundle_jar)
else:
with open_zip(binary_jar, "r") as src:
with open_zip(bundle_jar, "w", compression=ZIP_DEFLATED) as dest:
for item in src.infolist():
buffer = src.read(item.filename)
if Manifest.PATH == item.filename:
manifest = Manifest(buffer)
manifest.addentry(
Manifest.CLASS_PATH, " ".join(os.path.join("libs", jar) for jar in classpath)
)
buffer = manifest.contents()
dest.writestr(item, buffer)
for bundle in app.bundles:
for path, relpath in bundle.filemap.items():
bundlepath = os.path.join(bundledir, relpath)
safe_mkdir(os.path.dirname(bundlepath))
os.symlink(path, bundlepath)
return bundledir
def _extract(self, jarpath):
self.context.log.debug('Extracting idl jar to: %s' %
self._EXTRACT_BASE)
ZIP.extract(jarpath, self._EXTRACT_BASE)
with open_zip(jarpath) as jar:
sources = filter(lambda path: path.endswith('.thrift'),
jar.namelist())
self.context.log.debug('Found thrift IDL sources: %s' % sources)
return sources
def archive(self, basedir, outdir, name):
zippath = os.path.join(outdir, '%s.zip' % name)
with open_zip(zippath, 'w', compression=ZIP_DEFLATED) as zip:
for root, _, files in os.walk(basedir):
for file in files:
full_path = os.path.join(root, file)
relpath = os.path.relpath(full_path, basedir)
zip.write(full_path, relpath)
return zippath
def test_extract_pexinfo():
filename = None
with temporary_file() as fp:
filename = fp.name
with open_zip(filename, 'w') as zf:
zf.writestr('PEX-INFO', '{"build_properties":{"tag":"thermos_R31337"}}')
assert ExecutorVars.get_release_from_binary(filename) == 31337
assert ExecutorVars.get_release_from_binary(filename) == 'UNKNOWN'
assert ExecutorVars.get_release_from_binary('lololololo') == 'UNKNOWN'
def archive(self, basedir, outdir, name):
zippath = os.path.join(outdir, '%s.zip' % name)
with open_zip(zippath, 'w', compression=ZIP_DEFLATED) as zip:
for root, _, files in os.walk(basedir):
for file in files:
full_path = os.path.join(root, file)
relpath = os.path.relpath(full_path, basedir)
zip.write(full_path, relpath)
return zippath
def test_custom_manifest(self):
contents = b'Manifest-Version: 1.0\r\nCreated-By: test\r\n\r\n'
with self.jarfile() as existing_jarfile:
with self.jar_task.open_jar(existing_jarfile, overwrite=True) as jar:
jar.writestr('README', b'42')
with open_zip(existing_jarfile) as jar:
self.assert_listing(jar, 'README')
self.assertEquals('42', jar.read('README'))
self.assertNotEqual(contents, jar.read('META-INF/MANIFEST.MF'))
with self.jar_task.open_jar(existing_jarfile, overwrite=False) as jar:
jar.writestr('META-INF/MANIFEST.MF', contents)
with open_zip(existing_jarfile) as jar:
self.assert_listing(jar, 'README')
self.assertEquals('42', jar.read('README'))
self.assertEquals(contents, jar.read('META-INF/MANIFEST.MF'))
def test_extract_pexinfo():
filename = None
with temporary_file() as fp:
filename = fp.name
with open_zip(filename, 'w') as zf:
zf.writestr('PEX-INFO',
'{"build_properties":{"tag":"thermos_R31337"}}')
assert ExecutorVars.get_release_from_binary(filename) == 31337
assert ExecutorVars.get_release_from_binary(filename) == 'UNKNOWN'
assert ExecutorVars.get_release_from_binary('lololololo') == 'UNKNOWN'
def test_custom_manifest(self):
contents = b'Manifest-Version: 1.0\r\nCreated-By: test\r\n\r\n'
with self.jarfile() as existing_jarfile:
with self.jar_task.open_jar(existing_jarfile, overwrite=True) as jar:
jar.writestr('README', b'42')
with open_zip(existing_jarfile) as jar:
self.assert_listing(jar, 'README')
self.assertEquals('42', jar.read('README'))
self.assertNotEqual(contents, jar.read('META-INF/MANIFEST.MF'))
with self.jar_task.open_jar(existing_jarfile, overwrite=False) as jar:
jar.writestr('META-INF/MANIFEST.MF', contents)
with open_zip(existing_jarfile) as jar:
self.assert_listing(jar, 'README')
self.assertEquals('42', jar.read('README'))
self.assertEquals(contents, jar.read('META-INF/MANIFEST.MF'))
def _add_args_resources(self, target, resources_by_target, transitive):
classes_by_target = self.context.products.get_data('classes_by_target')
resource_dirs = set()
def collect_resource_dirs(tgt):
mapping = classes_by_target.get(tgt)
if mapping:
resource_dirs.update(
os.path.join(base, self.RESOURCE_RELDIR)
for base, _ in mapping.rel_paths())
if transitive:
target.walk(collect_resource_dirs)
else:
collect_resource_dirs(target)
lines = set()
for resource_dir in resource_dirs:
if os.path.exists(resource_dir):
for file_name in os.listdir(resource_dir):
if file_name.startswith(self.RESOURCE_BASENAME):
with open(os.path.join(resource_dir,
file_name)) as resource:
lines.update(resource.readlines())
if lines:
args = Args.for_target(target, transitive, classes_by_target)
lines = set(filter(args.matches, lines))
if transitive:
# Add args from any of our transitive external deps that have them.
resource_path = os.path.join(self.RESOURCE_RELDIR,
self.RESOURCE_BASENAME)
for base_dir, jar_path in self.list_external_jar_dependencies(
target):
jar_file = os.path.join(base_dir, jar_path)
try:
with open_zip(jar_file) as jar:
for zipinfo in jar.infolist():
if zipinfo.filename.startswith(resource_path):
lines.update(jar.open(zipinfo).readlines())
except zipfile.BadZipfile as e:
# Java itself allows non-jar files on the classpath, but
# doing do is only useful with a custom classloader that
# knows how to handle these files. This use of the
# classpath is innovative, bordering on the avant-garde.
self.context.log.info(
dedent("""
Skipping mapping of {file}. It is not a valid jar.
Continuing because invalid jars are ignored. {error}""").format(
file=jar_file, error=e))
self._addargs(lines, target, resources_by_target, transitive)
def assert_jar_contents(self, context, product_type, target, *contents):
jar_mapping = context.products.get(product_type).get(target)
self.assertEqual(1, len(jar_mapping))
for basedir, jars in jar_mapping.items():
self.assertEqual(1, len(jars))
with open_zip(os.path.join(basedir, jars[0])) as jar:
self.assertEqual(['META-INF/', 'META-INF/MANIFEST.MF'] + list(contents), jar.namelist())
for content in contents:
if not content.endswith('/'):
with closing(jar.open(content)) as fp:
self.assertEqual(os.path.basename(content), fp.read())
def extract(cls, path, outdir):
"""OS X's python 2.6.1 has a bug in zipfile that makes it unzip directories as regular files.
This method should work on for python 2.6-3.x.
"""
with open_zip(path) as zip:
for path in zip.namelist():
# While we're at it, we also perform this safety test.
if path.startswith(b'/') or path.startswith(b'..'):
raise ValueError('Zip file contains unsafe path: %s' % path)
# Ignore directories. extract() will create parent dirs as needed.
if not path.endswith(b'/'):
zip.extract(path, outdir)
def create(self, basedir, outdir, name, prefix=None):
zippath = os.path.join(outdir, '%s.zip' % name)
with open_zip(zippath, 'w', compression=ZIP_DEFLATED) as zip:
for root, _, files in os.walk(basedir):
root = root.decode('utf-8')
for file in files:
file = file.decode('utf-8')
full_path = os.path.join(root, file)
relpath = os.path.relpath(full_path, basedir)
if prefix:
relpath = os.path.join(prefix.decode('utf-8'), relpath)
zip.write(full_path, relpath)
return zippath
def create(self, basedir, outdir, name, prefix=None):
zippath = os.path.join(outdir, '%s.zip' % name)
with open_zip(zippath, 'w', compression=ZIP_DEFLATED) as zip:
for root, _, files in os.walk(basedir):
root = root.decode('utf-8')
for file in files:
file = file.decode('utf-8')
full_path = os.path.join(root, file)
relpath = os.path.relpath(full_path, basedir)
if prefix:
relpath = os.path.join(prefix.decode('utf-8'), relpath)
zip.write(full_path, relpath)
return zippath
def assert_jar_contents(self, context, product_type, target, *contents):
jar_mapping = context.products.get(product_type).get(target)
self.assertEqual(1, len(jar_mapping))
for basedir, jars in jar_mapping.items():
self.assertEqual(1, len(jars))
with open_zip(os.path.join(basedir, jars[0])) as jar:
self.assertEqual(['META-INF/', 'META-INF/MANIFEST.MF'] +
list(contents), jar.namelist())
for content in contents:
if not content.endswith('/'):
with closing(jar.open(content)) as fp:
self.assertEqual(os.path.basename(content),
fp.read())
def extract(cls, path, outdir):
"""OS X's python 2.6.1 has a bug in zipfile that makes it unzip directories as regular files.
This method should work on for python 2.6-3.x.
"""
with open_zip(path) as zip:
for path in zip.namelist():
# While we're at it, we also perform this safety test.
if path.startswith(b'/') or path.startswith(b'..'):
raise ValueError('Zip file contains unsafe path: %s' % path)
# Ignore directories. extract() will create parent dirs as needed.
if not path.endswith(b'/'):
zip.extract(path, outdir)
def test_overwrite_write(self):
with temporary_dir() as chroot:
_path = os.path.join(chroot, 'a/b/c')
safe_mkdir(_path)
data_file = os.path.join(_path, 'd.txt')
with open(data_file, 'w') as fd:
fd.write('e')
with self.jarfile() as existing_jarfile:
with self.jar_task.open_jar(existing_jarfile, overwrite=True) as jar:
jar.write(data_file, 'f/g/h')
with open_zip(existing_jarfile) as jar:
self.assert_listing(jar, 'f/', 'f/g/', 'f/g/h')
self.assertEquals('e', jar.read('f/g/h'))
def test_overwrite_write(self):
with temporary_dir() as chroot:
_path = os.path.join(chroot, 'a/b/c')
safe_mkdir(_path)
data_file = os.path.join(_path, 'd.txt')
with open(data_file, 'w') as fd:
fd.write('e')
with self.jarfile() as existing_jarfile:
with self.jar_task.open_jar(existing_jarfile, overwrite=True) as jar:
jar.write(data_file, 'f/g/h')
with open_zip(existing_jarfile) as jar:
self.assert_listing(jar, 'f/', 'f/g/', 'f/g/h')
self.assertEquals('e', jar.read('f/g/h'))
def _add_args_resources(self, target, resources_by_target, transitive):
classes_by_target = self.context.products.get_data('classes_by_target')
resource_dirs = set()
def collect_resource_dirs(tgt):
mapping = classes_by_target.get(tgt)
if mapping:
resource_dirs.update(os.path.join(base, self.RESOURCE_RELDIR)
for base, _ in mapping.rel_paths())
if transitive:
target.walk(collect_resource_dirs)
else:
collect_resource_dirs(target)
lines = set()
for resource_dir in resource_dirs:
if os.path.exists(resource_dir):
for file_name in os.listdir(resource_dir):
if file_name.startswith(self.RESOURCE_BASENAME):
with open(os.path.join(resource_dir, file_name)) as resource:
lines.update(resource.readlines())
if lines:
args = Args.for_target(target, transitive, classes_by_target)
lines = set(filter(args.matches, lines))
if transitive:
# Add args from any of our transitive external deps that have them.
resource_path = os.path.join(self.RESOURCE_RELDIR, self.RESOURCE_BASENAME)
for base_dir, jar_path in self.list_external_jar_dependencies(target):
jar_file = os.path.join(base_dir, jar_path)
try:
with open_zip(jar_file) as jar:
for zipinfo in jar.infolist():
if zipinfo.filename.startswith(resource_path):
lines.update(jar.open(zipinfo).readlines())
except zipfile.BadZipfile as e:
# Java itself allows non-jar files on the classpath, but
# doing do is only useful with a custom classloader that
# knows how to handle these files. This use of the
# classpath is innovative, bordering on the avant-garde.
self.context.log.info(dedent("""
Skipping mapping of {file}. It is not a valid jar.
Continuing because invalid jars are ignored. {error}""").format (file=jar_file, error=e))
self._addargs(lines, target, resources_by_target, transitive)
def _add_args_resources(self, target, resources_by_target, transitive):
classes_by_target = self.context.products.get_data('classes_by_target')
resource_dirs = set()
def collect_resource_dirs(tgt):
mapping = classes_by_target.get(tgt)
if mapping:
resource_dirs.update(
os.path.join(base, self.RESOURCE_RELDIR)
for base, _ in mapping.rel_paths())
if transitive:
target.walk(collect_resource_dirs)
else:
collect_resource_dirs(target)
lines = set()
for resource_dir in resource_dirs:
if os.path.exists(resource_dir):
for file_name in os.listdir(resource_dir):
if file_name.startswith(self.RESOURCE_BASENAME):
with open(os.path.join(resource_dir,
file_name)) as resource:
lines.update(resource.readlines())
if lines:
args = Args.for_target(target, transitive, classes_by_target)
lines = set(filter(args.matches, lines))
if transitive:
# Add args from any of our transitive external deps that have them.
resource_path = os.path.join(self.RESOURCE_RELDIR,
self.RESOURCE_BASENAME)
for base_dir, jar_path in self.list_external_jar_dependencies(
target):
with open_zip(os.path.join(base_dir, jar_path)) as jar:
for zipinfo in jar.infolist():
if zipinfo.filename.startswith(resource_path):
lines.update(jar.open(zipinfo).readlines())
self._addargs(lines, target, resources_by_target, transitive)
def _add_args_resources(self, target, resources_by_target, transitive):
classes_by_target = self.context.products.get_data('classes_by_target')
resource_dirs = set()
def collect_resource_dirs(tgt):
mapping = classes_by_target.get(tgt)
if mapping:
resource_dirs.update(os.path.join(base, self.RESOURCE_RELDIR)
for base, _ in mapping.rel_paths())
if transitive:
target.walk(collect_resource_dirs)
else:
collect_resource_dirs(target)
lines = set()
for resource_dir in resource_dirs:
if os.path.exists(resource_dir):
for file_name in os.listdir(resource_dir):
if file_name.startswith(self.RESOURCE_BASENAME):
with open(os.path.join(resource_dir, file_name)) as resource:
lines.update(resource.readlines())
if lines:
args = Args.for_target(target, transitive, classes_by_target)
lines = filter(args.matches, lines)
if transitive:
# Add args from any of our transitive external deps that have them.
resource_path = os.path.join(self.RESOURCE_RELDIR, self.RESOURCE_BASENAME)
for base_dir, jar_path in self.list_external_jar_dependencies(target):
with open_zip(os.path.join(base_dir, jar_path)) as jar:
for zipinfo in jar.infolist():
if zipinfo.filename.startswith(resource_path):
lines.update(jar.open(zipinfo).readlines())
self._addargs(lines, target, resources_by_target, transitive)
def _compute_classpath_elements_by_class(self, classpath):
# Don't consider loose classes dirs in our classpath. Those will be considered
# separately, by looking at products.
def non_product(path):
return not (path.startswith(self._pants_workdir) and os.path.isdir(path))
classpath_jars = filter(non_product, classpath)
if self._class_to_jarfile is None:
self._class_to_jarfile = {}
for jarpath in self.find_all_bootstrap_jars() + classpath_jars:
# Per the classloading spec, a 'jar' in this context can also be a .zip file.
if os.path.isfile(jarpath) and ((jarpath.endswith('.jar') or jarpath.endswith('.zip'))):
with open_zip(jarpath, 'r') as jar:
for cls in jar.namelist():
# First jar with a given class wins, just like when classloading.
if cls.endswith('.class') and not cls in self._class_to_jarfile:
self._class_to_jarfile[cls] = jarpath
elif os.path.isdir(jarpath):
for dirpath, _, filenames in os.walk(jarpath, followlinks=True):
for f in filter(lambda x: x.endswith('.class'), filenames):
cls = os.path.relpath(os.path.join(dirpath, f), jarpath)
if not cls in self._class_to_jarfile:
self._class_to_jarfile[cls] = jarpath
return self._class_to_jarfile
def _compute_classpath_elements_by_class(self, classpath):
# Don't consider loose classes dirs in our classes dir. Those will be considered
# separately, by looking at products.
def non_product(path):
return path != self._classes_dir
if self._upstream_class_to_path is None:
self._upstream_class_to_path = {}
classpath_entries = filter(non_product, classpath)
for cp_entry in self.find_all_bootstrap_jars() + classpath_entries:
# Per the classloading spec, a 'jar' in this context can also be a .zip file.
if os.path.isfile(cp_entry) and ((cp_entry.endswith('.jar') or cp_entry.endswith('.zip'))):
with open_zip(cp_entry, 'r') as jar:
for cls in jar.namelist():
# First jar with a given class wins, just like when classloading.
if cls.endswith(b'.class') and not cls in self._upstream_class_to_path:
self._upstream_class_to_path[cls] = cp_entry
elif os.path.isdir(cp_entry):
for dirpath, _, filenames in os.walk(cp_entry, followlinks=True):
for f in filter(lambda x: x.endswith('.class'), filenames):
cls = os.path.relpath(os.path.join(dirpath, f), cp_entry)
if not cls in self._upstream_class_to_path:
self._upstream_class_to_path[cls] = os.path.join(dirpath, f)
return self._upstream_class_to_path
def _compute_classpath_elements_by_class(self, classpath):
# Don't consider loose classes dirs in our classes dir. Those will be considered
# separately, by looking at products.
def non_product(path):
return path != self._classes_dir
if self._upstream_class_to_path is None:
self._upstream_class_to_path = {}
classpath_entries = filter(non_product, classpath)
for cp_entry in self.find_all_bootstrap_jars() + classpath_entries:
# Per the classloading spec, a 'jar' in this context can also be a .zip file.
if os.path.isfile(cp_entry) and ((cp_entry.endswith('.jar') or cp_entry.endswith('.zip'))):
with open_zip(cp_entry, 'r') as jar:
for cls in jar.namelist():
# First jar with a given class wins, just like when classloading.
if cls.endswith(b'.class') and not cls in self._upstream_class_to_path:
self._upstream_class_to_path[cls] = cp_entry
elif os.path.isdir(cp_entry):
for dirpath, _, filenames in os.walk(cp_entry, followlinks=True):
for f in filter(lambda x: x.endswith('.class'), filenames):
cls = os.path.relpath(os.path.join(dirpath, f), cp_entry)
if not cls in self._upstream_class_to_path:
self._upstream_class_to_path[cls] = os.path.join(dirpath, f)
return self._upstream_class_to_path
def open_jar(path, *args, **kwargs):
"""Yields a jar in a with context that will be closed when the context exits.
The yielded jar is a zipfile.ZipFile object with an additional mkdirs(arcpath) method that will
create a zip directory entry similar to unix `mkdir -p`. Additionally, the ZipFile.write and
ZipFile.writestr methods are enhanced to call mkdirs as needed to ensure all jar entries contain
a full complement of parent paths leading from each leaf to the root of the jar.
"""
with open_zip(path, *args, **kwargs) as jar:
real_write = jar.write
real_writestr = jar.writestr
made_dirs = set()
def mkdirs(arcpath):
if arcpath and arcpath not in made_dirs:
made_dirs.add(arcpath)
parent_path = os.path.dirname(arcpath)
mkdirs(parent_path)
zipinfo = zipfile.ZipInfo(arcpath if arcpath.endswith('/') else arcpath + '/')
# We store directories without compression since they have no contents and
# attempts to store them with compression lead to corrupted zip files as such:
# $ unzip -t junit-runner-0.0.19.jar
# Archive: junit-runner-0.0.19.jar
# testing: com/
# error: invalid compressed data to inflate
# testing: com/twitter/
# error: invalid compressed data to inflate
# testing: com/twitter/common/
# error: invalid compressed data to inflate
# testing: com/twitter/common/testing/
# error: invalid compressed data to inflate
# testing: com/twitter/common/testing/runner/
# error: invalid compressed data to inflate
# testing: com/twitter/common/testing/runner/StreamSource.class OK
zipinfo.compress_type = zipfile.ZIP_STORED
# PKZIP says external_attr is a 4 byte field that is host system dependant:
# http://www.pkware.com/documents/casestudies/APPNOTE.TXT
# These notes do mention the low order byte will carry DOS file attributes for DOS host
# system zips. The DOS file attribute bits are described here:
# http://www.xxcopy.com/xxcopy06.htm
#
# More details are only found reading source, for example in BSD:
# ftp://ftp-archive.freebsd.org/pub/FreeBSD-Archive/old-releases/i386/1.0-RELEASE/ports/info-zip/zipinfo/zipinfo.c
# These sources reveal the 2 high order bytes contain unix file attribute bits.
#
# In summary though the full 32 bit field layout is:
# TTTTsstrwxrwxrwx0000000000ADVSHR
# ^^^^____________________________ stat.h file type: S_IFXXX
# ^^^_________________________ setuid, setgid, sticky
# ^^^^^^^^^________________ permissions
# ^^^^^^^^________ ???
# ^^^^^^^^ DOS attribute bits
# Setup unix directory perm bits: drwxr-xr-x
zipinfo.external_attr = (
stat.S_IFDIR # file type dir
| stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR # u+rwx
| stat.S_IRGRP | stat.S_IXGRP # g+rx
| stat.S_IROTH | stat.S_IXOTH # o+rx
) << 16
# Add DOS directory bit
zipinfo.external_attr |= 0x10
real_writestr(zipinfo, '')
def write(path, arcname=None, **kwargs):
if os.path.isdir(path):
mkdirs(arcname or path)
else:
mkdirs(os.path.dirname(arcname or path))
real_write(path, arcname, **kwargs)
def writestr(zinfo_or_arcname, *args, **kwargs):
mkdirs(os.path.dirname(zinfo_or_arcname))
real_writestr(zinfo_or_arcname, *args, **kwargs)
jar.mkdirs = mkdirs
jar.write = write
jar.writestr = writestr
yield jar
def bundle(self, app):
"""Create a self-contained application bundle containing the target
classes, dependencies and resources.
"""
assert (isinstance(app, BundleCreate.App))
bundledir = os.path.join(self.outdir, '%s-bundle' % app.basename)
self.context.log.info('creating %s' %
os.path.relpath(bundledir, get_buildroot()))
safe_mkdir(bundledir, clean=True)
classpath = OrderedSet()
if not self.deployjar:
libdir = os.path.join(bundledir, 'libs')
os.mkdir(libdir)
# Add internal dependencies to the bundle.
def add_jars(target):
target_jars = self.context.products.get('jars').get(target)
if target_jars is not None:
for basedir, jars in target_jars.items():
for internaljar in jars:
os.symlink(os.path.join(basedir, internaljar),
os.path.join(libdir, internaljar))
classpath.add(internaljar)
app.binary.walk(add_jars, lambda t: t.is_internal)
# Add external dependencies to the bundle.
for basedir, externaljar in self.list_jar_dependencies(app.binary):
path = os.path.join(basedir, externaljar)
os.symlink(path, os.path.join(libdir, externaljar))
classpath.add(externaljar)
for basedir, jars in self.context.products.get('jars').get(
app.binary).items():
if len(jars) != 1:
raise TaskError(
'Expected 1 mapped binary for %s but found: %s' %
(app.binary, jars))
binary = jars[0]
binary_jar = os.path.join(basedir, binary)
bundle_jar = os.path.join(bundledir, binary)
if not classpath:
os.symlink(binary_jar, bundle_jar)
else:
with open_zip(binary_jar, 'r') as src:
with open_zip(bundle_jar, 'w',
compression=ZIP_DEFLATED) as dest:
for item in src.infolist():
buf = src.read(item.filename)
if Manifest.PATH == item.filename:
manifest = Manifest(buf)
manifest.addentry(
Manifest.CLASS_PATH, ' '.join(
os.path.join('libs', jar)
for jar in classpath))
buf = manifest.contents()
dest.writestr(item, buf)
for bundle in app.bundles:
for path, relpath in bundle.filemap.items():
bundlepath = os.path.join(bundledir, relpath)
safe_mkdir(os.path.dirname(bundlepath))
os.symlink(path, bundlepath)
return bundledir
def bundle(self, app):
"""Create a self-contained application bundle containing the target
classes, dependencies and resources.
"""
assert(isinstance(app, BundleCreate.App))
bundledir = os.path.join(self.outdir, '%s-bundle' % app.basename)
self.context.log.info('creating %s' % os.path.relpath(bundledir, get_buildroot()))
safe_mkdir(bundledir, clean=True)
classpath = OrderedSet()
if not self.deployjar:
libdir = os.path.join(bundledir, 'libs')
os.mkdir(libdir)
# Add internal dependencies to the bundle.
def add_jars(target):
target_jars = self.context.products.get('jars').get(target)
if target_jars is not None:
for basedir, jars in target_jars.items():
for internaljar in jars:
os.symlink(os.path.join(basedir, internaljar),
os.path.join(libdir, internaljar))
classpath.add(internaljar)
app.binary.walk(add_jars, lambda t: t.is_internal)
# Add external dependencies to the bundle.
for basedir, externaljar in self.list_jar_dependencies(app.binary):
path = os.path.join(basedir, externaljar)
os.symlink(path, os.path.join(libdir, externaljar))
classpath.add(externaljar)
for basedir, jars in self.context.products.get('jars').get(app.binary).items():
if len(jars) != 1:
raise TaskError('Expected 1 mapped binary for %s but found: %s' % (app.binary, jars))
binary = jars[0]
binary_jar = os.path.join(basedir, binary)
bundle_jar = os.path.join(bundledir, binary)
if not classpath:
os.symlink(binary_jar, bundle_jar)
else:
with open_zip(binary_jar, 'r') as src:
with open_zip(bundle_jar, 'w', compression=ZIP_DEFLATED) as dest:
for item in src.infolist():
buf = src.read(item.filename)
if Manifest.PATH == item.filename:
manifest = Manifest(buf)
manifest.addentry(Manifest.CLASS_PATH,
' '.join(os.path.join('libs', jar) for jar in classpath))
buf = manifest.contents()
dest.writestr(item, buf)
for bundle in app.bundles:
for path, relpath in bundle.filemap.items():
bundlepath = os.path.join(bundledir, relpath)
safe_mkdir(os.path.dirname(bundlepath))
os.symlink(path, bundlepath)
return bundledir
def open_jar(path, *args, **kwargs):
"""Yields a jar in a with context that will be closed when the context exits.
The yielded jar is a zipfile.ZipFile object with an additional mkdirs(arcpath) method that will
create a zip directory entry similar to unix `mkdir -p`. Additionally, the ZipFile.write and
ZipFile.writestr methods are enhanced to call mkdirs as needed to ensure all jar entries contain
a full complement of parent paths leading from each leaf to the root of the jar.
"""
with open_zip(path, *args, **kwargs) as jar:
real_write = jar.write
real_writestr = jar.writestr
made_dirs = set()
def mkdirs(arcpath):
if arcpath and arcpath not in made_dirs:
made_dirs.add(arcpath)
parent_path = os.path.dirname(arcpath)
mkdirs(parent_path)
zipinfo = zipfile.ZipInfo(
arcpath if arcpath.endswith('/') else arcpath + '/')
# We store directories without compression since they have no contents and
# attempts to store them with compression lead to corrupted zip files as such:
# $ unzip -t junit-runner-0.0.19.jar
# Archive: junit-runner-0.0.19.jar
# testing: com/
# error: invalid compressed data to inflate
# testing: com/twitter/
# error: invalid compressed data to inflate
# testing: com/twitter/common/
# error: invalid compressed data to inflate
# testing: com/twitter/common/testing/
# error: invalid compressed data to inflate
# testing: com/twitter/common/testing/runner/
# error: invalid compressed data to inflate
# testing: com/twitter/common/testing/runner/StreamSource.class OK
zipinfo.compress_type = zipfile.ZIP_STORED
# PKZIP says external_attr is a 4 byte field that is host system dependant:
# http://www.pkware.com/documents/casestudies/APPNOTE.TXT
# These notes do mention the low order byte will carry DOS file attributes for DOS host
# system zips. The DOS file attribute bits are described here:
# http://www.xxcopy.com/xxcopy06.htm
#
# More details are only found reading source, for example in BSD:
# ftp://ftp-archive.freebsd.org/pub/FreeBSD-Archive/old-releases/i386/1.0-RELEASE/ports/info-zip/zipinfo/zipinfo.c
# These sources reveal the 2 high order bytes contain unix file attribute bits.
#
# In summary though the full 32 bit field layout is:
# TTTTsstrwxrwxrwx0000000000ADVSHR
# ^^^^____________________________ stat.h file type: S_IFXXX
# ^^^_________________________ setuid, setgid, sticky
# ^^^^^^^^^________________ permissions
# ^^^^^^^^________ ???
# ^^^^^^^^ DOS attribute bits
# Setup unix directory perm bits: drwxr-xr-x
zipinfo.external_attr = (
stat.S_IFDIR # file type dir
| stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR # u+rwx
| stat.S_IRGRP | stat.S_IXGRP # g+rx
| stat.S_IROTH | stat.S_IXOTH # o+rx
) << 16
# Add DOS directory bit
zipinfo.external_attr |= 0x10
real_writestr(zipinfo, '')
def write(path, arcname=None, **kwargs):
if os.path.isdir(path):
mkdirs(arcname or path)
else:
mkdirs(os.path.dirname(arcname or path))
real_write(path, arcname, **kwargs)
def writestr(zinfo_or_arcname, *args, **kwargs):
mkdirs(os.path.dirname(zinfo_or_arcname))
real_writestr(zinfo_or_arcname, *args, **kwargs)
jar.mkdirs = mkdirs
jar.write = write
jar.writestr = writestr
yield jar
