def bundle(self, app): """Create a self-contained application bundle containing the target classes, dependencies and resources. """ assert(isinstance(app, BundleCreate.App)) bundledir = os.path.join(self.outdir, '%s-bundle' % app.basename) self.context.log.info('creating %s' % os.path.relpath(bundledir, get_buildroot())) safe_mkdir(bundledir, clean=True) classpath = OrderedSet() if not self.deployjar: libdir = os.path.join(bundledir, 'libs') os.mkdir(libdir) # Add external dependencies to the bundle. for basedir, externaljar in self.list_jar_dependencies(app.binary): path = os.path.join(basedir, externaljar) os.symlink(path, os.path.join(libdir, externaljar)) classpath.add(externaljar) # TODO: There should probably be a separate 'binary_jars' product type, # so we can more easily distinguish binary jars (that contain all the classes of their # transitive deps) and per-target jars. for basedir, jars in self.context.products.get('jars').get(app.binary).items(): if len(jars) != 1: raise TaskError('Expected 1 mapped binary for %s but found: %s' % (app.binary, jars)) binary = jars[0] binary_jar = os.path.join(basedir, binary) bundle_jar = os.path.join(bundledir, binary) # Add the internal classes into the bundle_jar. if not classpath: os.symlink(binary_jar, bundle_jar) else: # TODO: Can we copy the existing jar and inject the manifest in, instead of # laboriously copying the contents one by one? Would that be more efficient? with open_zip(binary_jar, 'r') as src: with open_zip(bundle_jar, 'w', compression=ZIP_DEFLATED) as dest: for item in src.infolist(): buf = src.read(item.filename) if Manifest.PATH == item.filename: manifest = Manifest(buf) manifest.addentry(Manifest.CLASS_PATH, ' '.join(os.path.join('libs', jar) for jar in classpath)) buf = manifest.contents() dest.writestr(item, buf) for bundle in app.bundles: for path, relpath in bundle.filemap.items(): bundlepath = os.path.join(bundledir, relpath) safe_mkdir(os.path.dirname(bundlepath)) os.symlink(path, bundlepath) return bundledir
def bundle(self, app): bundledir = os.path.join(self.outdir, '%s-bundle' % app.basename) self.context.log.info('creating %s' % os.path.relpath(bundledir, get_buildroot())) safe_mkdir(bundledir, clean=True) classpath = OrderedSet() if not self.deployjar: libdir = os.path.join(bundledir, 'libs') os.mkdir(libdir) for basedir, externaljar in self.list_jar_dependencies(app.binary): src = os.path.join(basedir, externaljar) link_name = os.path.join(libdir, externaljar) try: os.symlink(src, link_name) except OSError as e: if e.errno == errno.EEXIST: raise TaskError('Trying to symlink %s to %s, but it is already symlinked to %s. ' % (link_name, src, os.readlink(link_name)) + 'Does the bundled target depend on multiple jvm_binary targets?') else: raise classpath.add(externaljar) for basedir, jars in self.context.products.get('jars').get(app.binary).items(): if len(jars) != 1: raise TaskError('Expected 1 mapped binary but found: %s' % jars) binary = jars.pop() binary_jar = os.path.join(basedir, binary) bundle_jar = os.path.join(bundledir, binary) if not classpath: os.symlink(binary_jar, bundle_jar) else: with open_zip(binary_jar, 'r') as src: with open_zip(bundle_jar, 'w', compression=ZIP_DEFLATED) as dest: for item in src.infolist(): buffer = src.read(item.filename) if Manifest.PATH == item.filename: manifest = Manifest(buffer) manifest.addentry(Manifest.CLASS_PATH, ' '.join(os.path.join('libs', jar) for jar in classpath)) buffer = manifest.contents() dest.writestr(item, buffer) for bundle in app.bundles: for path, relpath in bundle.filemap.items(): bundlepath = os.path.join(bundledir, relpath) safe_mkdir(os.path.dirname(bundlepath)) os.symlink(path, bundlepath) return bundledir
def bundle(self, app): bundledir = os.path.join(self.outdir, '%s-bundle' % app.basename) self.context.log.info('creating %s' % os.path.relpath(bundledir, get_buildroot())) safe_mkdir(bundledir, clean=True) classpath = OrderedSet() if not self.deployjar: libdir = os.path.join(bundledir, 'libs') os.mkdir(libdir) for basedir, externaljar in self.list_jar_dependencies(app.binary): path = os.path.join(basedir, externaljar) os.symlink(path, os.path.join(libdir, externaljar)) classpath.add(externaljar) for basedir, jars in self.context.products.get('jars').get( app.binary).items(): if len(jars) != 1: raise TaskError('Expected 1 mapped binary but found: %s' % jars) binary = jars.pop() binary_jar = os.path.join(basedir, binary) bundle_jar = os.path.join(bundledir, binary) if not classpath: os.symlink(binary_jar, bundle_jar) else: with open_zip(binary_jar, 'r') as src: with open_zip(bundle_jar, 'w', compression=ZIP_DEFLATED) as dest: for item in src.infolist(): buffer = src.read(item.filename) if Manifest.PATH == item.filename: manifest = Manifest(buffer) manifest.addentry( Manifest.CLASS_PATH, ' '.join( os.path.join('libs', jar) for jar in classpath)) buffer = manifest.contents() dest.writestr(item, buffer) for bundle in app.bundles: for path, relpath in bundle.filemap.items(): bundlepath = os.path.join(bundledir, relpath) safe_mkdir(os.path.dirname(bundlepath)) os.symlink(path, bundlepath) return bundledir
def _extract(self, jarpath): self.context.log.debug('Extracting idl jar to: %s' % self._EXTRACT_BASE) ZIP.extract(jarpath, self._EXTRACT_BASE) with open_zip(jarpath) as jar: sources = filter(lambda path: path.endswith('.thrift'), jar.namelist()) self.context.log.debug('Found thrift IDL sources: %s' % sources) return sources
def _compute_classpath_elements_by_class(self, classpath): # Don't consider loose classes dirs in our classpath. Those will be considered # separately, by looking at products. def non_product(path): return not (path.startswith(self._pants_workdir) and os.path.isdir(path)) classpath_jars = filter(non_product, classpath) if self._class_to_jarfile is None: self._class_to_jarfile = {} for jarpath in self.find_all_bootstrap_jars() + classpath_jars: # Per the classloading spec, a 'jar' in this context can also be a .zip file. if os.path.isfile(jarpath) and ((jarpath.endswith('.jar') or jarpath.endswith('.zip'))): with open_zip(jarpath, 'r') as jar: for cls in jar.namelist(): # First jar with a given class wins, just like when classloading. if cls.endswith( b'.class' ) and not cls in self._class_to_jarfile: self._class_to_jarfile[cls] = jarpath elif os.path.isdir(jarpath): for dirpath, _, filenames in os.walk(jarpath, followlinks=True): for f in filter(lambda x: x.endswith('.class'), filenames): cls = os.path.relpath(os.path.join(dirpath, f), jarpath) if not cls in self._class_to_jarfile: self._class_to_jarfile[cls] = jarpath return self._class_to_jarfile
def assert_writestr(path, contents, *entries): with self.jarfile() as existing_jarfile: with self.jar_task.open_jar(existing_jarfile) as jar: jar.writestr(path, contents) with open_zip(existing_jarfile) as jar: self.assert_listing(jar, *entries) self.assertEquals(contents, jar.read(path))
def test_overwrite_writestr(self): with self.jarfile() as existing_jarfile: with self.jar_task.open_jar(existing_jarfile, overwrite=True) as jar: jar.writestr('README', b'42') with open_zip(existing_jarfile) as jar: self.assert_listing(jar, 'README') self.assertEquals('42', jar.read('README'))
def bundle(self, app): bundledir = os.path.join(self.outdir, "%s-bundle" % app.basename) self.context.log.info("creating %s" % os.path.relpath(bundledir, get_buildroot())) safe_mkdir(bundledir, clean=True) classpath = OrderedSet() if not self.deployjar: libdir = os.path.join(bundledir, "libs") os.mkdir(libdir) for basedir, externaljar in self.list_jar_dependencies(app.binary): path = os.path.join(basedir, externaljar) os.symlink(path, os.path.join(libdir, externaljar)) classpath.add(externaljar) for basedir, jars in self.context.products.get("jars").get(app.binary).items(): if len(jars) != 1: raise TaskError("Expected 1 mapped binary but found: %s" % jars) binary = jars.pop() binary_jar = os.path.join(basedir, binary) bundle_jar = os.path.join(bundledir, binary) if not classpath: os.symlink(binary_jar, bundle_jar) else: with open_zip(binary_jar, "r") as src: with open_zip(bundle_jar, "w", compression=ZIP_DEFLATED) as dest: for item in src.infolist(): buffer = src.read(item.filename) if Manifest.PATH == item.filename: manifest = Manifest(buffer) manifest.addentry( Manifest.CLASS_PATH, " ".join(os.path.join("libs", jar) for jar in classpath) ) buffer = manifest.contents() dest.writestr(item, buffer) for bundle in app.bundles: for path, relpath in bundle.filemap.items(): bundlepath = os.path.join(bundledir, relpath) safe_mkdir(os.path.dirname(bundlepath)) os.symlink(path, bundlepath) return bundledir
def archive(self, basedir, outdir, name): zippath = os.path.join(outdir, '%s.zip' % name) with open_zip(zippath, 'w', compression=ZIP_DEFLATED) as zip: for root, _, files in os.walk(basedir): for file in files: full_path = os.path.join(root, file) relpath = os.path.relpath(full_path, basedir) zip.write(full_path, relpath) return zippath
def test_extract_pexinfo(): filename = None with temporary_file() as fp: filename = fp.name with open_zip(filename, 'w') as zf: zf.writestr('PEX-INFO', '{"build_properties":{"tag":"thermos_R31337"}}') assert ExecutorVars.get_release_from_binary(filename) == 31337 assert ExecutorVars.get_release_from_binary(filename) == 'UNKNOWN' assert ExecutorVars.get_release_from_binary('lololololo') == 'UNKNOWN'
def test_custom_manifest(self): contents = b'Manifest-Version: 1.0\r\nCreated-By: test\r\n\r\n' with self.jarfile() as existing_jarfile: with self.jar_task.open_jar(existing_jarfile, overwrite=True) as jar: jar.writestr('README', b'42') with open_zip(existing_jarfile) as jar: self.assert_listing(jar, 'README') self.assertEquals('42', jar.read('README')) self.assertNotEqual(contents, jar.read('META-INF/MANIFEST.MF')) with self.jar_task.open_jar(existing_jarfile, overwrite=False) as jar: jar.writestr('META-INF/MANIFEST.MF', contents) with open_zip(existing_jarfile) as jar: self.assert_listing(jar, 'README') self.assertEquals('42', jar.read('README')) self.assertEquals(contents, jar.read('META-INF/MANIFEST.MF'))
def _add_args_resources(self, target, resources_by_target, transitive): classes_by_target = self.context.products.get_data('classes_by_target') resource_dirs = set() def collect_resource_dirs(tgt): mapping = classes_by_target.get(tgt) if mapping: resource_dirs.update( os.path.join(base, self.RESOURCE_RELDIR) for base, _ in mapping.rel_paths()) if transitive: target.walk(collect_resource_dirs) else: collect_resource_dirs(target) lines = set() for resource_dir in resource_dirs: if os.path.exists(resource_dir): for file_name in os.listdir(resource_dir): if file_name.startswith(self.RESOURCE_BASENAME): with open(os.path.join(resource_dir, file_name)) as resource: lines.update(resource.readlines()) if lines: args = Args.for_target(target, transitive, classes_by_target) lines = set(filter(args.matches, lines)) if transitive: # Add args from any of our transitive external deps that have them. resource_path = os.path.join(self.RESOURCE_RELDIR, self.RESOURCE_BASENAME) for base_dir, jar_path in self.list_external_jar_dependencies( target): jar_file = os.path.join(base_dir, jar_path) try: with open_zip(jar_file) as jar: for zipinfo in jar.infolist(): if zipinfo.filename.startswith(resource_path): lines.update(jar.open(zipinfo).readlines()) except zipfile.BadZipfile as e: # Java itself allows non-jar files on the classpath, but # doing do is only useful with a custom classloader that # knows how to handle these files. This use of the # classpath is innovative, bordering on the avant-garde. self.context.log.info( dedent(""" Skipping mapping of {file}. It is not a valid jar. Continuing because invalid jars are ignored. {error}""").format( file=jar_file, error=e)) self._addargs(lines, target, resources_by_target, transitive)
def assert_jar_contents(self, context, product_type, target, *contents): jar_mapping = context.products.get(product_type).get(target) self.assertEqual(1, len(jar_mapping)) for basedir, jars in jar_mapping.items(): self.assertEqual(1, len(jars)) with open_zip(os.path.join(basedir, jars[0])) as jar: self.assertEqual(['META-INF/', 'META-INF/MANIFEST.MF'] + list(contents), jar.namelist()) for content in contents: if not content.endswith('/'): with closing(jar.open(content)) as fp: self.assertEqual(os.path.basename(content), fp.read())
def extract(cls, path, outdir): """OS X's python 2.6.1 has a bug in zipfile that makes it unzip directories as regular files. This method should work on for python 2.6-3.x. """ with open_zip(path) as zip: for path in zip.namelist(): # While we're at it, we also perform this safety test. if path.startswith(b'/') or path.startswith(b'..'): raise ValueError('Zip file contains unsafe path: %s' % path) # Ignore directories. extract() will create parent dirs as needed. if not path.endswith(b'/'): zip.extract(path, outdir)
def create(self, basedir, outdir, name, prefix=None): zippath = os.path.join(outdir, '%s.zip' % name) with open_zip(zippath, 'w', compression=ZIP_DEFLATED) as zip: for root, _, files in os.walk(basedir): root = root.decode('utf-8') for file in files: file = file.decode('utf-8') full_path = os.path.join(root, file) relpath = os.path.relpath(full_path, basedir) if prefix: relpath = os.path.join(prefix.decode('utf-8'), relpath) zip.write(full_path, relpath) return zippath
def test_overwrite_write(self): with temporary_dir() as chroot: _path = os.path.join(chroot, 'a/b/c') safe_mkdir(_path) data_file = os.path.join(_path, 'd.txt') with open(data_file, 'w') as fd: fd.write('e') with self.jarfile() as existing_jarfile: with self.jar_task.open_jar(existing_jarfile, overwrite=True) as jar: jar.write(data_file, 'f/g/h') with open_zip(existing_jarfile) as jar: self.assert_listing(jar, 'f/', 'f/g/', 'f/g/h') self.assertEquals('e', jar.read('f/g/h'))
def _add_args_resources(self, target, resources_by_target, transitive): classes_by_target = self.context.products.get_data('classes_by_target') resource_dirs = set() def collect_resource_dirs(tgt): mapping = classes_by_target.get(tgt) if mapping: resource_dirs.update(os.path.join(base, self.RESOURCE_RELDIR) for base, _ in mapping.rel_paths()) if transitive: target.walk(collect_resource_dirs) else: collect_resource_dirs(target) lines = set() for resource_dir in resource_dirs: if os.path.exists(resource_dir): for file_name in os.listdir(resource_dir): if file_name.startswith(self.RESOURCE_BASENAME): with open(os.path.join(resource_dir, file_name)) as resource: lines.update(resource.readlines()) if lines: args = Args.for_target(target, transitive, classes_by_target) lines = set(filter(args.matches, lines)) if transitive: # Add args from any of our transitive external deps that have them. resource_path = os.path.join(self.RESOURCE_RELDIR, self.RESOURCE_BASENAME) for base_dir, jar_path in self.list_external_jar_dependencies(target): jar_file = os.path.join(base_dir, jar_path) try: with open_zip(jar_file) as jar: for zipinfo in jar.infolist(): if zipinfo.filename.startswith(resource_path): lines.update(jar.open(zipinfo).readlines()) except zipfile.BadZipfile as e: # Java itself allows non-jar files on the classpath, but # doing do is only useful with a custom classloader that # knows how to handle these files. This use of the # classpath is innovative, bordering on the avant-garde. self.context.log.info(dedent(""" Skipping mapping of {file}. It is not a valid jar. Continuing because invalid jars are ignored. {error}""").format (file=jar_file, error=e)) self._addargs(lines, target, resources_by_target, transitive)
def _add_args_resources(self, target, resources_by_target, transitive): classes_by_target = self.context.products.get_data('classes_by_target') resource_dirs = set() def collect_resource_dirs(tgt): mapping = classes_by_target.get(tgt) if mapping: resource_dirs.update( os.path.join(base, self.RESOURCE_RELDIR) for base, _ in mapping.rel_paths()) if transitive: target.walk(collect_resource_dirs) else: collect_resource_dirs(target) lines = set() for resource_dir in resource_dirs: if os.path.exists(resource_dir): for file_name in os.listdir(resource_dir): if file_name.startswith(self.RESOURCE_BASENAME): with open(os.path.join(resource_dir, file_name)) as resource: lines.update(resource.readlines()) if lines: args = Args.for_target(target, transitive, classes_by_target) lines = set(filter(args.matches, lines)) if transitive: # Add args from any of our transitive external deps that have them. resource_path = os.path.join(self.RESOURCE_RELDIR, self.RESOURCE_BASENAME) for base_dir, jar_path in self.list_external_jar_dependencies( target): with open_zip(os.path.join(base_dir, jar_path)) as jar: for zipinfo in jar.infolist(): if zipinfo.filename.startswith(resource_path): lines.update(jar.open(zipinfo).readlines()) self._addargs(lines, target, resources_by_target, transitive)
def _add_args_resources(self, target, resources_by_target, transitive): classes_by_target = self.context.products.get_data('classes_by_target') resource_dirs = set() def collect_resource_dirs(tgt): mapping = classes_by_target.get(tgt) if mapping: resource_dirs.update(os.path.join(base, self.RESOURCE_RELDIR) for base, _ in mapping.rel_paths()) if transitive: target.walk(collect_resource_dirs) else: collect_resource_dirs(target) lines = set() for resource_dir in resource_dirs: if os.path.exists(resource_dir): for file_name in os.listdir(resource_dir): if file_name.startswith(self.RESOURCE_BASENAME): with open(os.path.join(resource_dir, file_name)) as resource: lines.update(resource.readlines()) if lines: args = Args.for_target(target, transitive, classes_by_target) lines = filter(args.matches, lines) if transitive: # Add args from any of our transitive external deps that have them. resource_path = os.path.join(self.RESOURCE_RELDIR, self.RESOURCE_BASENAME) for base_dir, jar_path in self.list_external_jar_dependencies(target): with open_zip(os.path.join(base_dir, jar_path)) as jar: for zipinfo in jar.infolist(): if zipinfo.filename.startswith(resource_path): lines.update(jar.open(zipinfo).readlines()) self._addargs(lines, target, resources_by_target, transitive)
def _compute_classpath_elements_by_class(self, classpath): # Don't consider loose classes dirs in our classpath. Those will be considered # separately, by looking at products. def non_product(path): return not (path.startswith(self._pants_workdir) and os.path.isdir(path)) classpath_jars = filter(non_product, classpath) if self._class_to_jarfile is None: self._class_to_jarfile = {} for jarpath in self.find_all_bootstrap_jars() + classpath_jars: # Per the classloading spec, a 'jar' in this context can also be a .zip file. if os.path.isfile(jarpath) and ((jarpath.endswith('.jar') or jarpath.endswith('.zip'))): with open_zip(jarpath, 'r') as jar: for cls in jar.namelist(): # First jar with a given class wins, just like when classloading. if cls.endswith('.class') and not cls in self._class_to_jarfile: self._class_to_jarfile[cls] = jarpath elif os.path.isdir(jarpath): for dirpath, _, filenames in os.walk(jarpath, followlinks=True): for f in filter(lambda x: x.endswith('.class'), filenames): cls = os.path.relpath(os.path.join(dirpath, f), jarpath) if not cls in self._class_to_jarfile: self._class_to_jarfile[cls] = jarpath return self._class_to_jarfile
def _compute_classpath_elements_by_class(self, classpath): # Don't consider loose classes dirs in our classes dir. Those will be considered # separately, by looking at products. def non_product(path): return path != self._classes_dir if self._upstream_class_to_path is None: self._upstream_class_to_path = {} classpath_entries = filter(non_product, classpath) for cp_entry in self.find_all_bootstrap_jars() + classpath_entries: # Per the classloading spec, a 'jar' in this context can also be a .zip file. if os.path.isfile(cp_entry) and ((cp_entry.endswith('.jar') or cp_entry.endswith('.zip'))): with open_zip(cp_entry, 'r') as jar: for cls in jar.namelist(): # First jar with a given class wins, just like when classloading. if cls.endswith(b'.class') and not cls in self._upstream_class_to_path: self._upstream_class_to_path[cls] = cp_entry elif os.path.isdir(cp_entry): for dirpath, _, filenames in os.walk(cp_entry, followlinks=True): for f in filter(lambda x: x.endswith('.class'), filenames): cls = os.path.relpath(os.path.join(dirpath, f), cp_entry) if not cls in self._upstream_class_to_path: self._upstream_class_to_path[cls] = os.path.join(dirpath, f) return self._upstream_class_to_path
def open_jar(path, *args, **kwargs): """Yields a jar in a with context that will be closed when the context exits. The yielded jar is a zipfile.ZipFile object with an additional mkdirs(arcpath) method that will create a zip directory entry similar to unix `mkdir -p`. Additionally, the ZipFile.write and ZipFile.writestr methods are enhanced to call mkdirs as needed to ensure all jar entries contain a full complement of parent paths leading from each leaf to the root of the jar. """ with open_zip(path, *args, **kwargs) as jar: real_write = jar.write real_writestr = jar.writestr made_dirs = set() def mkdirs(arcpath): if arcpath and arcpath not in made_dirs: made_dirs.add(arcpath) parent_path = os.path.dirname(arcpath) mkdirs(parent_path) zipinfo = zipfile.ZipInfo(arcpath if arcpath.endswith('/') else arcpath + '/') # We store directories without compression since they have no contents and # attempts to store them with compression lead to corrupted zip files as such: # $ unzip -t junit-runner-0.0.19.jar # Archive: junit-runner-0.0.19.jar # testing: com/ # error: invalid compressed data to inflate # testing: com/twitter/ # error: invalid compressed data to inflate # testing: com/twitter/common/ # error: invalid compressed data to inflate # testing: com/twitter/common/testing/ # error: invalid compressed data to inflate # testing: com/twitter/common/testing/runner/ # error: invalid compressed data to inflate # testing: com/twitter/common/testing/runner/StreamSource.class OK zipinfo.compress_type = zipfile.ZIP_STORED # PKZIP says external_attr is a 4 byte field that is host system dependant: # http://www.pkware.com/documents/casestudies/APPNOTE.TXT # These notes do mention the low order byte will carry DOS file attributes for DOS host # system zips. The DOS file attribute bits are described here: # http://www.xxcopy.com/xxcopy06.htm # # More details are only found reading source, for example in BSD: # ftp://ftp-archive.freebsd.org/pub/FreeBSD-Archive/old-releases/i386/1.0-RELEASE/ports/info-zip/zipinfo/zipinfo.c # These sources reveal the 2 high order bytes contain unix file attribute bits. # # In summary though the full 32 bit field layout is: # TTTTsstrwxrwxrwx0000000000ADVSHR # ^^^^____________________________ stat.h file type: S_IFXXX # ^^^_________________________ setuid, setgid, sticky # ^^^^^^^^^________________ permissions # ^^^^^^^^________ ??? # ^^^^^^^^ DOS attribute bits # Setup unix directory perm bits: drwxr-xr-x zipinfo.external_attr = ( stat.S_IFDIR # file type dir | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR # u+rwx | stat.S_IRGRP | stat.S_IXGRP # g+rx | stat.S_IROTH | stat.S_IXOTH # o+rx ) << 16 # Add DOS directory bit zipinfo.external_attr |= 0x10 real_writestr(zipinfo, '') def write(path, arcname=None, **kwargs): if os.path.isdir(path): mkdirs(arcname or path) else: mkdirs(os.path.dirname(arcname or path)) real_write(path, arcname, **kwargs) def writestr(zinfo_or_arcname, *args, **kwargs): mkdirs(os.path.dirname(zinfo_or_arcname)) real_writestr(zinfo_or_arcname, *args, **kwargs) jar.mkdirs = mkdirs jar.write = write jar.writestr = writestr yield jar
def bundle(self, app): """Create a self-contained application bundle containing the target classes, dependencies and resources. """ assert (isinstance(app, BundleCreate.App)) bundledir = os.path.join(self.outdir, '%s-bundle' % app.basename) self.context.log.info('creating %s' % os.path.relpath(bundledir, get_buildroot())) safe_mkdir(bundledir, clean=True) classpath = OrderedSet() if not self.deployjar: libdir = os.path.join(bundledir, 'libs') os.mkdir(libdir) # Add internal dependencies to the bundle. def add_jars(target): target_jars = self.context.products.get('jars').get(target) if target_jars is not None: for basedir, jars in target_jars.items(): for internaljar in jars: os.symlink(os.path.join(basedir, internaljar), os.path.join(libdir, internaljar)) classpath.add(internaljar) app.binary.walk(add_jars, lambda t: t.is_internal) # Add external dependencies to the bundle. for basedir, externaljar in self.list_jar_dependencies(app.binary): path = os.path.join(basedir, externaljar) os.symlink(path, os.path.join(libdir, externaljar)) classpath.add(externaljar) for basedir, jars in self.context.products.get('jars').get( app.binary).items(): if len(jars) != 1: raise TaskError( 'Expected 1 mapped binary for %s but found: %s' % (app.binary, jars)) binary = jars[0] binary_jar = os.path.join(basedir, binary) bundle_jar = os.path.join(bundledir, binary) if not classpath: os.symlink(binary_jar, bundle_jar) else: with open_zip(binary_jar, 'r') as src: with open_zip(bundle_jar, 'w', compression=ZIP_DEFLATED) as dest: for item in src.infolist(): buf = src.read(item.filename) if Manifest.PATH == item.filename: manifest = Manifest(buf) manifest.addentry( Manifest.CLASS_PATH, ' '.join( os.path.join('libs', jar) for jar in classpath)) buf = manifest.contents() dest.writestr(item, buf) for bundle in app.bundles: for path, relpath in bundle.filemap.items(): bundlepath = os.path.join(bundledir, relpath) safe_mkdir(os.path.dirname(bundlepath)) os.symlink(path, bundlepath) return bundledir
def bundle(self, app): """Create a self-contained application bundle containing the target classes, dependencies and resources. """ assert(isinstance(app, BundleCreate.App)) bundledir = os.path.join(self.outdir, '%s-bundle' % app.basename) self.context.log.info('creating %s' % os.path.relpath(bundledir, get_buildroot())) safe_mkdir(bundledir, clean=True) classpath = OrderedSet() if not self.deployjar: libdir = os.path.join(bundledir, 'libs') os.mkdir(libdir) # Add internal dependencies to the bundle. def add_jars(target): target_jars = self.context.products.get('jars').get(target) if target_jars is not None: for basedir, jars in target_jars.items(): for internaljar in jars: os.symlink(os.path.join(basedir, internaljar), os.path.join(libdir, internaljar)) classpath.add(internaljar) app.binary.walk(add_jars, lambda t: t.is_internal) # Add external dependencies to the bundle. for basedir, externaljar in self.list_jar_dependencies(app.binary): path = os.path.join(basedir, externaljar) os.symlink(path, os.path.join(libdir, externaljar)) classpath.add(externaljar) for basedir, jars in self.context.products.get('jars').get(app.binary).items(): if len(jars) != 1: raise TaskError('Expected 1 mapped binary for %s but found: %s' % (app.binary, jars)) binary = jars[0] binary_jar = os.path.join(basedir, binary) bundle_jar = os.path.join(bundledir, binary) if not classpath: os.symlink(binary_jar, bundle_jar) else: with open_zip(binary_jar, 'r') as src: with open_zip(bundle_jar, 'w', compression=ZIP_DEFLATED) as dest: for item in src.infolist(): buf = src.read(item.filename) if Manifest.PATH == item.filename: manifest = Manifest(buf) manifest.addentry(Manifest.CLASS_PATH, ' '.join(os.path.join('libs', jar) for jar in classpath)) buf = manifest.contents() dest.writestr(item, buf) for bundle in app.bundles: for path, relpath in bundle.filemap.items(): bundlepath = os.path.join(bundledir, relpath) safe_mkdir(os.path.dirname(bundlepath)) os.symlink(path, bundlepath) return bundledir
def open_jar(path, *args, **kwargs): """Yields a jar in a with context that will be closed when the context exits. The yielded jar is a zipfile.ZipFile object with an additional mkdirs(arcpath) method that will create a zip directory entry similar to unix `mkdir -p`. Additionally, the ZipFile.write and ZipFile.writestr methods are enhanced to call mkdirs as needed to ensure all jar entries contain a full complement of parent paths leading from each leaf to the root of the jar. """ with open_zip(path, *args, **kwargs) as jar: real_write = jar.write real_writestr = jar.writestr made_dirs = set() def mkdirs(arcpath): if arcpath and arcpath not in made_dirs: made_dirs.add(arcpath) parent_path = os.path.dirname(arcpath) mkdirs(parent_path) zipinfo = zipfile.ZipInfo( arcpath if arcpath.endswith('/') else arcpath + '/') # We store directories without compression since they have no contents and # attempts to store them with compression lead to corrupted zip files as such: # $ unzip -t junit-runner-0.0.19.jar # Archive: junit-runner-0.0.19.jar # testing: com/ # error: invalid compressed data to inflate # testing: com/twitter/ # error: invalid compressed data to inflate # testing: com/twitter/common/ # error: invalid compressed data to inflate # testing: com/twitter/common/testing/ # error: invalid compressed data to inflate # testing: com/twitter/common/testing/runner/ # error: invalid compressed data to inflate # testing: com/twitter/common/testing/runner/StreamSource.class OK zipinfo.compress_type = zipfile.ZIP_STORED # PKZIP says external_attr is a 4 byte field that is host system dependant: # http://www.pkware.com/documents/casestudies/APPNOTE.TXT # These notes do mention the low order byte will carry DOS file attributes for DOS host # system zips. The DOS file attribute bits are described here: # http://www.xxcopy.com/xxcopy06.htm # # More details are only found reading source, for example in BSD: # ftp://ftp-archive.freebsd.org/pub/FreeBSD-Archive/old-releases/i386/1.0-RELEASE/ports/info-zip/zipinfo/zipinfo.c # These sources reveal the 2 high order bytes contain unix file attribute bits. # # In summary though the full 32 bit field layout is: # TTTTsstrwxrwxrwx0000000000ADVSHR # ^^^^____________________________ stat.h file type: S_IFXXX # ^^^_________________________ setuid, setgid, sticky # ^^^^^^^^^________________ permissions # ^^^^^^^^________ ??? # ^^^^^^^^ DOS attribute bits # Setup unix directory perm bits: drwxr-xr-x zipinfo.external_attr = ( stat.S_IFDIR # file type dir | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR # u+rwx | stat.S_IRGRP | stat.S_IXGRP # g+rx | stat.S_IROTH | stat.S_IXOTH # o+rx ) << 16 # Add DOS directory bit zipinfo.external_attr |= 0x10 real_writestr(zipinfo, '') def write(path, arcname=None, **kwargs): if os.path.isdir(path): mkdirs(arcname or path) else: mkdirs(os.path.dirname(arcname or path)) real_write(path, arcname, **kwargs) def writestr(zinfo_or_arcname, *args, **kwargs): mkdirs(os.path.dirname(zinfo_or_arcname)) real_writestr(zinfo_or_arcname, *args, **kwargs) jar.mkdirs = mkdirs jar.write = write jar.writestr = writestr yield jar