def find_plugins(self, plugin_names): """Returns a map from plugin name to plugin jar.""" plugin_names = set(plugin_names) plugins = {} buildroot = get_buildroot() # plugin_jars is the universe of all possible plugins and their transitive deps. # Here we select the ones to actually use. for jar in self.plugin_jars(): with open_zip64(jar, 'r') as jarfile: try: with closing(jarfile.open(_PLUGIN_INFO_FILE, 'r')) as plugin_info_file: plugin_info = ElementTree.parse( plugin_info_file).getroot() if plugin_info.tag != 'plugin': raise TaskError( 'File %s in %s is not a valid scalac plugin descriptor' % (_PLUGIN_INFO_FILE, jar)) name = plugin_info.find('name').text if name in plugin_names: if name in plugins: raise TaskError( 'Plugin %s defined in %s and in %s' % (name, plugins[name], jar)) # It's important to use relative paths, as the compiler flags get embedded in the zinc # analysis file, and we port those between systems via the artifact cache. plugins[name] = os.path.relpath(jar, buildroot) except KeyError: pass unresolved_plugins = plugin_names - set(plugins.keys()) if unresolved_plugins: raise TaskError('Could not find requested plugins: %s' % list(unresolved_plugins)) return plugins
def find_plugins(self, plugin_names): """Returns a map from plugin name to plugin jar.""" plugin_names = set(plugin_names) plugins = {} buildroot = get_buildroot() # plugin_jars is the universe of all possible plugins and their transitive deps. # Here we select the ones to actually use. for jar in self.plugin_jars(): with open_zip64(jar, 'r') as jarfile: try: with closing(jarfile.open(_PLUGIN_INFO_FILE, 'r')) as plugin_info_file: plugin_info = ElementTree.parse(plugin_info_file).getroot() if plugin_info.tag != 'plugin': raise TaskError( 'File %s in %s is not a valid scalac plugin descriptor' % (_PLUGIN_INFO_FILE, jar)) name = plugin_info.find('name').text if name in plugin_names: if name in plugins: raise TaskError('Plugin %s defined in %s and in %s' % (name, plugins[name], jar)) # It's important to use relative paths, as the compiler flags get embedded in the zinc # analysis file, and we port those between systems via the artifact cache. plugins[name] = os.path.relpath(jar, buildroot) except KeyError: pass unresolved_plugins = plugin_names - set(plugins.keys()) if unresolved_plugins: raise TaskError('Could not find requested plugins: %s' % list(unresolved_plugins)) return plugins
def _compute_classpath_elements_by_class(self, classpath): # Don't consider loose classes dirs in our classes dir. Those will be considered # separately, by looking at products. def non_product(path): return path != self._classes_dir if self._upstream_class_to_path is None: self._upstream_class_to_path = {} classpath_entries = filter(non_product, classpath) for cp_entry in self.find_all_bootstrap_jars() + classpath_entries: # Per the classloading spec, a 'jar' in this context can also be a .zip file. if os.path.isfile(cp_entry) and ( (cp_entry.endswith('.jar') or cp_entry.endswith('.zip'))): with open_zip64(cp_entry, 'r') as jar: for cls in jar.namelist(): # First jar with a given class wins, just like when classloading. if cls.endswith( b'.class' ) and not cls in self._upstream_class_to_path: self._upstream_class_to_path[cls] = cp_entry elif os.path.isdir(cp_entry): for dirpath, _, filenames in safe_walk(cp_entry, followlinks=True): for f in filter(lambda x: x.endswith('.class'), filenames): cls = os.path.relpath(os.path.join(dirpath, f), cp_entry) if not cls in self._upstream_class_to_path: self._upstream_class_to_path[ cls] = os.path.join(dirpath, f) return self._upstream_class_to_path
def create(self, basedir, outdir, name, prefix=None): zippath = os.path.join(outdir, '%s.zip' % name) with open_zip64(zippath, 'w', compression=ZIP_DEFLATED) as zip: for root, _, files in safe_walk(basedir): root = ensure_text(root) for file in files: file = ensure_text(file) full_path = os.path.join(root, file) relpath = os.path.relpath(full_path, basedir) if prefix: relpath = os.path.join(ensure_text(prefix), relpath) zip.write(full_path, relpath) return zippath
def _get_external_dependencies(self, binary_target): artifacts_by_file_name = defaultdict(set) for basedir, externaljar in self.list_external_jar_dependencies(binary_target): external_dep = os.path.join(basedir, externaljar) self.context.log.debug(' scanning %s' % external_dep) with open_zip64(external_dep) as dep_zip: for qualified_file_name in dep_zip.namelist(): # Zip entry names can come in any encoding and in practice we find some jars that have # utf-8 encoded entry names, some not. As a result we cannot simply decode in all cases # and need to do this to_bytes(...).decode('utf-8') dance to stay safe across all entry # name flavors and under all supported pythons. decoded_file_name = to_bytes(qualified_file_name).decode('utf-8') if os.path.basename(decoded_file_name).lower() in self._excludes: continue jar_name = os.path.basename(external_dep) if (not self._isdir(decoded_file_name)) and Manifest.PATH != decoded_file_name: artifacts_by_file_name[decoded_file_name].add(jar_name) return artifacts_by_file_name
def extract(cls, path, outdir, filter_func=None): """Extract from a zip file, with an optional filter :param string path: path to the zipfile to extract from :param string outdir: directory to extract files into :param function filter_func: optional filter with the filename as the parameter. Returns True if the file should be extracted. """ with open_zip64(path) as archive_file: for name in archive_file.namelist(): # While we're at it, we also perform this safety test. if name.startswith(b'/') or name.startswith(b'..'): raise ValueError('Zip file contains unsafe path: %s' % name) # Ignore directories. extract() will create parent dirs as needed. # OS X's python 2.6.1 has a bug in zipfile that makes it unzip directories as regular files. # This method should work on for python 2.6-3.x. # TODO(Eric Ayers) Pants no longer builds with python 2.6. Can this be removed? if not name.endswith(b'/'): if (not filter_func or filter_func(name)): archive_file.extract(name, outdir)
def _get_external_dependencies(self, binary_target): artifacts_by_file_name = defaultdict(set) for basedir, externaljar in self.list_external_jar_dependencies( binary_target): external_dep = os.path.join(basedir, externaljar) self.context.log.debug(' scanning %s' % external_dep) with open_zip64(external_dep) as dep_zip: for qualified_file_name in dep_zip.namelist(): # Zip entry names can come in any encoding and in practice we find some jars that have # utf-8 encoded entry names, some not. As a result we cannot simply decode in all cases # and need to do this to_bytes(...).decode('utf-8') dance to stay safe across all entry # name flavors and under all supported pythons. decoded_file_name = to_bytes(qualified_file_name).decode( 'utf-8') if os.path.basename( decoded_file_name).lower() in self._excludes: continue jar_name = os.path.basename(external_dep) if (not self._isdir(decoded_file_name) ) and Manifest.PATH != decoded_file_name: artifacts_by_file_name[decoded_file_name].add(jar_name) return artifacts_by_file_name
def _compute_classpath_elements_by_class(self, classpath): # Don't consider loose classes dirs in our classes dir. Those will be considered # separately, by looking at products. def non_product(path): return path != self._classes_dir if self._upstream_class_to_path is None: self._upstream_class_to_path = {} classpath_entries = filter(non_product, classpath) for cp_entry in self.find_all_bootstrap_jars() + classpath_entries: # Per the classloading spec, a 'jar' in this context can also be a .zip file. if os.path.isfile(cp_entry) and ((cp_entry.endswith('.jar') or cp_entry.endswith('.zip'))): with open_zip64(cp_entry, 'r') as jar: for cls in jar.namelist(): # First jar with a given class wins, just like when classloading. if cls.endswith(b'.class') and not cls in self._upstream_class_to_path: self._upstream_class_to_path[cls] = cp_entry elif os.path.isdir(cp_entry): for dirpath, _, filenames in safe_walk(cp_entry, followlinks=True): for f in filter(lambda x: x.endswith('.class'), filenames): cls = os.path.relpath(os.path.join(dirpath, f), cp_entry) if not cls in self._upstream_class_to_path: self._upstream_class_to_path[cls] = os.path.join(dirpath, f) return self._upstream_class_to_path
def test_open_zip64Default(self): with open_zip64('test', 'w') as zf: self.assertTrue(zf._allowZip64)
def test_open_zip64False(self): with temporary_dir() as tempdir: with open_zip64(os.path.join(tempdir, 'test'), 'w', allowZip64=False) as zf: self.assertFalse(zf._allowZip64)
def test_open_zip64Default(self): with temporary_dir() as tempdir: with open_zip64(os.path.join(tempdir, 'test'), 'w') as zf: self.assertTrue(zf._allowZip64)
def test_open_zip64False(self): with temporary_dir() as tempdir: with open_zip64(os.path.join(tempdir, 'test'), 'w', allowZip64=False) as zf: self.assertFalse(zf._allowZip64)
def generate_jar(path, *class_name): with open_zip64(generate_path(path), 'w') as zipfile: for clazz in class_name: zipfile.write(clazz) return zipfile.filename
def generate_jar(path, *class_name): with open_zip64(generate_path(path), 'w') as zipfile: for clazz in class_name: zipfile.write(clazz) return zipfile.filename
def list_jar(self, path): with open_zip64(path, 'r') as jar: return jar.namelist()
def test_open_zip64True(self): with open_zip64('test', 'w', allowZip64=True) as zf: self.assertTrue(zf._allowZip64)
def test_open_zip64False(self): with open_zip64('test', 'w', allowZip64=False) as zf: self.assertFalse(zf._allowZip64)
def test_open_zip64Default(self): with temporary_dir() as tempdir: with open_zip64(os.path.join(tempdir, 'test'), 'w') as zf: self.assertTrue(zf._allowZip64)