def find_plugins(self, plugin_names):
"""Returns a map from plugin name to plugin jar."""
plugin_names = set(plugin_names)
plugins = {}
buildroot = get_buildroot()
# plugin_jars is the universe of all possible plugins and their transitive deps.
# Here we select the ones to actually use.
for jar in self.plugin_jars():
with open_zip64(jar, 'r') as jarfile:
try:
with closing(jarfile.open(_PLUGIN_INFO_FILE,
'r')) as plugin_info_file:
plugin_info = ElementTree.parse(
plugin_info_file).getroot()
if plugin_info.tag != 'plugin':
raise TaskError(
'File %s in %s is not a valid scalac plugin descriptor'
% (_PLUGIN_INFO_FILE, jar))
name = plugin_info.find('name').text
if name in plugin_names:
if name in plugins:
raise TaskError(
'Plugin %s defined in %s and in %s' %
(name, plugins[name], jar))
# It's important to use relative paths, as the compiler flags get embedded in the zinc
# analysis file, and we port those between systems via the artifact cache.
plugins[name] = os.path.relpath(jar, buildroot)
except KeyError:
pass
unresolved_plugins = plugin_names - set(plugins.keys())
if unresolved_plugins:
raise TaskError('Could not find requested plugins: %s' %
list(unresolved_plugins))
return plugins
def find_plugins(self, plugin_names):
"""Returns a map from plugin name to plugin jar."""
plugin_names = set(plugin_names)
plugins = {}
buildroot = get_buildroot()
# plugin_jars is the universe of all possible plugins and their transitive deps.
# Here we select the ones to actually use.
for jar in self.plugin_jars():
with open_zip64(jar, 'r') as jarfile:
try:
with closing(jarfile.open(_PLUGIN_INFO_FILE, 'r')) as plugin_info_file:
plugin_info = ElementTree.parse(plugin_info_file).getroot()
if plugin_info.tag != 'plugin':
raise TaskError(
'File %s in %s is not a valid scalac plugin descriptor' % (_PLUGIN_INFO_FILE, jar))
name = plugin_info.find('name').text
if name in plugin_names:
if name in plugins:
raise TaskError('Plugin %s defined in %s and in %s' % (name, plugins[name], jar))
# It's important to use relative paths, as the compiler flags get embedded in the zinc
# analysis file, and we port those between systems via the artifact cache.
plugins[name] = os.path.relpath(jar, buildroot)
except KeyError:
pass
unresolved_plugins = plugin_names - set(plugins.keys())
if unresolved_plugins:
raise TaskError('Could not find requested plugins: %s' % list(unresolved_plugins))
return plugins
def _compute_classpath_elements_by_class(self, classpath):
# Don't consider loose classes dirs in our classes dir. Those will be considered
# separately, by looking at products.
def non_product(path):
return path != self._classes_dir
if self._upstream_class_to_path is None:
self._upstream_class_to_path = {}
classpath_entries = filter(non_product, classpath)
for cp_entry in self.find_all_bootstrap_jars() + classpath_entries:
# Per the classloading spec, a 'jar' in this context can also be a .zip file.
if os.path.isfile(cp_entry) and (
(cp_entry.endswith('.jar') or cp_entry.endswith('.zip'))):
with open_zip64(cp_entry, 'r') as jar:
for cls in jar.namelist():
# First jar with a given class wins, just like when classloading.
if cls.endswith(
b'.class'
) and not cls in self._upstream_class_to_path:
self._upstream_class_to_path[cls] = cp_entry
elif os.path.isdir(cp_entry):
for dirpath, _, filenames in safe_walk(cp_entry,
followlinks=True):
for f in filter(lambda x: x.endswith('.class'),
filenames):
cls = os.path.relpath(os.path.join(dirpath, f),
cp_entry)
if not cls in self._upstream_class_to_path:
self._upstream_class_to_path[
cls] = os.path.join(dirpath, f)
return self._upstream_class_to_path
def create(self, basedir, outdir, name, prefix=None):
zippath = os.path.join(outdir, '%s.zip' % name)
with open_zip64(zippath, 'w', compression=ZIP_DEFLATED) as zip:
for root, _, files in safe_walk(basedir):
root = ensure_text(root)
for file in files:
file = ensure_text(file)
full_path = os.path.join(root, file)
relpath = os.path.relpath(full_path, basedir)
if prefix:
relpath = os.path.join(ensure_text(prefix), relpath)
zip.write(full_path, relpath)
return zippath
def _get_external_dependencies(self, binary_target):
artifacts_by_file_name = defaultdict(set)
for basedir, externaljar in self.list_external_jar_dependencies(binary_target):
external_dep = os.path.join(basedir, externaljar)
self.context.log.debug(' scanning %s' % external_dep)
with open_zip64(external_dep) as dep_zip:
for qualified_file_name in dep_zip.namelist():
# Zip entry names can come in any encoding and in practice we find some jars that have
# utf-8 encoded entry names, some not. As a result we cannot simply decode in all cases
# and need to do this to_bytes(...).decode('utf-8') dance to stay safe across all entry
# name flavors and under all supported pythons.
decoded_file_name = to_bytes(qualified_file_name).decode('utf-8')
if os.path.basename(decoded_file_name).lower() in self._excludes:
continue
jar_name = os.path.basename(external_dep)
if (not self._isdir(decoded_file_name)) and Manifest.PATH != decoded_file_name:
artifacts_by_file_name[decoded_file_name].add(jar_name)
return artifacts_by_file_name
def extract(cls, path, outdir, filter_func=None):
"""Extract from a zip file, with an optional filter
:param string path: path to the zipfile to extract from
:param string outdir: directory to extract files into
:param function filter_func: optional filter with the filename as the parameter. Returns True if
the file should be extracted.
"""
with open_zip64(path) as archive_file:
for name in archive_file.namelist():
# While we're at it, we also perform this safety test.
if name.startswith(b'/') or name.startswith(b'..'):
raise ValueError('Zip file contains unsafe path: %s' % name)
# Ignore directories. extract() will create parent dirs as needed.
# OS X's python 2.6.1 has a bug in zipfile that makes it unzip directories as regular files.
# This method should work on for python 2.6-3.x.
# TODO(Eric Ayers) Pants no longer builds with python 2.6. Can this be removed?
if not name.endswith(b'/'):
if (not filter_func or filter_func(name)):
archive_file.extract(name, outdir)
def _get_external_dependencies(self, binary_target):
artifacts_by_file_name = defaultdict(set)
for basedir, externaljar in self.list_external_jar_dependencies(
binary_target):
external_dep = os.path.join(basedir, externaljar)
self.context.log.debug(' scanning %s' % external_dep)
with open_zip64(external_dep) as dep_zip:
for qualified_file_name in dep_zip.namelist():
# Zip entry names can come in any encoding and in practice we find some jars that have
# utf-8 encoded entry names, some not. As a result we cannot simply decode in all cases
# and need to do this to_bytes(...).decode('utf-8') dance to stay safe across all entry
# name flavors and under all supported pythons.
decoded_file_name = to_bytes(qualified_file_name).decode(
'utf-8')
if os.path.basename(
decoded_file_name).lower() in self._excludes:
continue
jar_name = os.path.basename(external_dep)
if (not self._isdir(decoded_file_name)
) and Manifest.PATH != decoded_file_name:
artifacts_by_file_name[decoded_file_name].add(jar_name)
return artifacts_by_file_name
def _compute_classpath_elements_by_class(self, classpath):
# Don't consider loose classes dirs in our classes dir. Those will be considered
# separately, by looking at products.
def non_product(path):
return path != self._classes_dir
if self._upstream_class_to_path is None:
self._upstream_class_to_path = {}
classpath_entries = filter(non_product, classpath)
for cp_entry in self.find_all_bootstrap_jars() + classpath_entries:
# Per the classloading spec, a 'jar' in this context can also be a .zip file.
if os.path.isfile(cp_entry) and ((cp_entry.endswith('.jar') or cp_entry.endswith('.zip'))):
with open_zip64(cp_entry, 'r') as jar:
for cls in jar.namelist():
# First jar with a given class wins, just like when classloading.
if cls.endswith(b'.class') and not cls in self._upstream_class_to_path:
self._upstream_class_to_path[cls] = cp_entry
elif os.path.isdir(cp_entry):
for dirpath, _, filenames in safe_walk(cp_entry, followlinks=True):
for f in filter(lambda x: x.endswith('.class'), filenames):
cls = os.path.relpath(os.path.join(dirpath, f), cp_entry)
if not cls in self._upstream_class_to_path:
self._upstream_class_to_path[cls] = os.path.join(dirpath, f)
return self._upstream_class_to_path
def test_open_zip64Default(self):
with open_zip64('test', 'w') as zf:
self.assertTrue(zf._allowZip64)
def test_open_zip64False(self):
with temporary_dir() as tempdir:
with open_zip64(os.path.join(tempdir, 'test'), 'w', allowZip64=False) as zf:
self.assertFalse(zf._allowZip64)
def test_open_zip64Default(self):
with temporary_dir() as tempdir:
with open_zip64(os.path.join(tempdir, 'test'), 'w') as zf:
self.assertTrue(zf._allowZip64)
def test_open_zip64False(self):
with temporary_dir() as tempdir:
with open_zip64(os.path.join(tempdir, 'test'),
'w',
allowZip64=False) as zf:
self.assertFalse(zf._allowZip64)
def generate_jar(path, *class_name):
with open_zip64(generate_path(path), 'w') as zipfile:
for clazz in class_name:
zipfile.write(clazz)
return zipfile.filename
def generate_jar(path, *class_name):
with open_zip64(generate_path(path), 'w') as zipfile:
for clazz in class_name:
zipfile.write(clazz)
return zipfile.filename
def list_jar(self, path):
with open_zip64(path, 'r') as jar:
return jar.namelist()
def test_open_zip64True(self):
with open_zip64('test', 'w', allowZip64=True) as zf:
self.assertTrue(zf._allowZip64)
def test_open_zip64False(self):
with open_zip64('test', 'w', allowZip64=False) as zf:
self.assertFalse(zf._allowZip64)
def test_open_zip64Default(self):
with temporary_dir() as tempdir:
with open_zip64(os.path.join(tempdir, 'test'), 'w') as zf:
self.assertTrue(zf._allowZip64)