def safe_classpath(classpath, synthetic_jar_dir, custom_name=None):
"""Bundles classpath into one synthetic jar that includes original classpath in its manifest.
This is to ensure classpath length never exceeds platform ARG_MAX.
:param list classpath: Classpath to be bundled.
:param string synthetic_jar_dir: directory to store the synthetic jar, if `None`
a temp directory will be provided and cleaned up upon process exit. Otherwise synthetic
jar will remain in the supplied directory, only for debugging purpose.
:param custom_name: filename of the synthetic jar to be created.
:returns: A classpath (singleton list with just the synthetic jar).
:rtype: list of strings
"""
if synthetic_jar_dir:
safe_mkdir(synthetic_jar_dir)
else:
synthetic_jar_dir = safe_mkdtemp()
# Quote the paths so that if they contain reserved characters can be safely passed to JVM classloader.
bundled_classpath = map(urllib.quote, relativize_classpath(classpath, synthetic_jar_dir))
manifest = Manifest()
manifest.addentry(Manifest.CLASS_PATH, ' '.join(bundled_classpath))
with temporary_file(root_dir=synthetic_jar_dir, cleanup=False, suffix='.jar') as jar_file:
with open_zip(jar_file, mode='w', compression=ZIP_STORED) as jar:
jar.writestr(Manifest.PATH, manifest.contents())
if custom_name:
custom_path = os.path.join(synthetic_jar_dir, custom_name)
safe_concurrent_rename(jar_file.name, custom_path)
return [custom_path]
else:
return [jar_file.name]
def execute(self):
pants_wd = self.get_options().pants_workdir
pants_trash = os.path.join(pants_wd, "trash")
# Creates, and eventually deletes, trash dir created in .pants_cleanall.
with temporary_dir(cleanup=False, root_dir=os.path.dirname(pants_wd), prefix=".pants_cleanall") as tmpdir:
logger.debug('Moving trash to {} for deletion'.format(tmpdir))
tmp_trash = os.path.join(tmpdir, "trash")
# Moves contents of .pants.d to cleanup dir.
safe_concurrent_rename(pants_wd, tmp_trash)
safe_concurrent_rename(tmpdir, pants_wd)
if self.get_options().async:
# The trash directory is deleted in a child process.
pid = os.fork()
if pid == 0:
try:
safe_rmtree(pants_trash)
except (IOError, OSError):
logger.warning("Async clean-all failed. Please try again.")
finally:
os._exit(0)
else:
logger.debug("Forked an asynchronous clean-all worker at pid: {}".format(pid))
else:
# Recursively removes pants cache; user waits patiently.
logger.info('For async removal, run `./pants clean-all --async`')
safe_rmtree(pants_trash)
def safe_classpath(classpath, synthetic_jar_dir, custom_name=None):
"""Bundles classpath into one synthetic jar that includes original classpath in its manifest.
This is to ensure classpath length never exceeds platform ARG_MAX.
:param list classpath: Classpath to be bundled.
:param string synthetic_jar_dir: directory to store the synthetic jar, if `None`
a temp directory will be provided and cleaned up upon process exit. Otherwise synthetic
jar will remain in the supplied directory, only for debugging purpose.
:param custom_name: filename of the synthetic jar to be created.
:returns: A classpath (singleton list with just the synthetic jar).
:rtype: list of strings
"""
if synthetic_jar_dir:
safe_mkdir(synthetic_jar_dir)
else:
synthetic_jar_dir = safe_mkdtemp()
# Quote the paths so that if they contain reserved characters can be safely passed to JVM classloader.
bundled_classpath = [parse.quote(cp) for cp in relativize_classpath(classpath, synthetic_jar_dir)]
manifest = Manifest()
manifest.addentry(Manifest.CLASS_PATH, ' '.join(bundled_classpath))
with temporary_file(root_dir=synthetic_jar_dir, cleanup=False, suffix='.jar') as jar_file:
with open_zip(jar_file, mode='w', compression=ZIP_STORED) as jar:
jar.writestr(Manifest.PATH, manifest.contents())
if custom_name:
custom_path = os.path.join(synthetic_jar_dir, custom_name)
safe_concurrent_rename(jar_file.name, custom_path)
return [custom_path]
else:
return [jar_file.name]
def execute(self):
pants_wd = self.get_options().pants_workdir
if self.get_options().pants_physical_workdir_base:
# If a physical workdir is in use, operate on it rather than on the symlink that points to it.
pants_wd = os.readlink(pants_wd)
pants_trash = os.path.join(pants_wd, "trash")
# Creates, and eventually deletes, trash dir created in .pants_cleanall.
with temporary_dir(cleanup=False, root_dir=os.path.dirname(pants_wd), prefix=".pants_cleanall") as tmpdir:
logger.debug('Moving trash to {} for deletion'.format(tmpdir))
tmp_trash = os.path.join(tmpdir, "trash")
# Moves contents of .pants.d to cleanup dir.
safe_concurrent_rename(pants_wd, tmp_trash)
safe_concurrent_rename(tmpdir, pants_wd)
if self.get_options()['async']:
# The trash directory is deleted in a child process.
pid = os.fork()
if pid == 0:
try:
safe_rmtree(pants_trash)
except (IOError, OSError):
logger.warning("Async clean-all failed. Please try again.")
finally:
os._exit(0)
else:
logger.debug("Forked an asynchronous clean-all worker at pid: {}".format(pid))
else:
# Recursively removes pants cache; user waits patiently.
logger.info('For async removal, run `./pants clean-all --async`')
safe_rmtree(pants_trash)
def _install_plugin(self, distribution: Distribution) -> str:
# We don't actually install the distribution. It's installed for us by the Pex resolver in
# a chroot. We just copy that chroot out of the Pex cache to a location we control.
# Historically though, Pex did not install wheels it resolved and we did this here by hand.
# We retain the terminology and, more importantly, the final resting "install" path and the
# contents of plugin-<hash>.txt files to keep the plugin cache forwards and backwards
# compatible between Pants releases.
#
# Concretely:
#
# 1. In the past Pex resolved the wheel file below and we installed it to the "-install"
# directory:
#
# ~/.cache/pants/plugins/
# requests-2.23.0-py2.py3-none-any.whl
# requests-2.23.0-py2.py3-none-any.whl-install/
#
# The plugins-<hash>.txt file that records plugin locations contained the un-installed
# wheel file path:
#
# $ cat ~/.cache/pants/plugins/plugins-418c36b574edbcf4720b266b0709750ad588c281.txt
# /home/jsirois/.cache/pants/plugins/requests-2.23.0-py2.py3-none-any.whl
#
# 2. Now Pex resolves an installed wheel chroot directory and we copy that directory to the
# "-install" directory:
#
# ~/.cache/pants/plugins/
# installed_wheels/6ce6cd759a2d13badb1f6b9e665e2aded7a012dd/requests-2.23.0-py2.py3-none-any.whl/
# requests-2.23.0-py2.py3-none-any.whl-install/
#
# The plugins-<hash>.txt file that records plugin locations now contains the final
# installed wheel path with the "-install" suffix omitted which leads to the same file
# contents as past Pants versions:
#
# $ cat ~/.cache/pants/plugins/plugins-418c36b574edbcf4720b266b0709750ad588c281.txt
# /home/jsirois/.cache/pants/plugins/requests-2.23.0-py2.py3-none-any.whl
#
# We add the suffix on after reading the file to find the actual installed wheel path.
wheel_basename = os.path.basename(distribution.location)
plugin_path = os.path.join(self.plugin_cache_dir, wheel_basename)
with temporary_dir() as td:
temp_install_dir = os.path.join(td, wheel_basename)
shutil.copytree(distribution.location, temp_install_dir)
safe_concurrent_rename(temp_install_dir,
self._plugin_location(plugin_path))
return plugin_path
def test_execute_java_no_error_weird_path(self):
"""
:API: public
"""
with temporary_file(suffix=".jar") as temp_path:
fetcher = Fetcher(get_buildroot())
try:
# Download a jar that echoes things.
fetcher.download(
"https://maven-central.storage-download.googleapis.com/repos/central/data/io/get-coursier/echo/1.0.0/echo-1.0.0.jar",
path_or_fd=temp_path.name,
timeout_secs=2,
)
except fetcher.Error:
self.fail("fail to download echo jar")
task = self.execute(self.context([]))
executor = task.create_java_executor()
# Executing the jar as is should work.
self.assertEqual(
0,
util.execute_java(
executor=executor,
classpath=[temp_path.name],
main="coursier.echo.Echo",
args=["Hello World"],
create_synthetic_jar=True,
),
)
# Rename the jar to contain reserved characters.
new_path = os.path.join(os.path.dirname(temp_path.name),
"%%!!!===++.jar")
safe_concurrent_rename(temp_path.name, new_path)
# Executing the new path should work.
self.assertEqual(
0,
util.execute_java(
executor=executor,
classpath=[new_path],
main="coursier.echo.Echo",
args=["Hello World"],
create_synthetic_jar=True,
),
)
def execute(self):
pants_wd = self.get_options().pants_workdir
pants_trash = os.path.join(pants_wd, "trash")
# Creates, and eventually deletes, trash dir created in .pants_cleanall.
with temporary_dir(cleanup=False, root_dir=os.path.dirname(pants_wd), prefix=".pants_cleanall") as tmpdir:
logger.debug('Moving trash to {} for deletion'.format(tmpdir))
tmp_trash = os.path.join(tmpdir, "trash")
# Moves contents of .pants.d to cleanup dir.
safe_concurrent_rename(pants_wd, tmp_trash)
safe_concurrent_rename(tmpdir, pants_wd)
if self.get_options().async:
# The trash directory is deleted in a child process.
pid = os.fork()
def extract(self, path, outdir, concurrency_safe=False, **kwargs):
"""Extracts an archive's contents to the specified outdir with an optional filter.
Keyword arguments are forwarded to the instance's self._extract() method.
:API: public
:param string path: path to the zipfile to extract from
:param string outdir: directory to extract files into
:param bool concurrency_safe: True to use concurrency safe method. Concurrency safe extraction
will be performed on a temporary directory and the extacted directory will then be renamed
atomically to the outdir. As a side effect, concurrency safe extraction will not allow
overlay of extracted contents onto an existing outdir.
"""
if concurrency_safe:
with temporary_dir() as temp_dir:
self._extract(path, temp_dir, **kwargs)
safe_concurrent_rename(temp_dir, outdir)
else:
# Leave the existing default behavior unchanged and allows overlay of contents.
self._extract(path, outdir, **kwargs)
def extract(self, path, outdir, concurrency_safe=False, **kwargs):
"""Extracts an archive's contents to the specified outdir with an optional filter.
Keyword arguments are forwarded to the instance's self._extract() method.
:API: public
:param string path: path to the zipfile to extract from
:param string outdir: directory to extract files into
:param bool concurrency_safe: True to use concurrency safe method. Concurrency safe extraction
will be performed on a temporary directory and the extacted directory will then be renamed
atomically to the outdir. As a side effect, concurrency safe extraction will not allow
overlay of extracted contents onto an existing outdir.
"""
if concurrency_safe:
with temporary_dir() as temp_dir:
self._extract(path, temp_dir, **kwargs)
safe_concurrent_rename(temp_dir, outdir)
else:
# Leave the existing default behavior unchanged and allows overlay of contents.
self._extract(path, outdir, **kwargs)
def extract(cls, path, outdir, filter_func=None, concurrency_safe=False):
"""Extracts an archive's contents to the specified outdir with an optional filter.
:API: public
:param string path: path to the zipfile to extract from
:param string outdir: directory to extract files into
:param function filter_func: optional filter with the filename as the parameter. Returns True
if the file should be extracted. Note that filter_func is ignored for non-zip archives.
:param bool concurrency_safe: True to use concurrency safe method. Concurrency safe extraction
will be performed on a temporary directory and the extacted directory will then be renamed
atomically to the outdir. As a side effect, concurrency safe extraction will not allow
overlay of extracted contents onto an existing outdir.
"""
if concurrency_safe:
with temporary_dir() as temp_dir:
cls._extract(path, temp_dir, filter_func=filter_func)
safe_concurrent_rename(temp_dir, outdir)
else:
# Leave the existing default behavior unchanged and allows overlay of contents.
cls._extract(path, outdir, filter_func=filter_func)
def test_execute_java_no_error_weird_path(self):
"""
:API: public
"""
with temporary_file(suffix='.jar') as temp_path:
fetcher = Fetcher(get_buildroot())
try:
# Download a jar that echoes things.
fetcher.download('https://repo1.maven.org/maven2/io/get-coursier/echo/1.0.0/echo-1.0.0.jar',
path_or_fd=temp_path.name,
timeout_secs=2)
except fetcher.Error:
self.fail("fail to download echo jar")
task = self.execute(self.context([]))
executor = task.create_java_executor()
# Executing the jar as is should work.
self.assertEquals(0, util.execute_java(
executor=executor,
classpath=[temp_path.name],
main='coursier.echo.Echo',
args=['Hello World'],
create_synthetic_jar=True))
# Rename the jar to contain reserved characters.
new_path = os.path.join(os.path.dirname(temp_path.name), "%%!!!===++.jar")
safe_concurrent_rename(temp_path.name, new_path)
# Executing the new path should work.
self.assertEquals(0, util.execute_java(
executor=executor,
classpath=[new_path],
main='coursier.echo.Echo',
args=['Hello World'],
create_synthetic_jar=True))
