def safe_classpath(classpath, synthetic_jar_dir, custom_name=None): """Bundles classpath into one synthetic jar that includes original classpath in its manifest. This is to ensure classpath length never exceeds platform ARG_MAX. :param list classpath: Classpath to be bundled. :param string synthetic_jar_dir: directory to store the synthetic jar, if `None` a temp directory will be provided and cleaned up upon process exit. Otherwise synthetic jar will remain in the supplied directory, only for debugging purpose. :param custom_name: filename of the synthetic jar to be created. :returns: A classpath (singleton list with just the synthetic jar). :rtype: list of strings """ if synthetic_jar_dir: safe_mkdir(synthetic_jar_dir) else: synthetic_jar_dir = safe_mkdtemp() # Quote the paths so that if they contain reserved characters can be safely passed to JVM classloader. bundled_classpath = map(urllib.quote, relativize_classpath(classpath, synthetic_jar_dir)) manifest = Manifest() manifest.addentry(Manifest.CLASS_PATH, ' '.join(bundled_classpath)) with temporary_file(root_dir=synthetic_jar_dir, cleanup=False, suffix='.jar') as jar_file: with open_zip(jar_file, mode='w', compression=ZIP_STORED) as jar: jar.writestr(Manifest.PATH, manifest.contents()) if custom_name: custom_path = os.path.join(synthetic_jar_dir, custom_name) safe_concurrent_rename(jar_file.name, custom_path) return [custom_path] else: return [jar_file.name]
def execute(self): pants_wd = self.get_options().pants_workdir pants_trash = os.path.join(pants_wd, "trash") # Creates, and eventually deletes, trash dir created in .pants_cleanall. with temporary_dir(cleanup=False, root_dir=os.path.dirname(pants_wd), prefix=".pants_cleanall") as tmpdir: logger.debug('Moving trash to {} for deletion'.format(tmpdir)) tmp_trash = os.path.join(tmpdir, "trash") # Moves contents of .pants.d to cleanup dir. safe_concurrent_rename(pants_wd, tmp_trash) safe_concurrent_rename(tmpdir, pants_wd) if self.get_options().async: # The trash directory is deleted in a child process. pid = os.fork() if pid == 0: try: safe_rmtree(pants_trash) except (IOError, OSError): logger.warning("Async clean-all failed. Please try again.") finally: os._exit(0) else: logger.debug("Forked an asynchronous clean-all worker at pid: {}".format(pid)) else: # Recursively removes pants cache; user waits patiently. logger.info('For async removal, run `./pants clean-all --async`') safe_rmtree(pants_trash)
def safe_classpath(classpath, synthetic_jar_dir, custom_name=None): """Bundles classpath into one synthetic jar that includes original classpath in its manifest. This is to ensure classpath length never exceeds platform ARG_MAX. :param list classpath: Classpath to be bundled. :param string synthetic_jar_dir: directory to store the synthetic jar, if `None` a temp directory will be provided and cleaned up upon process exit. Otherwise synthetic jar will remain in the supplied directory, only for debugging purpose. :param custom_name: filename of the synthetic jar to be created. :returns: A classpath (singleton list with just the synthetic jar). :rtype: list of strings """ if synthetic_jar_dir: safe_mkdir(synthetic_jar_dir) else: synthetic_jar_dir = safe_mkdtemp() # Quote the paths so that if they contain reserved characters can be safely passed to JVM classloader. bundled_classpath = [parse.quote(cp) for cp in relativize_classpath(classpath, synthetic_jar_dir)] manifest = Manifest() manifest.addentry(Manifest.CLASS_PATH, ' '.join(bundled_classpath)) with temporary_file(root_dir=synthetic_jar_dir, cleanup=False, suffix='.jar') as jar_file: with open_zip(jar_file, mode='w', compression=ZIP_STORED) as jar: jar.writestr(Manifest.PATH, manifest.contents()) if custom_name: custom_path = os.path.join(synthetic_jar_dir, custom_name) safe_concurrent_rename(jar_file.name, custom_path) return [custom_path] else: return [jar_file.name]
def execute(self): pants_wd = self.get_options().pants_workdir if self.get_options().pants_physical_workdir_base: # If a physical workdir is in use, operate on it rather than on the symlink that points to it. pants_wd = os.readlink(pants_wd) pants_trash = os.path.join(pants_wd, "trash") # Creates, and eventually deletes, trash dir created in .pants_cleanall. with temporary_dir(cleanup=False, root_dir=os.path.dirname(pants_wd), prefix=".pants_cleanall") as tmpdir: logger.debug('Moving trash to {} for deletion'.format(tmpdir)) tmp_trash = os.path.join(tmpdir, "trash") # Moves contents of .pants.d to cleanup dir. safe_concurrent_rename(pants_wd, tmp_trash) safe_concurrent_rename(tmpdir, pants_wd) if self.get_options()['async']: # The trash directory is deleted in a child process. pid = os.fork() if pid == 0: try: safe_rmtree(pants_trash) except (IOError, OSError): logger.warning("Async clean-all failed. Please try again.") finally: os._exit(0) else: logger.debug("Forked an asynchronous clean-all worker at pid: {}".format(pid)) else: # Recursively removes pants cache; user waits patiently. logger.info('For async removal, run `./pants clean-all --async`') safe_rmtree(pants_trash)
def _install_plugin(self, distribution: Distribution) -> str: # We don't actually install the distribution. It's installed for us by the Pex resolver in # a chroot. We just copy that chroot out of the Pex cache to a location we control. # Historically though, Pex did not install wheels it resolved and we did this here by hand. # We retain the terminology and, more importantly, the final resting "install" path and the # contents of plugin-<hash>.txt files to keep the plugin cache forwards and backwards # compatible between Pants releases. # # Concretely: # # 1. In the past Pex resolved the wheel file below and we installed it to the "-install" # directory: # # ~/.cache/pants/plugins/ # requests-2.23.0-py2.py3-none-any.whl # requests-2.23.0-py2.py3-none-any.whl-install/ # # The plugins-<hash>.txt file that records plugin locations contained the un-installed # wheel file path: # # $ cat ~/.cache/pants/plugins/plugins-418c36b574edbcf4720b266b0709750ad588c281.txt # /home/jsirois/.cache/pants/plugins/requests-2.23.0-py2.py3-none-any.whl # # 2. Now Pex resolves an installed wheel chroot directory and we copy that directory to the # "-install" directory: # # ~/.cache/pants/plugins/ # installed_wheels/6ce6cd759a2d13badb1f6b9e665e2aded7a012dd/requests-2.23.0-py2.py3-none-any.whl/ # requests-2.23.0-py2.py3-none-any.whl-install/ # # The plugins-<hash>.txt file that records plugin locations now contains the final # installed wheel path with the "-install" suffix omitted which leads to the same file # contents as past Pants versions: # # $ cat ~/.cache/pants/plugins/plugins-418c36b574edbcf4720b266b0709750ad588c281.txt # /home/jsirois/.cache/pants/plugins/requests-2.23.0-py2.py3-none-any.whl # # We add the suffix on after reading the file to find the actual installed wheel path. wheel_basename = os.path.basename(distribution.location) plugin_path = os.path.join(self.plugin_cache_dir, wheel_basename) with temporary_dir() as td: temp_install_dir = os.path.join(td, wheel_basename) shutil.copytree(distribution.location, temp_install_dir) safe_concurrent_rename(temp_install_dir, self._plugin_location(plugin_path)) return plugin_path
def test_execute_java_no_error_weird_path(self): """ :API: public """ with temporary_file(suffix=".jar") as temp_path: fetcher = Fetcher(get_buildroot()) try: # Download a jar that echoes things. fetcher.download( "https://maven-central.storage-download.googleapis.com/repos/central/data/io/get-coursier/echo/1.0.0/echo-1.0.0.jar", path_or_fd=temp_path.name, timeout_secs=2, ) except fetcher.Error: self.fail("fail to download echo jar") task = self.execute(self.context([])) executor = task.create_java_executor() # Executing the jar as is should work. self.assertEqual( 0, util.execute_java( executor=executor, classpath=[temp_path.name], main="coursier.echo.Echo", args=["Hello World"], create_synthetic_jar=True, ), ) # Rename the jar to contain reserved characters. new_path = os.path.join(os.path.dirname(temp_path.name), "%%!!!===++.jar") safe_concurrent_rename(temp_path.name, new_path) # Executing the new path should work. self.assertEqual( 0, util.execute_java( executor=executor, classpath=[new_path], main="coursier.echo.Echo", args=["Hello World"], create_synthetic_jar=True, ), )
def execute(self): pants_wd = self.get_options().pants_workdir pants_trash = os.path.join(pants_wd, "trash") # Creates, and eventually deletes, trash dir created in .pants_cleanall. with temporary_dir(cleanup=False, root_dir=os.path.dirname(pants_wd), prefix=".pants_cleanall") as tmpdir: logger.debug('Moving trash to {} for deletion'.format(tmpdir)) tmp_trash = os.path.join(tmpdir, "trash") # Moves contents of .pants.d to cleanup dir. safe_concurrent_rename(pants_wd, tmp_trash) safe_concurrent_rename(tmpdir, pants_wd) if self.get_options().async: # The trash directory is deleted in a child process. pid = os.fork()
def extract(self, path, outdir, concurrency_safe=False, **kwargs): """Extracts an archive's contents to the specified outdir with an optional filter. Keyword arguments are forwarded to the instance's self._extract() method. :API: public :param string path: path to the zipfile to extract from :param string outdir: directory to extract files into :param bool concurrency_safe: True to use concurrency safe method. Concurrency safe extraction will be performed on a temporary directory and the extacted directory will then be renamed atomically to the outdir. As a side effect, concurrency safe extraction will not allow overlay of extracted contents onto an existing outdir. """ if concurrency_safe: with temporary_dir() as temp_dir: self._extract(path, temp_dir, **kwargs) safe_concurrent_rename(temp_dir, outdir) else: # Leave the existing default behavior unchanged and allows overlay of contents. self._extract(path, outdir, **kwargs)
def extract(cls, path, outdir, filter_func=None, concurrency_safe=False): """Extracts an archive's contents to the specified outdir with an optional filter. :API: public :param string path: path to the zipfile to extract from :param string outdir: directory to extract files into :param function filter_func: optional filter with the filename as the parameter. Returns True if the file should be extracted. Note that filter_func is ignored for non-zip archives. :param bool concurrency_safe: True to use concurrency safe method. Concurrency safe extraction will be performed on a temporary directory and the extacted directory will then be renamed atomically to the outdir. As a side effect, concurrency safe extraction will not allow overlay of extracted contents onto an existing outdir. """ if concurrency_safe: with temporary_dir() as temp_dir: cls._extract(path, temp_dir, filter_func=filter_func) safe_concurrent_rename(temp_dir, outdir) else: # Leave the existing default behavior unchanged and allows overlay of contents. cls._extract(path, outdir, filter_func=filter_func)
def test_execute_java_no_error_weird_path(self): """ :API: public """ with temporary_file(suffix='.jar') as temp_path: fetcher = Fetcher(get_buildroot()) try: # Download a jar that echoes things. fetcher.download('https://repo1.maven.org/maven2/io/get-coursier/echo/1.0.0/echo-1.0.0.jar', path_or_fd=temp_path.name, timeout_secs=2) except fetcher.Error: self.fail("fail to download echo jar") task = self.execute(self.context([])) executor = task.create_java_executor() # Executing the jar as is should work. self.assertEquals(0, util.execute_java( executor=executor, classpath=[temp_path.name], main='coursier.echo.Echo', args=['Hello World'], create_synthetic_jar=True)) # Rename the jar to contain reserved characters. new_path = os.path.join(os.path.dirname(temp_path.name), "%%!!!===++.jar") safe_concurrent_rename(temp_path.name, new_path) # Executing the new path should work. self.assertEquals(0, util.execute_java( executor=executor, classpath=[new_path], main='coursier.echo.Echo', args=['Hello World'], create_synthetic_jar=True))