def java_home_exe(self): with distribution(executables=EXE('bin/java', version='1')) as jdk1_home: with distribution(executables=EXE('bin/java', version='2')) as jdk2_home: with temporary_file() as osx_java_home_exe: osx_java_home_exe.write(textwrap.dedent(""" #!/bin/sh echo '<?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> <plist version="1.0"> <array> <dict> <key>JVMHomePath</key> <string>{jdk1_home}</string> </dict> <dict> <key>JVMHomePath</key> <string>{jdk2_home}</string> </dict> </array> </plist> ' """.format(jdk1_home=jdk1_home, jdk2_home=jdk2_home)).strip()) osx_java_home_exe.close() chmod_plus_x(osx_java_home_exe.name) locator = _Locator(_UnknownEnvironment(_EnvVarEnvironment(), _OSXEnvironment(osx_java_home_exe.name))) yield locator, jdk1_home, jdk2_home
def capture(self, run_info_dict): # Force the scm discovery logging messages to appear before ours, so the startup delay # is properly associated in the user's mind with us and not with scm. logger.info(f'Capturing repro information to {self._path}') with open_tar(self._path, 'w:gz', dereference=True, compresslevel=6) as tarout: for relpath in os.listdir(self._buildroot): if relpath not in self._ignore: tarout.add(os.path.join(self._buildroot, relpath), relpath) with temporary_file(binary_mode=False) as tmpfile: tmpfile.write( '# Pants repro captured for the following build:\n') for k, v in sorted(run_info_dict.items()): tmpfile.write(f'# {k}: {v}\n') cmd_line = list(sys.argv) # Use 'pants' instead of whatever the full executable path was on the user's system. cmd_line[0] = 'pants' # Remove any repro-related flags. The repro-ing user won't want to call those. cmd_line = [ x for x in cmd_line if not x.startswith('--repro-') ] tmpfile.write("'" + "' '".join(cmd_line) + "'\n") tmpfile.flush() chmod_plus_x(tmpfile.name) tarout.add(tmpfile.name, 'repro.sh')
def distribution(installed_sdks=('18', '19'), installed_build_tools=('19.1.0', '20.0.0'), files=('android.jar',), executables=('aapt', 'zipalign')): """Mock Android SDK Distribution. :param tuple[strings] installed_sdks: SDK versions of the files being mocked. :param tuple[strings] installed_build_tools: Build tools version of any tools. :param tuple[strings] files: The files are to mock non-executables and one will be created for each installed_sdks version. :param tuple[strings] executables: Executables are any required tools and one is created for each installed_build_tools version. """ with temporary_dir() as sdk: for sdk_version in installed_sdks: for android_file in files: touch(os.path.join(sdk, 'platforms', 'android-' + sdk_version, android_file)) for version in installed_build_tools: for exe in maybe_list(executables or ()): path = os.path.join(sdk, 'build-tools', version, exe) touch(path) chmod_plus_x(path) dx_path = os.path.join(sdk, 'build-tools', version, 'lib/dx.jar') touch(dx_path) yield sdk
def fake_interpreter(python_tag: str, abi_tag: str, version: Tuple[int, int, int]): interpreter_dir = safe_mkdtemp() binary = os.path.join(interpreter_dir, "python") values = dict( binary=binary, python_tag=python_tag, abi_tag=abi_tag, platform_tag="", version=version, supported_tags=[], env_markers={}, ) id_str = json.dumps(values) with open(binary, "w") as fp: fp.write( dedent( f""" #!{PythonInterpreter.get().binary} from __future__ import print_function print({id_str!r}) """ ).strip() ) chmod_plus_x(binary) return PythonInterpreter.from_binary(binary)
def select_binary(self, supportdir, version, name): """Selects a binary matching the current os and architecture. :param string supportdir: The path the `name` binaries are stored under. :param string version: The version number of the binary to select. :param string name: The name of the binary to fetch. :raises: :class:`pants.binary_util.BinaryUtil.BinaryNotFound` if no binary of the given version and name could be found for the current platform. """ # TODO(John Sirois): finish doc of the path structure expected under base_path binary_path = self._select_binary_base_path(supportdir, version, name) bootstrap_dir = os.path.realpath( os.path.expanduser(self._pants_bootstrapdir)) bootstrapped_binary_path = os.path.join(bootstrap_dir, binary_path) if not os.path.exists(bootstrapped_binary_path): downloadpath = bootstrapped_binary_path + '~' try: with self._select_binary_stream(supportdir, version, name) as stream: with safe_open(downloadpath, 'wb') as bootstrapped_binary: bootstrapped_binary.write(stream()) os.rename(downloadpath, bootstrapped_binary_path) chmod_plus_x(bootstrapped_binary_path) finally: safe_delete(downloadpath) logger.debug('Selected {binary} binary bootstrapped to: {path}'.format( binary=name, path=bootstrapped_binary_path)) return bootstrapped_binary_path
def select_binary(self, base_path, version, name): """Selects a binary matching the current os and architecture. :raises: :class:`pants.binary_util.BinaryUtil.BinaryNotFound` if no binary of the given version and name could be found. """ # TODO(John Sirois): finish doc of the path structure expected under base_path bootstrap_dir = self._boostrap_dir binary_path = self.select_binary_base_path(base_path, version, name) bootstrapped_binary_path = os.path.join(bootstrap_dir, binary_path) if not os.path.exists(bootstrapped_binary_path): downloadpath = bootstrapped_binary_path + '~' try: with self.select_binary_stream(base_path, version, name) as stream: with safe_open(downloadpath, 'wb') as bootstrapped_binary: bootstrapped_binary.write(stream()) os.rename(downloadpath, bootstrapped_binary_path) chmod_plus_x(bootstrapped_binary_path) finally: safe_delete(downloadpath) logger.debug('Selected {binary} binary bootstrapped to: {path}'.format( binary=name, path=bootstrapped_binary_path)) return bootstrapped_binary_path
def java_home_exe(self): with distribution(executables=EXE('bin/java', version='1')) as jdk1_home: with distribution(executables=EXE('bin/java', version='2')) as jdk2_home: with temporary_dir() as tmpdir: osx_java_home_exe = os.path.join(tmpdir, 'java_home') with safe_open(osx_java_home_exe, 'w') as fp: fp.write(textwrap.dedent(""" #!/bin/sh echo '<?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> <plist version="1.0"> <array> <dict> <key>JVMHomePath</key> <string>{jdk1_home}</string> </dict> <dict> <key>JVMHomePath</key> <string>{jdk2_home}</string> </dict> </array> </plist> ' """.format(jdk1_home=jdk1_home, jdk2_home=jdk2_home)).strip()) chmod_plus_x(osx_java_home_exe) original_osx_java_home_exe = DistributionLocator._OSX_JAVA_HOME_EXE DistributionLocator._OSX_JAVA_HOME_EXE = osx_java_home_exe try: yield jdk1_home, jdk2_home finally: DistributionLocator._OSX_JAVA_HOME_EXE = original_osx_java_home_exe
def select_binary(self, name): """Selects a binary matching the current os and architecture. :param name: the name of the binary to fetch. :raises: :class:`pants.binary_util.BinaryUtil.BinaryNotFound` if no binary of the given version and name could be found. """ # TODO(John Sirois): finish doc of the path structure expected under base_path binary_path = BinaryUtil.select_binary_base_path(self._supportdir, self._version, name) bootstrap_dir = os.path.realpath(os.path.expanduser(self._pants_bootstrapdir)) bootstrapped_binary_path = os.path.join(bootstrap_dir, binary_path) if not os.path.exists(bootstrapped_binary_path): downloadpath = bootstrapped_binary_path + '~' try: with self.select_binary_stream(name) as stream: with safe_open(downloadpath, 'wb') as bootstrapped_binary: bootstrapped_binary.write(stream()) os.rename(downloadpath, bootstrapped_binary_path) chmod_plus_x(bootstrapped_binary_path) finally: safe_delete(downloadpath) logger.debug('Selected {binary} binary bootstrapped to: {path}' .format(binary=name, path=bootstrapped_binary_path)) return bootstrapped_binary_path
def distribution(installed_sdks=('18', '19'), installed_build_tools=('19.1.0', '20.0.0'), files=('android.jar', ), executables=('aapt', 'zipalign')): """Mock Android SDK Distribution. :param tuple[strings] installed_sdks: SDK versions of the files being mocked. :param tuple[strings] installed_build_tools: Build tools version of any tools. :param tuple[strings] files: The files are to mock non-executables and one will be created for each installed_sdks version. :param tuple[strings] executables: Executables are any required tools and one is created for each installed_build_tools version. """ with temporary_dir() as sdk: for sdk_version in installed_sdks: for android_file in files: touch( os.path.join(sdk, 'platforms', 'android-' + sdk_version, android_file)) for version in installed_build_tools: for exe in maybe_list(executables or ()): path = os.path.join(sdk, 'build-tools', version, exe) touch(path) chmod_plus_x(path) dx_path = os.path.join(sdk, 'build-tools', version, 'lib/dx.jar') touch(dx_path) yield sdk
def java_home_exe(self): with distribution( executables=EXE("bin/java", version="1")) as jdk1_home: with distribution( executables=EXE("bin/java", version="2")) as jdk2_home: with temporary_file(binary_mode=False) as osx_java_home_exe: osx_java_home_exe.write( textwrap.dedent(""" #!/bin/sh echo '<?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> <plist version="1.0"> <array> <dict> <key>JVMHomePath</key> <string>{jdk1_home}</string> </dict> <dict> <key>JVMHomePath</key> <string>{jdk2_home}</string> </dict> </array> </plist> ' """.format(jdk1_home=jdk1_home, jdk2_home=jdk2_home)).strip()) osx_java_home_exe.close() chmod_plus_x(osx_java_home_exe.name) locator = _Locator( _UnknownEnvironment( _EnvVarEnvironment(), _OSXEnvironment(osx_java_home_exe.name))) yield locator, jdk1_home, jdk2_home
def tool(self, name): with temporary_dir() as tool_root: tool_path = os.path.join(tool_root, name) touch(tool_path) chmod_plus_x(tool_path) new_path = os.pathsep.join([tool_root] + os.environ.get('PATH', '').split(os.pathsep)) with environment_as(PATH=new_path): yield tool_path
def tool(self, name): with temporary_dir() as tool_root: tool_path = os.path.join(tool_root, name) touch(tool_path) chmod_plus_x(tool_path) new_path = os.pathsep.join([tool_root] + os.environ.get("PATH", "").split(os.pathsep)) with environment_as(PATH=new_path): yield tool_path
def fake_distribution(self, version): with temporary_dir() as java_home: path = os.path.join(java_home, 'bin/java') with safe_open(path, 'w') as fp: fp.write(textwrap.dedent(""" #!/bin/sh echo java.version={version} """.format(version=version)).strip()) chmod_plus_x(path) yield java_home
def jre(self, env_var): with temporary_dir() as jre: path = os.path.join(jre, 'java') with safe_open(path, 'w') as fp: fp.write(textwrap.dedent(""" #!/bin/sh echo ${env_var} """.format(env_var=env_var)).strip()) chmod_plus_x(path) yield jre
def distribution(self, files=None, executables=None): with temporary_dir() as jdk: for f in maybe_list(files or ()): touch(os.path.join(jdk, f)) for exe in maybe_list(executables or (), expected_type=self.EXE): path = os.path.join(jdk, exe.name) with safe_open(path, 'w') as fp: fp.write(exe.contents or '') chmod_plus_x(path) yield jdk
def distribution(files=None, executables=None, java_home=None): with temporary_dir() as dist_root: for f in maybe_list(files or ()): touch(os.path.join(dist_root, f)) for executable in maybe_list(executables or (), expected_type=EXE): path = os.path.join(dist_root, executable.relpath) with safe_open(path, 'w') as fp: java_home = os.path.join(dist_root, java_home) if java_home else dist_root fp.write(executable.contents(java_home)) chmod_plus_x(path) yield dist_root
def jre(self, env_var): with temporary_dir() as jre: path = os.path.join(jre, "java") with safe_open(path, "w") as fp: fp.write( textwrap.dedent(""" #!/bin/sh echo ${env_var} >&2 echo "java.home={java_home}" """.format(env_var=env_var, java_home=jre)).strip()) chmod_plus_x(path) yield jre
def fake_interpreter(id_str): interpreter_dir = safe_mkdtemp() binary = os.path.join(interpreter_dir, 'binary') with open(binary, 'w') as fp: fp.write(dedent(""" #!{} from __future__ import print_function print({!r}) """.format(PythonInterpreter.get().binary, id_str)).strip()) chmod_plus_x(binary) return PythonInterpreter.from_binary(binary)
def distribution(files=None, executables=None, java_home=None): with subsystem_instance(DistributionLocator): with temporary_dir() as dist_root: with environment_as(DIST_ROOT=os.path.join(dist_root, java_home) if java_home else dist_root): for f in maybe_list(files or ()): touch(os.path.join(dist_root, f)) for executable in maybe_list(executables or (), expected_type=EXE): path = os.path.join(dist_root, executable.relpath) with safe_open(path, 'w') as fp: fp.write(executable.contents or '') chmod_plus_x(path) yield dist_root
def _fetch_binary(self, name, binary_path): bootstrap_dir = os.path.realpath(os.path.expanduser(self._pants_bootstrapdir)) bootstrapped_binary_path = os.path.join(bootstrap_dir, binary_path) if not os.path.exists(bootstrapped_binary_path): with safe_concurrent_creation(bootstrapped_binary_path) as downloadpath: with self._select_binary_stream(name, binary_path) as stream: with safe_open(downloadpath, 'wb') as bootstrapped_binary: bootstrapped_binary.write(stream()) os.rename(downloadpath, bootstrapped_binary_path) chmod_plus_x(bootstrapped_binary_path) logger.debug('Selected {binary} binary bootstrapped to: {path}' .format(binary=name, path=bootstrapped_binary_path)) return bootstrapped_binary_path
def distribution(self, installed_sdks=('18', '19'), installed_build_tools=('19.1.0', ), files='android.jar', executables='aapt'): with temporary_dir() as sdk: for sdks in installed_sdks: touch(os.path.join(sdk, 'platforms', 'android-' + sdks, files)) for build in installed_build_tools: for exe in maybe_list(executables or ()): path = os.path.join(sdk, 'build-tools', build, exe) with safe_open(path, 'w') as fp: fp.write('') chmod_plus_x(path) yield sdk
def select(self, binary_request): """Fetches a file, unpacking it if necessary.""" logger.debug("binary_request: {!r}".format(binary_request)) try: download_path = self._get_download_path(binary_request) except self.MissingMachineInfo as e: raise self.BinaryResolutionError(binary_request, e) try: url_generator = self._get_url_generator(binary_request) except self.NoBaseUrlsError as e: raise self.BinaryResolutionError(binary_request, e) urls = self._get_urls(url_generator, binary_request) if not isinstance(urls, list): # TODO: add test for this error! raise self.BinaryResolutionError( binary_request, TypeError("urls must be a list: was '{}'.".format(urls))) fetch_request = BinaryFetchRequest(download_path=download_path, urls=urls) logger.debug("fetch_request: {!r}".format(fetch_request)) try: downloaded_file = self._binary_tool_fetcher.fetch_binary( fetch_request) except BinaryToolFetcher.BinaryNotFound as e: raise self.BinaryResolutionError(binary_request, e) # NB: we mark the downloaded file executable if it is not an archive. archiver = binary_request.archiver if archiver is None: chmod_plus_x(downloaded_file) return downloaded_file download_dir = os.path.dirname(downloaded_file) # Use the 'name' given in the request as the directory name to extract to. unpacked_dirname = os.path.join(download_dir, binary_request.name) if not os.path.isdir(unpacked_dirname): logger.info("Extracting {} to {} .".format(downloaded_file, unpacked_dirname)) archiver.extract(downloaded_file, unpacked_dirname, concurrency_safe=True) return unpacked_dirname
def _fetch_binary(self, name, binary_path): bootstrap_dir = os.path.realpath(os.path.expanduser(self._pants_bootstrapdir)) bootstrapped_binary_path = os.path.join(bootstrap_dir, binary_path) if not os.path.exists(bootstrapped_binary_path): safe_mkdir_for(bootstrapped_binary_path) with temporary_dir(root_dir=os.path.dirname(bootstrapped_binary_path)) as tmpdir: downloadpath = os.path.join(tmpdir, 'download') with self._select_binary_stream(name, binary_path) as stream: with safe_open(downloadpath, 'wb') as bootstrapped_binary: bootstrapped_binary.write(stream()) os.rename(downloadpath, bootstrapped_binary_path) chmod_plus_x(bootstrapped_binary_path) logger.debug('Selected {binary} binary bootstrapped to: {path}' .format(binary=name, path=bootstrapped_binary_path)) return bootstrapped_binary_path
def select(self, binary_request): """Fetches a file, unpacking it if necessary.""" logger.debug("binary_request: {!r}".format(binary_request)) try: download_path = self._get_download_path(binary_request) except self.MissingMachineInfo as e: raise self.BinaryResolutionError(binary_request, e) try: url_generator = self._get_url_generator(binary_request) except self.NoBaseUrlsError as e: raise self.BinaryResolutionError(binary_request, e) urls = self._get_urls(url_generator, binary_request) if not isinstance(urls, list): # TODO: add test for this error! raise self.BinaryResolutionError( binary_request, TypeError("urls must be a list: was '{}'.".format(urls))) fetch_request = BinaryFetchRequest( download_path=download_path, urls=urls) logger.debug("fetch_request: {!r}".format(fetch_request)) try: downloaded_file = self._binary_tool_fetcher.fetch_binary(fetch_request) except BinaryToolFetcher.BinaryNotFound as e: raise self.BinaryResolutionError(binary_request, e) # NB: we mark the downloaded file executable if it is not an archive. archiver = binary_request.archiver if archiver is None: chmod_plus_x(downloaded_file) return downloaded_file download_dir = os.path.dirname(downloaded_file) # Use the 'name' given in the request as the directory name to extract to. unpacked_dirname = os.path.join(download_dir, binary_request.name) if not os.path.isdir(unpacked_dirname): logger.info("Extracting {} to {} .".format(downloaded_file, unpacked_dirname)) archiver.extract(downloaded_file, unpacked_dirname, concurrency_safe=True) return unpacked_dirname
def distribution(files=None, executables=None, java_home=None, dist_dir=None): # NB attempt to include the java version in the tmp dir name for better test failure messages. executables_as_list = ensure_list(executables or (), expected_type=EXE) if executables_as_list: dist_prefix = "jvm_{}_".format(executables_as_list[0]._version) else: dist_prefix = "jvm_na_" with temporary_dir(root_dir=dist_dir, prefix=dist_prefix) as dist_root: for f in ensure_str_list(files or ()): touch(os.path.join(dist_root, f)) for executable in executables_as_list: path = os.path.join(dist_root, executable.relpath) with safe_open(path, "w") as fp: java_home = os.path.join(dist_root, java_home) if java_home else dist_root fp.write(executable.contents(java_home)) chmod_plus_x(path) yield dist_root
def capture(self, run_info_dict): # Force the scm discovery logging messages to appear before ours, so the startup delay # is properly associated in the user's mind with us and not with scm. logger.info('Capturing repro information to {}'.format(self._path)) with open_tar(self._path, 'w:gz', dereference=True, compresslevel=6) as tarout: for relpath in os.listdir(self._buildroot): if relpath not in self._ignore: tarout.add(os.path.join(self._buildroot, relpath), relpath) with temporary_file() as tmpfile: tmpfile.write('# Pants repro captured for the following build:\n') for k, v in sorted(run_info_dict.items()): tmpfile.write('# {}: {}\n'.format(k, v)) cmd_line = list(sys.argv) # Use 'pants' instead of whatever the full executable path was on the user's system. cmd_line[0] = 'pants' # Remove any repro-related flags. The repro-ing user won't want to call those. cmd_line = [x for x in cmd_line if not x.startswith('--repro-')] tmpfile.write("'" +"' '".join(cmd_line) + "'\n") tmpfile.flush() chmod_plus_x(tmpfile.name) tarout.add(tmpfile.name, 'repro.sh')
def java_home_exe(self): with distribution( executables=exe('bin/java', version='1')) as jdk1_home: with distribution( executables=exe('bin/java', version='2')) as jdk2_home: with temporary_dir() as tmpdir: osx_java_home_exe = os.path.join(tmpdir, 'java_home') with safe_open(osx_java_home_exe, 'w') as fp: fp.write( textwrap.dedent(""" #!/bin/sh echo '<?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> <plist version="1.0"> <array> <dict> <key>JVMHomePath</key> <string>{jdk1_home}</string> </dict> <dict> <key>JVMHomePath</key> <string>{jdk2_home}</string> </dict> </array> </plist> ' """.format(jdk1_home=jdk1_home, jdk2_home=jdk2_home)).strip()) chmod_plus_x(osx_java_home_exe) original_osx_java_home_exe = DistributionLocator._OSX_JAVA_HOME_EXE DistributionLocator._OSX_JAVA_HOME_EXE = osx_java_home_exe try: yield jdk1_home, jdk2_home finally: DistributionLocator._OSX_JAVA_HOME_EXE = original_osx_java_home_exe
def executable_git(self): with self.unexecutable_git() as git: chmod_plus_x(git) yield git