def _fetch_homebrew(container_dir, distro_config): """Fetch homebrew and untar it in the container directory.""" try: os.stat(os.path.join(container_dir, "bin", "brew")) return container_for_directory(container_dir, distro_config) except OSError: with directory.Navigation(tempdir.TempDir().name): with TemporarilyDownloadedFile(_HOMEBREW_URL, filename="brew") as archive_file: with directory.Navigation(tempdir.TempDir().name) as extract: _extract_archive(archive_file, extract) first = os.path.join(extract, os.listdir(extract)[0]) files = [os.path.join(first, p) for p in os.listdir(first)] for filename in files: try: filename_base = os.path.basename(filename) shutil.move( filename, os.path.join(container_dir, filename_base)) except IOError: # suppress(pointless-except) # Ignore stuff that can't be moved for whatever # reason. These are all files that generally # don't matter. pass return OSXContainer(container_dir, distro_config["pkgsys"])
def install_packages(self, package_names): """Install all packages in list package_names. This works in a somewhat non-standard way. We will be updating the repository list as usual, but will be using a combination of apt-get download and dpkg manually to install packages into a local directory which we control. """ self._initialize_directories() from six.moves.urllib.parse import urlparse # suppress(import-error) root = self._executor.root_filesystem_directory() environment = { "APT_CONFIG": os.path.join(root, "etc", "apt", "apt.conf") } _run_task(self._executor, """Update repositories""", ["apt-get", "update", "-y", "--force-yes"], env=environment) # Separate out into packages that need to be downloaded with # apt-get and packages that can be downloaded directly # using download_file deb_packages = [p for p in package_names if urlparse(p).scheme] apt_packages = [p for p in package_names if not urlparse(p).scheme] # Clear out /var/cache/apt/archives archives = os.path.join(root, "var", "cache", "apt", "archives") if os.path.exists(archives): shutil.rmtree(archives) os.makedirs(archives) if len(deb_packages): with directory.Navigation(archives): _report_task("""Downloading user-specified packages""") for deb in deb_packages: download.download_file(deb) # Now use apt-get install -d to download the apt_packages and their # dependencies, but not install them if len(apt_packages): _run_task(self._executor, """Downloading APT packages and dependencies""", [ "apt-get", "-y", "--force-yes", "-d", "install", "--reinstall" ] + apt_packages, env=environment, detail=_format_package_list(apt_packages)) # Go back into our archives directory and unpack all our packages with directory.Navigation(archives): package_files = fnmatch.filter(os.listdir("."), "*.deb") for pkg in package_files: _run_task(self._executor, """Unpacking """, ["dpkg", "-x", pkg, root], detail=os.path.splitext(os.path.basename(pkg))[0])
def _fetch_choco(container_dir, distro_config): """Fetch chocolatey and install it in the container directory.""" try: os.stat(os.path.join(container_dir, "bin", "choco.exe")) return container_for_directory(container_dir, distro_config) except OSError: with tempdir.TempDir() as download_dir: with directory.Navigation(download_dir): _execute_no_output( ["setx", "ChocolateyInstall", container_dir]) # Also set the variable in the local environment # too, so that it gets propagated down to our # children os.environ["ChocolateyInstall"] = container_dir try: os.makedirs(container_dir) except OSError as error: if error.errno != errno.EEXIST: raise error _execute_no_output([ "powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", _CHOCO_INSTALL_CMD ]) # Reset variable back to original state to prevent # polluting the user's registry _execute_no_output(["setx", "ChocolateyInstall", ""]) return WindowsContainer(container_dir, distro_config["pkgsys"])
def test_enter_exist_dir(self): """Check that we can enter an existing dir.""" existing_dir = os.path.join(os.getcwd(), "existing") os.makedirs(existing_dir) self.addCleanup(lambda: shutil.rmtree(existing_dir)) with directory.Navigation(existing_dir) as entered: self.assertThat(entered, DirExists())
def _download_distro(details, path_to_distro_folder): """Download distribution and untar it in container root.""" distro_arch = details["arch"] download_url = details["url"].format(arch=distro_arch) with tempdir.TempDir() as download_dir: with directory.Navigation(download_dir): with TemporarilyDownloadedFile(download_url) as archive_file: _extract_distro_archive(archive_file, path_to_distro_folder)
def _download_proot(distribution_dir, arch): """Download arch build of proot into distribution.""" from psqtraviscontainer.download import download_file with directory.Navigation(os.path.join(distribution_dir, "bin")): proot_url = _PROOT_URL_BASE.format(arch=arch) path_to_proot = download_file(proot_url, "proot") os.chmod(path_to_proot, os.stat(path_to_proot).st_mode | stat.S_IXUSR) return path_to_proot
def add_repositories(self, repos): """Add a repository to the central packaging system.""" with tempdir.TempDir() as download_dir: with directory.Navigation(download_dir): for repo in repos: repo_file = download.download_file(repo) # Create a bash script to copy the downloaded repo file # over to /etc/yum/repos.d with tempfile.NamedTemporaryFile() as bash_script: copy_cmd = ("cp \"{0}\"" "/etc/yum/repos.d").format(repo_file) bash_script.write(six.b(copy_cmd)) bash_script.flush() self._executor.execute_success( ["bash", bash_script.name])
def install_packages(self, package_names): """Install all packages in list package_names.""" from six.moves import shlex_quote # suppress(import-error) from six.moves.urllib.parse import urlparse # suppress(import-error) # Drop directories which cause problems for brew taps hb_docs = os.path.join(self._executor.root_filesystem_directory(), "share", "doc", "homebrew") if os.path.exists(hb_docs): shutil.rmtree(hb_docs) # Separate out into packages that need to be downloaded with # brew and those that can be downloaded directly tar_packages = [p for p in package_names if urlparse(p).scheme] brew_packages = [p for p in package_names if not urlparse(p).scheme] if len(brew_packages): _run_task(self._executor, """Updating repositories""", ["brew", "update"]) _run_task(self._executor, """Install packages""", ["brew", "install"] + brew_packages, detail=_format_package_list(brew_packages)) for tar_pkg in tar_packages: _report_task("""Install {}""".format(tar_pkg)) with tempdir.TempDir() as download_dir: with directory.Navigation(download_dir): download.download_file(tar_pkg) extract_tarfile(os.path.basename(tar_pkg)) # The shell provides an easy way to do this, so just # use subprocess to call out to it. extracted_dir = [ d for d in os.listdir(download_dir) if d != os.path.basename(tar_pkg) ][0] subprocess.check_call("cp -r {src}/* {dst}".format( src=shlex_quote(extracted_dir), dst=self._executor.root_filesystem_directory()), shell=True)
def _download_qemu(distribution_dir, arch): """Download arch build of qemu and extract binaries.""" qemu_url = _QEMU_URL_BASE.format(arch=arch) with TemporarilyDownloadedFile(qemu_url, filename="qemu.deb") as qemu_deb: # Go into a separate subdirectory and extract the qemu deb # there, then copy out the requisite files, so that we don't # cause tons of pollution qemu_tmp = os.path.join(path_to_proot_dir, "_qemu_tmp") with directory.Navigation(qemu_tmp): qemu_binaries_path = os.path.join(qemu_tmp, "usr", "bin") _extract_qemu(qemu_deb.path(), qemu_tmp) _remove_unused_emulators(qemu_binaries_path) for filename in os.listdir(qemu_binaries_path): shutil.copy(os.path.join(qemu_binaries_path, filename), os.path.join(path_to_proot_dir, "bin")) shutil.rmtree(qemu_tmp) return os.path.join(distribution_dir, "bin", "qemu-{arch}")
def _fetch_proot_distribution(container_root, target_arch): """Fetch the initial proot distribution if it is not available. Touches /.have-proot-distribution when complete """ path_to_proot_check = constants.have_proot_distribution(container_root) path_to_proot_dir = constants.proot_distribution_dir(container_root) def _download_proot(distribution_dir, arch): """Download arch build of proot into distribution.""" from psqtraviscontainer.download import download_file with directory.Navigation(os.path.join(distribution_dir, "bin")): proot_url = _PROOT_URL_BASE.format(arch=arch) path_to_proot = download_file(proot_url, "proot") os.chmod(path_to_proot, os.stat(path_to_proot).st_mode | stat.S_IXUSR) return path_to_proot def _extract_qemu(qemu_deb_path, qemu_temp_dir): """Extract qemu.""" printer.unicode_safe( colored.magenta(("""-> Extracting {0}\n""" """""").format(qemu_deb_path), bold=True)) debian_package.extract_deb_data(qemu_deb_path, qemu_temp_dir) def _remove_unused_emulators(qemu_binaries_path): """Remove unused emulators from qemu distribution.""" distributions = distro.available_distributions() cur_arch = platform.machine() archs = [d["info"].kwargs["arch"] for d in distributions] archs = set([ architecture.Alias.qemu(a) for a in chain(*archs) if a != architecture.Alias.universal(cur_arch) ]) keep_binaries = ["qemu-" + a for a in archs] + ["proot"] for root, _, filenames in os.walk(qemu_binaries_path): for filename in filenames: if os.path.basename(filename) not in keep_binaries: os.remove(os.path.join(root, filename)) def _download_qemu(distribution_dir, arch): """Download arch build of qemu and extract binaries.""" qemu_url = _QEMU_URL_BASE.format(arch=arch) with TemporarilyDownloadedFile(qemu_url, filename="qemu.deb") as qemu_deb: # Go into a separate subdirectory and extract the qemu deb # there, then copy out the requisite files, so that we don't # cause tons of pollution qemu_tmp = os.path.join(path_to_proot_dir, "_qemu_tmp") with directory.Navigation(qemu_tmp): qemu_binaries_path = os.path.join(qemu_tmp, "usr", "bin") _extract_qemu(qemu_deb.path(), qemu_tmp) _remove_unused_emulators(qemu_binaries_path) for filename in os.listdir(qemu_binaries_path): shutil.copy(os.path.join(qemu_binaries_path, filename), os.path.join(path_to_proot_dir, "bin")) shutil.rmtree(qemu_tmp) return os.path.join(distribution_dir, "bin", "qemu-{arch}") try: os.stat(path_to_proot_check) printer.unicode_safe( colored.green( """-> """ """Using pre-existing proot """ """distribution\n""", bold=True)) except OSError: create_msg = """Creating distribution of proot in {}\n""" root_relative = os.path.relpath(container_root) printer.unicode_safe( colored.yellow(create_msg.format(root_relative), bold=True)) # Distro check does not exist - create the ./_proot directory # and download files for this architecture with directory.Navigation(path_to_proot_dir): proot_arch = architecture.Alias.universal(platform.machine()) _download_proot(path_to_proot_dir, proot_arch) # We may not need qemu if we're not going to emulate # anything. if (architecture.Alias.universal(platform.machine()) != architecture.Alias.universal(target_arch) or os.environ.get("_FORCE_DOWNLOAD_QEMU", None)): qemu_arch = architecture.Alias.debian(platform.machine()) _download_qemu(path_to_proot_dir, qemu_arch) with open(path_to_proot_check, "w+") as check_file: check_file.write("done") printer.unicode_safe( colored.green("""\N{check mark} """ """Successfully installed proot """ """distribution to """ """{}\n""".format(root_relative), bold=True)) return proot_distro_from_container(container_root)
def test_enter_create_dir(self): """Check that we create a dir when entering a non-existent one.""" does_not_exist = os.path.join(os.getcwd(), "does_not_exist") self.addCleanup(lambda: shutil.rmtree(does_not_exist)) with directory.Navigation(does_not_exist) as entered: self.assertThat(entered, DirExists())