def _fetch_homebrew(container_dir, distro_config):
"""Fetch homebrew and untar it in the container directory."""
try:
os.stat(os.path.join(container_dir, "bin", "brew"))
return container_for_directory(container_dir, distro_config)
except OSError:
with directory.Navigation(tempdir.TempDir().name):
with TemporarilyDownloadedFile(_HOMEBREW_URL,
filename="brew") as archive_file:
with directory.Navigation(tempdir.TempDir().name) as extract:
_extract_archive(archive_file, extract)
first = os.path.join(extract, os.listdir(extract)[0])
files = [os.path.join(first, p) for p in os.listdir(first)]
for filename in files:
try:
filename_base = os.path.basename(filename)
shutil.move(
filename,
os.path.join(container_dir, filename_base))
except IOError: # suppress(pointless-except)
# Ignore stuff that can't be moved for whatever
# reason. These are all files that generally
# don't matter.
pass
return OSXContainer(container_dir, distro_config["pkgsys"])
def install_packages(self, package_names):
"""Install all packages in list package_names.
This works in a somewhat non-standard way. We will be
updating the repository list as usual, but will be
using a combination of apt-get download and
dpkg manually to install packages into a local
directory which we control.
"""
self._initialize_directories()
from six.moves.urllib.parse import urlparse # suppress(import-error)
root = self._executor.root_filesystem_directory()
environment = {
"APT_CONFIG": os.path.join(root, "etc", "apt", "apt.conf")
}
_run_task(self._executor,
"""Update repositories""",
["apt-get", "update", "-y", "--force-yes"],
env=environment)
# Separate out into packages that need to be downloaded with
# apt-get and packages that can be downloaded directly
# using download_file
deb_packages = [p for p in package_names if urlparse(p).scheme]
apt_packages = [p for p in package_names if not urlparse(p).scheme]
# Clear out /var/cache/apt/archives
archives = os.path.join(root, "var", "cache", "apt", "archives")
if os.path.exists(archives):
shutil.rmtree(archives)
os.makedirs(archives)
if len(deb_packages):
with directory.Navigation(archives):
_report_task("""Downloading user-specified packages""")
for deb in deb_packages:
download.download_file(deb)
# Now use apt-get install -d to download the apt_packages and their
# dependencies, but not install them
if len(apt_packages):
_run_task(self._executor,
"""Downloading APT packages and dependencies""",
[
"apt-get", "-y", "--force-yes", "-d", "install",
"--reinstall"
] + apt_packages,
env=environment,
detail=_format_package_list(apt_packages))
# Go back into our archives directory and unpack all our packages
with directory.Navigation(archives):
package_files = fnmatch.filter(os.listdir("."), "*.deb")
for pkg in package_files:
_run_task(self._executor,
"""Unpacking """, ["dpkg", "-x", pkg, root],
detail=os.path.splitext(os.path.basename(pkg))[0])
def _fetch_choco(container_dir, distro_config):
"""Fetch chocolatey and install it in the container directory."""
try:
os.stat(os.path.join(container_dir, "bin", "choco.exe"))
return container_for_directory(container_dir, distro_config)
except OSError:
with tempdir.TempDir() as download_dir:
with directory.Navigation(download_dir):
_execute_no_output(
["setx", "ChocolateyInstall", container_dir])
# Also set the variable in the local environment
# too, so that it gets propagated down to our
# children
os.environ["ChocolateyInstall"] = container_dir
try:
os.makedirs(container_dir)
except OSError as error:
if error.errno != errno.EEXIST:
raise error
_execute_no_output([
"powershell", "-NoProfile", "-ExecutionPolicy", "Bypass",
"-Command", _CHOCO_INSTALL_CMD
])
# Reset variable back to original state to prevent
# polluting the user's registry
_execute_no_output(["setx", "ChocolateyInstall", ""])
return WindowsContainer(container_dir, distro_config["pkgsys"])
def test_enter_exist_dir(self):
"""Check that we can enter an existing dir."""
existing_dir = os.path.join(os.getcwd(), "existing")
os.makedirs(existing_dir)
self.addCleanup(lambda: shutil.rmtree(existing_dir))
with directory.Navigation(existing_dir) as entered:
self.assertThat(entered, DirExists())
def _download_distro(details, path_to_distro_folder):
"""Download distribution and untar it in container root."""
distro_arch = details["arch"]
download_url = details["url"].format(arch=distro_arch)
with tempdir.TempDir() as download_dir:
with directory.Navigation(download_dir):
with TemporarilyDownloadedFile(download_url) as archive_file:
_extract_distro_archive(archive_file,
path_to_distro_folder)
def _download_proot(distribution_dir, arch):
"""Download arch build of proot into distribution."""
from psqtraviscontainer.download import download_file
with directory.Navigation(os.path.join(distribution_dir, "bin")):
proot_url = _PROOT_URL_BASE.format(arch=arch)
path_to_proot = download_file(proot_url, "proot")
os.chmod(path_to_proot,
os.stat(path_to_proot).st_mode | stat.S_IXUSR)
return path_to_proot
def add_repositories(self, repos):
"""Add a repository to the central packaging system."""
with tempdir.TempDir() as download_dir:
with directory.Navigation(download_dir):
for repo in repos:
repo_file = download.download_file(repo)
# Create a bash script to copy the downloaded repo file
# over to /etc/yum/repos.d
with tempfile.NamedTemporaryFile() as bash_script:
copy_cmd = ("cp \"{0}\""
"/etc/yum/repos.d").format(repo_file)
bash_script.write(six.b(copy_cmd))
bash_script.flush()
self._executor.execute_success(
["bash", bash_script.name])
def install_packages(self, package_names):
"""Install all packages in list package_names."""
from six.moves import shlex_quote # suppress(import-error)
from six.moves.urllib.parse import urlparse # suppress(import-error)
# Drop directories which cause problems for brew taps
hb_docs = os.path.join(self._executor.root_filesystem_directory(),
"share", "doc", "homebrew")
if os.path.exists(hb_docs):
shutil.rmtree(hb_docs)
# Separate out into packages that need to be downloaded with
# brew and those that can be downloaded directly
tar_packages = [p for p in package_names if urlparse(p).scheme]
brew_packages = [p for p in package_names if not urlparse(p).scheme]
if len(brew_packages):
_run_task(self._executor, """Updating repositories""",
["brew", "update"])
_run_task(self._executor,
"""Install packages""",
["brew", "install"] + brew_packages,
detail=_format_package_list(brew_packages))
for tar_pkg in tar_packages:
_report_task("""Install {}""".format(tar_pkg))
with tempdir.TempDir() as download_dir:
with directory.Navigation(download_dir):
download.download_file(tar_pkg)
extract_tarfile(os.path.basename(tar_pkg))
# The shell provides an easy way to do this, so just
# use subprocess to call out to it.
extracted_dir = [
d for d in os.listdir(download_dir)
if d != os.path.basename(tar_pkg)
][0]
subprocess.check_call("cp -r {src}/* {dst}".format(
src=shlex_quote(extracted_dir),
dst=self._executor.root_filesystem_directory()),
shell=True)
def _download_qemu(distribution_dir, arch):
"""Download arch build of qemu and extract binaries."""
qemu_url = _QEMU_URL_BASE.format(arch=arch)
with TemporarilyDownloadedFile(qemu_url,
filename="qemu.deb") as qemu_deb:
# Go into a separate subdirectory and extract the qemu deb
# there, then copy out the requisite files, so that we don't
# cause tons of pollution
qemu_tmp = os.path.join(path_to_proot_dir, "_qemu_tmp")
with directory.Navigation(qemu_tmp):
qemu_binaries_path = os.path.join(qemu_tmp, "usr", "bin")
_extract_qemu(qemu_deb.path(), qemu_tmp)
_remove_unused_emulators(qemu_binaries_path)
for filename in os.listdir(qemu_binaries_path):
shutil.copy(os.path.join(qemu_binaries_path, filename),
os.path.join(path_to_proot_dir, "bin"))
shutil.rmtree(qemu_tmp)
return os.path.join(distribution_dir, "bin", "qemu-{arch}")
def _fetch_proot_distribution(container_root, target_arch):
"""Fetch the initial proot distribution if it is not available.
Touches /.have-proot-distribution when complete
"""
path_to_proot_check = constants.have_proot_distribution(container_root)
path_to_proot_dir = constants.proot_distribution_dir(container_root)
def _download_proot(distribution_dir, arch):
"""Download arch build of proot into distribution."""
from psqtraviscontainer.download import download_file
with directory.Navigation(os.path.join(distribution_dir, "bin")):
proot_url = _PROOT_URL_BASE.format(arch=arch)
path_to_proot = download_file(proot_url, "proot")
os.chmod(path_to_proot,
os.stat(path_to_proot).st_mode | stat.S_IXUSR)
return path_to_proot
def _extract_qemu(qemu_deb_path, qemu_temp_dir):
"""Extract qemu."""
printer.unicode_safe(
colored.magenta(("""-> Extracting {0}\n"""
"""""").format(qemu_deb_path),
bold=True))
debian_package.extract_deb_data(qemu_deb_path, qemu_temp_dir)
def _remove_unused_emulators(qemu_binaries_path):
"""Remove unused emulators from qemu distribution."""
distributions = distro.available_distributions()
cur_arch = platform.machine()
archs = [d["info"].kwargs["arch"] for d in distributions]
archs = set([
architecture.Alias.qemu(a) for a in chain(*archs)
if a != architecture.Alias.universal(cur_arch)
])
keep_binaries = ["qemu-" + a for a in archs] + ["proot"]
for root, _, filenames in os.walk(qemu_binaries_path):
for filename in filenames:
if os.path.basename(filename) not in keep_binaries:
os.remove(os.path.join(root, filename))
def _download_qemu(distribution_dir, arch):
"""Download arch build of qemu and extract binaries."""
qemu_url = _QEMU_URL_BASE.format(arch=arch)
with TemporarilyDownloadedFile(qemu_url,
filename="qemu.deb") as qemu_deb:
# Go into a separate subdirectory and extract the qemu deb
# there, then copy out the requisite files, so that we don't
# cause tons of pollution
qemu_tmp = os.path.join(path_to_proot_dir, "_qemu_tmp")
with directory.Navigation(qemu_tmp):
qemu_binaries_path = os.path.join(qemu_tmp, "usr", "bin")
_extract_qemu(qemu_deb.path(), qemu_tmp)
_remove_unused_emulators(qemu_binaries_path)
for filename in os.listdir(qemu_binaries_path):
shutil.copy(os.path.join(qemu_binaries_path, filename),
os.path.join(path_to_proot_dir, "bin"))
shutil.rmtree(qemu_tmp)
return os.path.join(distribution_dir, "bin", "qemu-{arch}")
try:
os.stat(path_to_proot_check)
printer.unicode_safe(
colored.green(
"""-> """
"""Using pre-existing proot """
"""distribution\n""",
bold=True))
except OSError:
create_msg = """Creating distribution of proot in {}\n"""
root_relative = os.path.relpath(container_root)
printer.unicode_safe(
colored.yellow(create_msg.format(root_relative), bold=True))
# Distro check does not exist - create the ./_proot directory
# and download files for this architecture
with directory.Navigation(path_to_proot_dir):
proot_arch = architecture.Alias.universal(platform.machine())
_download_proot(path_to_proot_dir, proot_arch)
# We may not need qemu if we're not going to emulate
# anything.
if (architecture.Alias.universal(platform.machine()) !=
architecture.Alias.universal(target_arch)
or os.environ.get("_FORCE_DOWNLOAD_QEMU", None)):
qemu_arch = architecture.Alias.debian(platform.machine())
_download_qemu(path_to_proot_dir, qemu_arch)
with open(path_to_proot_check, "w+") as check_file:
check_file.write("done")
printer.unicode_safe(
colored.green("""\N{check mark} """
"""Successfully installed proot """
"""distribution to """
"""{}\n""".format(root_relative),
bold=True))
return proot_distro_from_container(container_root)
def test_enter_create_dir(self):
"""Check that we create a dir when entering a non-existent one."""
does_not_exist = os.path.join(os.getcwd(), "does_not_exist")
self.addCleanup(lambda: shutil.rmtree(does_not_exist))
with directory.Navigation(does_not_exist) as entered:
self.assertThat(entered, DirExists())