def _extract_qemu(qemu_deb_path, qemu_temp_dir):
"""Extract qemu."""
printer.unicode_safe(
colored.magenta(("""-> Extracting {0}\n"""
"""""").format(qemu_deb_path),
bold=True))
debian_package.extract_deb_data(qemu_deb_path, qemu_temp_dir)
def _extract_distro_archive(distro_archive_file, distro_folder):
"""Extract distribution archive into distro_folder."""
with tarfile.open(name=distro_archive_file.path()) as archive:
msg = ("""-> Extracting """
"""{0}\n""").format(os.path.relpath(distro_archive_file.path()))
extract_members = [m for m in archive.getmembers() if not m.isdev()]
printer.unicode_safe(colored.magenta(msg, bold=True))
archive.extractall(members=extract_members, path=distro_folder)
# Set the permissions of the extracted archive so we can delete it
# if need be.
os.chmod(distro_folder, os.stat(distro_folder).st_mode | stat.S_IRWXU)
for root, directories, filenames in os.walk(distro_folder):
for distro_folder_directory in directories:
path = os.path.join(root, distro_folder_directory)
try:
os.chmod(path, os.stat(path).st_mode | stat.S_IRWXU)
except OSError: # suppress(pointless-except)
pass
for filename in filenames:
path = os.path.join(root, filename)
try:
os.chmod(path, os.stat(path).st_mode | stat.S_IRWXU)
except OSError: # suppress(pointless-except)
pass
def _extract_archive(archive_file, container_folder):
"""Extract distribution archive into container_folder."""
msg = ("""-> Extracting {0}\n""").format(archive_file.path())
printer.unicode_safe(colored.magenta(msg, bold=True))
with tarfile.open(name=archive_file.path()) as archive:
extract_members = archive.getmembers()
archive.extractall(members=extract_members, path=container_folder)
def _extract_archive(archive_file, container_folder):
"""Extract distribution archive into container_folder."""
with tarfile.open(name=archive_file.path()) as archive:
msg = ("""-> Extracting """
"""{0}\n""").format(archive_file.path())
extract_members = archive.getmembers()
printer.unicode_safe(colored.magenta(msg, bold=True))
archive.extractall(members=extract_members, path=container_folder)
def _print_distribution_details(details):
"""Print distribution details."""
output = bytearray()
output += ("\n" +
colored.white("""Configured Distribution:""", bold=True) +
"\n").encode()
output += _format_distribution_details(details, color=True).encode()
printer.unicode_safe(output.decode("utf-8"))
def _print_distribution_details(details):
"""Print distribution details."""
output = bytearray()
output += ("\n" +
colored.white("""Configured Distribution:""", bold=True) +
"\n").encode()
output += _format_distribution_details(details, color=True).encode()
printer.unicode_safe(output.decode("utf-8"))
def main(arguments=None):
"""Parse arguments and set up proot.
Parse arguments, fetches initial proot distribution and downloads
and sets up our proot.
"""
result = _parse_arguments(arguments=arguments)
container_dir = os.path.realpath(result.containerdir)
selected_distro = distro.lookup(vars(result))
try:
existing = distro.read_existing(result.containerdir)
for key, value in existing.items():
if selected_distro[key] != value:
details = _format_distribution_details(existing)
raise RuntimeError("""A distribution described by:\n"""
"""{details}\n"""
"""already exists in {containerdir}.\n"""
"""Use a different container directory """
"""or move this one out of the way"""
"""""".format(details=details,
containerdir=container_dir))
except distro.NoDistributionDetailsError: # suppress(pointless-except)
pass
_print_distribution_details(selected_distro)
# Now set up packages in the distribution. If more packages need
# to be installed or the installed packages need to be updated then
# the build cache should be cleared.
with selected_distro["info"].create_func(container_dir,
selected_distro) as container:
container.install_packages(result.repositories, result.packages)
distro.write_details(result.containerdir, selected_distro)
relative_containerdir = os.path.relpath(result.containerdir)
msg = """\N{check mark} Container has been set up in {0}\n"""
printer.unicode_safe(colored.green(msg.format(relative_containerdir),
bold=True))
def main(arguments=None):
"""Parse arguments and set up proot.
Parse arguments, fetches initial proot distribution and downloads
and sets up our proot.
"""
result = _parse_arguments(arguments=arguments)
container_dir = os.path.realpath(result.containerdir)
selected_distro = distro.lookup(vars(result))
try:
existing = distro.read_existing(result.containerdir)
for key, value in existing.items():
if selected_distro[key] != value:
details = _format_distribution_details(existing)
raise RuntimeError("""A distribution described by:\n"""
"""{details}\n"""
"""already exists in {containerdir}.\n"""
"""Use a different container directory """
"""or move this one out of the way"""
"""""".format(details=details,
containerdir=container_dir))
except distro.NoDistributionDetailsError: # suppress(pointless-except)
pass
_print_distribution_details(selected_distro)
# Now set up packages in the distribution. If more packages need
# to be installed or the installed packages need to be updated then
# the build cache should be cleared.
with selected_distro["info"].create_func(container_dir,
selected_distro) as container:
container.install_packages(result.repositories, result.packages)
distro.write_details(result.containerdir, selected_distro)
relative_containerdir = os.path.relpath(result.containerdir)
msg = """\N{check mark} Container has been set up in {0}\n"""
printer.unicode_safe(
colored.green(msg.format(relative_containerdir), bold=True))
def fetch_distribution(
container_root, # pylint:disable=R0913
proot_distro,
details):
"""Lazy-initialize distribution and return it."""
path_to_distro_folder = get_dir_for_distro(container_root, details)
def _download_distro(details, path_to_distro_folder):
"""Download distribution and untar it in container root."""
distro_arch = details["arch"]
download_url = details["url"].format(arch=distro_arch)
with tempdir.TempDir() as download_dir:
with directory.Navigation(download_dir):
with TemporarilyDownloadedFile(download_url) as archive_file:
_extract_distro_archive(archive_file,
path_to_distro_folder)
def _minimize_ubuntu(cont, root):
"""Reduce the install footprint of ubuntu as much as possible."""
required_packages = {
"precise":
set([
"apt", "base-files", "base-passwd", "bash", "bsdutils",
"coreutils", "dash", "debconf", "debianutils", "diffutils",
"dpkg", "findutils", "gcc-4.6-base", "gnupg", "gpgv", "grep",
"gzip", "libacl1", "libapt-pkg4.12", "libattr1", "libbz2-1.0",
"libc-bin", "libc6", "libdb5.1", "libffi6", "libgcc1",
"liblzma5", "libpam-modules", "libpam-modules-bin",
"libpam-runtime", "libpam0g", "libreadline6", "libselinux1",
"libstdc++6", "libtinfo5", "libusb-0.1-4", "makedev", "mawk",
"multiarch-support", "perl-base", "readline-common", "sed",
"sensible-utils", "tar", "tzdata", "ubuntu-keyring",
"xz-utils", "zlib1g"
]),
"trusty":
set([
"apt", "base-files", "base-passwd", "bash", "bsdutils",
"coreutils", "dash", "debconf", "debianutils", "diffutils",
"dh-python", "dpkg", "findutils", "gcc-4.8-base",
"gcc-4.9-base", "gnupg", "gpgv", "grep", "gzip", "libacl1",
"libapt-pkg4.12", "libaudit1", "libaudit-common", "libattr1",
"libbz2-1.0", "libc-bin", "libc6", "libcap2", "libdb5.3",
"libdebconfclient0", "libexpat1", "libmpdec2", "libffi6",
"libgcc1", "liblzma5", "libncursesw5", "libpcre3",
"libpam-modules", "libpam-modules-bin", "libpam-runtime",
"libpam0g", "libpython3-stdlib", "libpython3.4-stdlib",
"libpython3", "libpython3-minimal", "libpython3.4",
"libpython3.4-minimal", "libreadline6", "libselinux1",
"libssl1.0.0", "libstdc++6", "libsqlite3-0", "libtinfo5",
"libusb-0.1-4", "lsb-release", "makedev", "mawk",
"mime-support", "multiarch-support", "perl-base", "python3",
"python3-minimal", "python3.4", "python3.4-minimal",
"readline-common", "sed", "sensible-utils", "tar", "tzdata",
"ubuntu-keyring", "xz-utils", "zlib1g"
])
}
os.environ["SUDO_FORCE_REMOVE"] = "yes"
os.environ["DEBIAN_FRONTEND"] = "noninteractive"
if release in required_packages:
pkgs = set(
cont.execute([
"dpkg-query", "--admindir={}".format(
os.path.join(root, "var", "lib", "dpkg")), "-Wf",
"${Package}\n"
])[1].split("\n"))
release = details["release"]
remove = [
l for l in list(pkgs ^ required_packages[release]) if len(l)
]
if root != "/":
_clear_postrm_scripts_in_root(root)
if len(remove):
cont.execute_success([
"dpkg", "--root={}".format(root), "--purge", "--force-all"
] + remove,
minimal_bind=True)
with open(
os.path.join(get_dir_for_distro(container_root, details),
"etc", "apt", "apt.conf.d", "99container"),
"w") as apt_config:
apt_config.write("\n".join([
"APT::Install-Recommends \"0\";",
"APT::Install-Suggests \"0\";"
]))
# Container isn't safe to use until we've either verified that the
# path to the distro folder exists or we've downloaded a distro into it
linux_cont = LinuxContainer(proot_distro, path_to_distro_folder,
details["release"], details["arch"],
details["pkgsys"])
minimize_actions = defaultdict(lambda: lambda c, p: None)
try:
os.stat(path_to_distro_folder)
use_existing_msg = ("""\N{check mark} Using existing folder for """
"""proot distro """
"""{distro} {release} {arch}\n""")
printer.unicode_safe(
colored.green(use_existing_msg.format(**details), bold=True))
return (linux_cont, minimize_actions)
except OSError:
# Download the distribution tarball in the distro dir
_download_distro(details, path_to_distro_folder)
# Minimize the installed distribution, but only when it
# was just initially downloaded
minimize_actions = defaultdict(lambda: lambda c: None,
Ubuntu=_minimize_ubuntu)
return (linux_cont, minimize_actions)
def _fetch_proot_distribution(container_root, target_arch):
"""Fetch the initial proot distribution if it is not available.
Touches /.have-proot-distribution when complete
"""
path_to_proot_check = constants.have_proot_distribution(container_root)
path_to_proot_dir = constants.proot_distribution_dir(container_root)
def _download_proot(distribution_dir, arch):
"""Download arch build of proot into distribution."""
from psqtraviscontainer.download import download_file
with directory.Navigation(os.path.join(distribution_dir, "bin")):
proot_url = _PROOT_URL_BASE.format(arch=arch)
path_to_proot = download_file(proot_url, "proot")
os.chmod(path_to_proot,
os.stat(path_to_proot).st_mode | stat.S_IXUSR)
return path_to_proot
def _extract_qemu(qemu_deb_path, qemu_temp_dir):
"""Extract qemu."""
printer.unicode_safe(
colored.magenta(("""-> Extracting {0}\n"""
"""""").format(qemu_deb_path),
bold=True))
debian_package.extract_deb_data(qemu_deb_path, qemu_temp_dir)
def _remove_unused_emulators(qemu_binaries_path):
"""Remove unused emulators from qemu distribution."""
distributions = distro.available_distributions()
cur_arch = platform.machine()
archs = [d["info"].kwargs["arch"] for d in distributions]
archs = set([
architecture.Alias.qemu(a) for a in chain(*archs)
if a != architecture.Alias.universal(cur_arch)
])
keep_binaries = ["qemu-" + a for a in archs] + ["proot"]
for root, _, filenames in os.walk(qemu_binaries_path):
for filename in filenames:
if os.path.basename(filename) not in keep_binaries:
os.remove(os.path.join(root, filename))
def _download_qemu(distribution_dir, arch):
"""Download arch build of qemu and extract binaries."""
qemu_url = _QEMU_URL_BASE.format(arch=arch)
with TemporarilyDownloadedFile(qemu_url,
filename="qemu.deb") as qemu_deb:
# Go into a separate subdirectory and extract the qemu deb
# there, then copy out the requisite files, so that we don't
# cause tons of pollution
qemu_tmp = os.path.join(path_to_proot_dir, "_qemu_tmp")
with directory.Navigation(qemu_tmp):
qemu_binaries_path = os.path.join(qemu_tmp, "usr", "bin")
_extract_qemu(qemu_deb.path(), qemu_tmp)
_remove_unused_emulators(qemu_binaries_path)
for filename in os.listdir(qemu_binaries_path):
shutil.copy(os.path.join(qemu_binaries_path, filename),
os.path.join(path_to_proot_dir, "bin"))
shutil.rmtree(qemu_tmp)
return os.path.join(distribution_dir, "bin", "qemu-{arch}")
try:
os.stat(path_to_proot_check)
printer.unicode_safe(
colored.green(
"""-> """
"""Using pre-existing proot """
"""distribution\n""",
bold=True))
except OSError:
create_msg = """Creating distribution of proot in {}\n"""
root_relative = os.path.relpath(container_root)
printer.unicode_safe(
colored.yellow(create_msg.format(root_relative), bold=True))
# Distro check does not exist - create the ./_proot directory
# and download files for this architecture
with directory.Navigation(path_to_proot_dir):
proot_arch = architecture.Alias.universal(platform.machine())
_download_proot(path_to_proot_dir, proot_arch)
# We may not need qemu if we're not going to emulate
# anything.
if (architecture.Alias.universal(platform.machine()) !=
architecture.Alias.universal(target_arch)
or os.environ.get("_FORCE_DOWNLOAD_QEMU", None)):
qemu_arch = architecture.Alias.debian(platform.machine())
_download_qemu(path_to_proot_dir, qemu_arch)
with open(path_to_proot_check, "w+") as check_file:
check_file.write("done")
printer.unicode_safe(
colored.green("""\N{check mark} """
"""Successfully installed proot """
"""distribution to """
"""{}\n""".format(root_relative),
bold=True))
return proot_distro_from_container(container_root)