def _install_new_build_packages(cls, package_names):
package_names.sort()
logger.info(
'Installing build dependencies: %s', ' '.join(package_names))
env = os.environ.copy()
env.update({
'DEBIAN_FRONTEND': 'noninteractive',
'DEBCONF_NONINTERACTIVE_SEEN': 'true',
})
apt_command = ['sudo', 'apt-get',
'--no-install-recommends', '-y']
if not is_dumb_terminal():
apt_command.extend(['-o', 'Dpkg::Progress-Fancy=1'])
apt_command.append('install')
subprocess.check_call(apt_command + package_names, env=env)
try:
subprocess.check_call(['sudo', 'apt-mark', 'auto'] +
package_names, env=env)
except subprocess.CalledProcessError as e:
logger.warning(
'Impossible to mark packages as auto-installed: {}'
.format(e))
def _setup_apt(self, cache_dir):
# Do not install recommends
apt.apt_pkg.config.set("Apt::Install-Recommends", "False")
# Methods and solvers dir for when in the SNAP
if common.is_snap():
snap_dir = os.getenv("SNAP")
apt_dir = os.path.join(snap_dir, "usr", "lib", "apt")
apt.apt_pkg.config.set("Dir", apt_dir)
# yes apt is broken like that we need to append os.path.sep
methods_dir = os.path.join(apt_dir, "methods")
apt.apt_pkg.config.set("Dir::Bin::methods",
methods_dir + os.path.sep)
solvers_dir = os.path.join(apt_dir, "solvers")
apt.apt_pkg.config.set("Dir::Bin::solvers::",
solvers_dir + os.path.sep)
apt_key_path = os.path.join(snap_dir, "usr", "bin", "apt-key")
apt.apt_pkg.config.set("Dir::Bin::apt-key", apt_key_path)
gpgv_path = os.path.join(snap_dir, "usr", "bin", "gpgv")
apt.apt_pkg.config.set("Apt::Key::gpgvcommand", gpgv_path)
apt.apt_pkg.config.set("Dir::Etc::Trusted", "/etc/apt/trusted.gpg")
apt.apt_pkg.config.set("Dir::Etc::TrustedParts",
"/etc/apt/trusted.gpg.d/")
# Make sure we always use the system GPG configuration, even with
# apt.Cache(rootdir).
for key in "Dir::Etc::Trusted", "Dir::Etc::TrustedParts":
apt.apt_pkg.config.set(key, apt.apt_pkg.config.find_file(key))
# Clear up apt's Post-Invoke-Success as we are not running
# on the system.
apt.apt_pkg.config.clear("APT::Update::Post-Invoke-Success")
self.progress = apt.progress.text.AcquireProgress()
if is_dumb_terminal():
# Make output more suitable for logging.
self.progress.pulse = lambda owner: True
self.progress._width = 0
sources_list_file = os.path.join(cache_dir, "etc", "apt",
"sources.list")
os.makedirs(os.path.dirname(sources_list_file), exist_ok=True)
with open(sources_list_file, "w") as f:
f.write(self._collected_sources_list())
# dpkg also needs to be in the rootdir in order to support multiarch
# (apt calls dpkg --print-foreign-architectures).
dpkg_path = shutil.which("dpkg")
if dpkg_path:
# Symlink it into place
destination = os.path.join(cache_dir, dpkg_path[1:])
if not os.path.exists(destination):
os.makedirs(os.path.dirname(destination), exist_ok=True)
os.symlink(dpkg_path, destination)
else:
logger.warning(
"Cannot find 'dpkg' command needed to support multiarch")
return self._create_cache(cache_dir, sources_list_file)
def snap(project_options, directory=None, output=None):
if directory:
prime_dir = os.path.abspath(directory)
snap = _snap_data_from_dir(prime_dir)
else:
# make sure the full lifecycle is executed
prime_dir = project_options.prime_dir
execute('prime', project_options)
snap = _snap_data_from_dir(prime_dir)
snap_name = output or common.format_snap_name(snap)
# If a .snap-build exists at this point, when we are about to override
# the snap blob, it is stale. We rename it so user have a chance to
# recover accidentally lost assertions.
snap_build = snap_name + '-build'
if os.path.isfile(snap_build):
_new = '{}.{}'.format(snap_build, int(time.time()))
logger.warning('Renaming stale build assertion to {}'.format(_new))
os.rename(snap_build, _new)
# These options need to match the review tools:
# http://bazaar.launchpad.net/~click-reviewers/click-reviewers-tools/trunk/view/head:/clickreviews/common.py#L38
mksquashfs_args = ['-noappend', '-comp', 'xz', '-no-xattrs']
if snap['type'] != 'os':
mksquashfs_args.append('-all-root')
with Popen(['mksquashfs', prime_dir, snap_name] + mksquashfs_args,
stdout=PIPE,
stderr=STDOUT) as proc:
ret = None
if is_dumb_terminal():
logger.info('Snapping {!r} ...'.format(snap['name']))
ret = proc.wait()
else:
message = '\033[0;32m\rSnapping {!r}\033[0;32m '.format(
snap['name'])
progress_indicator = ProgressBar(
widgets=[message, AnimatedMarker()], maxval=7)
progress_indicator.start()
ret = proc.poll()
count = 0
while ret is None:
if count >= 7:
progress_indicator.start()
count = 0
progress_indicator.update(count)
count += 1
time.sleep(.2)
ret = proc.poll()
print('')
if ret != 0:
logger.error(proc.stdout.read().decode('utf-8'))
raise RuntimeError('Failed to create snap {!r}'.format(snap_name))
logger.debug(proc.stdout.read().decode('utf-8'))
return snap_name
def _configure_apt(self):
# Do not install recommends.
apt.apt_pkg.config.set("Apt::Install-Recommends", "False")
# Ensure repos are provided by trusted third-parties.
apt.apt_pkg.config.set("Acquire::AllowInsecureRepositories", "False")
# Methods and solvers dir for when in the SNAP.
snap_dir = os.getenv("SNAP")
if common.is_snap() and snap_dir and os.path.exists(snap_dir):
apt_dir = os.path.join(snap_dir, "usr", "lib", "apt")
apt.apt_pkg.config.set("Dir", apt_dir)
# yes apt is broken like that we need to append os.path.sep
methods_dir = os.path.join(apt_dir, "methods")
apt.apt_pkg.config.set("Dir::Bin::methods", methods_dir + os.path.sep)
solvers_dir = os.path.join(apt_dir, "solvers")
apt.apt_pkg.config.set("Dir::Bin::solvers::", solvers_dir + os.path.sep)
apt_key_path = os.path.join(snap_dir, "usr", "bin", "apt-key")
apt.apt_pkg.config.set("Dir::Bin::apt-key", apt_key_path)
gpgv_path = os.path.join(snap_dir, "usr", "bin", "gpgv")
apt.apt_pkg.config.set("Apt::Key::gpgvcommand", gpgv_path)
apt.apt_pkg.config.set("Dir::Etc::Trusted", "/etc/apt/trusted.gpg")
apt.apt_pkg.config.set("Dir::Etc::TrustedParts", "/etc/apt/trusted.gpg.d/")
# Clear up apt's Post-Invoke-Success as we are not running
# on the system.
apt.apt_pkg.config.clear("APT::Update::Post-Invoke-Success")
self.progress = apt.progress.text.AcquireProgress()
if is_dumb_terminal():
# Make output more suitable for logging.
self.progress.pulse = lambda owner: True
self.progress._width = 0
def install_build_packages(packages):
unique_packages = set(packages)
new_packages = []
with apt.Cache() as apt_cache:
for pkg in unique_packages:
try:
if not apt_cache[pkg].installed:
new_packages.append(pkg)
except KeyError as e:
raise EnvironmentError('Could not find a required package in '
'\'build-packages\': {}'.format(str(e)))
if new_packages:
new_packages.sort()
logger.info('Installing build dependencies: %s',
' '.join(new_packages))
env = os.environ.copy()
env.update({
'DEBIAN_FRONTEND': 'noninteractive',
'DEBCONF_NONINTERACTIVE_SEEN': 'true',
})
apt_command = ['sudo', 'apt-get', '--no-install-recommends', '-y']
if not is_dumb_terminal():
apt_command.extend(['-o', 'Dpkg::Progress-Fancy=1'])
apt_command.append('install')
subprocess.check_call(apt_command + new_packages, env=env)
try:
subprocess.check_call(['sudo', 'apt-mark', 'auto'] + new_packages,
env=env)
except subprocess.CalledProcessError as e:
logger.warning(
'Impossible to mark packages as auto-installed: {}'.format(e))
def install_build_packages(cls, package_names):
unique_packages = set(package_names)
new_packages = []
with apt.Cache() as apt_cache:
for pkg in unique_packages:
try:
if not apt_cache[pkg].installed:
new_packages.append(pkg)
except KeyError as e:
raise errors.BuildPackageNotFoundError(e) from e
if new_packages:
new_packages.sort()
logger.info(
'Installing build dependencies: %s', ' '.join(new_packages))
env = os.environ.copy()
env.update({
'DEBIAN_FRONTEND': 'noninteractive',
'DEBCONF_NONINTERACTIVE_SEEN': 'true',
})
apt_command = ['sudo', 'apt-get',
'--no-install-recommends', '-y']
if not is_dumb_terminal():
apt_command.extend(['-o', 'Dpkg::Progress-Fancy=1'])
apt_command.append('install')
subprocess.check_call(apt_command + new_packages, env=env)
try:
subprocess.check_call(['sudo', 'apt-mark', 'auto'] +
new_packages, env=env)
except subprocess.CalledProcessError as e:
logger.warning(
'Impossible to mark packages as auto-installed: {}'
.format(e))
def configure(logger_name=None, log_level=None):
if not log_level:
log_level = logging.INFO
stdout_handler = logging.StreamHandler(stream=sys.stdout)
stdout_handler.addFilter(_StdoutFilter())
stderr_handler = logging.StreamHandler(stream=sys.stderr)
stderr_handler.addFilter(_StderrFilter())
handlers = [stdout_handler, stderr_handler]
if is_dumb_terminal():
formatter = logging.Formatter(style='{')
else:
formatter = _ColoredFormatter(style='{')
logger = logging.getLogger(logger_name)
for handler in handlers:
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.setLevel(log_level)
# INFO by default for the requests lib as it is too noisy
if log_level == logging.DEBUG:
logging.getLogger("requests").setLevel(log_level)
else:
logging.getLogger("requests").setLevel(logging.WARNING)
def _install_packages(cls, package_names: List[str]) -> None:
logger.info("Installing build dependencies: %s",
" ".join(package_names))
env = os.environ.copy()
env.update({
"DEBIAN_FRONTEND": "noninteractive",
"DEBCONF_NONINTERACTIVE_SEEN": "true",
"DEBIAN_PRIORITY": "critical",
})
apt_command = [
"sudo",
"--preserve-env",
"apt-get",
"--no-install-recommends",
"-y",
"--allow-downgrades",
]
if not is_dumb_terminal():
apt_command.extend(["-o", "Dpkg::Progress-Fancy=1"])
apt_command.append("install")
try:
subprocess.check_call(apt_command + package_names, env=env)
except subprocess.CalledProcessError:
raise errors.BuildPackagesNotInstalledError(packages=package_names)
versionless_names = [get_pkg_name_parts(p)[0] for p in package_names]
try:
subprocess.check_call(["sudo", "apt-mark", "auto"] +
versionless_names,
env=env)
except subprocess.CalledProcessError as e:
logger.warning(
"Impossible to mark packages as auto-installed: {}".format(e))
def configure(logger_name=None, log_level=None):
if not log_level:
log_level = logging.INFO
stdout_handler = logging.StreamHandler(stream=sys.stdout)
stdout_handler.addFilter(_StdoutFilter())
stderr_handler = logging.StreamHandler(stream=sys.stderr)
stderr_handler.addFilter(_StderrFilter())
handlers = [stdout_handler, stderr_handler]
if is_dumb_terminal():
formatter = logging.Formatter(style="{")
else:
formatter = _ColoredFormatter(style="{")
logger = logging.getLogger(logger_name)
for handler in handlers:
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.setLevel(log_level)
# INFO by default for the requests lib as it is too noisy
if log_level == logging.DEBUG:
logging.getLogger("requests").setLevel(log_level)
else:
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("gnupg").setLevel(logging.WARNING)
def _install_new_build_packages(cls, package_names):
package_names.sort()
logger.info(
'Installing build dependencies: %s', ' '.join(package_names))
env = os.environ.copy()
env.update({
'DEBIAN_FRONTEND': 'noninteractive',
'DEBCONF_NONINTERACTIVE_SEEN': 'true',
})
apt_command = ['sudo', 'apt-get',
'--no-install-recommends', '-y']
if not is_dumb_terminal():
apt_command.extend(['-o', 'Dpkg::Progress-Fancy=1'])
apt_command.append('install')
subprocess.check_call(apt_command + package_names, env=env)
try:
subprocess.check_call(['sudo', 'apt-mark', 'auto'] +
package_names, env=env)
except subprocess.CalledProcessError as e:
logger.warning(
'Impossible to mark packages as auto-installed: {}'
.format(e))
def _install_new_build_packages(cls, package_names: List[str]) -> None:
package_names.sort()
logger.info("Installing build dependencies: %s",
" ".join(package_names))
env = os.environ.copy()
env.update({
"DEBIAN_FRONTEND": "noninteractive",
"DEBCONF_NONINTERACTIVE_SEEN": "true"
})
apt_command = ["sudo", "apt-get", "--no-install-recommends", "-y"]
if not is_dumb_terminal():
apt_command.extend(["-o", "Dpkg::Progress-Fancy=1"])
apt_command.append("install")
try:
subprocess.check_call(apt_command + package_names, env=env)
except subprocess.CalledProcessError:
raise errors.BuildPackagesNotInstalledError(packages=package_names)
try:
subprocess.check_call(["sudo", "apt-mark", "auto"] + package_names,
env=env)
except subprocess.CalledProcessError as e:
logger.warning(
"Impossible to mark packages as auto-installed: {}".format(e))
def snap(project_options, directory=None, output=None):
if directory:
snap_dir = os.path.abspath(directory)
snap = _snap_data_from_dir(snap_dir)
else:
# make sure the full lifecycle is executed
snap_dir = project_options.snap_dir
snap = execute('prime', project_options)
snap_name = output or common.format_snap_name(snap)
# If a .snap-build exists at this point, when we are about to override
# the snap blob, it is stale. We rename it so user have a chance to
# recover accidentally lost assertions.
snap_build = snap_name + '-build'
if os.path.isfile(snap_build):
_new = '{}.{}'.format(snap_build, int(time.time()))
logger.warning('Renaming stale build assertion to {}'.format(_new))
os.rename(snap_build, _new)
# These options need to match the review tools:
# http://bazaar.launchpad.net/~click-reviewers/click-reviewers-tools/trunk/view/head:/clickreviews/common.py#L38
mksquashfs_args = ['-noappend', '-comp', 'xz', '-no-xattrs']
if snap['type'] != 'os':
mksquashfs_args.append('-all-root')
with Popen(['mksquashfs', snap_dir, snap_name] + mksquashfs_args,
stdout=PIPE, stderr=STDOUT) as proc:
ret = None
if is_dumb_terminal():
logger.info('Snapping {!r} ...'.format(snap['name']))
ret = proc.wait()
else:
message = '\033[0;32m\rSnapping {!r}\033[0;32m '.format(
snap['name'])
progress_indicator = ProgressBar(
widgets=[message, AnimatedMarker()], maxval=7)
progress_indicator.start()
ret = proc.poll()
count = 0
while ret is None:
if count >= 7:
progress_indicator.start()
count = 0
progress_indicator.update(count)
count += 1
time.sleep(.2)
ret = proc.poll()
print('')
if ret != 0:
logger.error(proc.stdout.read().decode('utf-8'))
raise RuntimeError('Failed to create snap {!r}'.format(snap_name))
logger.debug(proc.stdout.read().decode('utf-8'))
logger.info('Snapped {}'.format(snap_name))
return snap_name
def configure(logger_name=None, log_level=None):
if not log_level:
log_level = logging.INFO
if not is_dumb_terminal() and not sys.stdout.line_buffering:
# Line buffering makes logs easier to handle.
sys.stdout.flush()
try:
sys.stdout = io.TextIOWrapper(sys.stdout.detach(),
encoding='UTF-8',
line_buffering=True)
except io.UnsupportedOperation:
# debuilding fails, we will just let it pass as if it were a
# isatty call
pass
stdout_handler = logging.StreamHandler(stream=sys.stdout)
stdout_handler.addFilter(_StdoutFilter())
stderr_handler = logging.StreamHandler(stream=sys.stderr)
stderr_handler.addFilter(_StderrFilter())
handlers = [stdout_handler, stderr_handler]
if is_dumb_terminal():
formatter = logging.Formatter(style='{')
else:
formatter = _ColoredFormatter(style='{')
logger = logging.getLogger(logger_name)
for handler in handlers:
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.setLevel(log_level)
# INFO by default for the requests lib as it is too noisy
if log_level == logging.DEBUG:
logging.getLogger("requests").setLevel(log_level)
else:
logging.getLogger("requests").setLevel(logging.WARNING)
def _run_mksquashfs(
mksquashfs_command, *, directory, snap_name, snap_type, output_snap_name
):
# These options need to match the review tools:
# http://bazaar.launchpad.net/~click-reviewers/click-reviewers-tools/trunk/view/head:/clickreviews/common.py#L38
mksquashfs_args = ["-noappend", "-comp", "xz", "-no-xattrs", "-no-fragments"]
if snap_type != "base":
mksquashfs_args.append("-all-root")
complete_command = [
mksquashfs_command,
directory,
output_snap_name,
] + mksquashfs_args
with Popen(complete_command, stdout=PIPE, stderr=STDOUT) as proc:
ret = None
if is_dumb_terminal():
logger.info("Snapping {!r} ...".format(snap_name))
ret = proc.wait()
else:
message = "\033[0;32m\rSnapping {!r}\033[0;32m ".format(snap_name)
progress_indicator = ProgressBar(
widgets=[message, AnimatedMarker()], maxval=7
)
progress_indicator.start()
ret = proc.poll()
count = 0
while ret is None:
if count >= 7:
progress_indicator.start()
count = 0
progress_indicator.update(count)
count += 1
time.sleep(0.2)
ret = proc.poll()
print("")
output = proc.stdout.read().decode("utf-8")
logger.debug(output)
if ret != 0:
raise RuntimeError(
"Failed to create snap {!r}, mksquashfs failed:\n{}".format(
output_snap_name, output
)
)
def _run_mksquashfs(
mksquashfs_command, *, directory, snap_name, snap_type, output_snap_name
):
# These options need to match the review tools:
# http://bazaar.launchpad.net/~click-reviewers/click-reviewers-tools/trunk/view/head:/clickreviews/common.py#L38
mksquashfs_args = ["-noappend", "-comp", "xz", "-no-xattrs", "-no-fragments"]
if snap_type not in ("os", "base"):
mksquashfs_args.append("-all-root")
complete_command = [
mksquashfs_command,
directory,
output_snap_name,
] + mksquashfs_args
with Popen(complete_command, stdout=PIPE, stderr=STDOUT) as proc:
ret = None
if is_dumb_terminal():
logger.info("Snapping {!r} ...".format(snap_name))
ret = proc.wait()
else:
message = "\033[0;32m\rSnapping {!r}\033[0;32m ".format(snap_name)
progress_indicator = ProgressBar(
widgets=[message, AnimatedMarker()], maxval=7
)
progress_indicator.start()
ret = proc.poll()
count = 0
while ret is None:
if count >= 7:
progress_indicator.start()
count = 0
progress_indicator.update(count)
count += 1
time.sleep(.2)
ret = proc.poll()
print("")
if ret != 0:
logger.error(proc.stdout.read().decode("utf-8"))
raise RuntimeError("Failed to create snap {!r}".format(output_snap_name))
logger.debug(proc.stdout.read().decode("utf-8"))
def _run_pack(snap_command: List[Union[str, pathlib.Path]]) -> str:
ret = None
stdout = ""
stderr = ""
with subprocess.Popen(
snap_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE
) as proc:
if indicators.is_dumb_terminal():
echo.info("Snapping...")
ret = proc.wait()
else:
message = f"\033[0;32mSnapping \033[0m"
progress_indicator = progressbar.ProgressBar(
widgets=[message, progressbar.AnimatedMarker()],
# From progressbar.ProgressBar.update(...).
maxval=progressbar.UnknownLength,
)
progress_indicator.start()
for counter in itertools.count():
progress_indicator.update(counter)
time.sleep(0.2)
ret = proc.poll()
if ret is not None:
break
progress_indicator.finish()
if proc.stdout is not None:
stdout = proc.stdout.read().decode()
if proc.stderr is not None:
stderr = proc.stderr.read().decode()
logger.debug(f"stdout: {stdout} | stderr: {stderr}")
if ret != 0:
raise RuntimeError(
f"Failed to create snap, snap command failed:\nstdout:\n{stdout}\nstderr:\n{stderr}"
)
try:
snap_filename = stdout.split(":")[1].strip()
except IndexError:
logger.debug("Failed to parse snap pack outpout: {stdout}")
snap_filename = stdout
return snap_filename
def _run_mksquashfs(mksquashfs_command, *, directory, snap_name, snap_type,
output_snap_name):
# These options need to match the review tools:
# http://bazaar.launchpad.net/~click-reviewers/click-reviewers-tools/trunk/view/head:/clickreviews/common.py#L38
mksquashfs_args = ['-noappend', '-comp', 'xz', '-no-xattrs',
'-no-fragments']
if snap_type != 'os':
mksquashfs_args.append('-all-root')
complete_command = [
mksquashfs_command, directory, output_snap_name] + mksquashfs_args
with Popen(complete_command, stdout=PIPE, stderr=STDOUT) as proc:
ret = None
if is_dumb_terminal():
logger.info('Snapping {!r} ...'.format(snap_name))
ret = proc.wait()
else:
message = '\033[0;32m\rSnapping {!r}\033[0;32m '.format(
snap_name)
progress_indicator = ProgressBar(
widgets=[message, AnimatedMarker()], maxval=7)
progress_indicator.start()
ret = proc.poll()
count = 0
while ret is None:
if count >= 7:
progress_indicator.start()
count = 0
progress_indicator.update(count)
count += 1
time.sleep(.2)
ret = proc.poll()
print('')
if ret != 0:
logger.error(proc.stdout.read().decode('utf-8'))
raise RuntimeError('Failed to create snap {!r}'.format(
output_snap_name))
logger.debug(proc.stdout.read().decode('utf-8'))
def install_build_packages(cls, package_names):
unique_packages = set(package_names)
new_packages = []
with apt.Cache() as apt_cache:
for pkg in unique_packages:
try:
pkg_name, version = repo.get_pkg_name_parts(pkg)
installed_version = apt_cache[pkg_name].installed
if not installed_version:
new_packages.append(pkg)
elif version and installed_version != version:
new_packages.append(pkg)
except KeyError as e:
raise errors.BuildPackageNotFoundError(e) from e
if new_packages:
new_packages.sort()
logger.info('Installing build dependencies: %s',
' '.join(new_packages))
env = os.environ.copy()
env.update({
'DEBIAN_FRONTEND': 'noninteractive',
'DEBCONF_NONINTERACTIVE_SEEN': 'true',
})
apt_command = ['sudo', 'apt-get', '--no-install-recommends', '-y']
if not is_dumb_terminal():
apt_command.extend(['-o', 'Dpkg::Progress-Fancy=1'])
apt_command.append('install')
subprocess.check_call(apt_command + new_packages, env=env)
try:
subprocess.check_call(['sudo', 'apt-mark', 'auto'] +
new_packages,
env=env)
except subprocess.CalledProcessError as e:
logger.warning(
'Impossible to mark packages as auto-installed: {}'.format(
e))
def _setup_apt(self, download_dir):
# Create the 'partial' subdir too (LP: #1578007).
os.makedirs(os.path.join(download_dir, 'partial'), exist_ok=True)
apt.apt_pkg.config.set("Dir::Cache::Archives", download_dir)
# Do not install recommends
apt.apt_pkg.config.set('Apt::Install-Recommends', 'False')
# Make sure we always use the system GPG configuration, even with
# apt.Cache(rootdir).
for key in 'Dir::Etc::Trusted', 'Dir::Etc::TrustedParts':
apt.apt_pkg.config.set(key, apt.apt_pkg.config.find_file(key))
# Clear up apt's Post-Invoke-Success as we are not running
# on the system.
apt.apt_pkg.config.clear('APT::Update::Post-Invoke-Success')
self.progress = apt.progress.text.AcquireProgress()
if is_dumb_terminal():
# Make output more suitable for logging.
self.progress.pulse = lambda owner: True
self.progress._width = 0
def _setup_apt(self, download_dir):
# Create the 'partial' subdir too (LP: #1578007).
os.makedirs(os.path.join(download_dir, 'partial'), exist_ok=True)
apt.apt_pkg.config.set("Dir::Cache::Archives", download_dir)
# Do not install recommends
apt.apt_pkg.config.set('Apt::Install-Recommends', 'False')
# Make sure we always use the system GPG configuration, even with
# apt.Cache(rootdir).
for key in 'Dir::Etc::Trusted', 'Dir::Etc::TrustedParts':
apt.apt_pkg.config.set(key, apt.apt_pkg.config.find_file(key))
# Clear up apt's Post-Invoke-Success as we are not running
# on the system.
apt.apt_pkg.config.clear('APT::Update::Post-Invoke-Success')
self.progress = apt.progress.text.AcquireProgress()
if is_dumb_terminal():
# Make output more suitable for logging.
self.progress.pulse = lambda owner: True
self.progress._width = 0
def _setup_apt(self, cache_dir):
# Do not install recommends
apt.apt_pkg.config.set('Apt::Install-Recommends', 'False')
# Methods and solvers dir for when in the SNAP
if common.is_snap():
snap_dir = os.getenv('SNAP')
apt_dir = os.path.join(snap_dir, 'apt')
apt.apt_pkg.config.set('Dir', apt_dir)
# yes apt is broken like that we need to append os.path.sep
apt.apt_pkg.config.set('Dir::Bin::methods',
apt_dir + os.path.sep)
apt.apt_pkg.config.set('Dir::Bin::solvers::',
apt_dir + os.path.sep)
apt_key_path = os.path.join(apt_dir, 'apt-key')
apt.apt_pkg.config.set('Dir::Bin::apt-key', apt_key_path)
gpgv_path = os.path.join(snap_dir, 'bin', 'gpgv')
apt.apt_pkg.config.set('Apt::Key::gpgvcommand', gpgv_path)
apt.apt_pkg.config.set('Dir::Etc::Trusted',
'/etc/apt/trusted.gpg')
apt.apt_pkg.config.set('Dir::Etc::TrustedParts',
'/etc/apt/trusted.gpg.d/')
# Make sure we always use the system GPG configuration, even with
# apt.Cache(rootdir).
for key in 'Dir::Etc::Trusted', 'Dir::Etc::TrustedParts':
apt.apt_pkg.config.set(key, apt.apt_pkg.config.find_file(key))
# Clear up apt's Post-Invoke-Success as we are not running
# on the system.
apt.apt_pkg.config.clear('APT::Update::Post-Invoke-Success')
self.progress = apt.progress.text.AcquireProgress()
if is_dumb_terminal():
# Make output more suitable for logging.
self.progress.pulse = lambda owner: True
self.progress._width = 0
sources_list_file = os.path.join(
cache_dir, 'etc', 'apt', 'sources.list')
os.makedirs(os.path.dirname(sources_list_file), exist_ok=True)
with open(sources_list_file, 'w') as f:
f.write(self._collected_sources_list())
# dpkg also needs to be in the rootdir in order to support multiarch
# (apt calls dpkg --print-foreign-architectures).
dpkg_path = shutil.which('dpkg')
if dpkg_path:
# Symlink it into place
destination = os.path.join(cache_dir, dpkg_path[1:])
if not os.path.exists(destination):
os.makedirs(os.path.dirname(destination), exist_ok=True)
os.symlink(dpkg_path, destination)
else:
logger.warning(
"Cannot find 'dpkg' command needed to support multiarch")
apt_cache = apt.Cache(rootdir=cache_dir, memonly=True)
apt_cache.update(fetch_progress=self.progress,
sources_list=sources_list_file)
return apt_cache
def _setup_apt(self, cache_dir):
# Do not install recommends
apt.apt_pkg.config.set('Apt::Install-Recommends', 'False')
# Methods and solvers dir for when in the SNAP
if common.is_snap():
snap_dir = os.getenv('SNAP')
apt_dir = os.path.join(snap_dir, 'apt')
apt.apt_pkg.config.set('Dir', apt_dir)
# yes apt is broken like that we need to append os.path.sep
apt.apt_pkg.config.set('Dir::Bin::methods',
apt_dir + os.path.sep)
apt.apt_pkg.config.set('Dir::Bin::solvers::',
apt_dir + os.path.sep)
apt_key_path = os.path.join(apt_dir, 'apt-key')
apt.apt_pkg.config.set('Dir::Bin::apt-key', apt_key_path)
gpgv_path = os.path.join(snap_dir, 'bin', 'gpgv')
apt.apt_pkg.config.set('Apt::Key::gpgvcommand', gpgv_path)
apt.apt_pkg.config.set('Dir::Etc::Trusted',
'/etc/apt/trusted.gpg')
apt.apt_pkg.config.set('Dir::Etc::TrustedParts',
'/etc/apt/trusted.gpg.d/')
# Make sure we always use the system GPG configuration, even with
# apt.Cache(rootdir).
for key in 'Dir::Etc::Trusted', 'Dir::Etc::TrustedParts':
apt.apt_pkg.config.set(key, apt.apt_pkg.config.find_file(key))
# Clear up apt's Post-Invoke-Success as we are not running
# on the system.
apt.apt_pkg.config.clear('APT::Update::Post-Invoke-Success')
self.progress = apt.progress.text.AcquireProgress()
if is_dumb_terminal():
# Make output more suitable for logging.
self.progress.pulse = lambda owner: True
self.progress._width = 0
sources_list_file = os.path.join(
cache_dir, 'etc', 'apt', 'sources.list')
os.makedirs(os.path.dirname(sources_list_file), exist_ok=True)
with open(sources_list_file, 'w') as f:
f.write(self._collected_sources_list())
# dpkg also needs to be in the rootdir in order to support multiarch
# (apt calls dpkg --print-foreign-architectures).
dpkg_path = shutil.which('dpkg')
if dpkg_path:
# Symlink it into place
destination = os.path.join(cache_dir, dpkg_path[1:])
if not os.path.exists(destination):
os.makedirs(os.path.dirname(destination), exist_ok=True)
os.symlink(dpkg_path, destination)
else:
logger.warning(
"Cannot find 'dpkg' command needed to support multiarch")
apt_cache = apt.Cache(rootdir=cache_dir, memonly=True)
apt_cache.update(fetch_progress=self.progress,
sources_list=sources_list_file)
return apt_cache
def test_vt100_terminal_environmment(self):
self.useFixture(fixtures.EnvironmentVariable('TERM', 'vt100'))
self.assertFalse(indicators.is_dumb_terminal())
def test_dumb_terminal_environment(self):
self.useFixture(fixtures.EnvironmentVariable('TERM', 'dumb'))
self.assertTrue(indicators.is_dumb_terminal())
def test_not_a_tty_terminal(self):
self.mock_os_isatty.return_value = False
self.assertFalse(indicators.is_dumb_terminal())
def test_tty_terminal(self):
self.assertTrue(indicators.is_dumb_terminal())
def test_tty_terminal(self):
self.assertTrue(indicators.is_dumb_terminal())
def _setup_apt(self, cache_dir):
# Do not install recommends
apt.apt_pkg.config.set("Apt::Install-Recommends", "False")
# Ensure repos are provided by trusted third-parties
apt.apt_pkg.config.set("Acquire::AllowInsecureRepositories", "False")
# Methods and solvers dir for when in the SNAP
if common.is_snap():
snap_dir = os.getenv("SNAP")
apt_dir = os.path.join(snap_dir, "usr", "lib", "apt")
apt.apt_pkg.config.set("Dir", apt_dir)
# yes apt is broken like that we need to append os.path.sep
methods_dir = os.path.join(apt_dir, "methods")
apt.apt_pkg.config.set("Dir::Bin::methods",
methods_dir + os.path.sep)
solvers_dir = os.path.join(apt_dir, "solvers")
apt.apt_pkg.config.set("Dir::Bin::solvers::",
solvers_dir + os.path.sep)
apt_key_path = os.path.join(snap_dir, "usr", "bin", "apt-key")
apt.apt_pkg.config.set("Dir::Bin::apt-key", apt_key_path)
gpgv_path = os.path.join(snap_dir, "usr", "bin", "gpgv")
apt.apt_pkg.config.set("Apt::Key::gpgvcommand", gpgv_path)
apt.apt_pkg.config.set("Dir::Etc::Trusted", "/etc/apt/trusted.gpg")
apt.apt_pkg.config.set("Dir::Etc::TrustedParts",
"/etc/apt/trusted.gpg.d/")
# Make sure we always use the system GPG configuration, even with
# apt.Cache(rootdir). However, we also want to be able to add keys to it
# without root, so symlink back to the system's, but maintain our own.
# We'll leave Trusted alone and just fiddle with TrustedParts (Trusted is the
# one modified by apt-key, so add-apt-repository should still work).
apt.apt_pkg.config.set(
"Dir::Etc::Trusted",
apt.apt_pkg.config.find_file("Dir::Etc::Trusted"))
apt_config_path = os.path.join(cache_dir, "etc", "apt", "apt.conf")
trusted_parts_path = apt.apt_pkg.config.find_file(
"Dir::Etc::TrustedParts")
if not trusted_parts_path.startswith(cache_dir):
cached_trusted_parts = os.path.join(cache_dir,
trusted_parts_path.lstrip("/"))
with contextlib.suppress(FileNotFoundError):
shutil.rmtree(cached_trusted_parts)
os.makedirs(cached_trusted_parts)
for trusted_part in os.scandir(trusted_parts_path):
os.symlink(
os.path.join(trusted_parts_path, trusted_part.name),
os.path.join(cached_trusted_parts, trusted_part.name),
)
apt.apt_pkg.config.set("Dir::Etc::TrustedParts",
cached_trusted_parts)
# The above config is all that is needed on bionic, but xenial
# requires this configuration file
os.makedirs(os.path.dirname(apt_config_path), exist_ok=True)
with open(apt_config_path, "w") as f:
f.write("Dir::Etc::TrustedParts {};\n".format(
cached_trusted_parts))
# Now copy in any requested keyrings
for keyring in self._keyrings:
shutil.copy2(keyring, cached_trusted_parts)
# Clear up apt's Post-Invoke-Success as we are not running
# on the system.
apt.apt_pkg.config.clear("APT::Update::Post-Invoke-Success")
self.progress = apt.progress.text.AcquireProgress()
if is_dumb_terminal():
# Make output more suitable for logging.
self.progress.pulse = lambda owner: True
self.progress._width = 0
sources_list_file = os.path.join(cache_dir, "etc", "apt",
"sources.list")
os.makedirs(os.path.dirname(sources_list_file), exist_ok=True)
with open(sources_list_file, "w") as f:
f.write(self._collected_sources_list())
# dpkg also needs to be in the rootdir in order to support multiarch
# (apt calls dpkg --print-foreign-architectures).
dpkg_path = shutil.which("dpkg")
if dpkg_path:
# Symlink it into place
destination = os.path.join(cache_dir, dpkg_path[1:])
if not os.path.exists(destination):
os.makedirs(os.path.dirname(destination), exist_ok=True)
os.symlink(dpkg_path, destination)
else:
logger.warning(
"Cannot find 'dpkg' command needed to support multiarch")
old_apt_config = os.environ.get("APT_CONFIG")
try:
# Only xenial needs this. If snapcraft changes to core18, this
# variable is unnecessary.
os.environ["APT_CONFIG"] = apt_config_path
return self._create_cache(cache_dir, sources_list_file)
finally:
if old_apt_config is None:
del os.environ["APT_CONFIG"]
else:
os.environ["APT_CONFIG"] = old_apt_config
def test_not_a_tty_terminal(self):
self.mock_os_isatty.return_value = False
self.assertFalse(indicators.is_dumb_terminal())
def test_dumb_terminal_environment(self):
self.useFixture(fixtures.EnvironmentVariable('TERM', 'dumb'))
self.assertTrue(indicators.is_dumb_terminal())
def test_vt100_terminal_environmment(self):
self.useFixture(fixtures.EnvironmentVariable('TERM', 'vt100'))
self.assertFalse(indicators.is_dumb_terminal())
