def _mark_install(cls, apt_cache: apt.Cache,
package_names: List[str]) -> None:
for name in package_names:
if name.endswith(":any"):
name = name[:-4]
if apt_cache.is_virtual_package(name):
name = apt_cache.get_providing_packages(name)[0].name
logger.debug(
"Marking {!r} (and its dependencies) to be fetched".format(
name))
name_arch, version = repo.get_pkg_name_parts(name)
try:
if version:
_set_pkg_version(apt_cache[name_arch], version)
# Disable automatic resolving of broken packages here
# because if that fails it raises a SystemError and the
# API doesn't expose enough information about he problem.
# Instead we let apt-get show a verbose error message later.
# Also, make sure this package is marked as auto-installed,
# which will propagate to its dependencies.
apt_cache[name_arch].mark_install(auto_fix=False,
from_user=False)
# Now mark this package as NOT automatically installed, which
# will leave its dependencies marked as auto-installed, which
# allows us to clean them up if necessary.
apt_cache[name_arch].mark_auto(False)
cls._verify_marked_install(apt_cache[name_arch])
except KeyError:
raise errors.PackageNotFoundError(name)
def get_installed_build_packages(cls, package_names):
build_packages = package_names[:]
installed_packages = []
with apt.Cache() as apt_cache:
while build_packages:
# Before we get to this point, the version specified in the
# yaml must have been installed. So we are ignoring the
# versions of the packages passed as arguments and we just use
# the versions installed.
# --elopio - 20170504
package_name, _ = repo.get_pkg_name_parts(
build_packages.pop(0))
if package_name.endswith(':any'):
package_name = package_name[:-4]
try:
installed_package = apt_cache[package_name].candidate
except KeyError as e:
providers = apt_cache.get_providing_packages(package_name)
if providers:
installed_package = providers[0].candidate
else:
raise errors.BuildPackageNotFoundError(e) from e
if str(installed_package) not in installed_packages:
installed_packages.append(str(installed_package))
for depends in installed_package.get_dependencies(
'Depends'):
# deps is a list of or dependencies. We are taking
# the first one that satisfies the dependency, which
# might or might not be problematic.
# --elopio - 20170504
build_packages.append(depends[0].name)
return installed_packages
def mark_pull_done(self):
pull_properties = self.code.get_pull_properties()
# Add the annotated list of build packages
part_build_packages = self._part_properties.get('build-packages', [])
build_packages = repo.Repo.get_installed_build_packages(
part_build_packages)
versioned_build_packages = []
for pkg in build_packages:
if pkg in part_build_packages:
versioned_build_packages.append(pkg)
else:
pkg_name, version = repo.get_pkg_name_parts(pkg)
if pkg_name in part_build_packages:
versioned_build_packages.append(pkg)
self.mark_done(
'pull',
states.PullState(
pull_properties,
part_properties=self._part_properties,
project=self._project_options,
stage_packages=self.stage_packages,
build_packages=versioned_build_packages,
source_details=self.source_handler.source_details))
def _mark_install(self, apt_cache, package_names):
for name in package_names:
logger.debug('Marking {!r} (and its dependencies) to be '
'fetched'.format(name))
name_arch, version = repo.get_pkg_name_parts(name)
try:
if version:
_set_pkg_version(apt_cache[name_arch], version)
apt_cache[name_arch].mark_install()
except KeyError:
raise errors.PackageNotFoundError(name)
def get_installed_build_packages(cls, package_names):
unique_packages = set(package_names)
pkg_list = []
with apt.Cache() as apt_cache:
for pkg in unique_packages:
try:
pkg_name, version = repo.get_pkg_name_parts(pkg)
pkg_list.append(str(apt_cache[pkg_name].candidate))
except KeyError as e:
raise errors.BuildPackageNotFoundError(e) from e
return pkg_list
def install_build_packages(cls, package_names: List[str]) -> List[str]:
"""Install packages on the host required to build.
:param package_names: a list of package names to install.
:type package_names: a list of strings.
:return: a list with the packages installed and their versions.
:rtype: list of strings.
:raises snapcraft.repo.errors.BuildPackageNotFoundError:
if one of the packages was not found.
:raises snapcraft.repo.errors.PackageBrokenError:
if dependencies for one of the packages cannot be resolved.
:raises snapcraft.repo.errors.BuildPackagesNotInstalledError:
if installing the packages on the host failed.
"""
logger.debug(f"Requested build-packages: {sorted(package_names)!r}")
# Make sure all packages are valid and remove already installed.
install_packages = list()
for name in sorted(package_names):
name, version = repo.get_pkg_name_parts(name)
try:
package = cls._get_resolved_package(name)
except errors.PackageNotFoundError:
raise errors.BuildPackageNotFoundError(name)
if name != package.name:
logger.info(
f"virtual build-package {name!r} resolved to {package.name!r}"
)
# Reconstruct resolved package name, if version used.
if version:
name = f"{package.name}={version}"
else:
name = package.name
if package.installed is None:
install_packages.append(name)
# Install packages, if any.
if install_packages:
cls._install_packages(install_packages)
# Return installed packages with version info.
return [
f"{name}={cls._get_installed_package_version(name)}"
for name in install_packages
]
def _mark_install(cls, apt_cache, package_names):
for name in package_names:
if name.endswith(':any'):
name = name[:-4]
if apt_cache.is_virtual_package(name):
name = apt_cache.get_providing_packages(name)[0].name
logger.debug('Marking {!r} (and its dependencies) to be '
'fetched'.format(name))
name_arch, version = repo.get_pkg_name_parts(name)
try:
if version:
_set_pkg_version(apt_cache[name_arch], version)
apt_cache[name_arch].mark_install()
except KeyError:
raise errors.PackageNotFoundError(name)
def _mark_install(cls, apt_cache, package_names):
for name in package_names:
if name.endswith(':any'):
name = name[:-4]
if apt_cache.is_virtual_package(name):
name = apt_cache.get_providing_packages(name)[0].name
logger.debug('Marking {!r} (and its dependencies) to be '
'fetched'.format(name))
name_arch, version = repo.get_pkg_name_parts(name)
try:
if version:
_set_pkg_version(apt_cache[name_arch], version)
apt_cache[name_arch].mark_install()
except KeyError:
raise errors.PackageNotFoundError(name)
def get_installed_build_packages(cls, package_names):
# It's important to preserve the order of packages to record on the
# state the same value received from the snapcraft.yaml
seen = set()
unique_packages = [
package for package in package_names
if not (package in seen or seen.add(package))
]
pkg_list = []
with apt.Cache() as apt_cache:
for pkg in unique_packages:
try:
pkg_name, version = repo.get_pkg_name_parts(pkg)
pkg_list.append(str(apt_cache[pkg_name].candidate))
except KeyError as e:
raise errors.BuildPackageNotFoundError(e) from e
return pkg_list
def _mark_install(cls, apt_cache: apt.Cache,
package_names: List[str]) -> None:
for name in package_names:
if name.endswith(':any'):
name = name[:-4]
if apt_cache.is_virtual_package(name):
name = apt_cache.get_providing_packages(name)[0].name
logger.debug('Marking {!r} (and its dependencies) to be '
'fetched'.format(name))
name_arch, version = repo.get_pkg_name_parts(name)
try:
if version:
_set_pkg_version(apt_cache[name_arch], version)
# Disable automatic resolving of broken packages here
# because if that fails it raises a SystemError and the
# API doesn't expose enough information about he problem.
# Instead we let apt-get show a verbose error message later.
apt_cache[name_arch].mark_install(auto_fix=False)
cls._verify_marked_install(apt_cache[name_arch])
except KeyError:
raise errors.PackageNotFoundError(name)
def install_build_packages(cls, package_names):
unique_packages = set(package_names)
new_packages = []
with apt.Cache() as apt_cache:
for pkg in unique_packages:
try:
pkg_name, version = repo.get_pkg_name_parts(pkg)
installed_version = apt_cache[pkg_name].installed
if not installed_version:
new_packages.append(pkg)
elif version and installed_version != version:
new_packages.append(pkg)
except KeyError as e:
raise errors.BuildPackageNotFoundError(e) from e
if new_packages:
new_packages.sort()
logger.info('Installing build dependencies: %s',
' '.join(new_packages))
env = os.environ.copy()
env.update({
'DEBIAN_FRONTEND': 'noninteractive',
'DEBCONF_NONINTERACTIVE_SEEN': 'true',
})
apt_command = ['sudo', 'apt-get', '--no-install-recommends', '-y']
if not is_dumb_terminal():
apt_command.extend(['-o', 'Dpkg::Progress-Fancy=1'])
apt_command.append('install')
subprocess.check_call(apt_command + new_packages, env=env)
try:
subprocess.check_call(['sudo', 'apt-mark', 'auto'] +
new_packages,
env=env)
except subprocess.CalledProcessError as e:
logger.warning(
'Impossible to mark packages as auto-installed: {}'.format(
e))
def install_build_packages(cls, package_names):
unique_packages = set(package_names)
new_packages = []
with apt.Cache() as apt_cache:
for pkg in unique_packages:
try:
pkg_name, version = repo.get_pkg_name_parts(pkg)
if pkg_name.endswith(':any'):
pkg_name = pkg_name[:-4]
installed_version = apt_cache[pkg_name].installed
if not installed_version:
new_packages.append(pkg)
elif version and installed_version != version:
new_packages.append(pkg)
except KeyError as e:
providers = apt_cache.get_providing_packages(pkg_name)
if providers:
new_packages.append(providers[0].name)
else:
raise errors.BuildPackageNotFoundError(e) from e
if new_packages:
cls._install_new_build_packages(new_packages)
def test_get_pkg_name_parts_no_arch(self):
name, version = repo.get_pkg_name_parts('hello=2.10-1')
self.assertEqual('hello', name)
self.assertEqual('2.10-1', version)
def test_get_pkg_name_parts_all(self):
name, version = repo.get_pkg_name_parts('hello:i386=2.10-1')
self.assertEqual('hello:i386', name)
self.assertEqual('2.10-1', version)
def test_get_pkg_name_parts_name_only(self):
name, version = repo.get_pkg_name_parts('hello')
self.assertEqual('hello', name)
self.assertEqual(None, version)
def test_get_pkg_name_parts_no_arch(self):
name, version = repo.get_pkg_name_parts("hello=2.10-1")
self.assertThat(name, Equals("hello"))
self.assertThat(version, Equals("2.10-1"))
def test_get_pkg_name_parts_all(self):
name, version = repo.get_pkg_name_parts("hello:i386=2.10-1")
self.assertThat(name, Equals("hello:i386"))
self.assertThat(version, Equals("2.10-1"))
def test_get_pkg_name_parts_name_only(self):
name, version = repo.get_pkg_name_parts("hello")
self.assertThat(name, Equals("hello"))
self.assertThat(version, Equals(None))
def test_get_pkg_name_parts_no_arch(self):
name, version = repo.get_pkg_name_parts("hello=2.10-1")
self.assertThat(name, Equals("hello"))
self.assertThat(version, Equals("2.10-1"))
def test_get_pkg_name_parts_all(self):
name, version = repo.get_pkg_name_parts("hello:i386=2.10-1")
self.assertThat(name, Equals("hello:i386"))
self.assertThat(version, Equals("2.10-1"))
def test_get_pkg_name_parts_name_only(self):
name, version = repo.get_pkg_name_parts("hello")
self.assertThat(name, Equals("hello"))
self.assertThat(version, Equals(None))
def install_stage_packages(
cls, *, package_names: List[str], install_dir: str
) -> List[str]:
marked_packages: Dict[str, apt.package.Version] = dict()
skipped_blacklisted: Set[str] = set()
skipped_essential: Set[str] = set()
logger.debug(f"Requested stage-packages: {sorted(package_names)!r}")
# First scan all packages and set desired version, if specified.
# We do this all at once in case it gets added as a dependency
# along the way.
for name in package_names:
name, specified_version = repo.get_pkg_name_parts(name)
package = cls._get_resolved_package(name)
if name != package.name:
logger.info(
f"virtual stage-package {name!r} resolved to {package.name!r}"
)
if specified_version:
cls._set_package_version(package, specified_version)
for name in package_names:
name, _ = repo.get_pkg_name_parts(name)
package = cls._get_resolved_package(name)
cls._mark_package_dependencies(
package=package,
marked_packages=marked_packages,
skipped_blacklisted=skipped_blacklisted,
skipped_essential=skipped_essential,
unfiltered_packages=package_names,
)
marked = sorted(marked_packages.keys())
logger.debug(f"Installing staged-packages {marked!r} to {install_dir!r}")
if skipped_blacklisted:
blacklisted = sorted(skipped_blacklisted)
logger.debug(f"Skipping blacklisted packages: {blacklisted!r}")
if skipped_essential:
essential = sorted(skipped_essential)
logger.debug(f"Skipping priority essential packages: {essential!r}")
for pkg_name, pkg_version in marked_packages.items():
try:
dl_path = pkg_version.fetch_binary(cls._cache_dir)
except apt.package.FetchError as e:
raise errors.PackageFetchError(str(e))
logger.debug(f"Extracting stage package: {pkg_name}")
with tempfile.TemporaryDirectory() as temp_dir:
# Extract deb package.
cls._extract_deb(dl_path, temp_dir)
# Mark source of files.
marked_name = f"{pkg_name}:{pkg_version.version}"
cls._mark_origin_stage_package(temp_dir, marked_name)
# Stage files to install_dir.
file_utils.link_or_copy_tree(temp_dir, install_dir)
cls.normalize(install_dir)
return [
f"{pkg_name}={pkg_version}"
for pkg_name, pkg_version in marked_packages.items()
]