def run_translate(g: guestfs.GuestFS):
if (g.exists('/etc/redhat-release')
and 'Red Hat' in g.cat('/etc/redhat-release')):
distro = Distro.RHEL
else:
distro = Distro.CENTOS
use_rhel_gce_license = utils.GetMetadataAttribute('use_rhel_gce_license')
el_release = utils.GetMetadataAttribute('el_release')
install_gce = utils.GetMetadataAttribute('install_gce_packages')
spec = TranslateSpec(g=g,
use_rhel_gce_license=use_rhel_gce_license == 'true',
distro=distro,
el_release=el_release,
install_gce=install_gce == 'true')
try:
DistroSpecific(spec)
except BaseException as raised:
logging.debug('Translation failed due to: {}'.format(raised))
for check in checks:
msg = check(spec)
if msg:
raise RuntimeError('{} {}'.format(msg, raised)) from raised
raise raised
def check_inspection(self, support_threading):
if not support_threading:
return False
try:
# Check we can open the Python guestfs module.
from guestfs import GuestFS
g = GuestFS()
# Check for the first version which fixed Python GIL bug.
version = g.version()
if version["major"] == 1: # major must be 1
if version["minor"] == 8:
if version["release"] >= 6: # >= 1.8.6
return True
elif version["minor"] == 10:
if version["release"] >= 1: # >= 1.10.1
return True
elif version["minor"] == 11:
if version["release"] >= 2: # >= 1.11.2
return True
elif version["minor"] >= 12: # >= 1.12, 1.13, etc.
return True
except:
pass
return False
def check_inspection(self, support_threading):
if not support_threading:
return False
try:
# Check we can open the Python guestfs module.
from guestfs import GuestFS
g = GuestFS()
# Check for the first version which fixed Python GIL bug.
version = g.version()
if version["major"] == 1: # major must be 1
if version["minor"] == 8:
if version["release"] >= 6: # >= 1.8.6
return True
elif version["minor"] == 10:
if version["release"] >= 1: # >= 1.10.1
return True
elif version["minor"] == 11:
if version["release"] >= 2: # >= 1.11.2
return True
elif version["minor"] >= 12: # >= 1.12, 1.13, etc.
return True
except:
pass
return False
def _reset_network(g: guestfs.GuestFS):
"""Update network to use DHCP."""
logging.info('Updating network to use DHCP.')
if g.exists('/etc/resolv.conf'):
g.sh('echo "" > /etc/resolv.conf')
g.write(
'/etc/sysconfig/network/ifcfg-eth0', '\n'.join(
('BOOTPROTO=dhcp', 'STARTMODE=auto', 'DHCLIENT_SET_HOSTNAME=yes')))
def _install_packages(g: guestfs.GuestFS, pkgs: typing.List[str]):
if not pkgs:
return
try:
g.sh('zypper --debug --non-interactive install --no-recommends '
+ ' '.join(pkgs))
except Exception as e:
raise ValueError('Failed to install {}: {}'.format(pkgs, e))
def _unmount_guest(g: guestfs.GuestFS, chroot: Path, guest_runtime_dir: Path):
mounts = [['umount', '-l', 'proc/'], ['umount', '-l', 'sys/'],
['umount', '-l', 'dev/'],
['umount', '-l', guest_runtime_dir.as_posix()]]
for m in mounts:
print('unmounting %s', m)
subprocess.run(m, check=True, cwd=chroot.as_posix())
g.umount_local()
def _update_grub(g: guestfs.GuestFS):
"""Update and rebuild grub to ensure the image is bootable on GCP.
See https://cloud.google.com/compute/docs/import/import-existing-image
"""
g.write('/etc/default/grub', configs.update_grub_conf(
g.cat('/etc/default/grub'),
GRUB_CMDLINE_LINUX_DEFAULT='console=ttyS0,38400n8',
GRUB_CMDLINE_LINUX='',
))
g.command(['grub2-mkconfig', '-o', '/boot/grub2/grub.cfg'])
def _install_product(g: guestfs.GuestFS, release: _SuseRelease):
"""Executes SuseConnect -p for each product on `release`.
Raises:
ValueError: If there was a failure adding the subscription.
"""
if release.cloud_product:
try:
g.command(['SUSEConnect', '--debug', '-p', release.cloud_product])
except Exception as e:
raise _disambiguate_suseconnect_product_error(
g, release.cloud_product, e)
def install_cloud_sdk(g: guestfs.GuestFS, ubuntu_release: str) -> None:
""" Installs Google Cloud SDK, supporting apt and snap.
Args:
g: A mounted GuestFS instance.
ubuntu_release: The release nickname (eg: trusty).
"""
try:
run(g, 'gcloud --version')
logging.info(
'Found gcloud. Skipping installation of Google Cloud SDK.')
return
except RuntimeError:
logging.info('Did not find previous install of gcloud.')
if g.gcp_image_major < '18':
g.write('/etc/apt/sources.list.d/partner.list',
apt_cloud_sdk.format(ubuntu_release=ubuntu_release))
utils.update_apt(g)
utils.install_apt_packages(g, 'google-cloud-sdk')
logging.info('Installed Google Cloud SDK with apt.')
return
# Starting at 18.04, Canonical installs the sdk using snap.
# Running `snap install` directly is not an option here since it
# requires the snapd daemon to be running on the guest.
g.write('/etc/cloud/cloud.cfg.d/91-google-cloud-sdk.cfg',
cloud_init_cloud_sdk)
logging.info(
'Google Cloud SDK will be installed using snap with cloud-init.')
# Include /snap/bin in the PATH for startup and shutdown scripts.
# This was present in the old guest agent, but lost in the new guest
# agent.
for p in [
'/lib/systemd/system/google-shutdown-scripts.service',
'/lib/systemd/system/google-startup-scripts.service'
]:
logging.debug('[%s] Checking whether /bin/snap is on PATH.', p)
if not g.exists(p):
logging.debug('[%s] Skipping: Unit not found.', p)
continue
original_unit = g.cat(p)
# Check whether the PATH is already set; if so, skip patching to avoid
# overwriting existing directive.
match = re.search('Environment=[\'"]?PATH.*',
original_unit,
flags=re.IGNORECASE)
if match:
logging.debug(
'[%s] Skipping: PATH already defined in unit file: %s.', p,
match.group())
continue
# Add Environment directive to unit file, and show diff in debug log.
patched_unit = original_unit.replace('[Service]', snap_env_directive)
g.write(p, patched_unit)
diff = '\n'.join(Differ().compare(original_unit.splitlines(),
patched_unit.splitlines()))
logging.debug('[%s] PATH not defined. Added:\n%s', p, diff)
def cleanup(g: guestfs.GuestFS):
"""Shutdown and close the guestfs.GuestFS handle, retrying on failure."""
success = False
for i in range(6):
try:
logging.debug('g.shutdown(): {}'.format(g.shutdown()))
logging.debug('g.close(): {}'.format(g.close()))
success = True
break
except BaseException as raised:
logging.debug('cleanup failed due to: {}'.format(raised))
logging.debug('try again in 10 seconds')
time.sleep(10)
if not success:
logging.debug('Shutdown failed. Continuing anyway.')
def check_inspection(self):
try:
# Check we can open the Python guestfs module.
from guestfs import GuestFS # pylint: disable=import-error
g = GuestFS(close_on_exit=False)
return bool(getattr(g, "add_libvirt_dom", None))
except Exception:
return False
def _stage_etc_hosts(g: guestfs.GuestFS, dst: Path):
"""Stage copy of guest's /etc/hosts at dst with metadata server added."""
original = ''
if g.exists('/etc/hosts'):
original = g.cat('/etc/hosts')
metadata_host_line = None
with open('/etc/hosts') as worker_etc_hosts:
for line in worker_etc_hosts:
if 'metadata.google.internal' in line:
metadata_host_line = line
break
if not metadata_host_line:
raise AssertionError(
'Did not find metadata host in worker\'s /etc/hosts')
dst.write_text('{}\n{}\n'.format(original, metadata_host_line))
def check_inspection(self):
try:
# Check we can open the Python guestfs module.
from guestfs import GuestFS # pylint: disable=import-error
GuestFS(close_on_exit=False)
return True
except:
return False
def setup_cloud_init(g: guestfs.GuestFS):
""" Install cloud-init if not present, and configure to the cloud provider.
Args:
g: A mounted GuestFS instance.
"""
a = Apt(run)
curr_version = a.get_package_version(g, 'cloud-init')
available_versions = a.list_available_versions(g, 'cloud-init')
# Try to avoid installing 21.3-1, which conflicts which the guest agent.
# On first boot, systemd reaches a deadlock deadlock and doest start
# its unit. If a version other than 21.3-1 isn't explicitly found, *and*
# cloud-init isn't currently installed, then this allows apt to pick the
# version to install.
version_to_install = Apt.determine_version_to_install(
curr_version, available_versions, {'21.3-1'})
pkg_to_install = ''
if version_to_install:
pkg_to_install = 'cloud-init=' + version_to_install
elif curr_version == '':
pkg_to_install = 'cloud-init'
# If this block doesn't execute, it means that cloud-init was found
# on the system, but there wasn't an upgrade candidate. Therefore
# leave the version that's currently installed.
if pkg_to_install:
logging.info(pkg_to_install)
utils.install_apt_packages(g, pkg_to_install)
# Ubuntu 14.04's version of cloud-init doesn't have `clean`.
if g.gcp_image_major > '14':
run(g, 'cloud-init clean')
# Remove cloud-init configs that may conflict with GCE's.
#
# - subiquity disables automatic network configuration
# https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/1871975
for cfg in [
'azure', 'curtin', 'waagent', 'walinuxagent', 'aws', 'amazon',
'subiquity'
]:
run(g, 'rm -f /etc/cloud/cloud.cfg.d/*%s*' % cfg)
g.write('/etc/cloud/cloud.cfg.d/91-gce-system.cfg', cloud_init_config)
def install_cloud_sdk(g: guestfs.GuestFS, ubuntu_release: str) -> None:
""" Installs Google Cloud SDK, supporting apt and snap.
Args:
g: A mounted GuestFS instance.
ubuntu_release: The release nickname (eg: trusty).
"""
try:
g.sh('gcloud --version')
logging.info(
'Found gcloud. Skipping installation of Google Cloud SDK.')
return
except RuntimeError:
logging.info('Did not find previous install of gcloud.')
if g.gcp_image_major >= '18':
# Starting at 18.04, the ubuntu-os-cloud images install the sdk
# using snap. Running `snap install` directly is not an option
# here since it requires the snapd daemon to be running on the guest.
g.write('/etc/cloud/cloud.cfg.d/91-google-cloud-sdk.cfg',
cloud_init_cloud_sdk)
logging.info(
'Google Cloud SDK will be installed using snap with cloud-init.')
else:
g.write('/etc/apt/sources.list.d/partner.list',
apt_cloud_sdk.format(ubuntu_release=ubuntu_release))
utils.update_apt(g)
utils.install_apt_packages(g, 'google-cloud-sdk')
logging.info('Installed Google Cloud SDK with apt.')
def _remove_existing_subscriptions(g: guestfs.GuestFS):
"""Remove existing subscriptions from guest.
Implements steps from https://www.suse.com/support/kb/doc/?id=000019085
"""
for cmd in [
'rm -f /etc/SUSEConnect',
'rm -f /etc/zypp/{repos,services,credentials}.d/*'
]:
g.sh(cmd)
updated = []
skip = False
for line in g.cat('/etc/hosts').splitlines():
if skip:
skip = False
elif '# Added by SMT reg' in line:
log.info('Removing previous SMT host from /etc/hosts')
skip = True
else:
updated.append(line)
g.write('/etc/hosts', '\n'.join(updated))
def _write_staged_to_guest(g: guestfs.GuestFS, staged_runtime_dir: Path,
staging_fs_root: Path):
"""Write the staged files to the guest.
Returns:
Path to mounted guest, path to runtime directory inside guest
"""
chroot = Path(tempfile.mkdtemp(prefix='guest_chroot'))
runtime_dir = Path(chroot, g.mkdtemp('/tmp/gceXXXXXX')[1:])
log.info('writing network files')
g.copy_in((staging_fs_root / 'etc/hosts').as_posix(), '/etc')
g.copy_in((staging_fs_root / 'etc/resolv.conf').as_posix(), '/etc')
log.info('writing network files...done')
log.info('setting up chroot')
g.mount_local(chroot.as_posix())
thread = threading.Thread(target=g.mount_local_run, daemon=True)
thread.start()
log.info('setting up chroot...done')
log.info('setting up mounts')
host_runtime = staged_runtime_dir.as_posix()
chroot_runtime = runtime_dir.relative_to(chroot).as_posix()
mounts = [['mount', '-t', 'proc', '/proc', 'proc/'],
['mount', '--rbind', '/sys', 'sys/'],
['mount', '--rbind', '/dev', 'dev/'],
['mount', '--rbind', host_runtime, chroot_runtime]]
for cmd in mounts:
print('mounting {}'.format(cmd))
subprocess.run(cmd, check=True, cwd=chroot.as_posix())
log.info('setting up mounts...done')
_validate_directory_structure(fs_root=chroot,
runtime_dir=runtime_dir,
check_os_mounts=True)
return chroot, runtime_dir
class FileSystem:
"""Convenience wrapper over GuestFS instance.
Simplifies some common routines.
Automatically translates paths according to the contained File System.
"""
def __init__(self, disk_path):
self._root = None
self._handler = GuestFS()
self._disk_path = disk_path
def __enter__(self):
self.mount()
return self
def __exit__(self, *_):
self.umount()
def __getattr__(self, attr):
return getattr(self._handler, attr)
@property
def osname(self):
"""Returns the Operating System name."""
return self._handler.inspect_get_type(self._root)
@property
def fsroot(self):
"""Returns the file system root."""
if self.osname == "windows":
return "{}:\\".format(self._handler.inspect_get_drive_mappings(self._root)[0][0])
else:
return self._handler.inspect_get_mountpoints(self._root)[0][0]
def mount(self, readonly=True):
"""Mounts the given disk.
It must be called before any other method.
"""
self._handler.add_drive_opts(self._disk_path, readonly=True)
self._handler.launch()
for mountpoint, device in self._inspect_disk():
if readonly:
self._handler.mount_ro(device, mountpoint)
else:
self._handler.mount(device, mountpoint)
if self._handler.inspect_get_type(self._root) == "windows":
self.path = self._windows_path
else:
self.path = posix_path
def _inspect_disk(self):
"""Inspects the disk and returns the mountpoints mapping
as a list which order is the supposed one for correct mounting.
"""
roots = self._handler.inspect_os()
if roots:
self._root = roots[0]
return sorted(self._handler.inspect_get_mountpoints(self._root), key=lambda m: len(m[0]))
else:
raise RuntimeError("No OS found on the given disk image.")
def umount(self):
"""Unmounts the disk.
After this method is called no further action is allowed.
"""
self._handler.close()
def download(self, source, destination):
"""Downloads the file on the disk at source into destination."""
self._handler.download(posix_path(source), destination)
def ls(self, path):
"""Lists the content at the given path."""
return self._handler.ls(posix_path(path))
def nodes(self, path):
"""Iterates over the files and directories contained within the disk
starting from the given path.
Yields the path of the nodes.
"""
path = posix_path(path)
yield from (self.path(path, e) for e in self._handler.find(path))
def checksum(self, path, hashtype="sha1"):
"""Returns the checksum of the given path."""
return self._handler.checksum(hashtype, posix_path(path))
def checksums(self, path, hashtype="sha1"):
"""Iterates over the files hashes contained within the disk
starting from the given path.
The hashtype keyword allows to choose the file hashing algorithm.
Yields the following values:
"C:\\Windows\\System32\\NTUSER.DAT", "hash" for windows
"/home/user/text.txt", "hash" for other FS
"""
with NamedTemporaryFile(buffering=0) as tempfile:
self._handler.checksums_out(hashtype, posix_path(path), tempfile.name)
yield from (
(self.path(f[1].lstrip(".")), f[0]) for f in (l.decode("utf8").strip().split(None, 1) for l in tempfile)
)
def stat(self, path):
"""Retrieves the status of the node at the given path.
Returns a dictionary.
"""
return self._handler.stat(posix_path(path))
def file(self, path):
"""Analogous to Unix file command.
Returns the type of node at the given path.
"""
return self._handler.file(posix_path(path))
def exists(self, path):
"""Returns whether the path exists."""
return self._handler.exists(posix_path(path))
def path(self, *segments):
"""Normalizes the path returned by guestfs in the File System format."""
raise NotImplementedError("FileSystem needs to be mounted first")
def _windows_path(self, *segments):
drive = self._handler.inspect_get_drive_mappings(self._root)[0][0]
return "%s:%s" % (drive, os.path.join(*segments).replace("/", "\\"))
def DistroSpecific(g: guestfs.GuestFS):
el_release = utils.GetMetadataAttribute('el_release')
install_gce = utils.GetMetadataAttribute('install_gce_packages')
rhel_license = utils.GetMetadataAttribute('use_rhel_gce_license')
# This must be performed prior to making network calls from the guest.
# Otherwise, if /etc/resolv.conf is present, and has an immutable attribute,
# guestfs will fail with:
#
# rename: /sysroot/etc/resolv.conf to
# /sysroot/etc/i9r7obu6: Operation not permitted
utils.common.ClearEtcResolv(g)
# Some imported images haven't contained `/etc/yum.repos.d`.
if not g.exists('/etc/yum.repos.d'):
g.mkdir('/etc/yum.repos.d')
if rhel_license == 'true':
if 'Red Hat' in g.cat('/etc/redhat-release'):
g.command(['yum', 'remove', '-y', '*rhui*'])
logging.info('Adding in GCE RHUI package.')
g.write('/etc/yum.repos.d/google-cloud.repo',
repo_compute % el_release)
yum_install(g, 'google-rhui-client-rhel' + el_release)
if install_gce == 'true':
logging.info('Installing GCE packages.')
g.write('/etc/yum.repos.d/google-cloud.repo',
repo_compute % el_release)
if el_release == '6':
if 'CentOS' in g.cat('/etc/redhat-release'):
logging.info('Installing CentOS SCL.')
g.command(['rm', '-f', '/etc/yum.repos.d/CentOS-SCL.repo'])
yum_install(g, 'centos-release-scl')
# Install Google Cloud SDK from the upstream tar and create links for the
# python27 SCL environment.
logging.info('Installing python27 from SCL.')
yum_install(g, 'python27')
logging.info('Installing Google Cloud SDK from tar.')
sdk_base_url = 'https://dl.google.com/dl/cloudsdk/channels/rapid'
sdk_base_tar = '%s/google-cloud-sdk.tar.gz' % sdk_base_url
tar = utils.HttpGet(sdk_base_tar)
g.write('/tmp/google-cloud-sdk.tar.gz', tar)
g.command(
['tar', 'xzf', '/tmp/google-cloud-sdk.tar.gz', '-C', '/tmp'])
sdk_version = g.cat('/tmp/google-cloud-sdk/VERSION').strip()
logging.info('Getting Cloud SDK Version %s', sdk_version)
sdk_version_tar = 'google-cloud-sdk-%s-linux-x86_64.tar.gz' % sdk_version
sdk_version_tar_url = '%s/downloads/%s' % (sdk_base_url,
sdk_version_tar)
logging.info('Getting versioned Cloud SDK tar file from %s',
sdk_version_tar_url)
tar = utils.HttpGet(sdk_version_tar_url)
sdk_version_tar_file = os.path.join('/tmp', sdk_version_tar)
g.write(sdk_version_tar_file, tar)
g.mkdir_p('/usr/local/share/google')
g.command([
'tar', 'xzf', sdk_version_tar_file, '-C',
'/usr/local/share/google', '--no-same-owner'
])
logging.info('Creating CloudSDK SCL symlinks.')
sdk_bin_path = '/usr/local/share/google/google-cloud-sdk/bin'
g.ln_s(os.path.join(sdk_bin_path, 'git-credential-gcloud.sh'),
os.path.join('/usr/bin', 'git-credential-gcloud.sh'))
for binary in ['bq', 'gcloud', 'gsutil']:
binary_path = os.path.join(sdk_bin_path, binary)
new_bin_path = os.path.join('/usr/bin', binary)
bin_str = '#!/bin/bash\nsource /opt/rh/python27/enable\n%s $@' % \
binary_path
g.write(new_bin_path, bin_str)
g.chmod(0o755, new_bin_path)
else:
g.write_append('/etc/yum.repos.d/google-cloud.repo',
repo_sdk % el_release)
yum_install(g, 'google-cloud-sdk')
yum_install(g, 'google-compute-engine', 'google-osconfig-agent')
logging.info('Updating initramfs')
for kver in g.ls('/lib/modules'):
# Although each directory in /lib/modules typically corresponds to a
# kernel version [1], that may not always be true.
# kernel-abi-whitelists, for example, creates extra directories in
# /lib/modules.
#
# Skip building initramfs if the directory doesn't look like a
# kernel version. Emulates the version matching from depmod [2].
#
# 1. https://tldp.org/LDP/Linux-Filesystem-Hierarchy/html/lib.html
# 2. https://kernel.googlesource.com/pub/scm/linux/kernel/git/mmarek/kmod
# /+/tip/tools/depmod.c#2537
if not re.match(r'^\d+.\d+', kver):
logging.debug(
'/lib/modules/{} doesn\'t look like a kernel directory. '
'Skipping creation of initramfs for it'.format(kver))
continue
if not g.exists(os.path.join('/lib/modules', kver, 'modules.dep')):
try:
g.command(['depmod', kver])
except RuntimeError as e:
logging.info(
'Failed to write initramfs for {kver}. If image fails to '
'boot, verify that depmod /lib/modules/{kver} runs on '
'the original machine'.format(kver=kver))
logging.debug('depmod error: {}'.format(e))
continue
if el_release == '6':
# Version 6 doesn't have option --kver
g.command(['dracut', '-v', '-f', kver])
else:
g.command(['dracut', '--stdlog=1', '-f', '--kver', kver])
logging.info('Update grub configuration')
if el_release == '6':
# Version 6 doesn't have grub2, file grub.conf needs to be updated by hand
g.write('/tmp/grub_gce_generated', grub_cfg)
g.sh(r'grep -P "^[\t ]*initrd|^[\t ]*root|^[\t ]*kernel|^[\t ]*title" '
r'/boot/grub/grub.conf >> /tmp/grub_gce_generated;'
r'sed -i "s/console=ttyS0[^ ]*//g" /tmp/grub_gce_generated;'
r'sed -i "/^[\t ]*kernel/s/$/ console=ttyS0,38400n8/" '
r'/tmp/grub_gce_generated;'
r'mv /tmp/grub_gce_generated /boot/grub/grub.conf')
else:
g.write('/etc/default/grub', grub2_cfg)
g.command(['grub2-mkconfig', '-o', '/boot/grub2/grub.cfg'])
# Reset network for DHCP.
logging.info('Resetting network to DHCP for eth0.')
# Remove NetworkManager-config-server if it's present. The package configures
# NetworkManager to *not* use DHCP.
# https://access.redhat.com/solutions/894763
g.command(['yum', 'remove', '-y', 'NetworkManager-config-server'])
g.write('/etc/sysconfig/network-scripts/ifcfg-eth0', ifcfg_eth0)
def _process(self, conn, vm, vmuuid):
g = GuestFS()
prettyvm = conn.get_uri() + ":" + vm.get_name()
ignore = vmuuid
disks = []
for disk in vm.get_disk_devices():
if (disk.path and
(disk.type == "block" or disk.type == "file") and
not disk.device == "cdrom"):
disks.append(disk)
if not disks:
logging.debug("%s: nothing to inspect", prettyvm)
return
# Add the disks. Note they *must* be added with readonly flag set.
for disk in disks:
path = disk.path
driver_type = disk.driver_type
if not (os.path.exists(path) and os.access(path, os.R_OK)):
logging.debug("%s: cannot access '%s', skipping inspection",
prettyvm, path)
return
g.add_drive_opts(path, readonly=1, format=driver_type)
g.launch()
# Inspect the operating system.
roots = g.inspect_os()
if len(roots) == 0:
logging.debug("%s: no operating systems found", prettyvm)
return
# Arbitrarily pick the first root device.
root = roots[0]
# Inspection results.
typ = g.inspect_get_type(root) # eg. "linux"
distro = g.inspect_get_distro(root) # eg. "fedora"
major_version = g.inspect_get_major_version(root) # eg. 14
minor_version = g.inspect_get_minor_version(root) # eg. 0
hostname = g.inspect_get_hostname(root) # string
product_name = g.inspect_get_product_name(root) # string
# Added in libguestfs 1.9.13:
product_variant = None
if hasattr(g, "inspect_get_product_variant"):
product_variant = g.inspect_get_product_variant(root) # string
# For inspect_list_applications and inspect_get_icon we
# require that the guest filesystems are mounted. However
# don't fail if this is not possible (I'm looking at you,
# FreeBSD).
filesystems_mounted = False
try:
# Mount up the disks, like guestfish --ro -i.
# Sort keys by length, shortest first, so that we end up
# mounting the filesystems in the correct order.
mps = g.inspect_get_mountpoints(root)
def compare(a, b):
if len(a[0]) > len(b[0]):
return 1
elif len(a[0]) == len(b[0]):
return 0
else:
return -1
mps.sort(compare)
for mp_dev in mps:
try:
g.mount_ro(mp_dev[1], mp_dev[0])
except:
logging.exception("%s: exception mounting %s on %s "
"(ignored)",
prettyvm, mp_dev[1], mp_dev[0])
filesystems_mounted = True
except:
logging.exception("%s: exception while mounting disks (ignored)",
prettyvm)
icon = None
apps = None
if filesystems_mounted:
# Added in libguestfs 1.11.12:
if hasattr(g, "inspect_get_icon"):
# string containing PNG data
icon = g.inspect_get_icon(root, favicon=0, highquality=1)
if icon == "":
icon = None
# Inspection applications.
apps = g.inspect_list_applications(root)
# Force the libguestfs handle to close right now.
del g
# Log what we found.
logging.debug("%s: detected operating system: %s %s %d.%d (%s)",
prettyvm, typ, distro, major_version, minor_version,
product_name)
logging.debug("hostname: %s", hostname)
if icon:
logging.debug("icon: %d bytes", len(icon))
if apps:
logging.debug("# apps: %d", len(apps))
data = vmmInspectionData()
data.type = str(type)
data.distro = str(distro)
data.major_version = int(major_version)
data.minor_version = int(minor_version)
data.hostname = str(hostname)
data.product_name = str(product_name)
data.product_variant = str(product_variant)
data.icon = str(icon)
data.applications = list(apps)
self._set_vm_inspection_data(vm, data)
def _install_virtio_drivers(g: guestfs.GuestFS):
"""Rebuilds initramfs to ensure that virtio drivers are present."""
logging.info('Installing virtio drivers.')
for kernel in g.ls('/lib/modules'):
g.command(['dracut', '-v', '-f', '--kver', kernel])
def _refresh_zypper(g: guestfs.GuestFS):
try:
g.command(['zypper', '--debug', 'refresh'])
except Exception as e:
raise ValueError('Failed to call zypper refresh', e)
class FileSystem:
"""Convenience wrapper over GuestFS instance.
Simplifies some common routines.
Automatically translates paths according to the contained File System.
"""
def __init__(self, disk_path):
self._root = None
self._handler = GuestFS()
self.disk_path = disk_path
def __enter__(self):
self.mount()
return self
def __exit__(self, *_):
self.umount()
def __getattr__(self, attr):
return getattr(self._handler, attr)
@property
def osname(self):
"""Returns the Operating System name."""
return self._handler.inspect_get_type(self._root)
@property
def fsroot(self):
"""Returns the file system root."""
if self.osname == 'windows':
return '{}:\\'.format(
self._handler.inspect_get_drive_mappings(self._root)[0][0])
else:
return self._handler.inspect_get_mountpoints(self._root)[0][0]
def mount(self, readonly=True):
"""Mounts the given disk.
It must be called before any other method.
"""
self._handler.add_drive_opts(self.disk_path, readonly=True)
self._handler.launch()
for mountpoint, device in self._inspect_disk():
if readonly:
self._handler.mount_ro(device, mountpoint)
else:
self._handler.mount(device, mountpoint)
if self._handler.inspect_get_type(self._root) == 'windows':
self.path = self._windows_path
else:
self.path = posix_path
def _inspect_disk(self):
"""Inspects the disk and returns the mountpoints mapping
as a list which order is the supposed one for correct mounting.
"""
roots = self._handler.inspect_os()
if roots:
self._root = roots[0]
return sorted(self._handler.inspect_get_mountpoints(self._root),
key=lambda m: len(m[0]))
else:
raise RuntimeError("No OS found on the given disk image.")
def umount(self):
"""Unmounts the disk.
After this method is called no further action is allowed.
"""
self._handler.close()
def download(self, source, destination):
"""Downloads the file on the disk at source into destination."""
self._handler.download(posix_path(source), destination)
def ls(self, path):
"""Lists the content at the given path."""
return self._handler.ls(posix_path(path))
def nodes(self, path):
"""Iterates over the files and directories contained within the disk
starting from the given path.
Yields the path of the nodes.
"""
path = posix_path(path)
yield from (self.path(path, e) for e in self._handler.find(path))
def checksum(self, path, hashtype='sha1'):
"""Returns the checksum of the given path."""
return self._handler.checksum(hashtype, posix_path(path))
def checksums(self, path, hashtype='sha1'):
"""Iterates over the files hashes contained within the disk
starting from the given path.
The hashtype keyword allows to choose the file hashing algorithm.
Yields the following values:
"C:\\Windows\\System32\\NTUSER.DAT", "hash" for windows
"/home/user/text.txt", "hash" for other FS
"""
with NamedTemporaryFile(buffering=0) as tempfile:
self._handler.checksums_out(hashtype, posix_path(path),
tempfile.name)
yield from ((self.path(f[1].lstrip('.')), f[0])
for f in (l.decode('utf8').strip().split(None, 1)
for l in tempfile))
def stat(self, path):
"""Retrieves the status of the node at the given path.
Returns a dictionary.
"""
return self._handler.stat(posix_path(path))
def file(self, path):
"""Analogous to Unix file command.
Returns the type of node at the given path.
"""
return self._handler.file(posix_path(path))
def exists(self, path):
"""Returns whether the path exists."""
return self._handler.exists(posix_path(path))
def path(self, *segments):
"""Normalizes the path returned by guestfs in the File System format."""
raise NotImplementedError("FileSystem needs to be mounted first")
def _windows_path(self, *segments):
drive = self._handler.inspect_get_drive_mappings(self._root)[0][0]
return "%s:%s" % (drive, os.path.join(*segments).replace('/', '\\'))
def __init__(self, disk_path):
self._root = None
self._handler = GuestFS()
self.disk_path = disk_path
def _disambiguate_suseconnect_product_error(
g: guestfs.GuestFS, product: str, error: Exception) -> Exception:
"""Creates a user-debuggable error after failing to add a product
using SUSEConnect.
Args:
g: Mounted GuestFS instance.
product: The product that failed to be added.
error: The error returned from `SUSEConnect -p`.
"""
statuses = []
try:
statuses = json.loads(g.command(['SUSEConnect', '--status']))
except Exception as e:
return ValueError(
'Unable to communicate with SCC. Ensure the import '
'is running in a network that allows internet access.', e)
# `SUSEConnect --status` returns a list of status objects,
# where the triple of (identifier, version, arch) uniquely
# identifies a product in SCC. Below are two examples.
#
# Example 1: SLES for SAP 12.2, No subscription
# [
# {
# "identifier":"SLES_SAP",
# "version":"12.2",
# "arch":"x86_64",
# "status":"Not Registered"
# }
# ]
#
# Example 2: SLES 15.1, Active
# [
# {
# "status":"Registered",
# "version":"15.1",
# "arch":"x86_64",
# "identifier":"SLES",
# "subscription_status":"ACTIVE"
# },
# {
# "status":"Registered",
# "version":"15.1",
# "arch":"x86_64",
# "identifier":"sle-module-basesystem"
# }
# ]
for status in statuses:
if status.get('identifier') not in ('SLES', 'SLES_SAP'):
continue
if status.get('subscription_status') == 'ACTIVE':
return ValueError(
'Unable to add product "%s" using SUSEConnect. Please ensure that '
'your subscription includes access to this product.' % product,
error)
return ValueError(
'Unable to find an active SLES subscription. '
'SCC returned: %s' % statuses)
def DistroSpecific(g: guestfs.GuestFS):
el_release = utils.GetMetadataAttribute('el_release')
install_gce = utils.GetMetadataAttribute('install_gce_packages')
rhel_license = utils.GetMetadataAttribute('use_rhel_gce_license')
# This must be performed prior to making network calls from the guest.
# Otherwise, if /etc/resolv.conf is present, and has an immutable attribute,
# guestfs will fail with:
#
# rename: /sysroot/etc/resolv.conf to
# /sysroot/etc/i9r7obu6: Operation not permitted
utils.common.ClearEtcResolv(g)
# Some imported images haven't contained `/etc/yum.repos.d`.
if not g.exists('/etc/yum.repos.d'):
g.mkdir('/etc/yum.repos.d')
if rhel_license == 'true':
if 'Red Hat' in g.cat('/etc/redhat-release'):
g.command(['yum', 'remove', '-y', '*rhui*'])
logging.info('Adding in GCE RHUI package.')
g.write('/etc/yum.repos.d/google-cloud.repo', repo_compute % el_release)
yum_install(g, 'google-rhui-client-rhel' + el_release)
if install_gce == 'true':
logging.info('Installing GCE packages.')
g.write('/etc/yum.repos.d/google-cloud.repo', repo_compute % el_release)
if el_release == '6':
if 'CentOS' in g.cat('/etc/redhat-release'):
logging.info('Installing CentOS SCL.')
g.command(['rm', '-f', '/etc/yum.repos.d/CentOS-SCL.repo'])
yum_install(g, 'centos-release-scl')
# Install Google Cloud SDK from the upstream tar and create links for the
# python27 SCL environment.
logging.info('Installing python27 from SCL.')
yum_install(g, 'python27')
logging.info('Installing Google Cloud SDK from tar.')
sdk_base_url = 'https://dl.google.com/dl/cloudsdk/channels/rapid'
sdk_base_tar = '%s/google-cloud-sdk.tar.gz' % sdk_base_url
tar = utils.HttpGet(sdk_base_tar)
g.write('/tmp/google-cloud-sdk.tar.gz', tar)
g.command(['tar', 'xzf', '/tmp/google-cloud-sdk.tar.gz', '-C', '/tmp'])
sdk_version = g.cat('/tmp/google-cloud-sdk/VERSION').strip()
logging.info('Getting Cloud SDK Version %s', sdk_version)
sdk_version_tar = 'google-cloud-sdk-%s-linux-x86_64.tar.gz' % sdk_version
sdk_version_tar_url = '%s/downloads/%s' % (sdk_base_url, sdk_version_tar)
logging.info('Getting versioned Cloud SDK tar file from %s',
sdk_version_tar_url)
tar = utils.HttpGet(sdk_version_tar_url)
sdk_version_tar_file = os.path.join('/tmp', sdk_version_tar)
g.write(sdk_version_tar_file, tar)
g.mkdir_p('/usr/local/share/google')
g.command(['tar', 'xzf', sdk_version_tar_file, '-C',
'/usr/local/share/google', '--no-same-owner'])
logging.info('Creating CloudSDK SCL symlinks.')
sdk_bin_path = '/usr/local/share/google/google-cloud-sdk/bin'
g.ln_s(os.path.join(sdk_bin_path, 'git-credential-gcloud.sh'),
os.path.join('/usr/bin', 'git-credential-gcloud.sh'))
for binary in ['bq', 'gcloud', 'gsutil']:
binary_path = os.path.join(sdk_bin_path, binary)
new_bin_path = os.path.join('/usr/bin', binary)
bin_str = '#!/bin/bash\nsource /opt/rh/python27/enable\n%s $@' % \
binary_path
g.write(new_bin_path, bin_str)
g.chmod(0o755, new_bin_path)
else:
g.write_append(
'/etc/yum.repos.d/google-cloud.repo', repo_sdk % el_release)
yum_install(g, 'google-cloud-sdk')
yum_install(g, 'google-compute-engine', 'google-osconfig-agent')
logging.info('Updating initramfs')
for kver in g.ls('/lib/modules'):
if not g.exists(os.path.join('/lib/modules', kver, 'modules.dep')):
g.command(['depmod', kver])
if el_release == '6':
# Version 6 doesn't have option --kver
g.command(['dracut', '-v', '-f', kver])
else:
g.command(['dracut', '--stdlog=1', '-f', '--kver', kver])
logging.info('Update grub configuration')
if el_release == '6':
# Version 6 doesn't have grub2, file grub.conf needs to be updated by hand
g.write('/tmp/grub_gce_generated', grub_cfg)
g.sh(
r'grep -P "^[\t ]*initrd|^[\t ]*root|^[\t ]*kernel|^[\t ]*title" '
r'/boot/grub/grub.conf >> /tmp/grub_gce_generated;'
r'sed -i "s/console=ttyS0[^ ]*//g" /tmp/grub_gce_generated;'
r'sed -i "/^[\t ]*kernel/s/$/ console=ttyS0,38400n8/" '
r'/tmp/grub_gce_generated;'
r'mv /tmp/grub_gce_generated /boot/grub/grub.conf')
else:
g.write('/etc/default/grub', grub2_cfg)
g.command(['grub2-mkconfig', '-o', '/boot/grub2/grub.cfg'])
# Reset network for DHCP.
logging.info('Resetting network to DHCP for eth0.')
# Remove NetworkManager-config-server if it's present. The package configures
# NetworkManager to *not* use DHCP.
# https://access.redhat.com/solutions/894763
g.command(['yum', 'remove', '-y', 'NetworkManager-config-server'])
g.write('/etc/sysconfig/network-scripts/ifcfg-eth0', ifcfg_eth0)
def __init__(self, disk_path):
self._root = None
self._handler = GuestFS()
self._disk_path = disk_path
def _process(self, conn, vm, vmuuid):
if re.search(r"^guestfs-", vm.get_name()):
logging.debug("ignore libvirt/guestfs temporary VM %s",
vm.get_name())
return
g = GuestFS()
prettyvm = conn.get_uri() + ":" + vm.get_name()
ignore = vmuuid
disks = []
for disk in vm.get_disk_devices():
if (disk.path and (disk.type == "block" or disk.type == "file")
and not disk.device == "cdrom"):
disks.append(disk)
if not disks:
logging.debug("%s: nothing to inspect", prettyvm)
return
# Add the disks. Note they *must* be added with readonly flag set.
for disk in disks:
path = disk.path
driver_type = disk.driver_type
if not (os.path.exists(path) and os.access(path, os.R_OK)):
logging.debug("%s: cannot access '%s', skipping inspection",
prettyvm, path)
return
g.add_drive_opts(path, readonly=1, format=driver_type)
g.launch()
# Inspect the operating system.
roots = g.inspect_os()
if len(roots) == 0:
logging.debug("%s: no operating systems found", prettyvm)
return
# Arbitrarily pick the first root device.
root = roots[0]
# Inspection results.
typ = g.inspect_get_type(root) # eg. "linux"
distro = g.inspect_get_distro(root) # eg. "fedora"
major_version = g.inspect_get_major_version(root) # eg. 14
minor_version = g.inspect_get_minor_version(root) # eg. 0
hostname = g.inspect_get_hostname(root) # string
product_name = g.inspect_get_product_name(root) # string
product_variant = g.inspect_get_product_variant(root) # string
# For inspect_list_applications and inspect_get_icon we
# require that the guest filesystems are mounted. However
# don't fail if this is not possible (I'm looking at you,
# FreeBSD).
filesystems_mounted = False
try:
# Mount up the disks, like guestfish --ro -i.
# Sort keys by length, shortest first, so that we end up
# mounting the filesystems in the correct order.
mps = list(g.inspect_get_mountpoints(root))
def compare(a, b):
if len(a[0]) > len(b[0]):
return 1
elif len(a[0]) == len(b[0]):
return 0
else:
return -1
mps.sort(compare)
for mp_dev in mps:
try:
g.mount_ro(mp_dev[1], mp_dev[0])
except:
logging.exception(
"%s: exception mounting %s on %s "
"(ignored)", prettyvm, mp_dev[1], mp_dev[0])
filesystems_mounted = True
except:
logging.exception("%s: exception while mounting disks (ignored)",
prettyvm)
icon = None
apps = None
if filesystems_mounted:
# string containing PNG data
icon = g.inspect_get_icon(root, favicon=0, highquality=1)
if icon == "":
icon = None
# Inspection applications.
apps = g.inspect_list_applications(root)
# Force the libguestfs handle to close right now.
del g
# Log what we found.
logging.debug("%s: detected operating system: %s %s %d.%d (%s)",
prettyvm, typ, distro, major_version, minor_version,
product_name)
logging.debug("hostname: %s", hostname)
if icon:
logging.debug("icon: %d bytes", len(icon))
if apps:
logging.debug("# apps: %d", len(apps))
data = vmmInspectionData()
data.type = str(type)
data.distro = str(distro)
data.major_version = int(major_version)
data.minor_version = int(minor_version)
data.hostname = str(hostname)
data.product_name = str(product_name)
data.product_variant = str(product_variant)
data.icon = icon
data.applications = list(apps)
self._set_vm_inspection_data(vm, data)
def _process(self, conn, vm):
if re.search(r"^guestfs-", vm.get_name()):
logging.debug("ignore libvirt/guestfs temporary VM %s",
vm.get_name())
return None
g = GuestFS(close_on_exit=False)
prettyvm = conn.get_uri() + ":" + vm.get_name()
g.add_libvirt_dom(vm.get_backend(), readonly=1)
g.launch()
# Inspect the operating system.
roots = g.inspect_os()
if len(roots) == 0:
logging.debug("%s: no operating systems found", prettyvm)
return None
# Arbitrarily pick the first root device.
root = roots[0]
# Inspection results.
typ = g.inspect_get_type(root) # eg. "linux"
distro = g.inspect_get_distro(root) # eg. "fedora"
major_version = g.inspect_get_major_version(root) # eg. 14
minor_version = g.inspect_get_minor_version(root) # eg. 0
hostname = g.inspect_get_hostname(root) # string
product_name = g.inspect_get_product_name(root) # string
product_variant = g.inspect_get_product_variant(root) # string
# For inspect_list_applications and inspect_get_icon we
# require that the guest filesystems are mounted. However
# don't fail if this is not possible (I'm looking at you,
# FreeBSD).
filesystems_mounted = False
try:
# Mount up the disks, like guestfish --ro -i.
# Sort keys by length, shortest first, so that we end up
# mounting the filesystems in the correct order.
mps = list(g.inspect_get_mountpoints(root))
def compare(a, b):
if len(a[0]) > len(b[0]):
return 1
elif len(a[0]) == len(b[0]):
return 0
else:
return -1
mps.sort(compare)
for mp_dev in mps:
try:
g.mount_ro(mp_dev[1], mp_dev[0])
except:
logging.exception("%s: exception mounting %s on %s "
"(ignored)",
prettyvm, mp_dev[1], mp_dev[0])
filesystems_mounted = True
except:
logging.exception("%s: exception while mounting disks (ignored)",
prettyvm)
icon = None
apps = None
if filesystems_mounted:
# string containing PNG data
icon = g.inspect_get_icon(root, favicon=0, highquality=1)
if icon == "":
icon = None
# Inspection applications.
apps = g.inspect_list_applications(root)
# Force the libguestfs handle to close right now.
del g
# Log what we found.
logging.debug("%s: detected operating system: %s %s %d.%d (%s)",
prettyvm, typ, distro, major_version, minor_version,
product_name)
logging.debug("hostname: %s", hostname)
if icon:
logging.debug("icon: %d bytes", len(icon))
if apps:
logging.debug("# apps: %d", len(apps))
data = vmmInspectionData()
data.type = str(type)
data.distro = str(distro)
data.major_version = int(major_version)
data.minor_version = int(minor_version)
data.hostname = str(hostname)
data.product_name = str(product_name)
data.product_variant = str(product_variant)
data.icon = icon
data.applications = list(apps)
data.error = False
return data
