def run_translate(g: guestfs.GuestFS):
if (g.exists('/etc/redhat-release')
and 'Red Hat' in g.cat('/etc/redhat-release')):
distro = Distro.RHEL
else:
distro = Distro.CENTOS
use_rhel_gce_license = utils.GetMetadataAttribute('use_rhel_gce_license')
el_release = utils.GetMetadataAttribute('el_release')
install_gce = utils.GetMetadataAttribute('install_gce_packages')
spec = TranslateSpec(g=g,
use_rhel_gce_license=use_rhel_gce_license == 'true',
distro=distro,
el_release=el_release,
install_gce=install_gce == 'true')
try:
DistroSpecific(spec)
except BaseException as raised:
logging.debug('Translation failed due to: {}'.format(raised))
for check in checks:
msg = check(spec)
if msg:
raise RuntimeError('{} {}'.format(msg, raised)) from raised
raise raised
def install_cloud_sdk(g: guestfs.GuestFS, ubuntu_release: str) -> None:
""" Installs Google Cloud SDK, supporting apt and snap.
Args:
g: A mounted GuestFS instance.
ubuntu_release: The release nickname (eg: trusty).
"""
try:
run(g, 'gcloud --version')
logging.info(
'Found gcloud. Skipping installation of Google Cloud SDK.')
return
except RuntimeError:
logging.info('Did not find previous install of gcloud.')
if g.gcp_image_major < '18':
g.write('/etc/apt/sources.list.d/partner.list',
apt_cloud_sdk.format(ubuntu_release=ubuntu_release))
utils.update_apt(g)
utils.install_apt_packages(g, 'google-cloud-sdk')
logging.info('Installed Google Cloud SDK with apt.')
return
# Starting at 18.04, Canonical installs the sdk using snap.
# Running `snap install` directly is not an option here since it
# requires the snapd daemon to be running on the guest.
g.write('/etc/cloud/cloud.cfg.d/91-google-cloud-sdk.cfg',
cloud_init_cloud_sdk)
logging.info(
'Google Cloud SDK will be installed using snap with cloud-init.')
# Include /snap/bin in the PATH for startup and shutdown scripts.
# This was present in the old guest agent, but lost in the new guest
# agent.
for p in [
'/lib/systemd/system/google-shutdown-scripts.service',
'/lib/systemd/system/google-startup-scripts.service'
]:
logging.debug('[%s] Checking whether /bin/snap is on PATH.', p)
if not g.exists(p):
logging.debug('[%s] Skipping: Unit not found.', p)
continue
original_unit = g.cat(p)
# Check whether the PATH is already set; if so, skip patching to avoid
# overwriting existing directive.
match = re.search('Environment=[\'"]?PATH.*',
original_unit,
flags=re.IGNORECASE)
if match:
logging.debug(
'[%s] Skipping: PATH already defined in unit file: %s.', p,
match.group())
continue
# Add Environment directive to unit file, and show diff in debug log.
patched_unit = original_unit.replace('[Service]', snap_env_directive)
g.write(p, patched_unit)
diff = '\n'.join(Differ().compare(original_unit.splitlines(),
patched_unit.splitlines()))
logging.debug('[%s] PATH not defined. Added:\n%s', p, diff)
def _reset_network(g: guestfs.GuestFS):
"""Update network to use DHCP."""
logging.info('Updating network to use DHCP.')
if g.exists('/etc/resolv.conf'):
g.sh('echo "" > /etc/resolv.conf')
g.write(
'/etc/sysconfig/network/ifcfg-eth0', '\n'.join(
('BOOTPROTO=dhcp', 'STARTMODE=auto', 'DHCLIENT_SET_HOSTNAME=yes')))
def _stage_etc_hosts(g: guestfs.GuestFS, dst: Path):
"""Stage copy of guest's /etc/hosts at dst with metadata server added."""
original = ''
if g.exists('/etc/hosts'):
original = g.cat('/etc/hosts')
metadata_host_line = None
with open('/etc/hosts') as worker_etc_hosts:
for line in worker_etc_hosts:
if 'metadata.google.internal' in line:
metadata_host_line = line
break
if not metadata_host_line:
raise AssertionError(
'Did not find metadata host in worker\'s /etc/hosts')
dst.write_text('{}\n{}\n'.format(original, metadata_host_line))
class FileSystem:
"""Convenience wrapper over GuestFS instance.
Simplifies some common routines.
Automatically translates paths according to the contained File System.
"""
def __init__(self, disk_path):
self._root = None
self._handler = GuestFS()
self._disk_path = disk_path
def __enter__(self):
self.mount()
return self
def __exit__(self, *_):
self.umount()
def __getattr__(self, attr):
return getattr(self._handler, attr)
@property
def osname(self):
"""Returns the Operating System name."""
return self._handler.inspect_get_type(self._root)
@property
def fsroot(self):
"""Returns the file system root."""
if self.osname == "windows":
return "{}:\\".format(self._handler.inspect_get_drive_mappings(self._root)[0][0])
else:
return self._handler.inspect_get_mountpoints(self._root)[0][0]
def mount(self, readonly=True):
"""Mounts the given disk.
It must be called before any other method.
"""
self._handler.add_drive_opts(self._disk_path, readonly=True)
self._handler.launch()
for mountpoint, device in self._inspect_disk():
if readonly:
self._handler.mount_ro(device, mountpoint)
else:
self._handler.mount(device, mountpoint)
if self._handler.inspect_get_type(self._root) == "windows":
self.path = self._windows_path
else:
self.path = posix_path
def _inspect_disk(self):
"""Inspects the disk and returns the mountpoints mapping
as a list which order is the supposed one for correct mounting.
"""
roots = self._handler.inspect_os()
if roots:
self._root = roots[0]
return sorted(self._handler.inspect_get_mountpoints(self._root), key=lambda m: len(m[0]))
else:
raise RuntimeError("No OS found on the given disk image.")
def umount(self):
"""Unmounts the disk.
After this method is called no further action is allowed.
"""
self._handler.close()
def download(self, source, destination):
"""Downloads the file on the disk at source into destination."""
self._handler.download(posix_path(source), destination)
def ls(self, path):
"""Lists the content at the given path."""
return self._handler.ls(posix_path(path))
def nodes(self, path):
"""Iterates over the files and directories contained within the disk
starting from the given path.
Yields the path of the nodes.
"""
path = posix_path(path)
yield from (self.path(path, e) for e in self._handler.find(path))
def checksum(self, path, hashtype="sha1"):
"""Returns the checksum of the given path."""
return self._handler.checksum(hashtype, posix_path(path))
def checksums(self, path, hashtype="sha1"):
"""Iterates over the files hashes contained within the disk
starting from the given path.
The hashtype keyword allows to choose the file hashing algorithm.
Yields the following values:
"C:\\Windows\\System32\\NTUSER.DAT", "hash" for windows
"/home/user/text.txt", "hash" for other FS
"""
with NamedTemporaryFile(buffering=0) as tempfile:
self._handler.checksums_out(hashtype, posix_path(path), tempfile.name)
yield from (
(self.path(f[1].lstrip(".")), f[0]) for f in (l.decode("utf8").strip().split(None, 1) for l in tempfile)
)
def stat(self, path):
"""Retrieves the status of the node at the given path.
Returns a dictionary.
"""
return self._handler.stat(posix_path(path))
def file(self, path):
"""Analogous to Unix file command.
Returns the type of node at the given path.
"""
return self._handler.file(posix_path(path))
def exists(self, path):
"""Returns whether the path exists."""
return self._handler.exists(posix_path(path))
def path(self, *segments):
"""Normalizes the path returned by guestfs in the File System format."""
raise NotImplementedError("FileSystem needs to be mounted first")
def _windows_path(self, *segments):
drive = self._handler.inspect_get_drive_mappings(self._root)[0][0]
return "%s:%s" % (drive, os.path.join(*segments).replace("/", "\\"))
def DistroSpecific(g: guestfs.GuestFS):
el_release = utils.GetMetadataAttribute('el_release')
install_gce = utils.GetMetadataAttribute('install_gce_packages')
rhel_license = utils.GetMetadataAttribute('use_rhel_gce_license')
# This must be performed prior to making network calls from the guest.
# Otherwise, if /etc/resolv.conf is present, and has an immutable attribute,
# guestfs will fail with:
#
# rename: /sysroot/etc/resolv.conf to
# /sysroot/etc/i9r7obu6: Operation not permitted
utils.common.ClearEtcResolv(g)
# Some imported images haven't contained `/etc/yum.repos.d`.
if not g.exists('/etc/yum.repos.d'):
g.mkdir('/etc/yum.repos.d')
if rhel_license == 'true':
if 'Red Hat' in g.cat('/etc/redhat-release'):
g.command(['yum', 'remove', '-y', '*rhui*'])
logging.info('Adding in GCE RHUI package.')
g.write('/etc/yum.repos.d/google-cloud.repo',
repo_compute % el_release)
yum_install(g, 'google-rhui-client-rhel' + el_release)
if install_gce == 'true':
logging.info('Installing GCE packages.')
g.write('/etc/yum.repos.d/google-cloud.repo',
repo_compute % el_release)
if el_release == '6':
if 'CentOS' in g.cat('/etc/redhat-release'):
logging.info('Installing CentOS SCL.')
g.command(['rm', '-f', '/etc/yum.repos.d/CentOS-SCL.repo'])
yum_install(g, 'centos-release-scl')
# Install Google Cloud SDK from the upstream tar and create links for the
# python27 SCL environment.
logging.info('Installing python27 from SCL.')
yum_install(g, 'python27')
logging.info('Installing Google Cloud SDK from tar.')
sdk_base_url = 'https://dl.google.com/dl/cloudsdk/channels/rapid'
sdk_base_tar = '%s/google-cloud-sdk.tar.gz' % sdk_base_url
tar = utils.HttpGet(sdk_base_tar)
g.write('/tmp/google-cloud-sdk.tar.gz', tar)
g.command(
['tar', 'xzf', '/tmp/google-cloud-sdk.tar.gz', '-C', '/tmp'])
sdk_version = g.cat('/tmp/google-cloud-sdk/VERSION').strip()
logging.info('Getting Cloud SDK Version %s', sdk_version)
sdk_version_tar = 'google-cloud-sdk-%s-linux-x86_64.tar.gz' % sdk_version
sdk_version_tar_url = '%s/downloads/%s' % (sdk_base_url,
sdk_version_tar)
logging.info('Getting versioned Cloud SDK tar file from %s',
sdk_version_tar_url)
tar = utils.HttpGet(sdk_version_tar_url)
sdk_version_tar_file = os.path.join('/tmp', sdk_version_tar)
g.write(sdk_version_tar_file, tar)
g.mkdir_p('/usr/local/share/google')
g.command([
'tar', 'xzf', sdk_version_tar_file, '-C',
'/usr/local/share/google', '--no-same-owner'
])
logging.info('Creating CloudSDK SCL symlinks.')
sdk_bin_path = '/usr/local/share/google/google-cloud-sdk/bin'
g.ln_s(os.path.join(sdk_bin_path, 'git-credential-gcloud.sh'),
os.path.join('/usr/bin', 'git-credential-gcloud.sh'))
for binary in ['bq', 'gcloud', 'gsutil']:
binary_path = os.path.join(sdk_bin_path, binary)
new_bin_path = os.path.join('/usr/bin', binary)
bin_str = '#!/bin/bash\nsource /opt/rh/python27/enable\n%s $@' % \
binary_path
g.write(new_bin_path, bin_str)
g.chmod(0o755, new_bin_path)
else:
g.write_append('/etc/yum.repos.d/google-cloud.repo',
repo_sdk % el_release)
yum_install(g, 'google-cloud-sdk')
yum_install(g, 'google-compute-engine', 'google-osconfig-agent')
logging.info('Updating initramfs')
for kver in g.ls('/lib/modules'):
# Although each directory in /lib/modules typically corresponds to a
# kernel version [1], that may not always be true.
# kernel-abi-whitelists, for example, creates extra directories in
# /lib/modules.
#
# Skip building initramfs if the directory doesn't look like a
# kernel version. Emulates the version matching from depmod [2].
#
# 1. https://tldp.org/LDP/Linux-Filesystem-Hierarchy/html/lib.html
# 2. https://kernel.googlesource.com/pub/scm/linux/kernel/git/mmarek/kmod
# /+/tip/tools/depmod.c#2537
if not re.match(r'^\d+.\d+', kver):
logging.debug(
'/lib/modules/{} doesn\'t look like a kernel directory. '
'Skipping creation of initramfs for it'.format(kver))
continue
if not g.exists(os.path.join('/lib/modules', kver, 'modules.dep')):
try:
g.command(['depmod', kver])
except RuntimeError as e:
logging.info(
'Failed to write initramfs for {kver}. If image fails to '
'boot, verify that depmod /lib/modules/{kver} runs on '
'the original machine'.format(kver=kver))
logging.debug('depmod error: {}'.format(e))
continue
if el_release == '6':
# Version 6 doesn't have option --kver
g.command(['dracut', '-v', '-f', kver])
else:
g.command(['dracut', '--stdlog=1', '-f', '--kver', kver])
logging.info('Update grub configuration')
if el_release == '6':
# Version 6 doesn't have grub2, file grub.conf needs to be updated by hand
g.write('/tmp/grub_gce_generated', grub_cfg)
g.sh(r'grep -P "^[\t ]*initrd|^[\t ]*root|^[\t ]*kernel|^[\t ]*title" '
r'/boot/grub/grub.conf >> /tmp/grub_gce_generated;'
r'sed -i "s/console=ttyS0[^ ]*//g" /tmp/grub_gce_generated;'
r'sed -i "/^[\t ]*kernel/s/$/ console=ttyS0,38400n8/" '
r'/tmp/grub_gce_generated;'
r'mv /tmp/grub_gce_generated /boot/grub/grub.conf')
else:
g.write('/etc/default/grub', grub2_cfg)
g.command(['grub2-mkconfig', '-o', '/boot/grub2/grub.cfg'])
# Reset network for DHCP.
logging.info('Resetting network to DHCP for eth0.')
# Remove NetworkManager-config-server if it's present. The package configures
# NetworkManager to *not* use DHCP.
# https://access.redhat.com/solutions/894763
g.command(['yum', 'remove', '-y', 'NetworkManager-config-server'])
g.write('/etc/sysconfig/network-scripts/ifcfg-eth0', ifcfg_eth0)
def DistroSpecific(g: guestfs.GuestFS):
el_release = utils.GetMetadataAttribute('el_release')
install_gce = utils.GetMetadataAttribute('install_gce_packages')
rhel_license = utils.GetMetadataAttribute('use_rhel_gce_license')
# This must be performed prior to making network calls from the guest.
# Otherwise, if /etc/resolv.conf is present, and has an immutable attribute,
# guestfs will fail with:
#
# rename: /sysroot/etc/resolv.conf to
# /sysroot/etc/i9r7obu6: Operation not permitted
utils.common.ClearEtcResolv(g)
# Some imported images haven't contained `/etc/yum.repos.d`.
if not g.exists('/etc/yum.repos.d'):
g.mkdir('/etc/yum.repos.d')
if rhel_license == 'true':
if 'Red Hat' in g.cat('/etc/redhat-release'):
g.command(['yum', 'remove', '-y', '*rhui*'])
logging.info('Adding in GCE RHUI package.')
g.write('/etc/yum.repos.d/google-cloud.repo', repo_compute % el_release)
yum_install(g, 'google-rhui-client-rhel' + el_release)
if install_gce == 'true':
logging.info('Installing GCE packages.')
g.write('/etc/yum.repos.d/google-cloud.repo', repo_compute % el_release)
if el_release == '6':
if 'CentOS' in g.cat('/etc/redhat-release'):
logging.info('Installing CentOS SCL.')
g.command(['rm', '-f', '/etc/yum.repos.d/CentOS-SCL.repo'])
yum_install(g, 'centos-release-scl')
# Install Google Cloud SDK from the upstream tar and create links for the
# python27 SCL environment.
logging.info('Installing python27 from SCL.')
yum_install(g, 'python27')
logging.info('Installing Google Cloud SDK from tar.')
sdk_base_url = 'https://dl.google.com/dl/cloudsdk/channels/rapid'
sdk_base_tar = '%s/google-cloud-sdk.tar.gz' % sdk_base_url
tar = utils.HttpGet(sdk_base_tar)
g.write('/tmp/google-cloud-sdk.tar.gz', tar)
g.command(['tar', 'xzf', '/tmp/google-cloud-sdk.tar.gz', '-C', '/tmp'])
sdk_version = g.cat('/tmp/google-cloud-sdk/VERSION').strip()
logging.info('Getting Cloud SDK Version %s', sdk_version)
sdk_version_tar = 'google-cloud-sdk-%s-linux-x86_64.tar.gz' % sdk_version
sdk_version_tar_url = '%s/downloads/%s' % (sdk_base_url, sdk_version_tar)
logging.info('Getting versioned Cloud SDK tar file from %s',
sdk_version_tar_url)
tar = utils.HttpGet(sdk_version_tar_url)
sdk_version_tar_file = os.path.join('/tmp', sdk_version_tar)
g.write(sdk_version_tar_file, tar)
g.mkdir_p('/usr/local/share/google')
g.command(['tar', 'xzf', sdk_version_tar_file, '-C',
'/usr/local/share/google', '--no-same-owner'])
logging.info('Creating CloudSDK SCL symlinks.')
sdk_bin_path = '/usr/local/share/google/google-cloud-sdk/bin'
g.ln_s(os.path.join(sdk_bin_path, 'git-credential-gcloud.sh'),
os.path.join('/usr/bin', 'git-credential-gcloud.sh'))
for binary in ['bq', 'gcloud', 'gsutil']:
binary_path = os.path.join(sdk_bin_path, binary)
new_bin_path = os.path.join('/usr/bin', binary)
bin_str = '#!/bin/bash\nsource /opt/rh/python27/enable\n%s $@' % \
binary_path
g.write(new_bin_path, bin_str)
g.chmod(0o755, new_bin_path)
else:
g.write_append(
'/etc/yum.repos.d/google-cloud.repo', repo_sdk % el_release)
yum_install(g, 'google-cloud-sdk')
yum_install(g, 'google-compute-engine', 'google-osconfig-agent')
logging.info('Updating initramfs')
for kver in g.ls('/lib/modules'):
if not g.exists(os.path.join('/lib/modules', kver, 'modules.dep')):
g.command(['depmod', kver])
if el_release == '6':
# Version 6 doesn't have option --kver
g.command(['dracut', '-v', '-f', kver])
else:
g.command(['dracut', '--stdlog=1', '-f', '--kver', kver])
logging.info('Update grub configuration')
if el_release == '6':
# Version 6 doesn't have grub2, file grub.conf needs to be updated by hand
g.write('/tmp/grub_gce_generated', grub_cfg)
g.sh(
r'grep -P "^[\t ]*initrd|^[\t ]*root|^[\t ]*kernel|^[\t ]*title" '
r'/boot/grub/grub.conf >> /tmp/grub_gce_generated;'
r'sed -i "s/console=ttyS0[^ ]*//g" /tmp/grub_gce_generated;'
r'sed -i "/^[\t ]*kernel/s/$/ console=ttyS0,38400n8/" '
r'/tmp/grub_gce_generated;'
r'mv /tmp/grub_gce_generated /boot/grub/grub.conf')
else:
g.write('/etc/default/grub', grub2_cfg)
g.command(['grub2-mkconfig', '-o', '/boot/grub2/grub.cfg'])
# Reset network for DHCP.
logging.info('Resetting network to DHCP for eth0.')
# Remove NetworkManager-config-server if it's present. The package configures
# NetworkManager to *not* use DHCP.
# https://access.redhat.com/solutions/894763
g.command(['yum', 'remove', '-y', 'NetworkManager-config-server'])
g.write('/etc/sysconfig/network-scripts/ifcfg-eth0', ifcfg_eth0)
class FileSystem:
"""Convenience wrapper over GuestFS instance.
Simplifies some common routines.
Automatically translates paths according to the contained File System.
"""
def __init__(self, disk_path):
self._root = None
self._handler = GuestFS()
self.disk_path = disk_path
def __enter__(self):
self.mount()
return self
def __exit__(self, *_):
self.umount()
def __getattr__(self, attr):
return getattr(self._handler, attr)
@property
def osname(self):
"""Returns the Operating System name."""
return self._handler.inspect_get_type(self._root)
@property
def fsroot(self):
"""Returns the file system root."""
if self.osname == 'windows':
return '{}:\\'.format(
self._handler.inspect_get_drive_mappings(self._root)[0][0])
else:
return self._handler.inspect_get_mountpoints(self._root)[0][0]
def mount(self, readonly=True):
"""Mounts the given disk.
It must be called before any other method.
"""
self._handler.add_drive_opts(self.disk_path, readonly=True)
self._handler.launch()
for mountpoint, device in self._inspect_disk():
if readonly:
self._handler.mount_ro(device, mountpoint)
else:
self._handler.mount(device, mountpoint)
if self._handler.inspect_get_type(self._root) == 'windows':
self.path = self._windows_path
else:
self.path = posix_path
def _inspect_disk(self):
"""Inspects the disk and returns the mountpoints mapping
as a list which order is the supposed one for correct mounting.
"""
roots = self._handler.inspect_os()
if roots:
self._root = roots[0]
return sorted(self._handler.inspect_get_mountpoints(self._root),
key=lambda m: len(m[0]))
else:
raise RuntimeError("No OS found on the given disk image.")
def umount(self):
"""Unmounts the disk.
After this method is called no further action is allowed.
"""
self._handler.close()
def download(self, source, destination):
"""Downloads the file on the disk at source into destination."""
self._handler.download(posix_path(source), destination)
def ls(self, path):
"""Lists the content at the given path."""
return self._handler.ls(posix_path(path))
def nodes(self, path):
"""Iterates over the files and directories contained within the disk
starting from the given path.
Yields the path of the nodes.
"""
path = posix_path(path)
yield from (self.path(path, e) for e in self._handler.find(path))
def checksum(self, path, hashtype='sha1'):
"""Returns the checksum of the given path."""
return self._handler.checksum(hashtype, posix_path(path))
def checksums(self, path, hashtype='sha1'):
"""Iterates over the files hashes contained within the disk
starting from the given path.
The hashtype keyword allows to choose the file hashing algorithm.
Yields the following values:
"C:\\Windows\\System32\\NTUSER.DAT", "hash" for windows
"/home/user/text.txt", "hash" for other FS
"""
with NamedTemporaryFile(buffering=0) as tempfile:
self._handler.checksums_out(hashtype, posix_path(path),
tempfile.name)
yield from ((self.path(f[1].lstrip('.')), f[0])
for f in (l.decode('utf8').strip().split(None, 1)
for l in tempfile))
def stat(self, path):
"""Retrieves the status of the node at the given path.
Returns a dictionary.
"""
return self._handler.stat(posix_path(path))
def file(self, path):
"""Analogous to Unix file command.
Returns the type of node at the given path.
"""
return self._handler.file(posix_path(path))
def exists(self, path):
"""Returns whether the path exists."""
return self._handler.exists(posix_path(path))
def path(self, *segments):
"""Normalizes the path returned by guestfs in the File System format."""
raise NotImplementedError("FileSystem needs to be mounted first")
def _windows_path(self, *segments):
drive = self._handler.inspect_get_drive_mappings(self._root)[0][0]
return "%s:%s" % (drive, os.path.join(*segments).replace('/', '\\'))