def install_cloud_sdk(g: guestfs.GuestFS, ubuntu_release: str) -> None:
""" Installs Google Cloud SDK, supporting apt and snap.
Args:
g: A mounted GuestFS instance.
ubuntu_release: The release nickname (eg: trusty).
"""
try:
run(g, 'gcloud --version')
logging.info(
'Found gcloud. Skipping installation of Google Cloud SDK.')
return
except RuntimeError:
logging.info('Did not find previous install of gcloud.')
if g.gcp_image_major < '18':
g.write('/etc/apt/sources.list.d/partner.list',
apt_cloud_sdk.format(ubuntu_release=ubuntu_release))
utils.update_apt(g)
utils.install_apt_packages(g, 'google-cloud-sdk')
logging.info('Installed Google Cloud SDK with apt.')
return
# Starting at 18.04, Canonical installs the sdk using snap.
# Running `snap install` directly is not an option here since it
# requires the snapd daemon to be running on the guest.
g.write('/etc/cloud/cloud.cfg.d/91-google-cloud-sdk.cfg',
cloud_init_cloud_sdk)
logging.info(
'Google Cloud SDK will be installed using snap with cloud-init.')
# Include /snap/bin in the PATH for startup and shutdown scripts.
# This was present in the old guest agent, but lost in the new guest
# agent.
for p in [
'/lib/systemd/system/google-shutdown-scripts.service',
'/lib/systemd/system/google-startup-scripts.service'
]:
logging.debug('[%s] Checking whether /bin/snap is on PATH.', p)
if not g.exists(p):
logging.debug('[%s] Skipping: Unit not found.', p)
continue
original_unit = g.cat(p)
# Check whether the PATH is already set; if so, skip patching to avoid
# overwriting existing directive.
match = re.search('Environment=[\'"]?PATH.*',
original_unit,
flags=re.IGNORECASE)
if match:
logging.debug(
'[%s] Skipping: PATH already defined in unit file: %s.', p,
match.group())
continue
# Add Environment directive to unit file, and show diff in debug log.
patched_unit = original_unit.replace('[Service]', snap_env_directive)
g.write(p, patched_unit)
diff = '\n'.join(Differ().compare(original_unit.splitlines(),
patched_unit.splitlines()))
logging.debug('[%s] PATH not defined. Added:\n%s', p, diff)
def remove_azure_agents(g):
try:
run(g, ['apt-get', 'remove', '-y', '-f', 'walinuxagent'])
except Exception as e:
logging.debug(str(e))
try:
run(g, ['apt-get', 'remove', '-y', '-f', 'waagent'])
except Exception as e:
logging.debug(str(e))
def reset_network_for_dhcp(spec: TranslateSpec):
logging.info('Resetting network to DHCP for eth0.')
spec.g.write('/etc/sysconfig/network-scripts/ifcfg-eth0', ifcfg_eth0)
# Remove NetworkManager-config-server if it's present. The package configures
# NetworkManager to *not* use DHCP.
# https://access.redhat.com/solutions/894763
pkg = 'NetworkManager-config-server'
if package_is_installed(spec, pkg):
if yum_is_on_path(spec):
run(spec.g, ['yum', 'remove', '-y', pkg])
else:
run(spec.g, ['rpm', '--erase', pkg])
def check_repos(spec: TranslateSpec) -> str:
"""Check for unreachable repos.
YUM fails if any of its repos are unreachable. Running `yum updateinfo`
will have a non-zero return code when it fail to update any of its repos.
"""
if run(spec.g, 'yum --help | grep updateinfo').code != 0:
logging.debug('command `yum updateinfo` not available. skipping test.')
return ''
v = 'yum updateinfo -v'
p = run(spec.g, v)
logging.debug('yum updateinfo -v: {}'.format(p))
if p.code != 0:
return 'Ensure all configured repos are reachable.'
def yum_install(g, *packages):
"""Install one or more packages using YUM.
Args:
g (guestfs.GuestFS): A mounted GuestFS instance.
*packages (list of str): The YUM packages to be installed.
Raises:
RuntimeError: If there is a failure during installation.
"""
p = None
for i in range(6):
# There's no sleep on the first iteration since `i` is zero.
time.sleep(i**2)
# Bypass HTTP proxies configured in the guest image to allow
# import to continue when the proxy is unreachable.
# no_proxy="*": Disables proxies set by using the `http_proxy`
# environment variable.
# proxy=_none_: Disables proxies set in /etc/yum.conf.
cmd = 'no_proxy="*" yum install --setopt=proxy=_none_ -y ' + ' '.join(
'"{0}"'.format(p) for p in packages)
p = run(g, cmd, raiseOnError=False)
if p.code == 0:
return
logging.debug('Yum install failed: {}'.format(p))
raise RuntimeError('Failed to install {}. Details: {}.'.format(
', '.join(packages), p))
def check_yum_on_path(spec: TranslateSpec) -> str:
"""Check whether the `yum` command is available.
If `yum` isn't found, errs is updated.
"""
p = run(spec.g, 'yum --version', raiseOnError=False)
logging.debug('yum --version: {}'.format(p))
if p.code != 0:
return 'Verify the disk\'s OS: `yum` not found.'
def check_repos(spec: TranslateSpec) -> str:
"""Check for unreachable repos.
YUM fails if any of its repos are unreachable. Running `yum updateinfo`
will have a non-zero return code when it fail to update any of its repos.
"""
v = 'yum updateinfo -v'
p = run(spec.g, v)
logging.debug('yum updateinfo -v: {}'.format(p))
if p.code != 0:
return 'Ensure all configured repos are reachable.'
def setup_cloud_init(g: guestfs.GuestFS):
""" Install cloud-init if not present, and configure to the cloud provider.
Args:
g: A mounted GuestFS instance.
"""
a = Apt(run)
curr_version = a.get_package_version(g, 'cloud-init')
available_versions = a.list_available_versions(g, 'cloud-init')
# Try to avoid installing 21.3-1, which conflicts which the guest agent.
# On first boot, systemd reaches a deadlock deadlock and doest start
# its unit. If a version other than 21.3-1 isn't explicitly found, *and*
# cloud-init isn't currently installed, then this allows apt to pick the
# version to install.
version_to_install = Apt.determine_version_to_install(
curr_version, available_versions, {'21.3-1'})
pkg_to_install = ''
if version_to_install:
pkg_to_install = 'cloud-init=' + version_to_install
elif curr_version == '':
pkg_to_install = 'cloud-init'
# If this block doesn't execute, it means that cloud-init was found
# on the system, but there wasn't an upgrade candidate. Therefore
# leave the version that's currently installed.
if pkg_to_install:
logging.info(pkg_to_install)
utils.install_apt_packages(g, pkg_to_install)
# Ubuntu 14.04's version of cloud-init doesn't have `clean`.
if g.gcp_image_major > '14':
run(g, 'cloud-init clean')
# Remove cloud-init configs that may conflict with GCE's.
#
# - subiquity disables automatic network configuration
# https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/1871975
for cfg in [
'azure', 'curtin', 'waagent', 'walinuxagent', 'aws', 'amazon',
'subiquity'
]:
run(g, 'rm -f /etc/cloud/cloud.cfg.d/*%s*' % cfg)
g.write('/etc/cloud/cloud.cfg.d/91-gce-system.cfg', cloud_init_config)
def check_rhel_license(spec: TranslateSpec) -> str:
"""Check for an active RHEL license.
If a license isn't found, errs is updated.
"""
if spec.distro != Distro.RHEL or spec.use_rhel_gce_license:
return ''
p = run(spec.g, 'subscription-manager status', raiseOnError=False)
logging.debug('subscription-manager: {}'.format(p))
if p.code != 0:
return 'subscription-manager did not find an active subscription. ' \
'Omit `-byol` to register with on-demand licensing.'
def DistroSpecific(g):
ubuntu_release = utils.GetMetadataAttribute('ubuntu_release')
install_gce = utils.GetMetadataAttribute('install_gce_packages')
# If present, remove any hard coded DNS settings in resolvconf.
# This is a common workaround to include permanent changes:
# https://askubuntu.com/questions/157154
if g.exists('/etc/resolvconf/resolv.conf.d/base'):
logging.info('Resetting resolvconf base.')
run(g, 'echo "" > /etc/resolvconf/resolv.conf.d/base')
# Reset the network to DHCP.
if ubuntu_release == 'trusty':
g.write('/etc/network/interfaces', network_trusty)
elif ubuntu_release == 'xenial':
g.write('/etc/network/interfaces', network_xenial)
elif g.is_dir('/etc/netplan'):
run(g, 'rm -f /etc/netplan/*.yaml')
g.write('/etc/netplan/config.yaml', network_netplan)
run(g, 'netplan apply')
if install_gce == 'true':
utils.update_apt(g)
setup_cloud_init(g)
remove_azure_agents(g)
if g.gcp_image_major > '14':
install_osconfig_agent(g)
utils.install_apt_packages(g, 'gce-compute-image-packages')
install_cloud_sdk(g, ubuntu_release)
# Update grub config to log to console.
run(g, [
'sed', '-i',
r's#^\(GRUB_CMDLINE_LINUX=".*\)"$#\1 console=ttyS0,38400n8"#',
'/etc/default/grub'
])
run(g, ['update-grub2'])
def translate():
"""Mounts the disk, runs translation steps, then unmounts the disk."""
include_gce_packages = utils.GetMetadataAttribute(
'install_gce_packages', 'true').lower() == 'true'
subscription_model = utils.GetMetadataAttribute(
'subscription_model', 'byol').lower()
g = diskutils.MountDisk('/dev/sdb')
release = _get_release(g)
pkgs = release.gce_packages if include_gce_packages else []
if subscription_model == 'gce':
logging.info('Converting to on-demand')
migrate.migrate(
g=g, tar_url=release.on_demand_rpms.url,
tar_sha256=release.on_demand_rpms.sha256,
cloud_product=release.cloud_product,
post_convert_packages=pkgs
)
else:
_install_product(g, release)
_refresh_zypper(g)
_install_packages(g, pkgs)
_install_virtio_drivers(g)
if include_gce_packages:
logging.info('Enabling google services.')
for unit in g.ls('/usr/lib/systemd/system/'):
if unit.startswith('google-'):
run(g, ['systemctl', 'enable', '/usr/lib/systemd/system/' + unit],
raiseOnError=True)
_reset_network(g)
_update_grub(g)
utils.CommonRoutines(g)
diskutils.UnmountDisk(g)
def test_capture_stdout(self):
cmd = 'echo abc123'
result = run(_make_local_guestfs(), cmd, raiseOnError=False)
assert result == CompletedProcess('abc123\n', '', 0, cmd)
def test_support_array_args():
result = run(_make_local_guestfs(), ['echo', 'hi'])
assert result == CompletedProcess('hi\n', '', 0, 'echo hi')
def test_capture_runtime_errors():
result = run(_make_local_guestfs(), 'not-a-command')
assert result.code != 0
assert 'not-a-command' in result.stderr
def test_raise_error_when_failure(self):
cmd = '>&2 echo stderr msg; exit 1'
with pytest.raises(RuntimeError, match='stderr msg'):
run(_make_local_guestfs(), cmd)
def test_return_completed_process_when_success(self):
cmd = 'echo abc123'
result = run(_make_local_guestfs(), cmd)
assert result == CompletedProcess('abc123\n', '', 0, cmd)
def test_capture_output_when_non_zero_return(self):
cmd = 'printf content; printf err > /dev/stderr; exit 1'
result = run(_make_local_guestfs(), cmd, raiseOnError=False)
assert result == CompletedProcess('content', 'err', 1, cmd)
def test_capture_runtime_errors(self):
result = run(_make_local_guestfs(),
'not-a-command',
raiseOnError=False)
assert result.code != 0
assert 'not-a-command' in result.stderr
def test_escape_array_members(self):
result = run(_make_local_guestfs(), ['echo', 'hello', '; ls *'],
raiseOnError=False)
assert result == CompletedProcess('hello ; ls *\n', '', 0,
"echo hello '; ls *'")
def test_support_array_args(self):
result = run(_make_local_guestfs(), ['echo', 'hi'], raiseOnError=False)
assert result == CompletedProcess('hi\n', '', 0, 'echo hi')
def test_support_positive_code(self):
cmd = 'exit 100'
result = run(_make_local_guestfs(), cmd, raiseOnError=False)
assert result == CompletedProcess('', '', 100, cmd)
def test_support_positive_code():
cmd = 'exit 100'
result = run(_make_local_guestfs(), cmd)
assert result == CompletedProcess('', '', 100, cmd)
def yum_is_on_path(spec: TranslateSpec) -> bool:
"""Check whether the `yum` command is available."""
p = run(spec.g, 'yum --version', raiseOnError=False)
logging.debug('yum --version: {}'.format(p))
return p.code == 0
def DistroSpecific(spec: TranslateSpec):
g = spec.g
el_release = spec.el_release
# This must be performed prior to making network calls from the guest.
# Otherwise, if /etc/resolv.conf is present, and has an immutable attribute,
# guestfs will fail with:
#
# rename: /sysroot/etc/resolv.conf to
# /sysroot/etc/i9r7obu6: Operation not permitted
utils.common.ClearEtcResolv(g)
# Some imported images haven't contained `/etc/yum.repos.d`.
if not g.exists('/etc/yum.repos.d'):
g.mkdir('/etc/yum.repos.d')
if spec.distro == Distro.RHEL:
if spec.use_rhel_gce_license:
run(g, ['yum', 'remove', '-y', '*rhui*'])
logging.info('Adding in GCE RHUI package.')
g.write('/etc/yum.repos.d/google-cloud.repo',
repo_compute % el_release)
yum_install(g, 'google-rhui-client-rhel' + el_release)
# Historically, translations have failed for corrupt dbcache and rpmdb.
run(g, 'yum clean -y all')
if spec.install_gce:
logging.info('Installing GCE packages.')
g.write('/etc/yum.repos.d/google-cloud.repo',
repo_compute % el_release)
if el_release == '6':
# yum operations fail when the epel repo is used with stale
# ca-certificates, causing translation to fail. To avoid that,
# update ca-certificates when the epel repo is found.
#
# The `--disablerepo` flag does the following:
# 1. Skip the epel repo for *this* operation only.
# 2. Block update if the epel repo isn't found.
p = run(g,
'yum update -y ca-certificates --disablerepo=epel',
raiseOnError=False)
logging.debug('Attempted conditional update of '
'ca-certificates. Success expected only '
'if epel repo is installed. Result={}'.format(p))
# Install Google Cloud SDK from the upstream tar and create links for the
# python27 SCL environment.
logging.info('Installing python27 from SCL.')
yum_install(g, 'python27')
logging.info('Installing Google Cloud SDK from tar.')
sdk_base_url = 'https://dl.google.com/dl/cloudsdk/channels/rapid'
sdk_base_tar = '%s/google-cloud-sdk.tar.gz' % sdk_base_url
tar = utils.HttpGet(sdk_base_tar)
g.write('/tmp/google-cloud-sdk.tar.gz', tar)
run(g,
['tar', 'xzf', '/tmp/google-cloud-sdk.tar.gz', '-C', '/tmp'])
sdk_version = g.cat('/tmp/google-cloud-sdk/VERSION').strip()
logging.info('Getting Cloud SDK Version %s', sdk_version)
sdk_version_tar = 'google-cloud-sdk-%s-linux-x86_64.tar.gz' % sdk_version
sdk_version_tar_url = '%s/downloads/%s' % (sdk_base_url,
sdk_version_tar)
logging.info('Getting versioned Cloud SDK tar file from %s',
sdk_version_tar_url)
tar = utils.HttpGet(sdk_version_tar_url)
sdk_version_tar_file = os.path.join('/tmp', sdk_version_tar)
g.write(sdk_version_tar_file, tar)
g.mkdir_p('/usr/local/share/google')
run(g, [
'tar', 'xzf', sdk_version_tar_file, '-C',
'/usr/local/share/google', '--no-same-owner'
])
logging.info('Creating CloudSDK SCL symlinks.')
sdk_bin_path = '/usr/local/share/google/google-cloud-sdk/bin'
g.ln_s(os.path.join(sdk_bin_path, 'git-credential-gcloud.sh'),
os.path.join('/usr/bin', 'git-credential-gcloud.sh'))
for binary in ['bq', 'gcloud', 'gsutil']:
binary_path = os.path.join(sdk_bin_path, binary)
new_bin_path = os.path.join('/usr/bin', binary)
bin_str = '#!/bin/bash\nsource /opt/rh/python27/enable\n%s $@' % \
binary_path
g.write(new_bin_path, bin_str)
g.chmod(0o755, new_bin_path)
else:
g.write_append('/etc/yum.repos.d/google-cloud.repo',
repo_sdk % el_release)
yum_install(g, 'google-cloud-sdk')
yum_install(g, 'google-compute-engine', 'google-osconfig-agent')
logging.info('Updating initramfs')
for kver in g.ls('/lib/modules'):
# Although each directory in /lib/modules typically corresponds to a
# kernel version [1], that may not always be true.
# kernel-abi-whitelists, for example, creates extra directories in
# /lib/modules.
#
# Skip building initramfs if the directory doesn't look like a
# kernel version. Emulates the version matching from depmod [2].
#
# 1. https://tldp.org/LDP/Linux-Filesystem-Hierarchy/html/lib.html
# 2. https://kernel.googlesource.com/pub/scm/linux/kernel/git/mmarek/kmod
# /+/tip/tools/depmod.c#2537
if not re.match(r'^\d+.\d+', kver):
logging.debug(
'/lib/modules/{} doesn\'t look like a kernel directory. '
'Skipping creation of initramfs for it'.format(kver))
continue
if not g.exists(os.path.join('/lib/modules', kver, 'modules.dep')):
try:
run(g, ['depmod', kver])
except RuntimeError as e:
logging.info(
'Failed to write initramfs for {kver}. If image fails to '
'boot, verify that depmod /lib/modules/{kver} runs on '
'the original machine'.format(kver=kver))
logging.debug('depmod error: {}'.format(e))
continue
if el_release == '6':
# Version 6 doesn't have option --kver
run(g, ['dracut', '-v', '-f', kver])
else:
run(g, ['dracut', '--stdlog=1', '-f', '--kver', kver])
logging.info('Update grub configuration')
if el_release == '6':
# Version 6 doesn't have grub2, file grub.conf needs to be updated by hand
g.write('/tmp/grub_gce_generated', grub_cfg)
run(
g,
r'grep -P "^[\t ]*initrd|^[\t ]*root|^[\t ]*kernel|^[\t ]*title" '
r'/boot/grub/grub.conf >> /tmp/grub_gce_generated;'
r'sed -i "s/console=ttyS0[^ ]*//g" /tmp/grub_gce_generated;'
r'sed -i "/^[\t ]*kernel/s/$/ console=ttyS0,38400n8/" '
r'/tmp/grub_gce_generated;'
r'mv /tmp/grub_gce_generated /boot/grub/grub.conf')
else:
g.write('/etc/default/grub', grub2_cfg)
run(g, ['grub2-mkconfig', '-o', '/boot/grub2/grub.cfg'])
# Reset network for DHCP.
logging.info('Resetting network to DHCP for eth0.')
# Remove NetworkManager-config-server if it's present. The package configures
# NetworkManager to *not* use DHCP.
# https://access.redhat.com/solutions/894763
run(g, ['yum', 'remove', '-y', 'NetworkManager-config-server'])
g.write('/etc/sysconfig/network-scripts/ifcfg-eth0', ifcfg_eth0)
def package_is_installed(spec: TranslateSpec, package: str) -> bool:
"""Check whether package is installed."""
p = run(spec.g, ['rpm', '-q', package], raiseOnError=False)
logging.debug('rpm -q: {}'.format(p))
return p.code == 0
def DistroSpecific(g):
ubuntu_release = utils.GetMetadataAttribute('ubuntu_release')
install_gce = utils.GetMetadataAttribute('install_gce_packages')
# If present, remove any hard coded DNS settings in resolvconf.
# This is a common workaround to include permanent changes:
# https://askubuntu.com/questions/157154
if g.exists('/etc/resolvconf/resolv.conf.d/base'):
logging.info('Resetting resolvconf base.')
run(g, 'echo "" > /etc/resolvconf/resolv.conf.d/base')
# Reset the network to DHCP.
if ubuntu_release == 'trusty':
g.write('/etc/network/interfaces', network_trusty)
elif ubuntu_release == 'xenial':
g.write('/etc/network/interfaces', network_xenial)
elif g.is_dir('/etc/netplan'):
run(g, 'rm -f /etc/netplan/*.yaml')
g.write('/etc/netplan/config.yaml', network_netplan)
run(g, 'netplan apply')
if install_gce == 'true':
utils.update_apt(g)
logging.info('Installing cloud-init.')
utils.install_apt_packages(g, 'cloud-init')
# Ubuntu 14.04's version of cloud-init doesn't have `clean`.
if g.gcp_image_major > '14':
run(g, 'cloud-init clean')
# Remove cloud-init configs that may conflict with GCE's.
#
# - subiquity disables automatic network configuration
# https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/1871975
for cfg in [
'azure', 'curtin', 'waagent', 'walinuxagent', 'aws', 'amazon',
'subiquity'
]:
run(g, 'rm -f /etc/cloud/cloud.cfg.d/*%s*' % cfg)
remove_azure_agents(g)
g.write('/etc/cloud/cloud.cfg.d/91-gce-system.cfg', cloud_init_repos)
if g.gcp_image_major > '14':
install_osconfig_agent(g)
utils.install_apt_packages(g, 'gce-compute-image-packages')
install_cloud_sdk(g, ubuntu_release)
# Update grub config to log to console.
run(g, [
'sed', '-i',
r's#^\(GRUB_CMDLINE_LINUX=".*\)"$#\1 console=ttyS0,38400n8"#',
'/etc/default/grub'
])
run(g, ['update-grub2'])
def test_capture_stderr(self):
cmd = 'echo error msg > /dev/stderr'
result = run(_make_local_guestfs(), cmd, raiseOnError=False)
assert result == CompletedProcess('', 'error msg\n', 0, cmd)
def DistroSpecific(g):
install_gce = utils.GetMetadataAttribute('install_gce_packages')
deb_release = utils.GetMetadataAttribute('debian_release')
if install_gce == 'true':
logging.info('Installing GCE packages.')
utils.update_apt(g)
utils.install_apt_packages(g, 'gnupg')
run(g, [
'wget', 'https://packages.cloud.google.com/apt/doc/apt-key.gpg',
'-O', '/tmp/gce_key'
])
run(g, ['apt-key', 'add', '/tmp/gce_key'])
g.rm('/tmp/gce_key')
g.write('/etc/apt/sources.list.d/google-cloud.list',
google_cloud.format(deb_release=deb_release))
# Remove Azure agent.
try:
run(g,
['apt-get', 'remove', '-y', '-f', 'waagent', 'walinuxagent'])
except Exception as e:
logging.debug(str(e))
logging.warn('Could not uninstall Azure agent. Continuing anyway.')
utils.update_apt(g)
pkgs = [
'google-cloud-packages-archive-keyring', 'google-compute-engine'
]
# Debian 8 differences:
# 1. No NGE
# 2. No Cloud SDK, since it requires Python 3.5+.
# 3. No OS config agent.
if deb_release == 'jessie':
# Debian 8 doesn't support the new guest agent, so we need to install
# the legacy Python version.
pkgs += [
'python-google-compute-engine', 'python3-google-compute-engine'
]
logging.info('Skipping installation of OS Config agent. '
'Requires Debian 9 or newer.')
else:
pkgs += ['google-cloud-sdk', 'google-osconfig-agent']
utils.install_apt_packages(g, *pkgs)
# Update grub config to log to console.
run(g, [
'sed', '-i""', r'/GRUB_CMDLINE_LINUX/s#"$# console=ttyS0,38400n8"#',
'/etc/default/grub'
])
# Disable predictive network interface naming in 9+.
if deb_release != 'jessie':
run(g, [
'sed', '-i',
r's#^\(GRUB_CMDLINE_LINUX=".*\)"$#\1 net.ifnames=0 biosdevname=0"#',
'/etc/default/grub'
])
run(g, ['update-grub2'])
# Reset network for DHCP.
logging.info('Resetting network to DHCP for eth0.')
g.write('/etc/network/interfaces', interfaces)
def test_capture_stdout():
cmd = 'echo abc123'
result = run(_make_local_guestfs(), cmd)
assert result == CompletedProcess('abc123\n', '', 0, cmd)
