def run_translate(g: guestfs.GuestFS):
    if (g.exists('/etc/redhat-release')
            and 'Red Hat' in g.cat('/etc/redhat-release')):
        distro = Distro.RHEL
    else:
        distro = Distro.CENTOS

    use_rhel_gce_license = utils.GetMetadataAttribute('use_rhel_gce_license')
    el_release = utils.GetMetadataAttribute('el_release')
    install_gce = utils.GetMetadataAttribute('install_gce_packages')
    spec = TranslateSpec(g=g,
                         use_rhel_gce_license=use_rhel_gce_license == 'true',
                         distro=distro,
                         el_release=el_release,
                         install_gce=install_gce == 'true')

    try:
        DistroSpecific(spec)
    except BaseException as raised:
        logging.debug('Translation failed due to: {}'.format(raised))
        for check in checks:
            msg = check(spec)
            if msg:
                raise RuntimeError('{} {}'.format(msg, raised)) from raised
        raise raised
Esempio n. 2
0
def install_cloud_sdk(g: guestfs.GuestFS, ubuntu_release: str) -> None:
    """ Installs Google Cloud SDK, supporting apt and snap.

  Args:
    g: A mounted GuestFS instance.
    ubuntu_release: The release nickname (eg: trusty).
  """
    try:
        run(g, 'gcloud --version')
        logging.info(
            'Found gcloud. Skipping installation of Google Cloud SDK.')
        return
    except RuntimeError:
        logging.info('Did not find previous install of gcloud.')

    if g.gcp_image_major < '18':
        g.write('/etc/apt/sources.list.d/partner.list',
                apt_cloud_sdk.format(ubuntu_release=ubuntu_release))
        utils.update_apt(g)
        utils.install_apt_packages(g, 'google-cloud-sdk')
        logging.info('Installed Google Cloud SDK with apt.')
        return

    # Starting at 18.04, Canonical installs the sdk using snap.
    # Running `snap install` directly is not an option here since it
    # requires the snapd daemon to be running on the guest.
    g.write('/etc/cloud/cloud.cfg.d/91-google-cloud-sdk.cfg',
            cloud_init_cloud_sdk)
    logging.info(
        'Google Cloud SDK will be installed using snap with cloud-init.')

    # Include /snap/bin in the PATH for startup and shutdown scripts.
    # This was present in the old guest agent, but lost in the new guest
    # agent.
    for p in [
            '/lib/systemd/system/google-shutdown-scripts.service',
            '/lib/systemd/system/google-startup-scripts.service'
    ]:
        logging.debug('[%s] Checking whether /bin/snap is on PATH.', p)
        if not g.exists(p):
            logging.debug('[%s] Skipping: Unit not found.', p)
            continue
        original_unit = g.cat(p)
        # Check whether the PATH is already set; if so, skip patching to avoid
        # overwriting existing directive.
        match = re.search('Environment=[\'"]?PATH.*',
                          original_unit,
                          flags=re.IGNORECASE)
        if match:
            logging.debug(
                '[%s] Skipping: PATH already defined in unit file: %s.', p,
                match.group())
            continue
        # Add Environment directive to unit file, and show diff in debug log.
        patched_unit = original_unit.replace('[Service]', snap_env_directive)
        g.write(p, patched_unit)
        diff = '\n'.join(Differ().compare(original_unit.splitlines(),
                                          patched_unit.splitlines()))
        logging.debug('[%s] PATH not defined. Added:\n%s', p, diff)
Esempio n. 3
0
def _reset_network(g: guestfs.GuestFS):
    """Update network to use DHCP."""
    logging.info('Updating network to use DHCP.')
    if g.exists('/etc/resolv.conf'):
        g.sh('echo "" > /etc/resolv.conf')
    g.write(
        '/etc/sysconfig/network/ifcfg-eth0', '\n'.join(
            ('BOOTPROTO=dhcp', 'STARTMODE=auto', 'DHCLIENT_SET_HOSTNAME=yes')))
Esempio n. 4
0
def _stage_etc_hosts(g: guestfs.GuestFS, dst: Path):
    """Stage copy of guest's /etc/hosts at dst with metadata server added."""
    original = ''
    if g.exists('/etc/hosts'):
        original = g.cat('/etc/hosts')

    metadata_host_line = None
    with open('/etc/hosts') as worker_etc_hosts:
        for line in worker_etc_hosts:
            if 'metadata.google.internal' in line:
                metadata_host_line = line
                break
    if not metadata_host_line:
        raise AssertionError(
            'Did not find metadata host in worker\'s /etc/hosts')

    dst.write_text('{}\n{}\n'.format(original, metadata_host_line))
Esempio n. 5
0
class FileSystem:
    """Convenience wrapper over GuestFS instance.

    Simplifies some common routines.

    Automatically translates paths according to the contained File System.

    """

    def __init__(self, disk_path):
        self._root = None
        self._handler = GuestFS()
        self._disk_path = disk_path

    def __enter__(self):
        self.mount()

        return self

    def __exit__(self, *_):
        self.umount()

    def __getattr__(self, attr):
        return getattr(self._handler, attr)

    @property
    def osname(self):
        """Returns the Operating System name."""
        return self._handler.inspect_get_type(self._root)

    @property
    def fsroot(self):
        """Returns the file system root."""
        if self.osname == "windows":
            return "{}:\\".format(self._handler.inspect_get_drive_mappings(self._root)[0][0])
        else:
            return self._handler.inspect_get_mountpoints(self._root)[0][0]

    def mount(self, readonly=True):
        """Mounts the given disk.
        It must be called before any other method.

        """
        self._handler.add_drive_opts(self._disk_path, readonly=True)
        self._handler.launch()

        for mountpoint, device in self._inspect_disk():
            if readonly:
                self._handler.mount_ro(device, mountpoint)
            else:
                self._handler.mount(device, mountpoint)

        if self._handler.inspect_get_type(self._root) == "windows":
            self.path = self._windows_path
        else:
            self.path = posix_path

    def _inspect_disk(self):
        """Inspects the disk and returns the mountpoints mapping
        as a list which order is the supposed one for correct mounting.

        """
        roots = self._handler.inspect_os()

        if roots:
            self._root = roots[0]
            return sorted(self._handler.inspect_get_mountpoints(self._root), key=lambda m: len(m[0]))
        else:
            raise RuntimeError("No OS found on the given disk image.")

    def umount(self):
        """Unmounts the disk.

        After this method is called no further action is allowed.

        """
        self._handler.close()

    def download(self, source, destination):
        """Downloads the file on the disk at source into destination."""
        self._handler.download(posix_path(source), destination)

    def ls(self, path):
        """Lists the content at the given path."""
        return self._handler.ls(posix_path(path))

    def nodes(self, path):
        """Iterates over the files and directories contained within the disk
        starting from the given path.

        Yields the path of the nodes.

        """
        path = posix_path(path)

        yield from (self.path(path, e) for e in self._handler.find(path))

    def checksum(self, path, hashtype="sha1"):
        """Returns the checksum of the given path."""
        return self._handler.checksum(hashtype, posix_path(path))

    def checksums(self, path, hashtype="sha1"):
        """Iterates over the files hashes contained within the disk
        starting from the given path.

        The hashtype keyword allows to choose the file hashing algorithm.

        Yields the following values:

            "C:\\Windows\\System32\\NTUSER.DAT", "hash" for windows
            "/home/user/text.txt", "hash" for other FS

        """
        with NamedTemporaryFile(buffering=0) as tempfile:
            self._handler.checksums_out(hashtype, posix_path(path), tempfile.name)

            yield from (
                (self.path(f[1].lstrip(".")), f[0]) for f in (l.decode("utf8").strip().split(None, 1) for l in tempfile)
            )

    def stat(self, path):
        """Retrieves the status of the node at the given path.

        Returns a dictionary.

        """
        return self._handler.stat(posix_path(path))

    def file(self, path):
        """Analogous to Unix file command.
        Returns the type of node at the given path.

        """
        return self._handler.file(posix_path(path))

    def exists(self, path):
        """Returns whether the path exists."""
        return self._handler.exists(posix_path(path))

    def path(self, *segments):
        """Normalizes the path returned by guestfs in the File System format."""
        raise NotImplementedError("FileSystem needs to be mounted first")

    def _windows_path(self, *segments):
        drive = self._handler.inspect_get_drive_mappings(self._root)[0][0]

        return "%s:%s" % (drive, os.path.join(*segments).replace("/", "\\"))
Esempio n. 6
0
def DistroSpecific(g: guestfs.GuestFS):
    el_release = utils.GetMetadataAttribute('el_release')
    install_gce = utils.GetMetadataAttribute('install_gce_packages')
    rhel_license = utils.GetMetadataAttribute('use_rhel_gce_license')

    # This must be performed prior to making network calls from the guest.
    # Otherwise, if /etc/resolv.conf is present, and has an immutable attribute,
    # guestfs will fail with:
    #
    #   rename: /sysroot/etc/resolv.conf to
    #     /sysroot/etc/i9r7obu6: Operation not permitted
    utils.common.ClearEtcResolv(g)

    # Some imported images haven't contained `/etc/yum.repos.d`.
    if not g.exists('/etc/yum.repos.d'):
        g.mkdir('/etc/yum.repos.d')

    if rhel_license == 'true':
        if 'Red Hat' in g.cat('/etc/redhat-release'):
            g.command(['yum', 'remove', '-y', '*rhui*'])
            logging.info('Adding in GCE RHUI package.')
            g.write('/etc/yum.repos.d/google-cloud.repo',
                    repo_compute % el_release)
            yum_install(g, 'google-rhui-client-rhel' + el_release)

    if install_gce == 'true':
        logging.info('Installing GCE packages.')
        g.write('/etc/yum.repos.d/google-cloud.repo',
                repo_compute % el_release)
        if el_release == '6':
            if 'CentOS' in g.cat('/etc/redhat-release'):
                logging.info('Installing CentOS SCL.')
                g.command(['rm', '-f', '/etc/yum.repos.d/CentOS-SCL.repo'])
                yum_install(g, 'centos-release-scl')
            # Install Google Cloud SDK from the upstream tar and create links for the
            # python27 SCL environment.
            logging.info('Installing python27 from SCL.')
            yum_install(g, 'python27')
            logging.info('Installing Google Cloud SDK from tar.')
            sdk_base_url = 'https://dl.google.com/dl/cloudsdk/channels/rapid'
            sdk_base_tar = '%s/google-cloud-sdk.tar.gz' % sdk_base_url
            tar = utils.HttpGet(sdk_base_tar)
            g.write('/tmp/google-cloud-sdk.tar.gz', tar)
            g.command(
                ['tar', 'xzf', '/tmp/google-cloud-sdk.tar.gz', '-C', '/tmp'])
            sdk_version = g.cat('/tmp/google-cloud-sdk/VERSION').strip()

            logging.info('Getting Cloud SDK Version %s', sdk_version)
            sdk_version_tar = 'google-cloud-sdk-%s-linux-x86_64.tar.gz' % sdk_version
            sdk_version_tar_url = '%s/downloads/%s' % (sdk_base_url,
                                                       sdk_version_tar)
            logging.info('Getting versioned Cloud SDK tar file from %s',
                         sdk_version_tar_url)
            tar = utils.HttpGet(sdk_version_tar_url)
            sdk_version_tar_file = os.path.join('/tmp', sdk_version_tar)
            g.write(sdk_version_tar_file, tar)
            g.mkdir_p('/usr/local/share/google')
            g.command([
                'tar', 'xzf', sdk_version_tar_file, '-C',
                '/usr/local/share/google', '--no-same-owner'
            ])

            logging.info('Creating CloudSDK SCL symlinks.')
            sdk_bin_path = '/usr/local/share/google/google-cloud-sdk/bin'
            g.ln_s(os.path.join(sdk_bin_path, 'git-credential-gcloud.sh'),
                   os.path.join('/usr/bin', 'git-credential-gcloud.sh'))
            for binary in ['bq', 'gcloud', 'gsutil']:
                binary_path = os.path.join(sdk_bin_path, binary)
                new_bin_path = os.path.join('/usr/bin', binary)
                bin_str = '#!/bin/bash\nsource /opt/rh/python27/enable\n%s $@' % \
                    binary_path
                g.write(new_bin_path, bin_str)
                g.chmod(0o755, new_bin_path)
        else:
            g.write_append('/etc/yum.repos.d/google-cloud.repo',
                           repo_sdk % el_release)
            yum_install(g, 'google-cloud-sdk')
        yum_install(g, 'google-compute-engine', 'google-osconfig-agent')

    logging.info('Updating initramfs')
    for kver in g.ls('/lib/modules'):
        # Although each directory in /lib/modules typically corresponds to a
        # kernel version  [1], that may not always be true.
        # kernel-abi-whitelists, for example, creates extra directories in
        # /lib/modules.
        #
        # Skip building initramfs if the directory doesn't look like a
        # kernel version. Emulates the version matching from depmod [2].
        #
        # 1. https://tldp.org/LDP/Linux-Filesystem-Hierarchy/html/lib.html
        # 2. https://kernel.googlesource.com/pub/scm/linux/kernel/git/mmarek/kmod
        # /+/tip/tools/depmod.c#2537
        if not re.match(r'^\d+.\d+', kver):
            logging.debug(
                '/lib/modules/{} doesn\'t look like a kernel directory. '
                'Skipping creation of initramfs for it'.format(kver))
            continue
        if not g.exists(os.path.join('/lib/modules', kver, 'modules.dep')):
            try:
                g.command(['depmod', kver])
            except RuntimeError as e:
                logging.info(
                    'Failed to write initramfs for {kver}. If image fails to '
                    'boot, verify that depmod /lib/modules/{kver} runs on '
                    'the original machine'.format(kver=kver))
                logging.debug('depmod error: {}'.format(e))
                continue
        if el_release == '6':
            # Version 6 doesn't have option --kver
            g.command(['dracut', '-v', '-f', kver])
        else:
            g.command(['dracut', '--stdlog=1', '-f', '--kver', kver])

    logging.info('Update grub configuration')
    if el_release == '6':
        # Version 6 doesn't have grub2, file grub.conf needs to be updated by hand
        g.write('/tmp/grub_gce_generated', grub_cfg)
        g.sh(r'grep -P "^[\t ]*initrd|^[\t ]*root|^[\t ]*kernel|^[\t ]*title" '
             r'/boot/grub/grub.conf >> /tmp/grub_gce_generated;'
             r'sed -i "s/console=ttyS0[^ ]*//g" /tmp/grub_gce_generated;'
             r'sed -i "/^[\t ]*kernel/s/$/ console=ttyS0,38400n8/" '
             r'/tmp/grub_gce_generated;'
             r'mv /tmp/grub_gce_generated /boot/grub/grub.conf')
    else:
        g.write('/etc/default/grub', grub2_cfg)
        g.command(['grub2-mkconfig', '-o', '/boot/grub2/grub.cfg'])

    # Reset network for DHCP.
    logging.info('Resetting network to DHCP for eth0.')
    # Remove NetworkManager-config-server if it's present. The package configures
    # NetworkManager to *not* use DHCP.
    #  https://access.redhat.com/solutions/894763
    g.command(['yum', 'remove', '-y', 'NetworkManager-config-server'])
    g.write('/etc/sysconfig/network-scripts/ifcfg-eth0', ifcfg_eth0)
Esempio n. 7
0
def DistroSpecific(g: guestfs.GuestFS):
  el_release = utils.GetMetadataAttribute('el_release')
  install_gce = utils.GetMetadataAttribute('install_gce_packages')
  rhel_license = utils.GetMetadataAttribute('use_rhel_gce_license')

  # This must be performed prior to making network calls from the guest.
  # Otherwise, if /etc/resolv.conf is present, and has an immutable attribute,
  # guestfs will fail with:
  #
  #   rename: /sysroot/etc/resolv.conf to
  #     /sysroot/etc/i9r7obu6: Operation not permitted
  utils.common.ClearEtcResolv(g)

  # Some imported images haven't contained `/etc/yum.repos.d`.
  if not g.exists('/etc/yum.repos.d'):
    g.mkdir('/etc/yum.repos.d')

  if rhel_license == 'true':
    if 'Red Hat' in g.cat('/etc/redhat-release'):
      g.command(['yum', 'remove', '-y', '*rhui*'])
      logging.info('Adding in GCE RHUI package.')
      g.write('/etc/yum.repos.d/google-cloud.repo', repo_compute % el_release)
      yum_install(g, 'google-rhui-client-rhel' + el_release)

  if install_gce == 'true':
    logging.info('Installing GCE packages.')
    g.write('/etc/yum.repos.d/google-cloud.repo', repo_compute % el_release)
    if el_release == '6':
      if 'CentOS' in g.cat('/etc/redhat-release'):
        logging.info('Installing CentOS SCL.')
        g.command(['rm', '-f', '/etc/yum.repos.d/CentOS-SCL.repo'])
        yum_install(g, 'centos-release-scl')
      # Install Google Cloud SDK from the upstream tar and create links for the
      # python27 SCL environment.
      logging.info('Installing python27 from SCL.')
      yum_install(g, 'python27')
      logging.info('Installing Google Cloud SDK from tar.')
      sdk_base_url = 'https://dl.google.com/dl/cloudsdk/channels/rapid'
      sdk_base_tar = '%s/google-cloud-sdk.tar.gz' % sdk_base_url
      tar = utils.HttpGet(sdk_base_tar)
      g.write('/tmp/google-cloud-sdk.tar.gz', tar)
      g.command(['tar', 'xzf', '/tmp/google-cloud-sdk.tar.gz', '-C', '/tmp'])
      sdk_version = g.cat('/tmp/google-cloud-sdk/VERSION').strip()

      logging.info('Getting Cloud SDK Version %s', sdk_version)
      sdk_version_tar = 'google-cloud-sdk-%s-linux-x86_64.tar.gz' % sdk_version
      sdk_version_tar_url = '%s/downloads/%s' % (sdk_base_url, sdk_version_tar)
      logging.info('Getting versioned Cloud SDK tar file from %s',
                   sdk_version_tar_url)
      tar = utils.HttpGet(sdk_version_tar_url)
      sdk_version_tar_file = os.path.join('/tmp', sdk_version_tar)
      g.write(sdk_version_tar_file, tar)
      g.mkdir_p('/usr/local/share/google')
      g.command(['tar', 'xzf', sdk_version_tar_file, '-C',
                 '/usr/local/share/google', '--no-same-owner'])

      logging.info('Creating CloudSDK SCL symlinks.')
      sdk_bin_path = '/usr/local/share/google/google-cloud-sdk/bin'
      g.ln_s(os.path.join(sdk_bin_path, 'git-credential-gcloud.sh'),
             os.path.join('/usr/bin', 'git-credential-gcloud.sh'))
      for binary in ['bq', 'gcloud', 'gsutil']:
        binary_path = os.path.join(sdk_bin_path, binary)
        new_bin_path = os.path.join('/usr/bin', binary)
        bin_str = '#!/bin/bash\nsource /opt/rh/python27/enable\n%s $@' % \
            binary_path
        g.write(new_bin_path, bin_str)
        g.chmod(0o755, new_bin_path)
    else:
      g.write_append(
          '/etc/yum.repos.d/google-cloud.repo', repo_sdk % el_release)
      yum_install(g, 'google-cloud-sdk')
    yum_install(g, 'google-compute-engine', 'google-osconfig-agent')

  logging.info('Updating initramfs')
  for kver in g.ls('/lib/modules'):
    if not g.exists(os.path.join('/lib/modules', kver, 'modules.dep')):
      g.command(['depmod', kver])
    if el_release == '6':
      # Version 6 doesn't have option --kver
      g.command(['dracut', '-v', '-f', kver])
    else:
      g.command(['dracut', '--stdlog=1', '-f', '--kver', kver])

  logging.info('Update grub configuration')
  if el_release == '6':
    # Version 6 doesn't have grub2, file grub.conf needs to be updated by hand
    g.write('/tmp/grub_gce_generated', grub_cfg)
    g.sh(
        r'grep -P "^[\t ]*initrd|^[\t ]*root|^[\t ]*kernel|^[\t ]*title" '
        r'/boot/grub/grub.conf >> /tmp/grub_gce_generated;'
        r'sed -i "s/console=ttyS0[^ ]*//g" /tmp/grub_gce_generated;'
        r'sed -i "/^[\t ]*kernel/s/$/ console=ttyS0,38400n8/" '
        r'/tmp/grub_gce_generated;'
        r'mv /tmp/grub_gce_generated /boot/grub/grub.conf')
  else:
    g.write('/etc/default/grub', grub2_cfg)
    g.command(['grub2-mkconfig', '-o', '/boot/grub2/grub.cfg'])

  # Reset network for DHCP.
  logging.info('Resetting network to DHCP for eth0.')
  # Remove NetworkManager-config-server if it's present. The package configures
  # NetworkManager to *not* use DHCP.
  #  https://access.redhat.com/solutions/894763
  g.command(['yum', 'remove', '-y', 'NetworkManager-config-server'])
  g.write('/etc/sysconfig/network-scripts/ifcfg-eth0', ifcfg_eth0)
Esempio n. 8
0
class FileSystem:
    """Convenience wrapper over GuestFS instance.

    Simplifies some common routines.

    Automatically translates paths according to the contained File System.

    """
    def __init__(self, disk_path):
        self._root = None
        self._handler = GuestFS()

        self.disk_path = disk_path

    def __enter__(self):
        self.mount()

        return self

    def __exit__(self, *_):
        self.umount()

    def __getattr__(self, attr):
        return getattr(self._handler, attr)

    @property
    def osname(self):
        """Returns the Operating System name."""
        return self._handler.inspect_get_type(self._root)

    @property
    def fsroot(self):
        """Returns the file system root."""
        if self.osname == 'windows':
            return '{}:\\'.format(
                self._handler.inspect_get_drive_mappings(self._root)[0][0])
        else:
            return self._handler.inspect_get_mountpoints(self._root)[0][0]

    def mount(self, readonly=True):
        """Mounts the given disk.
        It must be called before any other method.

        """
        self._handler.add_drive_opts(self.disk_path, readonly=True)
        self._handler.launch()

        for mountpoint, device in self._inspect_disk():
            if readonly:
                self._handler.mount_ro(device, mountpoint)
            else:
                self._handler.mount(device, mountpoint)

        if self._handler.inspect_get_type(self._root) == 'windows':
            self.path = self._windows_path
        else:
            self.path = posix_path

    def _inspect_disk(self):
        """Inspects the disk and returns the mountpoints mapping
        as a list which order is the supposed one for correct mounting.

        """
        roots = self._handler.inspect_os()

        if roots:
            self._root = roots[0]
            return sorted(self._handler.inspect_get_mountpoints(self._root),
                          key=lambda m: len(m[0]))
        else:
            raise RuntimeError("No OS found on the given disk image.")

    def umount(self):
        """Unmounts the disk.

        After this method is called no further action is allowed.

        """
        self._handler.close()

    def download(self, source, destination):
        """Downloads the file on the disk at source into destination."""
        self._handler.download(posix_path(source), destination)

    def ls(self, path):
        """Lists the content at the given path."""
        return self._handler.ls(posix_path(path))

    def nodes(self, path):
        """Iterates over the files and directories contained within the disk
        starting from the given path.

        Yields the path of the nodes.

        """
        path = posix_path(path)

        yield from (self.path(path, e) for e in self._handler.find(path))

    def checksum(self, path, hashtype='sha1'):
        """Returns the checksum of the given path."""
        return self._handler.checksum(hashtype, posix_path(path))

    def checksums(self, path, hashtype='sha1'):
        """Iterates over the files hashes contained within the disk
        starting from the given path.

        The hashtype keyword allows to choose the file hashing algorithm.

        Yields the following values:

            "C:\\Windows\\System32\\NTUSER.DAT", "hash" for windows
            "/home/user/text.txt", "hash" for other FS

        """
        with NamedTemporaryFile(buffering=0) as tempfile:
            self._handler.checksums_out(hashtype, posix_path(path),
                                        tempfile.name)

            yield from ((self.path(f[1].lstrip('.')), f[0])
                        for f in (l.decode('utf8').strip().split(None, 1)
                                  for l in tempfile))

    def stat(self, path):
        """Retrieves the status of the node at the given path.

        Returns a dictionary.

        """
        return self._handler.stat(posix_path(path))

    def file(self, path):
        """Analogous to Unix file command.
        Returns the type of node at the given path.

        """
        return self._handler.file(posix_path(path))

    def exists(self, path):
        """Returns whether the path exists."""
        return self._handler.exists(posix_path(path))

    def path(self, *segments):
        """Normalizes the path returned by guestfs in the File System format."""
        raise NotImplementedError("FileSystem needs to be mounted first")

    def _windows_path(self, *segments):
        drive = self._handler.inspect_get_drive_mappings(self._root)[0][0]

        return "%s:%s" % (drive, os.path.join(*segments).replace('/', '\\'))