def DistroSpecific(g):
el_release = utils.GetMetadataAttribute('el_release')
install_gce = utils.GetMetadataAttribute('install_gce_packages')
rhel_license = utils.GetMetadataAttribute('use_rhel_gce_license')
# This must be performed prior to making network calls from the guest.
# Otherwise, if /etc/resolv.conf is present, and has an immutable attribute,
# guestfs will fail with:
#
# rename: /sysroot/etc/resolv.conf to
# /sysroot/etc/i9r7obu6: Operation not permitted
utils.common.ClearEtcResolv(g)
if rhel_license == 'true':
if 'Red Hat' in g.cat('/etc/redhat-release'):
g.command(['yum', 'remove', '-y', '*rhui*'])
logging.info('Adding in GCE RHUI package.')
g.write('/etc/yum.repos.d/google-cloud.repo',
repo_compute % el_release)
yum_install(g, 'google-rhui-client-rhel' + el_release)
if install_gce == 'true':
logging.info('Installing GCE packages.')
g.write('/etc/yum.repos.d/google-cloud.repo',
repo_compute % el_release)
if el_release == '6':
if 'CentOS' in g.cat('/etc/redhat-release'):
logging.info('Installing CentOS SCL.')
g.command(['rm', '-f', '/etc/yum.repos.d/CentOS-SCL.repo'])
yum_install(g, 'centos-release-scl')
# Install Google Cloud SDK from the upstream tar and create links for the
# python27 SCL environment.
logging.info('Installing python27 from SCL.')
yum_install(g, 'python27')
g.command(
['scl', 'enable', 'python27', 'pip2.7 install --upgrade pip'])
g.command([
'scl', 'enable', 'python27',
'pip2.7 install --upgrade google_compute_engine'
])
logging.info('Installing Google Cloud SDK from tar.')
sdk_base_url = 'https://dl.google.com/dl/cloudsdk/channels/rapid'
sdk_base_tar = '%s/google-cloud-sdk.tar.gz' % sdk_base_url
tar = utils.HttpGet(sdk_base_tar)
g.write('/tmp/google-cloud-sdk.tar.gz', tar)
g.command(
['tar', 'xzf', '/tmp/google-cloud-sdk.tar.gz', '-C', '/tmp'])
sdk_version = g.cat('/tmp/google-cloud-sdk/VERSION').strip()
logging.info('Getting Cloud SDK Version %s', sdk_version)
sdk_version_tar = 'google-cloud-sdk-%s-linux-x86_64.tar.gz' % sdk_version
sdk_version_tar_url = '%s/downloads/%s' % (sdk_base_url,
sdk_version_tar)
logging.info('Getting versioned Cloud SDK tar file from %s',
sdk_version_tar_url)
tar = utils.HttpGet(sdk_version_tar_url)
sdk_version_tar_file = os.path.join('/tmp', sdk_version_tar)
g.write(sdk_version_tar_file, tar)
g.mkdir_p('/usr/local/share/google')
g.command([
'tar', 'xzf', sdk_version_tar_file, '-C',
'/usr/local/share/google', '--no-same-owner'
])
logging.info('Creating CloudSDK SCL symlinks.')
sdk_bin_path = '/usr/local/share/google/google-cloud-sdk/bin'
g.ln_s(os.path.join(sdk_bin_path, 'git-credential-gcloud.sh'),
os.path.join('/usr/bin', 'git-credential-gcloud.sh'))
for binary in ['bq', 'gcloud', 'gsutil']:
binary_path = os.path.join(sdk_bin_path, binary)
new_bin_path = os.path.join('/usr/bin', binary)
bin_str = '#!/bin/bash\nsource /opt/rh/python27/enable\n%s $@' % \
binary_path
g.write(new_bin_path, bin_str)
g.chmod(0o755, new_bin_path)
else:
g.write_append('/etc/yum.repos.d/google-cloud.repo',
repo_sdk % el_release)
yum_install(g, 'google-cloud-sdk')
if el_release == '8':
yum_install(g, 'google-compute-engine',
'python3-google-compute-engine')
else:
yum_install(g, 'google-compute-engine',
'python-google-compute-engine')
logging.info('Updating initramfs')
for kver in g.ls('/lib/modules'):
if not g.exists(os.path.join('/lib/modules', kver, 'modules.dep')):
g.command(['depmod', kver])
if el_release == '6':
# Version 6 doesn't have option --kver
g.command(['dracut', '-v', '-f', kver])
else:
g.command(['dracut', '--stdlog=1', '-f', '--kver', kver])
logging.info('Update grub configuration')
if el_release == '6':
# Version 6 doesn't have grub2, file grub.conf needs to be updated by hand
g.write('/tmp/grub_gce_generated', grub_cfg)
g.sh(r'grep -P "^[\t ]*initrd|^[\t ]*root|^[\t ]*kernel|^[\t ]*title" '
r'/boot/grub/grub.conf >> /tmp/grub_gce_generated;'
r'sed -i "s/console=ttyS0[^ ]*//g" /tmp/grub_gce_generated;'
r'sed -i "/^[\t ]*kernel/s/$/ console=ttyS0,38400n8/" '
r'/tmp/grub_gce_generated;'
r'mv /tmp/grub_gce_generated /boot/grub/grub.conf')
else:
g.write('/etc/default/grub', grub2_cfg)
g.command(['grub2-mkconfig', '-o', '/boot/grub2/grub.cfg'])
# Reset network for DHCP.
logging.info('Resetting network to DHCP for eth0.')
# Remove NetworkManager-config-server if it's present. The package configures
# NetworkManager to *not* use DHCP.
# https://access.redhat.com/solutions/894763
g.command(['yum', 'remove', '-y', 'NetworkManager-config-server'])
g.write('/etc/sysconfig/network-scripts/ifcfg-eth0', ifcfg_eth0)
def main():
# Get parameters from instance metadata.
image_id = utils.GetMetadataAttribute('image_id')
image_name = utils.GetMetadataAttribute('image_name')
image_family = utils.GetMetadataAttribute('image_family')
distribution = utils.GetMetadataAttribute('distribution',
raise_on_not_found=True)
uefi = utils.GetMetadataAttribute('uefi', 'false').lower() == 'true'
outs_path = utils.GetMetadataAttribute('daisy-outs-path')
logging.info('Creating upload metadata of the image and packages.')
utc_time = datetime.datetime.now(datetime.timezone.utc)
image_version = image_name.rsplit("v")[-1]
publish_date = utc_time.astimezone().isoformat()
image = {
'id': image_id,
'name': image_name,
'family': image_family,
'version': image_version,
'publish_date': publish_date,
'packages': [],
}
# All the guest environment packages maintained by guest-os team.
guest_packages = [
'google-compute-engine',
'google-compute-engine-oslogin',
'google-guest-agent',
'google-osconfig-agent',
'gce-disk-expand',
]
# This assumes that:
# 1. /dev/sdb1 is the EFI system partition.
# 2. /dev/sdb2 is the root mount for the installed system.
# Except for debian 10, which has out-of-order partitions.
if uefi and 'debian-10' not in image_family:
mount_disk = '/dev/sdb2'
else:
mount_disk = '/dev/sdb1'
subprocess.run(['mount', mount_disk, '/mnt'], check=False)
logging.info('Mount %s device to /mnt', mount_disk)
if distribution == 'enterprise_linux':
# chroot prevents access to /dev/random and /dev/urandom (as designed).
# The rpm required those random bits to initialize GnuTLS otherwise
# error: Failed to initialize NSS library.
subprocess.run(['mount', '-o', 'bind', '/dev', '/mnt/dev'],
check=False)
if distribution == 'debian':
# This package is debian-only.
guest_packages.append('google-cloud-packages-archive-keyring')
cmd_prefix = [
'chroot', '/mnt', 'dpkg-query', '-W', '--showformat',
'${Package}\n\n${Version}\n${Git}'
]
elif distribution == 'enterprise_linux':
cmd_prefix = [
'chroot', '/mnt', 'rpm', '-q', '--queryformat',
'%{NAME}\n%{EPOCH}\n%{VERSION}-%{RELEASE}\n%{VCS}'
]
else:
logging.error('Unknown Linux distribution.')
return
for package in guest_packages:
try:
process = subprocess.run(cmd_prefix + [package],
capture_output=True,
check=True)
except subprocess.CalledProcessError as e:
logging.info('failed to execute cmd: %s stdout: %s stderr: %s', e,
e.stdout, e.stderr)
continue
stdout = process.stdout.decode()
try:
package, epoch, version, commit_hash = stdout.split('\n', 3)
except ValueError:
logging.info('command result was malformed: %s', stdout)
continue
md = make_pkg_metadata(package, version, epoch, commit_hash)
image['packages'].append(md)
# Write image metadata to a file.
with tempfile.NamedTemporaryFile(mode='w', dir='/tmp', delete=False) as f:
f.write(json.dumps(image))
# We upload the result to the daisy outs path as well, to aid in
# troubleshooting.
logging.info('Uploading image metadata to daisy outs path.')
try:
utils.UploadFile(f.name, outs_path + "/metadata.json")
except Exception as e:
logging.error('Failed uploading metadata file %s', e)
return
logging.success('Export metadata was successful!')
def main():
# Get Parameters.
bvz_manifest = utils.GetMetadataAttribute('bootstrap_vz_manifest',
raise_on_not_found=True)
bvz_version = utils.GetMetadataAttribute('bootstrap_vz_version',
raise_on_not_found=True)
repo = utils.GetMetadataAttribute('google_cloud_repo',
raise_on_not_found=True).strip()
image_dest = utils.GetMetadataAttribute('image_dest',
raise_on_not_found=True)
outs_path = utils.GetMetadataAttribute('daisy-outs-path',
raise_on_not_found=True)
if repo not in REPOS:
raise ValueError('Metadata "google_cloud_repo" must be one of %s.' %
REPOS)
logging.info('Bootstrap_vz manifest: %s' % bvz_manifest)
logging.info('Bootstrap_vz version: %s' % bvz_version)
logging.info('Google Cloud repo: %s' % repo)
# Download and setup bootstrap_vz.
bvz_url = 'https://github.com/andsens/bootstrap-vz/archive/%s.zip'
bvz_url %= bvz_version
bvz_zip_dir = 'bvz_zip'
logging.info('Downloading bootstrap-vz at commit %s' % bvz_version)
urllib.request.urlretrieve(bvz_url, 'bvz.zip')
with zipfile.ZipFile('bvz.zip', 'r') as z:
z.extractall(bvz_zip_dir)
logging.info('Downloaded and extracted %s to bvz.zip.' % bvz_url)
bvz_zip_contents = [d for d in os.listdir(bvz_zip_dir)]
bvz_zip_subdir = os.path.join(bvz_zip_dir, bvz_zip_contents[0])
utils.Execute(['mv', bvz_zip_subdir, BVZ_DIR])
logging.info('Moved bootstrap_vz from %s to %s.' %
(bvz_zip_subdir, BVZ_DIR))
bvz_bin = os.path.join(BVZ_DIR, 'bootstrap-vz')
utils.MakeExecutable(bvz_bin)
logging.info('Made %s executable.' % bvz_bin)
bvz_manifest_file = os.path.join(BVZ_DIR, 'manifests', bvz_manifest)
# Inject Google Cloud test repo plugin if using staging or unstable repos.
# This is used to test new package releases in images.
if repo != 'stable':
logging.info('Adding Google Cloud test repos plugin for bootstrapvz.')
repo_plugin_dir = '/files/google_cloud_test_repos'
bvz_plugins = os.path.join(BVZ_DIR, 'bootstrapvz', 'plugins')
shutil.move(repo_plugin_dir, bvz_plugins)
with open(bvz_manifest_file, 'r+') as manifest_file:
manifest_data = yaml.load(manifest_file)
manifest_plugins = manifest_data['plugins']
manifest_plugins['google_cloud_test_repos'] = {repo: True}
manifest_yaml = yaml.dump(manifest_data, default_flow_style=False)
manifest_file.write(manifest_yaml)
# Run bootstrap_vz build.
cmd = [bvz_bin, '--debug', bvz_manifest_file]
logging.info('Starting build in %s with params: %s' % (BVZ_DIR, str(cmd)))
utils.Execute(cmd, cwd=BVZ_DIR)
# Upload tar.
image_tar_gz = '/target/disk.tar.gz'
if os.path.exists(image_tar_gz):
logging.info('Saving %s to %s' % (image_tar_gz, image_dest))
utils.UploadFile(image_tar_gz, image_dest)
# Create and upload the synopsis of the image.
logging.info('Creating image synopsis.')
synopsis = {}
packages = collections.OrderedDict()
_, output = utils.Execute(['dpkg-query', '-W'], capture_output=True)
for line in output.split('\n')[:-1]: # Last line is an empty line.
parts = line.split()
packages[parts[0]] = parts[1]
synopsis['installed_packages'] = packages
with open('/tmp/synopsis.json', 'w') as f:
f.write(json.dumps(synopsis))
logging.info('Uploading image synopsis.')
synopsis_dest = os.path.join(outs_path, 'synopsis.json')
utils.UploadFile('/tmp/synopsis.json', synopsis_dest)
def main():
# Get Parameters
repo = utils.GetMetadataAttribute('google_cloud_repo',
raise_on_not_found=True)
release = utils.GetMetadataAttribute('el_release', raise_on_not_found=True)
daisy_logs_path = utils.GetMetadataAttribute('daisy-logs-path',
raise_on_not_found=True)
savelogs = utils.GetMetadataAttribute('el_savelogs') == 'true'
byos = utils.GetMetadataAttribute('rhel_byos') == 'true'
sap = utils.GetMetadataAttribute('rhel_sap') == 'true'
uefi = utils.GetMetadataAttribute('rhel_uefi') == 'true'
logging.info('EL Release: %s' % release)
logging.info('Google Cloud repo: %s' % repo)
logging.info('Build working directory: %s' % os.getcwd())
iso_file = '/files/installer.iso'
# Necessary libs and tools to build the installer disk.
utils.AptGetInstall(['dosfstools', 'rsync'])
# Build the kickstart file.
ks_content = ks_helpers.BuildKsConfig(release, repo, byos, sap, uefi)
ks_cfg = 'ks.cfg'
utils.WriteFile(ks_cfg, ks_content)
# Save the generated kickstart file to the build logs.
utils.UploadFile(ks_cfg, '%s/ks.cfg' % daisy_logs_path)
# Write the installer disk. Write GPT label, create partition,
# copy installer boot files over.
logging.info('Writing installer disk.')
utils.Execute(['parted', '/dev/sdb', 'mklabel', 'gpt'])
utils.Execute(['sync'])
utils.Execute(['parted', '/dev/sdb', 'mkpart', 'primary', 'fat32', '1MB',
'1024MB'])
utils.Execute(['sync'])
utils.Execute(['parted', '/dev/sdb', 'mkpart', 'primary', 'ext2', '1024MB',
'100%'])
utils.Execute(['sync'])
utils.Execute(['parted', '/dev/sdb', 'set', '1', 'boot', 'on'])
utils.Execute(['sync'])
utils.Execute(['parted', '/dev/sdb', 'set', '1', 'esp', 'on'])
utils.Execute(['sync'])
utils.Execute(['mkfs.vfat', '-F', '32', '/dev/sdb1'])
utils.Execute(['sync'])
utils.Execute(['fatlabel', '/dev/sdb1', 'ESP'])
utils.Execute(['sync'])
utils.Execute(['mkfs.ext2', '-L', 'INSTALLER', '/dev/sdb2'])
utils.Execute(['sync'])
utils.Execute(['mkdir', '-vp', 'iso', 'installer', 'boot'])
utils.Execute(['mount', '-o', 'ro,loop', '-t', 'iso9660', iso_file, 'iso'])
utils.Execute(['mount', '-t', 'vfat', '/dev/sdb1', 'boot'])
utils.Execute(['mount', '-t', 'ext2', '/dev/sdb2', 'installer'])
utils.Execute(['rsync', '-Pav', 'iso/EFI', 'iso/images', 'boot/'])
utils.Execute(['cp', iso_file, 'installer/'])
utils.Execute(['cp', ks_cfg, 'installer/'])
# Modify boot config.
with open('boot/EFI/BOOT/grub.cfg', 'r+') as f:
oldcfg = f.read()
cfg = re.sub(r'-l .RHEL.*', r"""-l 'ESP'""", oldcfg)
cfg = re.sub(r'timeout=60', 'timeout=1', cfg)
cfg = re.sub(r'set default=.*', 'set default="0"', cfg)
cfg = re.sub(r'load_video\n',
r'serial --speed=38400 --unit=0 --word=8 --parity=no\n'
'terminal_input serial\nterminal_output serial\n', cfg)
# Change boot args.
args = ' '.join([
'text', 'ks=hd:LABEL=INSTALLER:/%s' % ks_cfg,
'console=ttyS0,38400n8', 'inst.gpt', 'loglevel=debug'
])
# Tell Anaconda not to store its logs in the installed image,
# unless requested to keep them for debugging.
if not savelogs:
args += ' inst.nosave=all'
cfg = re.sub(r'inst\.stage2.*', r'\g<0> %s' % args, cfg)
# Change labels to explicit partitions.
if release.startswith(('centos7', 'rhel7', 'rhel-7', 'oraclelinux7',
'centos8', 'rhel8', 'rhel-8')):
cfg = re.sub(r'LABEL=[^ ]+', 'LABEL=INSTALLER', cfg)
# Print out a the modifications.
diff = difflib.Differ().compare(
oldcfg.splitlines(1),
cfg.splitlines(1))
logging.info('Modified grub.cfg:\n%s' % '\n'.join(diff))
f.seek(0)
f.write(cfg)
f.truncate()
utils.Execute(['umount', 'installer'])
utils.Execute(['umount', 'iso'])
utils.Execute(['umount', 'boot'])
def main():
# Get Parameters.
build_date = utils.GetMetadataAttribute(
'build_date', raise_on_not_found=True)
debian_cloud_images_version = utils.GetMetadataAttribute(
'debian_cloud_images_version', raise_on_not_found=True)
debian_version = utils.GetMetadataAttribute(
'debian_version', raise_on_not_found=True)
image_dest = utils.GetMetadataAttribute('image_dest',
raise_on_not_found=True)
outs_path = utils.GetMetadataAttribute('daisy-outs-path',
raise_on_not_found=True)
logging.info('debian-cloud-images version: %s' % debian_cloud_images_version)
logging.info('debian version: %s' % debian_version)
# force an apt-get update before next install
utils.AptGetInstall.first_run = True
debian_host_version = utils.Execute(['cat', '/etc/debian_version'],
capture_output=True)
# the FAI's version in stretch does not satisfy our need, so the version from
# stretch-backports is needed.
if debian_host_version[1].startswith('9'):
utils.AptGetInstall(['fai-server', 'fai-setup-storage'],
'stretch-backports')
else:
utils.AptGetInstall(['fai-server', 'fai-setup-storage'])
# Download and setup debian's debian-cloud-images scripts.
url_params = {
'project': 'debian-cloud-images',
'version': debian_cloud_images_version,
}
url_params['filename'] = '%(project)s-%(version)s' % url_params
url = "https://salsa.debian.org/cloud-team/" + \
"%(project)s/-/archive/%(version)s/%(filename)s.tar.gz" % url_params
logging.info('Downloading %(project)s at version %(version)s' % url_params)
urllib.request.urlretrieve(url, 'fci.tar.gz')
with tarfile.open('fci.tar.gz') as tar:
tar.extractall()
logging.info('Downloaded and extracted %s.' % url)
# Config fai-tool
work_dir = url_params['filename']
fai_classes = ['DEBIAN', 'CLOUD', 'GCE', 'GCE_SDK', 'AMD64',
'GRUB_CLOUD_AMD64', 'LINUX_IMAGE_CLOUD']
if debian_version == 'stretch':
fai_classes += ['STRETCH', 'BACKPORTS', 'BACKPORTS_LINUX']
elif debian_version == 'buster':
fai_classes += ['BUSTER']
elif debian_version == 'sid':
fai_classes += ['SID']
image_size = '10G'
disk_name = 'disk.raw'
config_space = os.getcwd() + work_dir + '/config_space/'
# Copy GCE_SPECIFIC fai class
utils.Execute(['cp', '/files/fai_config/packages/GCE_SPECIFIC',
config_space + 'package_config/GCE_SPECIFIC'])
fai_classes += ['GCE_SPECIFIC']
# Run fai-tool.
cmd = ['fai-diskimage', '--verbose', '--hostname', 'debian', '--class',
','.join(fai_classes), '--size', image_size, '--cspace', config_space,
disk_name]
logging.info('Starting build in %s with params: %s' % (
work_dir, ' '.join(cmd))
)
utils.Execute(cmd, cwd=work_dir, capture_output=True)
# Packs a gzipped tar file with disk.raw inside
disk_tar_gz = 'debian-{}-{}.tar.gz'.format(debian_version, build_date)
logging.info('Compressing it into tarball %s' % disk_tar_gz)
tar = tarfile.open(disk_tar_gz, "w:gz")
tar.add('%s/disk.raw' % work_dir, arcname="disk.raw")
tar.close()
# Upload tar.
logging.info('Saving %s to %s' % (disk_tar_gz, image_dest))
utils.UploadFile(disk_tar_gz, image_dest)
# Create and upload the synopsis of the image.
logging.info('Creating image synopsis.')
synopsis = {}
packages = collections.OrderedDict()
_, output = utils.Execute(['dpkg-query', '-W'], capture_output=True)
for line in output.split('\n')[:-1]: # Last line is an empty line.
parts = line.split()
packages[parts[0]] = parts[1]
synopsis['installed_packages'] = packages
with open('/tmp/synopsis.json', 'w') as f:
f.write(json.dumps(synopsis))
logging.info('Uploading image synopsis.')
synopsis_dest = os.path.join(outs_path, 'synopsis.json')
utils.UploadFile('/tmp/synopsis.json', synopsis_dest)
def DistroSpecific(g):
ubu_release = utils.GetMetadataAttribute('ubuntu_release')
install_gce = utils.GetMetadataAttribute('install_gce_packages')
# Remove any hard coded DNS settings in resolvconf.
logging.info('Resetting resolvconf base.')
g.sh('echo "" > /etc/resolvconf/resolv.conf.d/base')
# Try to reset the network to DHCP.
if ubu_release == 'trusty':
g.write('/etc/network/interfaces', trusty_network)
elif ubu_release == 'xenial':
g.write('/etc/network/interfaces', xenial_network)
if install_gce == 'true':
g.command(['apt-get', 'update'])
logging.info('Installing cloud-init.')
g.sh(
'DEBIAN_FRONTEND=noninteractive apt-get install -y'
' --no-install-recommends cloud-init')
# Try to remove azure or aws configs so cloud-init has a chance.
g.sh('rm -f /etc/cloud/cloud.cfg.d/*azure*')
g.sh('rm -f /etc/cloud/cloud.cfg.d/*waagent*')
g.sh('rm -f /etc/cloud/cloud.cfg.d/*walinuxagent*')
g.sh('rm -f /etc/cloud/cloud.cfg.d/*aws*')
g.sh('rm -f /etc/cloud/cloud.cfg.d/*amazon*')
# Remove Azure agent.
try:
g.command(['apt-get', 'remove', '-y', '-f', 'waagent', 'walinuxagent'])
except Exception as e:
logging.debug(str(e))
logging.warn('Could not uninstall Azure agent. Continuing anyway.')
g.write(
'/etc/apt/sources.list.d/partner.list',
partner_list.format(ubu_release=ubu_release))
g.write('/etc/cloud/cloud.cfg.d/91-gce-system.cfg', gce_system)
# Use host machine as http proxy so cloud-init can access GCE API
with open('/etc/tinyproxy/tinyproxy.conf', 'w') as cfg:
cfg.write(tinyproxy_cfg)
utils.Execute(['/etc/init.d/tinyproxy', 'restart'])
default_gw = g.sh("ip route | awk '/default/ { printf $3 }'")
logging.debug(
g.sh('http_proxy="http://%s:8888" cloud-init -d init' % default_gw))
logging.info('Installing GCE packages.')
g.command(['apt-get', 'update'])
g.sh(
'DEBIAN_FRONTEND=noninteractive apt-get install -y'
' --no-install-recommends gce-compute-image-packages google-cloud-sdk')
# Update grub config to log to console.
g.command(
['sed', '-i',
r's#^\(GRUB_CMDLINE_LINUX=".*\)"$#\1 console=ttyS0,38400n8"#',
'/etc/default/grub'])
g.command(['update-grub2'])
def DistroSpecific(g):
ubu_release = utils.GetMetadataAttribute('ubuntu_release')
install_gce = utils.GetMetadataAttribute('install_gce_packages')
# Remove any hard coded DNS settings in resolvconf.
if ubu_release != 'bionic':
logging.info('Resetting resolvconf base.')
g.sh('echo "" > /etc/resolvconf/resolv.conf.d/base')
# Try to reset the network to DHCP.
if ubu_release == 'trusty':
g.write('/etc/network/interfaces', trusty_network)
elif ubu_release == 'xenial':
g.write('/etc/network/interfaces', xenial_network)
if install_gce == 'true':
utils.update_apt(g)
logging.info('Installing cloud-init.')
utils.install_apt_package(g, 'cloud-init')
# Try to remove azure or aws configs so cloud-init has a chance.
g.sh('rm -f /etc/cloud/cloud.cfg.d/*azure*')
g.sh('rm -f /etc/cloud/cloud.cfg.d/*curtin*')
g.sh('rm -f /etc/cloud/cloud.cfg.d/*waagent*')
g.sh('rm -f /etc/cloud/cloud.cfg.d/*walinuxagent*')
g.sh('rm -f /etc/cloud/cloud.cfg.d/*aws*')
g.sh('rm -f /etc/cloud/cloud.cfg.d/*amazon*')
if ubu_release == 'bionic':
g.sh('rm -f /etc/netplan/*')
logging.debug(g.sh('cloud-init clean'))
remove_azure_agents(g)
g.write('/etc/apt/sources.list.d/partner.list',
partner_list.format(ubu_release=ubu_release))
g.write('/etc/cloud/cloud.cfg.d/91-gce-system.cfg', gce_system)
# Use host machine as http proxy so cloud-init can access GCE API
with open('/etc/tinyproxy/tinyproxy.conf', 'w') as cfg:
cfg.write(tinyproxy_cfg)
utils.Execute(['/etc/init.d/tinyproxy', 'restart'])
default_gw = g.sh("ip route | awk '/default/ { printf $3 }'")
try:
logging.debug(
g.sh('http_proxy="http://%s:8888" cloud-init -d init' %
default_gw))
except Exception as e:
logging.debug('Failed to run cloud-init. Details: {}.'.format(e))
raise RuntimeError(
'Failed to run cloud-init. Connect to a shell in the original VM '
'and ensure that the following command executes successfully: '
'apt-get install -y --no-install-recommends cloud-init '
'&& cloud-init -d init')
logging.info('Installing GCE packages.')
utils.update_apt(g)
utils.install_apt_package(g, 'gce-compute-image-packages')
utils.install_apt_package(g, 'google-cloud-sdk')
# Update grub config to log to console.
g.command([
'sed', '-i',
r's#^\(GRUB_CMDLINE_LINUX=".*\)"$#\1 console=ttyS0,38400n8"#',
'/etc/default/grub'
])
g.command(['update-grub2'])
def main():
# Get Parameters
repo = utils.GetMetadataAttribute('google_cloud_repo',
raise_on_not_found=True)
release = utils.GetMetadataAttribute('el_release', raise_on_not_found=True)
daisy_logs_path = utils.GetMetadataAttribute('daisy-logs-path',
raise_on_not_found=True)
savelogs = utils.GetMetadataAttribute('el_savelogs',
raise_on_not_found=False) == 'true'
byos = utils.GetMetadataAttribute('rhel_byos',
raise_on_not_found=False) == 'true'
sap = utils.GetMetadataAttribute('rhel_sap',
raise_on_not_found=False) == 'true'
logging.info('EL Release: %s' % release)
logging.info('Google Cloud repo: %s' % repo)
logging.info('Build working directory: %s' % os.getcwd())
iso_file = '/files/installer.iso'
# Necessary libs and tools to build the installer disk.
utils.AptGetInstall(['extlinux', 'rsync'])
# Build the kickstart file.
uefi = False
ks_content = ks_helpers.BuildKsConfig(release, repo, byos, sap, uefi)
ks_cfg = 'ks.cfg'
utils.WriteFile(ks_cfg, ks_content)
# Save the generated kickstart file to the build logs.
utils.UploadFile(ks_cfg, '%s/ks.cfg' % daisy_logs_path)
# Write the installer disk. Write extlinux MBR, create partition,
# copy installer ISO and ISO boot files over.
logging.info('Writing installer disk.')
utils.Execute(['parted', '/dev/sdb', 'mklabel', 'msdos'])
utils.Execute(['sync'])
utils.Execute(['parted', '/dev/sdb', 'mkpart', 'primary', '1MB', '100%'])
utils.Execute(['sync'])
utils.Execute(['parted', '/dev/sdb', 'set', '1', 'boot', 'on'])
utils.Execute(['sync'])
utils.Execute(['dd', 'if=/usr/lib/EXTLINUX/mbr.bin', 'of=/dev/sdb'])
utils.Execute(['sync'])
utils.Execute(['mkfs.ext2', '-L', 'INSTALLER', '/dev/sdb1'])
utils.Execute(['sync'])
utils.Execute(['mkdir', 'iso', 'installer'])
utils.Execute(['mount', '-o', 'ro,loop', '-t', 'iso9660', iso_file, 'iso'])
utils.Execute(['mount', '-t', 'ext2', '/dev/sdb1', 'installer'])
utils.Execute(
['rsync', '-Pav', 'iso/images', 'iso/isolinux', 'installer/'])
utils.Execute(['cp', iso_file, 'installer/'])
utils.Execute(['cp', ks_cfg, 'installer/'])
# Modify boot files on installer disk.
utils.Execute(['mv', 'installer/isolinux', 'installer/extlinux'])
utils.Execute([
'mv', 'installer/extlinux/isolinux.cfg',
'installer/extlinux/extlinux.conf'
])
# Modify boot config.
with open('installer/extlinux/extlinux.conf', 'r+') as f:
oldcfg = f.read()
cfg = re.sub(r'^default.*', r'default linux', oldcfg, count=1)
# Change boot args.
args = ' '.join([
'text',
'ks=hd:/dev/sda1:/%s' % ks_cfg, 'console=ttyS0,38400n8',
'loglevel=debug'
])
# Tell Anaconda not to store its logs in the installed image,
# unless requested to keep them for debugging.
if not savelogs:
args += ' inst.nosave=all'
cfg = re.sub(r'append initrd=initrd\.img.*', r'\g<0> %s' % args, cfg)
# Change labels to explicit partitions.
if release.startswith(('centos7', 'rhel7', 'rhel-7', 'oraclelinux7',
'centos8', 'rhel8')):
cfg = re.sub(r'LABEL=[^ ]+', 'LABEL=INSTALLER', cfg)
# Print out a the modifications.
diff = difflib.Differ().compare(oldcfg.splitlines(1),
cfg.splitlines(1))
logging.info('Modified extlinux.conf:\n%s' % '\n'.join(diff))
f.seek(0)
f.write(cfg)
f.truncate()
# Activate extlinux.
utils.Execute(['extlinux', '--install', 'installer/extlinux'])
def main():
# Get Parameters.
build_date = utils.GetMetadataAttribute(
'build_date', raise_on_not_found=True)
debian_cloud_images_version = utils.GetMetadataAttribute(
'debian_cloud_images_version', raise_on_not_found=True)
debian_version = utils.GetMetadataAttribute(
'debian_version', raise_on_not_found=True)
uefi = utils.GetMetadataAttribute('uefi', raise_on_not_found=True)
image_dest = utils.GetMetadataAttribute('image_dest',
raise_on_not_found=True)
outs_path = utils.GetMetadataAttribute('daisy-outs-path',
raise_on_not_found=True)
logging.info('debian-cloud-images version: %s' % debian_cloud_images_version)
logging.info('debian version: %s' % debian_version)
# First, install fai-client from fai-project repository
key_url = 'https://fai-project.org/download/2BF8D9FE074BCDE4.asc'
urllib.request.urlretrieve(key_url, 'key.asc')
utils.Execute(['apt-key', 'add', 'key.asc'])
with open('/etc/apt/sources.list.d/fai-project.list', 'w') as fai_list:
fai_list.write('deb https://fai-project.org/download stretch koeln')
# force an apt-get update before next install
utils.AptGetInstall.first_run = True
utils.AptGetInstall(['fai-server', 'fai-setup-storage'])
# Download and setup debian's debian-cloud-images scripts.
url_params = {
'project': 'debian-cloud-images',
'commit': debian_cloud_images_version,
}
url_params['filename'] = '%(project)s-%(commit)s' % url_params
url = "https://salsa.debian.org/cloud-team/" + \
"%(project)s/-/archive/%(commit)s/%(filename)s.tar.gz" % url_params
logging.info('Downloading %(project)s at commit %(commit)s' % url_params)
urllib.request.urlretrieve(url, 'fci.tar.gz')
with tarfile.open('fci.tar.gz') as tar:
tar.extractall()
logging.info('Downloaded and extracted %s.' % url)
# Run fai-tool.
work_dir = url_params['filename']
fai_bin = 'bin/build'
arch = 'amd64-efi' if uefi else 'amd64'
cmd = [fai_bin, debian_version, 'gce', arch, 'disk', build_date]
logging.info('Starting build in %s with params: %s' % (
work_dir, ' '.join(cmd))
)
utils.Execute(cmd, cwd=work_dir, capture_output=True)
# Packs a gzipped tar file with disk.raw inside
disk_tar_gz = 'disk.tar.gz'
logging.info('Compressing it into tarball %s' % disk_tar_gz)
tar = tarfile.open(disk_tar_gz, "w:gz")
tar.add('%s/disk.raw' % work_dir, arcname="disk.raw")
tar.close()
# Upload tar.
logging.info('Saving %s to %s' % (disk_tar_gz, image_dest))
utils.UploadFile(disk_tar_gz, image_dest)
# Create and upload the synopsis of the image.
logging.info('Creating image synopsis.')
synopsis = {}
packages = collections.OrderedDict()
_, output = utils.Execute(['dpkg-query', '-W'], capture_output=True)
for line in output.split('\n')[:-1]: # Last line is an empty line.
parts = line.split()
packages[parts[0]] = parts[1]
synopsis['installed_packages'] = packages
with open('/tmp/synopsis.json', 'w') as f:
f.write(json.dumps(synopsis))
logging.info('Uploading image synopsis.')
synopsis_dest = os.path.join(outs_path, 'synopsis.json')
utils.UploadFile('/tmp/synopsis.json', synopsis_dest)
def main():
# Get parameters from instance metadata.
metadata_dest = utils.GetMetadataAttribute('metadata_dest',
raise_on_not_found=True)
image_id = utils.GetMetadataAttribute('image_id')
image_name = utils.GetMetadataAttribute('image_name')
image_family = utils.GetMetadataAttribute('image_family')
distribution = utils.GetMetadataAttribute('distribution',
raise_on_not_found=True)
uefi = utils.GetMetadataAttribute('uefi', 'false').lower() == 'true'
logging.info('Creating upload metadata of the image and packages.')
utc_time = datetime.datetime.now(datetime.timezone.utc)
image_version = utc_time.strftime('%Y%m%d')
build_date = utc_time.astimezone().isoformat()
image = {
'id': image_id,
'name': image_name,
'family': image_family,
'version': image_version,
'build_date': build_date,
'packages': [],
}
# All the guest environment packages maintained by guest-os team.
guest_packages = [
'google-cloud-packages-archive-keyring',
'google-compute-engine',
'google-compute-engine-oslogin',
'google-guest-agent',
'google-osconfig-agent',
'gce-disk-expand',
]
# This assumes that:
# 1. /dev/sdb1 is the EFI system partition.
# 2. /dev/sdb2 is the root mount for the installed system.
if uefi:
mount_disk = '/dev/sdb2'
else:
mount_disk = '/dev/sdb1'
subprocess.run(['mount', mount_disk, '/mnt'], check=False)
logging.info('Mount %s device to /mnt', mount_disk)
if distribution == 'enterprise_linux':
# chroot prevents access to /dev/random and /dev/urandom (as designed).
# The rpm required those random bits to initialize GnuTLS otherwise
# error: Failed to initialize NSS library.
subprocess.run(['mount', '-o', 'bind', '/dev', '/mnt/dev'],
check=False)
has_commit_hash = True
if distribution == 'debian':
cmd_prefix = [
'chroot', '/mnt', 'dpkg-query', '-W', '--showformat',
'${Package}\n${Version}\n${Git}'
]
elif distribution == 'enterprise_linux':
if 'centos-6' in image_family or 'rhel-6' in image_family:
# centos-6 and rhel-6 doesn't support vcs tag
cmd_prefix = [
'chroot', '/mnt', 'rpm', '-q', '--queryformat',
'%{NAME}\n%{VERSION}-%{RELEASE}'
]
has_commit_hash = False
else:
cmd_prefix = [
'chroot', '/mnt', 'rpm', '-q', '--queryformat',
'%{NAME}\n%{VERSION}-%{RELEASE}\n%{VCS}'
]
else:
logging.error('Unknown Linux distribution.')
return Exception
version, commit_hash = '', ''
for package in guest_packages:
cmd = cmd_prefix + [package]
try:
stdout = subprocess.run(cmd, stdout=subprocess.PIPE,
check=True).stdout
stdout = stdout.decode()
logging.info('Package metadata is %s', stdout)
except subprocess.CalledProcessError as e:
logging.warning('Fail to execute cmd. %s', e)
continue
if has_commit_hash:
package, version, commit_hash = stdout.split('\n', 2)
else:
package, version = stdout.split('\n', 1)
package_metadata = {
'name': package,
'version': version,
'commit_hash': commit_hash,
}
image['packages'].append(package_metadata)
# Write image metadata to a file.
with tempfile.NamedTemporaryFile(mode='w', dir='/tmp', delete=False) as f:
f.write(json.dumps(image))
logging.info('Uploading image metadata.')
try:
utils.UploadFile(f.name, metadata_dest)
except ValueError as e:
logging.exception('ExportFailed: Failed uploading metadata file %s', e)
sys.exit(1)
logging.info('ExportSuccess: Export metadata was successful!')
def main():
# Get Parameters.
build_date = utils.GetMetadataAttribute('build_date',
raise_on_not_found=True)
debian_cloud_images_version = utils.GetMetadataAttribute(
'debian_cloud_images_version', raise_on_not_found=True)
debian_version = utils.GetMetadataAttribute('debian_version',
raise_on_not_found=True)
google_cloud_repo = utils.GetMetadataAttribute('google_cloud_repo',
raise_on_not_found=True)
outs_path = utils.GetMetadataAttribute('daisy-outs-path',
raise_on_not_found=True)
logging.info('debian-cloud-images version: %s' %
debian_cloud_images_version)
logging.info('debian version: %s' % debian_version)
# force an apt-get update before next install
utils.AptGetInstall.first_run = True
utils.AptGetInstall(['fai-server', 'fai-setup-storage'])
# Download and setup debian's debian-cloud-images scripts.
url_params = {
'project': 'debian-cloud-images',
'version': debian_cloud_images_version,
}
url_params['filename'] = '%(project)s-%(version)s' % url_params
url = ('https://salsa.debian.org/cloud-team/'
'%(project)s/-/archive/%(version)s/%(filename)s.tar.gz' %
url_params)
logging.info('Downloading %(project)s at version %(version)s' % url_params)
urllib.request.urlretrieve(url, 'fci.tar.gz')
with tarfile.open('fci.tar.gz') as tar:
tar.extractall()
logging.info('Downloaded and extracted %s.' % url)
# Config fai-tool
work_dir = url_params['filename']
fai_classes = [
'DEBIAN', 'CLOUD', 'GCE', 'GCE_SDK', 'AMD64', 'GRUB_CLOUD_AMD64',
'LINUX_IMAGE_CLOUD'
]
if debian_version == 'buster':
fai_classes += ['BUSTER', 'BACKPORTS']
elif debian_version == "bullseye":
fai_classes += ['BULLSEYE']
elif debian_version == 'sid':
fai_classes += ['SID']
image_size = '10G'
disk_name = 'disk.raw'
config_space = os.getcwd() + work_dir + '/config_space/'
apt_sources_base = 'files/etc/apt/sources.list.d/'
# Copy GCE_SPECIFIC fai classes.
CopyToConfigSpace('/files/fai_config/packages/GCE_SPECIFIC',
'package_config/GCE_SPECIFIC', config_space)
os.mkdir(config_space + apt_sources_base + 'google-cloud.list')
CopyToConfigSpace('/files/fai_config/sources/GCE_SPECIFIC',
apt_sources_base + 'google-cloud.list/GCE_SPECIFIC',
config_space)
CopyToConfigSpace('/files/fai_config/sources/file_modes',
apt_sources_base + '/google-cloud.list/file_modes',
config_space)
CopyToConfigSpace('/files/fai_config/sources/repository.GCE_SPECIFIC',
'hooks/repository.GCE_SPECIFIC', config_space)
fai_classes += ['GCE_SPECIFIC']
# GCE staging package repo.
if google_cloud_repo == 'staging' or google_cloud_repo == 'unstable':
os.mkdir(config_space + apt_sources_base + 'google-cloud-staging.list')
CopyToConfigSpace(
'/files/fai_config/sources/GCE_STAGING',
apt_sources_base + 'google-cloud-staging.list/GCE_STAGING',
config_space)
CopyToConfigSpace(
'/files/fai_config/sources/file_modes',
apt_sources_base + 'google-cloud-staging.list/file_modes',
config_space)
CopyToConfigSpace('/files/fai_config/sources/repository.GCE_STAGING',
'hooks/repository.GCE_STAGING', config_space)
fai_classes += ['GCE_STAGING']
# GCE unstable package repo.
if google_cloud_repo == 'unstable':
os.mkdir(config_space + apt_sources_base +
'google-cloud-unstable.list')
CopyToConfigSpace(
'/files/fai_config/sources/GCE_UNSTABLE',
apt_sources_base + 'google-cloud-unstable.list/GCE_UNSTABLE',
config_space)
CopyToConfigSpace(
'/files/fai_config/sources/file_modes',
apt_sources_base + 'google-cloud-unstable.list/file_modes',
config_space)
CopyToConfigSpace('/files/fai_config/sources/file_modes',
'hooks/repository.GCE_UNSTABLE', config_space)
fai_classes += ['GCE_UNSTABLE']
# Cleanup class for GCE.
os.mkdir(config_space + 'scripts/GCE_CLEAN')
CopyToConfigSpace('/files/fai_config/scripts/10-gce-clean',
'scripts/GCE_CLEAN/10-gce-clean', config_space)
os.chmod(config_space + 'scripts/GCE_CLEAN/10-gce-clean', 0o755)
fai_classes += ['GCE_CLEAN']
# Remove failing test method for now.
os.remove(config_space + 'hooks/tests.CLOUD')
# Run fai-tool.
cmd = [
'fai-diskimage', '--verbose', '--hostname', 'debian', '--class',
','.join(fai_classes), '--size', image_size, '--cspace', config_space,
disk_name
]
logging.info('Starting build in %s with params: %s' %
(work_dir, ' '.join(cmd)))
utils.Execute(cmd, cwd=work_dir, capture_output=True)
# Packs a gzipped tar file with disk.raw inside
disk_tar_gz = 'debian-{}-{}.tar.gz'.format(debian_version, build_date)
logging.info('Compressing it into tarball %s' % disk_tar_gz)
tar = tarfile.open(disk_tar_gz, 'w:gz')
tar.add('%s/disk.raw' % work_dir, arcname='disk.raw')
tar.close()
# Upload tar.
image_dest = os.path.join(outs_path, 'root.tar.gz')
logging.info('Saving %s to %s' % (disk_tar_gz, image_dest))
utils.UploadFile(disk_tar_gz, image_dest)
def main():
# Get Parameters
release = utils.GetMetadataAttribute('el_release', raise_on_not_found=True)
savelogs = utils.GetMetadataAttribute('el_savelogs') == 'true'
logging.info('EL Release: %s' % release)
logging.info('Build working directory: %s' % os.getcwd())
iso_file = '/files/installer.iso'
ks_cfg = '/files/ks.cfg'
utils.AptGetInstall(['rsync'])
# Write the installer disk. Write GPT label, create partition,
# copy installer boot files over.
logging.info('Writing installer disk.')
utils.Execute(['parted', '/dev/sdb', 'mklabel', 'gpt'])
utils.Execute(['sync'])
utils.Execute(
['parted', '/dev/sdb', 'mkpart', 'primary', 'fat32', '1MB', '1024MB'])
utils.Execute(['sync'])
utils.Execute(
['parted', '/dev/sdb', 'mkpart', 'primary', 'ext2', '1024MB', '100%'])
utils.Execute(['sync'])
utils.Execute(['parted', '/dev/sdb', 'set', '1', 'boot', 'on'])
utils.Execute(['sync'])
utils.Execute(['parted', '/dev/sdb', 'set', '1', 'esp', 'on'])
utils.Execute(['sync'])
utils.Execute(['mkfs.vfat', '-F', '32', '/dev/sdb1'])
utils.Execute(['sync'])
utils.Execute(['fatlabel', '/dev/sdb1', 'ESP'])
utils.Execute(['sync'])
utils.Execute(['mkfs.ext2', '-L', 'INSTALLER', '/dev/sdb2'])
utils.Execute(['sync'])
utils.Execute(['mkdir', '-vp', 'iso', 'installer', 'boot'])
utils.Execute(['mount', '-o', 'ro,loop', '-t', 'iso9660', iso_file, 'iso'])
utils.Execute(['mount', '-t', 'vfat', '/dev/sdb1', 'boot'])
utils.Execute(['mount', '-t', 'ext2', '/dev/sdb2', 'installer'])
utils.Execute(['rsync', '-Pav', 'iso/EFI', 'iso/images', 'boot/'])
utils.Execute(['cp', iso_file, 'installer/'])
utils.Execute(['cp', ks_cfg, 'installer/'])
# Modify boot config.
with open('boot/EFI/BOOT/grub.cfg', 'r+') as f:
oldcfg = f.read()
cfg = re.sub(r'-l .RHEL.*', r"""-l 'ESP'""", oldcfg)
cfg = re.sub(r'timeout=60', 'timeout=1', cfg)
cfg = re.sub(r'set default=.*', 'set default="0"', cfg)
cfg = re.sub(
r'load_video\n',
r'serial --speed=38400 --unit=0 --word=8 --parity=no\n'
'terminal_input serial\nterminal_output serial\n', cfg)
# Change boot args.
args = ' '.join([
'inst.text',
'inst.ks=hd:LABEL=INSTALLER:/%s' % ks_cfg, 'console=ttyS0,38400n8',
'inst.gpt', 'inst.loglevel=debug'
])
# Tell Anaconda not to store its logs in the installed image,
# unless requested to keep them for debugging.
if not savelogs:
args += ' inst.nosave=all'
cfg = re.sub(r'inst\.stage2.*', r'\g<0> %s' % args, cfg)
# Change labels to explicit partitions.
cfg = re.sub(r'LABEL=[^ ]+', 'LABEL=INSTALLER', cfg)
# Print out a the modifications.
diff = difflib.Differ().compare(oldcfg.splitlines(1),
cfg.splitlines(1))
logging.info('Modified grub.cfg:\n%s' % '\n'.join(diff))
f.seek(0)
f.write(cfg)
f.truncate()
utils.Execute(['umount', 'installer'])
utils.Execute(['umount', 'iso'])
utils.Execute(['umount', 'boot'])
def DistroSpecific(g):
install_gce = utils.GetMetadataAttribute('install_gce_packages')
deb_release = utils.GetMetadataAttribute('debian_release')
if install_gce == 'true':
logging.info('Installing GCE packages.')
utils.update_apt(g)
utils.install_apt_packages(g, 'gnupg')
run(g, [
'wget', 'https://packages.cloud.google.com/apt/doc/apt-key.gpg',
'-O', '/tmp/gce_key'
])
run(g, ['apt-key', 'add', '/tmp/gce_key'])
g.rm('/tmp/gce_key')
g.write('/etc/apt/sources.list.d/google-cloud.list',
google_cloud.format(deb_release=deb_release))
# Remove Azure agent.
try:
run(g,
['apt-get', 'remove', '-y', '-f', 'waagent', 'walinuxagent'])
except Exception as e:
logging.debug(str(e))
logging.warn('Could not uninstall Azure agent. Continuing anyway.')
utils.update_apt(g)
pkgs = [
'google-cloud-packages-archive-keyring', 'google-compute-engine'
]
# Debian 8 differences:
# 1. No NGE
# 2. No Cloud SDK, since it requires Python 3.5+.
# 3. No OS config agent.
if deb_release == 'jessie':
# Debian 8 doesn't support the new guest agent, so we need to install
# the legacy Python version.
pkgs += [
'python-google-compute-engine', 'python3-google-compute-engine'
]
logging.info('Skipping installation of OS Config agent. '
'Requires Debian 9 or newer.')
else:
pkgs += ['google-cloud-sdk', 'google-osconfig-agent']
utils.install_apt_packages(g, *pkgs)
# Update grub config to log to console.
run(g, [
'sed', '-i""', r'/GRUB_CMDLINE_LINUX/s#"$# console=ttyS0,38400n8"#',
'/etc/default/grub'
])
# Disable predictive network interface naming in Stretch and Buster.
if deb_release in ['stretch', 'buster']:
run(g, [
'sed', '-i',
r's#^\(GRUB_CMDLINE_LINUX=".*\)"$#\1 net.ifnames=0 biosdevname=0"#',
'/etc/default/grub'
])
run(g, ['update-grub2'])
# Reset network for DHCP.
logging.info('Resetting network to DHCP for eth0.')
g.write('/etc/network/interfaces', interfaces)
def main():
# Get Parameters.
build_date = utils.GetMetadataAttribute(
'build_date', raise_on_not_found=True)
debian_cloud_images_version = '69783f7417aefb332d5d7250ba242adeca444131'
debian_version = utils.GetMetadataAttribute(
'debian_version', raise_on_not_found=True)
outs_path = utils.GetMetadataAttribute('daisy-outs-path',
raise_on_not_found=True)
logging.info('debian-cloud-images version: %s' % debian_cloud_images_version)
logging.info('debian version: %s' % debian_version)
# force an apt-get update before next install
utils.AptGetInstall.first_run = True
utils.AptGetInstall(['fai-server', 'fai-setup-storage'])
# Download and setup debian's debian-cloud-images scripts.
url_params = {
'project': 'debian-cloud-images',
'version': debian_cloud_images_version,
}
url_params['filename'] = '%(project)s-%(version)s' % url_params
url = ('https://salsa.debian.org/cloud-team/'
'%(project)s/-/archive/%(version)s/%(filename)s.tar.gz' % url_params)
logging.info('Downloading %(project)s at version %(version)s', url_params)
urllib.request.urlretrieve(url, 'fci.tar.gz')
with tarfile.open('fci.tar.gz') as tar:
tar.extractall()
logging.info('Downloaded and extracted %s.', url)
work_dir = url_params['filename']
config_space = os.getcwd() + '/' + work_dir + '/config_space/'
# We are going to replace this with our variant
os.remove(config_space + 'class/BULLSEYE.var')
# Remove failing test method for now.
os.remove(config_space + 'hooks/tests.CLOUD')
# Copy our classes to the FAI config space
mycopytree('/files/fai_config', config_space)
# Set scripts executable (daisy doesn't preserve this)
os.chmod(config_space + 'scripts/GCE_CLEAN/10-gce-clean', 0o755)
os.chmod(config_space + 'scripts/GCE_SPECIFIC/12-sshd', 0o755)
# Config fai-tool
# Base classes
fai_classes = ['DEBIAN', 'CLOUD', 'GCE', 'GCE_SDK', 'LINUX_IMAGE_CLOUD',
'GCE_SPECIFIC', 'GCE_CLEAN']
# Arch-specific classes
if platform.machine() == 'aarch64':
fai_classes += ['ARM64', 'GRUB_EFI_ARM64', 'BACKPORTS_LINUX']
else:
fai_classes += ['AMD64', 'GRUB_CLOUD_AMD64']
# Version-specific classes
if debian_version == 'buster':
fai_classes += ['BUSTER']
elif debian_version == 'bullseye':
fai_classes += ['BULLSEYE']
elif debian_version == 'sid':
fai_classes += ['SID']
image_size = '10G'
disk_name = 'disk.raw'
# Run fai-tool.
cmd = ['fai-diskimage', '--verbose', '--hostname', 'debian', '--class',
','.join(fai_classes), '--size', image_size, '--cspace',
config_space, disk_name]
logging.info('Starting build in %s with params: %s', work_dir, ' '.join(cmd))
returncode, output = utils.Execute(
cmd, cwd=work_dir, capture_output=True, raise_errors=False)
# Verbose printing to console for debugging.
for line in output.splitlines():
print(line)
if returncode != 0:
raise subprocess.CalledProcessError(returncode, cmd)
# Packs a gzipped tar file with disk.raw inside
disk_tar_gz = 'debian-{}-{}.tar.gz'.format(debian_version, build_date)
logging.info('Compressing it into tarball %s', disk_tar_gz)
tar = tarfile.open(disk_tar_gz, 'w:gz', format=tarfile.GNU_FORMAT)
tar.add('%s/%s' % (work_dir, disk_name), arcname=disk_name)
tar.close()
# Upload tar.
image_dest = os.path.join(outs_path, 'root.tar.gz')
logging.info('Saving %s to %s', disk_tar_gz, image_dest)
utils.UploadFile(disk_tar_gz, image_dest)
def DistroSpecific(g):
el_release = utils.GetMetadataAttribute('el_release')
install_gce = utils.GetMetadataAttribute('install_gce_packages')
rhel_license = utils.GetMetadataAttribute('use_rhel_gce_license')
if rhel_license == 'true':
if 'Red Hat' in g.cat('/etc/redhat-release'):
g.command(['yum', 'remove', '-y', '*rhui*'])
logging.info('Adding in GCE RHUI package.')
g.write('/etc/yum.repos.d/google-cloud.repo',
repo_compute % el_release)
g.command([
'yum', 'install', '-y',
'google-rhui-client-rhel%s' % el_release
])
if install_gce == 'true':
logging.info('Installing GCE packages.')
g.write('/etc/yum.repos.d/google-cloud.repo',
repo_compute % el_release)
if el_release == '7':
g.write_append('/etc/yum.repos.d/google-cloud.repo',
repo_sdk % el_release)
g.command(['yum', '-y', 'install', 'google-cloud-sdk'])
if el_release == '6':
if 'CentOS' in g.cat('/etc/redhat-release'):
logging.info('Installing CentOS SCL.')
g.command(['rm', '-f', '/etc/yum.repos.d/CentOS-SCL.repo'])
g.command(['yum', '-y', 'install', 'centos-release-scl'])
# Install Google Cloud SDK from the upstream tar and create links for the
# python27 SCL environment.
logging.info('Installing python27 from SCL.')
g.command(['yum', '-y', 'install', 'python27'])
g.command([
'scl', 'enable', 'python27',
'pip2.7 install --upgrade google_compute_engine'
])
logging.info('Installing Google Cloud SDK from tar.')
sdk_base_url = 'https://dl.google.com/dl/cloudsdk/channels/rapid'
sdk_base_tar = '%s/google-cloud-sdk.tar.gz' % sdk_base_url
tar = utils.HttpGet(sdk_base_tar)
g.write('/tmp/google-cloud-sdk.tar.gz', tar)
g.command(
['tar', 'xzf', '/tmp/google-cloud-sdk.tar.gz', '-C', '/tmp'])
sdk_version = g.cat('/tmp/google-cloud-sdk/VERSION').strip()
logging.info('Getting Cloud SDK Version %s', sdk_version)
sdk_version_tar = 'google-cloud-sdk-%s-linux-x86_64.tar.gz' % sdk_version
sdk_version_tar_url = '%s/downloads/%s' % (sdk_base_url,
sdk_version_tar)
logging.info('Getting versioned Cloud SDK tar file from %s',
sdk_version_tar_url)
tar = utils.HttpGet(sdk_version_tar_url)
sdk_version_tar_file = os.path.join('/tmp', sdk_version_tar)
g.write(sdk_version_tar_file, tar)
g.mkdir_p('/usr/local/share/google')
g.command([
'tar', 'xzf', sdk_version_tar_file, '-C',
'/usr/local/share/google', '--no-same-owner'
])
logging.info('Creating CloudSDK SCL symlinks.')
sdk_bin_path = '/usr/local/share/google/google-cloud-sdk/bin'
g.ln_s(os.path.join(sdk_bin_path, 'git-credential-gcloud.sh'),
os.path.join('/usr/bin', 'git-credential-gcloud.sh'))
for binary in ['bq', 'gcloud', 'gsutil']:
binary_path = os.path.join(sdk_bin_path, binary)
new_bin_path = os.path.join('/usr/bin', binary)
bin_str = '#!/bin/bash\nsource /opt/rh/python27/enable\n%s $@' % binary_path
g.write(new_bin_path, bin_str)
g.chmod(0755, new_bin_path)
g.command([
'yum', '-y', 'install', 'google-compute-engine',
'python-google-compute-engine'
])
logging.info('Updating initramfs')
for kver in g.ls('/lib/modules'):
if el_release == '6':
# Version 6 doesn't have option --kver
g.command(['dracut', '-v', '-f', kver])
else:
g.command(['dracut', '-v', '-f', '--kver', kver])
logging.info('Update grub configuration')
if el_release == '6':
# Version 6 doesn't have grub2, file grub.conf needs to be updated by hand
g.write('/tmp/grub_gce_generated', grub_cfg)
g.sh(r'grep -P "^[\t ]*initrd|^[\t ]*root|^[\t ]*kernel|^[\t ]*title" '
r'/boot/grub/grub.conf >> /tmp/grub_gce_generated;'
r'sed -i "s/console=ttyS0[^ ]*//g" /tmp/grub_gce_generated;'
r'sed -i "/^[\t ]*kernel/s/$/ console=ttyS0,38400n8/" '
r'/tmp/grub_gce_generated;'
r'mv /tmp/grub_gce_generated /boot/grub/grub.conf')
else:
g.write('/etc/default/grub', grub2_cfg)
g.command(['grub2-mkconfig', '-o', '/boot/grub2/grub.cfg'])
# Reset network for DHCP.
logging.info('Resetting network to DHCP for eth0.')
g.write('/etc/sysconfig/network-scripts/ifcfg-eth0', ifcfg_eth0)
def DistroSpecific(g: guestfs.GuestFS):
el_release = utils.GetMetadataAttribute('el_release')
install_gce = utils.GetMetadataAttribute('install_gce_packages')
rhel_license = utils.GetMetadataAttribute('use_rhel_gce_license')
# This must be performed prior to making network calls from the guest.
# Otherwise, if /etc/resolv.conf is present, and has an immutable attribute,
# guestfs will fail with:
#
# rename: /sysroot/etc/resolv.conf to
# /sysroot/etc/i9r7obu6: Operation not permitted
utils.common.ClearEtcResolv(g)
# Some imported images haven't contained `/etc/yum.repos.d`.
if not g.exists('/etc/yum.repos.d'):
g.mkdir('/etc/yum.repos.d')
if rhel_license == 'true':
if 'Red Hat' in g.cat('/etc/redhat-release'):
g.command(['yum', 'remove', '-y', '*rhui*'])
logging.info('Adding in GCE RHUI package.')
g.write('/etc/yum.repos.d/google-cloud.repo',
repo_compute % el_release)
yum_install(g, 'google-rhui-client-rhel' + el_release)
if install_gce == 'true':
logging.info('Installing GCE packages.')
g.write('/etc/yum.repos.d/google-cloud.repo',
repo_compute % el_release)
if el_release == '6':
if 'CentOS' in g.cat('/etc/redhat-release'):
logging.info('Installing CentOS SCL.')
g.command(['rm', '-f', '/etc/yum.repos.d/CentOS-SCL.repo'])
yum_install(g, 'centos-release-scl')
# Install Google Cloud SDK from the upstream tar and create links for the
# python27 SCL environment.
logging.info('Installing python27 from SCL.')
yum_install(g, 'python27')
logging.info('Installing Google Cloud SDK from tar.')
sdk_base_url = 'https://dl.google.com/dl/cloudsdk/channels/rapid'
sdk_base_tar = '%s/google-cloud-sdk.tar.gz' % sdk_base_url
tar = utils.HttpGet(sdk_base_tar)
g.write('/tmp/google-cloud-sdk.tar.gz', tar)
g.command(
['tar', 'xzf', '/tmp/google-cloud-sdk.tar.gz', '-C', '/tmp'])
sdk_version = g.cat('/tmp/google-cloud-sdk/VERSION').strip()
logging.info('Getting Cloud SDK Version %s', sdk_version)
sdk_version_tar = 'google-cloud-sdk-%s-linux-x86_64.tar.gz' % sdk_version
sdk_version_tar_url = '%s/downloads/%s' % (sdk_base_url,
sdk_version_tar)
logging.info('Getting versioned Cloud SDK tar file from %s',
sdk_version_tar_url)
tar = utils.HttpGet(sdk_version_tar_url)
sdk_version_tar_file = os.path.join('/tmp', sdk_version_tar)
g.write(sdk_version_tar_file, tar)
g.mkdir_p('/usr/local/share/google')
g.command([
'tar', 'xzf', sdk_version_tar_file, '-C',
'/usr/local/share/google', '--no-same-owner'
])
logging.info('Creating CloudSDK SCL symlinks.')
sdk_bin_path = '/usr/local/share/google/google-cloud-sdk/bin'
g.ln_s(os.path.join(sdk_bin_path, 'git-credential-gcloud.sh'),
os.path.join('/usr/bin', 'git-credential-gcloud.sh'))
for binary in ['bq', 'gcloud', 'gsutil']:
binary_path = os.path.join(sdk_bin_path, binary)
new_bin_path = os.path.join('/usr/bin', binary)
bin_str = '#!/bin/bash\nsource /opt/rh/python27/enable\n%s $@' % \
binary_path
g.write(new_bin_path, bin_str)
g.chmod(0o755, new_bin_path)
else:
g.write_append('/etc/yum.repos.d/google-cloud.repo',
repo_sdk % el_release)
yum_install(g, 'google-cloud-sdk')
yum_install(g, 'google-compute-engine', 'google-osconfig-agent')
logging.info('Updating initramfs')
for kver in g.ls('/lib/modules'):
# Although each directory in /lib/modules typically corresponds to a
# kernel version [1], that may not always be true.
# kernel-abi-whitelists, for example, creates extra directories in
# /lib/modules.
#
# Skip building initramfs if the directory doesn't look like a
# kernel version. Emulates the version matching from depmod [2].
#
# 1. https://tldp.org/LDP/Linux-Filesystem-Hierarchy/html/lib.html
# 2. https://kernel.googlesource.com/pub/scm/linux/kernel/git/mmarek/kmod
# /+/tip/tools/depmod.c#2537
if not re.match(r'^\d+.\d+', kver):
logging.debug(
'/lib/modules/{} doesn\'t look like a kernel directory. '
'Skipping creation of initramfs for it'.format(kver))
continue
if not g.exists(os.path.join('/lib/modules', kver, 'modules.dep')):
try:
g.command(['depmod', kver])
except RuntimeError as e:
logging.info(
'Failed to write initramfs for {kver}. If image fails to '
'boot, verify that depmod /lib/modules/{kver} runs on '
'the original machine'.format(kver=kver))
logging.debug('depmod error: {}'.format(e))
continue
if el_release == '6':
# Version 6 doesn't have option --kver
g.command(['dracut', '-v', '-f', kver])
else:
g.command(['dracut', '--stdlog=1', '-f', '--kver', kver])
logging.info('Update grub configuration')
if el_release == '6':
# Version 6 doesn't have grub2, file grub.conf needs to be updated by hand
g.write('/tmp/grub_gce_generated', grub_cfg)
g.sh(r'grep -P "^[\t ]*initrd|^[\t ]*root|^[\t ]*kernel|^[\t ]*title" '
r'/boot/grub/grub.conf >> /tmp/grub_gce_generated;'
r'sed -i "s/console=ttyS0[^ ]*//g" /tmp/grub_gce_generated;'
r'sed -i "/^[\t ]*kernel/s/$/ console=ttyS0,38400n8/" '
r'/tmp/grub_gce_generated;'
r'mv /tmp/grub_gce_generated /boot/grub/grub.conf')
else:
g.write('/etc/default/grub', grub2_cfg)
g.command(['grub2-mkconfig', '-o', '/boot/grub2/grub.cfg'])
# Reset network for DHCP.
logging.info('Resetting network to DHCP for eth0.')
# Remove NetworkManager-config-server if it's present. The package configures
# NetworkManager to *not* use DHCP.
# https://access.redhat.com/solutions/894763
g.command(['yum', 'remove', '-y', 'NetworkManager-config-server'])
g.write('/etc/sysconfig/network-scripts/ifcfg-eth0', ifcfg_eth0)
