def InstallComputeImagePackages(packages_dir):
utils.LogStep('Install compute-image-packages')
utils.Run([
"egrep -lRZ 'python' %s | "
"xargs -0 -l sed -i -e '/#!.*python/c\#!/usr/bin/env python2'" %
packages_dir
],
shell=True)
utils.CopyFiles(os.path.join(packages_dir, 'google-daemon', '*'), '/')
utils.CopyFiles(os.path.join(packages_dir, 'google-startup-scripts', '*'),
'/')
utils.SecureDeleteFile('/README.md')
# TODO: Fix gcimagebundle does not work with Arch yet.
#InstallGcimagebundle(packages_dir)
# Patch Google services to run after the network is actually available.
PatchGoogleSystemdService(
'/usr/lib/systemd/system/google-startup-scripts.service')
PatchGoogleSystemdService(
'/usr/lib/systemd/system/google-accounts-manager.service')
PatchGoogleSystemdService(
'/usr/lib/systemd/system/google-address-manager.service')
PatchGoogleSystemdService('/usr/lib/systemd/system/google.service')
utils.EnableService('google-accounts-manager.service')
utils.EnableService('google-address-manager.service')
utils.EnableService('google.service')
utils.EnableService('google-startup-scripts.service')
utils.DeleteDirectory(packages_dir)
def AddImageToComputeEngineProject(image_name, gs_path, description):
utils.LogStep('Add image to project')
utils.Run(['gcloud', 'compute', 'images', 'delete', image_name, '-q'])
utils.Run([
'gcloud', 'compute', 'images', 'create', image_name, '-q',
'--source-uri', gs_path, '--description', description
])
def ConfigureSerialPortOutput():
# https://wiki.archlinux.org/index.php/working_with_the_serial_console
# Try this: http://wiki.alpinelinux.org/wiki/Enable_Serial_Console_on_Boot
utils.LogStep('Configure Serial Port Output')
utils.Sed('/boot/syslinux/syslinux.cfg', '/DEFAULT/aserial 0 38400')
utils.ReplaceLine('/boot/syslinux/syslinux.cfg', 'TIMEOUT', 'TIMEOUT 1')
def InstallGoogleCloudSdk():
# TODO: There's a google-cloud-sdk in AUR which should be used
# but it's not optimal for cloud use. The image is too large.
utils.LogStep('Install Google Cloud SDK')
usr_share_google = '/usr/share/google'
archive = os.path.join(usr_share_google, 'google-cloud-sdk.zip')
unzip_dir = os.path.join(usr_share_google, 'google-cloud-sdk')
utils.CreateDirectory(usr_share_google)
utils.DownloadFile(
'https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.zip',
archive)
utils.Run(['unzip', archive, '-d', usr_share_google])
utils.AppendFile('/etc/bash.bashrc',
'export CLOUDSDK_PYTHON=/usr/bin/python2')
utils.Run([
os.path.join(unzip_dir, 'install.sh'), '--usage-reporting', 'false',
'--bash-completion', 'true', '--disable-installation-options',
'--rc-path', '/etc/bash.bashrc', '--path-update', 'true'
],
cwd=unzip_dir,
env={'CLOUDSDK_PYTHON': '/usr/bin/python2'})
utils.Symlink(os.path.join(unzip_dir, 'bin/gcloud'), '/usr/bin/gcloud')
utils.Symlink(os.path.join(unzip_dir, 'bin/gcutil'), '/usr/bin/gcutil')
utils.Symlink(os.path.join(unzip_dir, 'bin/gsutil'), '/usr/bin/gsutil')
utils.SecureDeleteFile(archive)
def UploadImage(image_path, gs_path, make_public=False):
utils.LogStep('Upload Image to Cloud Storage')
utils.SecureDeleteFile('~/.gsutil/*.url')
utils.Run(['gsutil', 'rm', gs_path])
utils.Run(['gsutil', 'cp', image_path, gs_path])
if make_public:
utils.Run(['gsutil', 'acl', 'set', 'public-read', gs_path])
def CreateBlankImage(image_path, size_gb=10, fs_type='ext4'):
utils.LogStep('Create Image')
utils.Run(['rm', '-f', image_path])
utils.Run(['truncate', image_path, '--size=%sG' % size_gb])
utils.Run(['parted', image_path, 'mklabel', 'msdos'])
utils.Run(['parted', image_path, 'mkpart', 'primary',
fs_type, '1', str(int(size_gb) * 1024)])
def SaveImage(arch_root, image_filename):
utils.LogStep('Save Arch Linux Image in GCE format')
source_image_raw = os.path.join(arch_root, 'disk.raw')
image_raw = os.path.join(os.getcwd(), 'disk.raw')
image_file = os.path.join(os.getcwd(), image_filename)
utils.Run(['cp', '--sparse=always', source_image_raw, image_raw])
utils.Run(['tar', '-Szcf', image_file, 'disk.raw'])
return image_file
def SetupFail2ban():
utils.LogStep('Configure fail2ban')
# http://flexion.org/posts/2012-11-ssh-brute-force-defence.html
utils.Pacman(['-S', 'fail2ban'])
utils.WriteFile('/etc/fail2ban/jail.local', ETC_FAIL2BAN_JAIL_LOCAL)
utils.WriteFile('/etc/fail2ban/jail.d/sshd.conf',
ETC_FAIL2BAN_JAIL_D_SSHD_CONF)
utils.EnableService('syslog-ng')
utils.EnableService('fail2ban.service')
def AddImageToComputeEngineProject(image_name, gs_path, description):
utils.LogStep('Add image to project')
utils.Run(['gcloud', 'compute', 'images', 'delete', image_name, '-q'],
env={'CLOUDSDK_PYTHON': '/usr/bin/python2'})
utils.Run([
'gcloud', 'compute', 'images', 'create', image_name, '-q',
'--source-uri', gs_path, '--description', description
],
env={'CLOUDSDK_PYTHON': '/usr/bin/python2'})
def SetupNetwork():
utils.LogStep('Setup Networking')
utils.SecureDeleteFile('/etc/hostname')
utils.WriteFile('/etc/hosts', ETC_HOSTS)
utils.WriteFile('/etc/sysctl.d/70-disable-ipv6.conf',
ETC_SYSCTL_D_70_DISABLE_IPV6_CONF)
utils.EnableService('dhcpcd.service')
utils.EnableService('systemd-networkd.service')
utils.EnableService('systemd-networkd-wait-online.service')
def ConfigureKernel():
utils.LogStep('Configure Kernel')
utils.Replace(
'/etc/mkinitcpio.conf', 'MODULES=""',
'MODULES="virtio virtio_blk virtio_pci virtio_scsi virtio_net"')
utils.Replace('/etc/mkinitcpio.conf', 'autodetect ', '')
utils.Run([
'mkinitcpio', '-g', '/boot/initramfs-linux.img', '-k',
'/boot/vmlinuz-linux', '-c', '/etc/mkinitcpio.conf'
])
def SetupAccounts(args):
accounts = args['accounts']
if accounts:
utils.LogStep('Add Accounts')
for account in accounts:
username, password = account.split(':')
logging.info(' - %s', username)
utils.Run(['useradd', username, '-m', '-s', '/bin/bash',
'-G', 'adm,video'])
utils.Run('echo %s:%s | chpasswd' % (username, password), shell=True)
def UploadImage(image_path, gs_path, make_public=False):
utils.LogStep('Upload Image to Cloud Storage')
utils.SecureDeleteFile('~/.gsutil/*.url')
utils.Run(['gsutil', 'rm', gs_path],
env={'CLOUDSDK_PYTHON': '/usr/bin/python2'})
utils.Run(['gsutil', 'cp', image_path, gs_path],
env={'CLOUDSDK_PYTHON': '/usr/bin/python2'})
if make_public:
utils.Run(['gsutil', 'acl', 'set', 'public-read', gs_path],
env={'CLOUDSDK_PYTHON': '/usr/bin/python2'})
def SetupNetwork():
utils.LogStep('Setup Networking')
utils.SecureDeleteFile('/etc/hostname')
utils.WriteFile('/etc/hosts', ETC_HOSTS)
utils.WriteFile('/etc/sysctl.d/70-disable-ipv6.conf',
ETC_SYSCTL_D_70_DISABLE_IPV6_CONF)
# https://wiki.archlinux.org/index.php/Network_configuration#Reverting_to_traditional_device_names
utils.Symlink('/dev/null', '/etc/udev/rules.d/80-net-setup-link.rules')
utils.EnableService('dhcpcd.service')
utils.EnableService('systemd-networkd.service')
utils.EnableService('systemd-networkd-wait-online.service')
def SetupSsh():
utils.LogStep('Configure SSH')
utils.WriteFile('/etc/ssh/sshd_not_to_be_run', 'GOOGLE')
utils.SecureDeleteFile('/etc/ssh/ssh_host_key')
utils.SecureDeleteFile('/etc/ssh/ssh_host_rsa_key*')
utils.SecureDeleteFile('/etc/ssh/ssh_host_dsa_key*')
utils.SecureDeleteFile('/etc/ssh/ssh_host_ecdsa_key*')
utils.WriteFile('/etc/ssh/ssh_config', ETC_SSH_SSH_CONFIG)
utils.Chmod('/etc/ssh/ssh_config', 644)
utils.WriteFile('/etc/ssh/sshd_config', ETC_SSH_SSHD_CONFIG)
utils.Chmod('/etc/ssh/sshd_config', 644)
utils.EnableService('sshd.service')
def SetupFileSystem(base_dir, image_mapping_path):
utils.LogStep('File Systems')
_, fstab_contents, _ = utils.Run(['genfstab', '-p', base_dir],
capture_output=True)
utils.WriteFile(os.path.join(base_dir, 'etc', 'fstab'), fstab_contents)
_, disk_uuid, _ = utils.Run(
['blkid', '-s', 'UUID', '-o', 'value', image_mapping_path],
capture_output=True)
disk_uuid = disk_uuid.strip()
utils.WriteFile(os.path.join(base_dir, 'etc', 'fstab'),
'UUID=%s / ext4 defaults 0 1' % disk_uuid)
utils.Run(['tune2fs', '-i', '1', '-U', disk_uuid, image_mapping_path])
return disk_uuid
def SetupNetwork():
utils.LogStep('Setup Networking')
utils.SecureDeleteFile('/etc/hostname')
utils.WriteFile('/etc/hosts', ETC_HOSTS)
utils.WriteFile('/etc/sysctl.d/70-disable-ipv6.conf',
ETC_SYSCTL_D_70_DISABLE_IPV6_CONF)
utils.EnableService('dhcpcd.service')
utils.EnableService('systemd-networkd.service')
utils.EnableService('systemd-networkd-wait-online.service')
# Set Google Compute specific MTU
# https://cloud.google.com/compute/docs/troubleshooting#packetfragmentation
utils.WriteFile('/etc/systemd/system/[email protected]', ETC_SYSTEM_D_SET_MTU)
utils.CreateDirectory('/etc/conf.d/')
utils.WriteFile('/etc/conf.d/setmtu', ETC_CONF_D_SET_MTU)
utils.EnableService('*****@*****.**')
def InstallBootloader(device, uuid, debugmode):
utils.LogStep('Install Syslinux bootloader')
'''
utils.Run(['syslinux-install_update', '-i', '-a', '-m'])
'''
utils.Run(['blkid', '-s', 'PTTYPE', '-o', 'value', device])
utils.CreateDirectory('/boot/syslinux')
utils.CopyFiles('/usr/lib/syslinux/bios/*.c32', '/boot/syslinux/')
utils.Run(['extlinux', '--install', '/boot/syslinux'])
utils.Replace('/boot/syslinux/syslinux.cfg', 'sda3', 'sda1')
utils.Run(['fdisk', '-l', device])
utils.Run([
'dd', 'bs=440', 'count=1', 'conv=notrunc',
'if=/usr/lib/syslinux/bios/mbr.bin',
'of=%s' % device
])
boot_params = [
'console=ttyS0,38400',
'CONFIG_KVM_GUEST=y',
'CONFIG_KVM_CLOCK=y',
'CONFIG_VIRTIO_PCI=y',
'CONFIG_SCSI_VIRTIO=y',
'CONFIG_VIRTIO_NET=y',
'CONFIG_STRICT_DEVMEM=y',
'CONFIG_DEVKMEM=n',
'CONFIG_DEFAULT_MMAP_MIN_ADDR=65536',
'CONFIG_DEBUG_RODATA=y',
'CONFIG_DEBUG_SET_MODULE_RONX=y',
'CONFIG_CC_STACKPROTECTOR=y',
'CONFIG_COMPAT_VDSO=n',
'CONFIG_COMPAT_BRK=n',
'CONFIG_X86_PAE=y',
'CONFIG_SYN_COOKIES=y',
'CONFIG_SECURITY_YAMA=y',
'CONFIG_SECURITY_YAMA_STACKED=y',
]
if debugmode:
boot_params += [
'systemd.log_level=debug',
'systemd.log_target=console',
'systemd.journald.forward_to_syslog=yes',
'systemd.journald.forward_to_kmsg=yes',
'systemd.journald.forward_to_console=yes',
]
boot_params = ' '.join(boot_params)
boot_spec = ' APPEND root=UUID=%s rw append %s' % (uuid, boot_params)
utils.ReplaceLine('/boot/syslinux/syslinux.cfg', 'APPEND root=', boot_spec)
def PrepareBootstrap(workspace_dir, mirror_server, use_pacman_keys):
utils.LogStep('Setting up Bootstrap Environment')
arch_root = os.path.join(workspace_dir, os.listdir(workspace_dir)[0])
mirrorlist = 'Server = {MIRROR_SERVER}'.format(MIRROR_SERVER=mirror_server)
utils.AppendFile(os.path.join(arch_root, 'etc/pacman.d/mirrorlist'),
mirrorlist)
utils.CreateDirectory(os.path.join(arch_root, 'run/shm'))
if use_pacman_keys:
utils.RunChroot(arch_root, 'pacman-key --init')
utils.RunChroot(arch_root, 'pacman-key --populate archlinux')
else:
utils.ReplaceLine(os.path.join(arch_root, 'etc/pacman.conf'),
'SigLevel', 'SigLevel = Never')
# Install the most basic utilities for the bootstrapper.
utils.RunChroot(arch_root, 'pacman --noconfirm -Sy python3 sed')
return arch_root
def DownloadArchBootstrap(bootstrap_tarball):
utils.LogStep('Download Arch Linux Bootstrap')
if bootstrap_tarball:
url = bootstrap_tarball
sha1sum = None
else:
url, sha1sum = GetLatestBootstrapUrl()
logging.debug('Downloading %s', url)
local_bootstrap = os.path.join(os.getcwd(), os.path.basename(url))
if os.path.isfile(local_bootstrap):
logging.debug('Using local file instead.')
if sha1sum and utils.Sha1Sum(local_bootstrap) == sha1sum:
return local_bootstrap
utils.DownloadFile(url, local_bootstrap)
if not sha1sum or utils.Sha1Sum(local_bootstrap) != sha1sum:
raise ValueError('Bad checksum')
return local_bootstrap
def ConfigureArchInstall(args, mount_path, parent_path, disk_uuid):
relative_builder_path = utils.CopyBuilder(mount_path)
utils.LogStep('Download compute-image-packages')
packages_dir = utils.CreateTempDirectory(mount_path)
utils.Run(['git', 'clone', COMPUTE_IMAGE_PACKAGES_GIT_URL, packages_dir])
utils.CreateDirectory(os.path.join(mount_path, ''))
packages_dir = os.path.relpath(packages_dir, mount_path)
params = {
'packages_dir': '/%s' % packages_dir,
'device': parent_path,
'disk_uuid': disk_uuid,
'accounts': args['accounts'],
'debugmode': args['debugmode'],
}
params.update(args)
config_arch_py = os.path.join('/', relative_builder_path, 'arch-image.py')
utils.RunChroot(mount_path,
'%s "%s"' % (config_arch_py, utils.EncodeArgs(params)),
use_custom_path=False)
utils.DeleteDirectory(os.path.join(mount_path, relative_builder_path))
def ConfigureSecurity():
utils.LogStep('Compute Engine Security Recommendations')
utils.WriteFile('/etc/sysctl.d/70-gce-security-strongly-recommended.conf',
ETC_SYSCTL_D_70_GCE_SECURITY_STRONGLY_RECOMMENDED_CONF)
utils.WriteFile('/etc/sysctl.d/70-gce-security-recommended.conf',
ETC_SYSCTL_D_70_GCE_SECURITY_RECOMMENDED_CONF)
utils.LogStep('Lock Root User Account')
utils.Run(['usermod', '-L', 'root'])
utils.LogStep('PAM Security Settings')
utils.WriteFile('/etc/pam.d/passwd', ETC_PAM_D_PASSWD)
utils.LogStep('Disable CAP_SYS_MODULE')
utils.WriteFile('/proc/sys/kernel/modules_disabled', '1')
utils.LogStep('Remove the kernel symbol table')
utils.SecureDeleteFile('/boot/System.map')
utils.LogStep('Sudo Access')
utils.WriteFile('/etc/sudoers.d/add-group-adm', ETC_SUDOERS_D_ADD_GROUP_ADM)
utils.Run(['chown', 'root:root', '/etc/sudoers.d/add-group-adm'])
utils.Run(['chmod', '0440', '/etc/sudoers.d/add-group-adm'])
def ConfigMessageOfTheDay():
utils.LogStep('Configure Message of the Day')
utils.WriteFile('/etc/motd', ETC_MOTD)
def SaveImage(disk_image_file, image_filename):
utils.LogStep('Save Arch Linux Image in GCE format')
image_raw = os.path.join(os.getcwd(), IMAGE_FILE)
gce_image_file = os.path.join(os.getcwd(), image_filename)
utils.Run(['tar', '-Szcf', image_filename, IMAGE_FILE])
return gce_image_file
def ShrinkDisk(image_mapping_path):
utils.LogStep('Shrink Disk')
utils.Run(['zerofree', image_mapping_path])
def OptimizePackages():
utils.LogStep('Cleanup Cached Package Data')
utils.Pacman(['-Syu'])
utils.Pacman(['-Sc'])
utils.Run(['pacman-optimize'])
def SetupLocale():
utils.LogStep('Set Locale to US English (UTF-8)')
utils.SetupArchLocale()
def SetupNtpServer():
utils.LogStep('Configure NTP')
utils.WriteFile('/etc/ntp.conf', 'server metadata.google.internal iburst')
def ForwardSystemdToConsole():
utils.LogStep('Installing syslinux bootloader')
utils.AppendFile('/etc/systemd/journald.conf', 'ForwardToConsole=yes')
def ConfigureTimeZone():
utils.LogStep('Set Timezone to UTC')
utils.Run(['ln', '-sf', '/usr/share/zoneinfo/UTC', '/etc/localtime'])
