def InstallComputeImagePackages(packages_dir): utils.LogStep('Install compute-image-packages') utils.Run([ "egrep -lRZ 'python' %s | " "xargs -0 -l sed -i -e '/#!.*python/c\#!/usr/bin/env python2'" % packages_dir ], shell=True) utils.CopyFiles(os.path.join(packages_dir, 'google-daemon', '*'), '/') utils.CopyFiles(os.path.join(packages_dir, 'google-startup-scripts', '*'), '/') utils.SecureDeleteFile('/README.md') # TODO: Fix gcimagebundle does not work with Arch yet. #InstallGcimagebundle(packages_dir) # Patch Google services to run after the network is actually available. PatchGoogleSystemdService( '/usr/lib/systemd/system/google-startup-scripts.service') PatchGoogleSystemdService( '/usr/lib/systemd/system/google-accounts-manager.service') PatchGoogleSystemdService( '/usr/lib/systemd/system/google-address-manager.service') PatchGoogleSystemdService('/usr/lib/systemd/system/google.service') utils.EnableService('google-accounts-manager.service') utils.EnableService('google-address-manager.service') utils.EnableService('google.service') utils.EnableService('google-startup-scripts.service') utils.DeleteDirectory(packages_dir)
def AddImageToComputeEngineProject(image_name, gs_path, description): utils.LogStep('Add image to project') utils.Run(['gcloud', 'compute', 'images', 'delete', image_name, '-q']) utils.Run([ 'gcloud', 'compute', 'images', 'create', image_name, '-q', '--source-uri', gs_path, '--description', description ])
def ConfigureSerialPortOutput(): # https://wiki.archlinux.org/index.php/working_with_the_serial_console # Try this: http://wiki.alpinelinux.org/wiki/Enable_Serial_Console_on_Boot utils.LogStep('Configure Serial Port Output') utils.Sed('/boot/syslinux/syslinux.cfg', '/DEFAULT/aserial 0 38400') utils.ReplaceLine('/boot/syslinux/syslinux.cfg', 'TIMEOUT', 'TIMEOUT 1')
def InstallGoogleCloudSdk(): # TODO: There's a google-cloud-sdk in AUR which should be used # but it's not optimal for cloud use. The image is too large. utils.LogStep('Install Google Cloud SDK') usr_share_google = '/usr/share/google' archive = os.path.join(usr_share_google, 'google-cloud-sdk.zip') unzip_dir = os.path.join(usr_share_google, 'google-cloud-sdk') utils.CreateDirectory(usr_share_google) utils.DownloadFile( 'https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.zip', archive) utils.Run(['unzip', archive, '-d', usr_share_google]) utils.AppendFile('/etc/bash.bashrc', 'export CLOUDSDK_PYTHON=/usr/bin/python2') utils.Run([ os.path.join(unzip_dir, 'install.sh'), '--usage-reporting', 'false', '--bash-completion', 'true', '--disable-installation-options', '--rc-path', '/etc/bash.bashrc', '--path-update', 'true' ], cwd=unzip_dir, env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) utils.Symlink(os.path.join(unzip_dir, 'bin/gcloud'), '/usr/bin/gcloud') utils.Symlink(os.path.join(unzip_dir, 'bin/gcutil'), '/usr/bin/gcutil') utils.Symlink(os.path.join(unzip_dir, 'bin/gsutil'), '/usr/bin/gsutil') utils.SecureDeleteFile(archive)
def UploadImage(image_path, gs_path, make_public=False): utils.LogStep('Upload Image to Cloud Storage') utils.SecureDeleteFile('~/.gsutil/*.url') utils.Run(['gsutil', 'rm', gs_path]) utils.Run(['gsutil', 'cp', image_path, gs_path]) if make_public: utils.Run(['gsutil', 'acl', 'set', 'public-read', gs_path])
def CreateBlankImage(image_path, size_gb=10, fs_type='ext4'): utils.LogStep('Create Image') utils.Run(['rm', '-f', image_path]) utils.Run(['truncate', image_path, '--size=%sG' % size_gb]) utils.Run(['parted', image_path, 'mklabel', 'msdos']) utils.Run(['parted', image_path, 'mkpart', 'primary', fs_type, '1', str(int(size_gb) * 1024)])
def SaveImage(arch_root, image_filename): utils.LogStep('Save Arch Linux Image in GCE format') source_image_raw = os.path.join(arch_root, 'disk.raw') image_raw = os.path.join(os.getcwd(), 'disk.raw') image_file = os.path.join(os.getcwd(), image_filename) utils.Run(['cp', '--sparse=always', source_image_raw, image_raw]) utils.Run(['tar', '-Szcf', image_file, 'disk.raw']) return image_file
def SetupFail2ban(): utils.LogStep('Configure fail2ban') # http://flexion.org/posts/2012-11-ssh-brute-force-defence.html utils.Pacman(['-S', 'fail2ban']) utils.WriteFile('/etc/fail2ban/jail.local', ETC_FAIL2BAN_JAIL_LOCAL) utils.WriteFile('/etc/fail2ban/jail.d/sshd.conf', ETC_FAIL2BAN_JAIL_D_SSHD_CONF) utils.EnableService('syslog-ng') utils.EnableService('fail2ban.service')
def AddImageToComputeEngineProject(image_name, gs_path, description): utils.LogStep('Add image to project') utils.Run(['gcloud', 'compute', 'images', 'delete', image_name, '-q'], env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) utils.Run([ 'gcloud', 'compute', 'images', 'create', image_name, '-q', '--source-uri', gs_path, '--description', description ], env={'CLOUDSDK_PYTHON': '/usr/bin/python2'})
def SetupNetwork(): utils.LogStep('Setup Networking') utils.SecureDeleteFile('/etc/hostname') utils.WriteFile('/etc/hosts', ETC_HOSTS) utils.WriteFile('/etc/sysctl.d/70-disable-ipv6.conf', ETC_SYSCTL_D_70_DISABLE_IPV6_CONF) utils.EnableService('dhcpcd.service') utils.EnableService('systemd-networkd.service') utils.EnableService('systemd-networkd-wait-online.service')
def ConfigureKernel(): utils.LogStep('Configure Kernel') utils.Replace( '/etc/mkinitcpio.conf', 'MODULES=""', 'MODULES="virtio virtio_blk virtio_pci virtio_scsi virtio_net"') utils.Replace('/etc/mkinitcpio.conf', 'autodetect ', '') utils.Run([ 'mkinitcpio', '-g', '/boot/initramfs-linux.img', '-k', '/boot/vmlinuz-linux', '-c', '/etc/mkinitcpio.conf' ])
def SetupAccounts(args): accounts = args['accounts'] if accounts: utils.LogStep('Add Accounts') for account in accounts: username, password = account.split(':') logging.info(' - %s', username) utils.Run(['useradd', username, '-m', '-s', '/bin/bash', '-G', 'adm,video']) utils.Run('echo %s:%s | chpasswd' % (username, password), shell=True)
def UploadImage(image_path, gs_path, make_public=False): utils.LogStep('Upload Image to Cloud Storage') utils.SecureDeleteFile('~/.gsutil/*.url') utils.Run(['gsutil', 'rm', gs_path], env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) utils.Run(['gsutil', 'cp', image_path, gs_path], env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) if make_public: utils.Run(['gsutil', 'acl', 'set', 'public-read', gs_path], env={'CLOUDSDK_PYTHON': '/usr/bin/python2'})
def SetupNetwork(): utils.LogStep('Setup Networking') utils.SecureDeleteFile('/etc/hostname') utils.WriteFile('/etc/hosts', ETC_HOSTS) utils.WriteFile('/etc/sysctl.d/70-disable-ipv6.conf', ETC_SYSCTL_D_70_DISABLE_IPV6_CONF) # https://wiki.archlinux.org/index.php/Network_configuration#Reverting_to_traditional_device_names utils.Symlink('/dev/null', '/etc/udev/rules.d/80-net-setup-link.rules') utils.EnableService('dhcpcd.service') utils.EnableService('systemd-networkd.service') utils.EnableService('systemd-networkd-wait-online.service')
def SetupSsh(): utils.LogStep('Configure SSH') utils.WriteFile('/etc/ssh/sshd_not_to_be_run', 'GOOGLE') utils.SecureDeleteFile('/etc/ssh/ssh_host_key') utils.SecureDeleteFile('/etc/ssh/ssh_host_rsa_key*') utils.SecureDeleteFile('/etc/ssh/ssh_host_dsa_key*') utils.SecureDeleteFile('/etc/ssh/ssh_host_ecdsa_key*') utils.WriteFile('/etc/ssh/ssh_config', ETC_SSH_SSH_CONFIG) utils.Chmod('/etc/ssh/ssh_config', 644) utils.WriteFile('/etc/ssh/sshd_config', ETC_SSH_SSHD_CONFIG) utils.Chmod('/etc/ssh/sshd_config', 644) utils.EnableService('sshd.service')
def SetupFileSystem(base_dir, image_mapping_path): utils.LogStep('File Systems') _, fstab_contents, _ = utils.Run(['genfstab', '-p', base_dir], capture_output=True) utils.WriteFile(os.path.join(base_dir, 'etc', 'fstab'), fstab_contents) _, disk_uuid, _ = utils.Run( ['blkid', '-s', 'UUID', '-o', 'value', image_mapping_path], capture_output=True) disk_uuid = disk_uuid.strip() utils.WriteFile(os.path.join(base_dir, 'etc', 'fstab'), 'UUID=%s / ext4 defaults 0 1' % disk_uuid) utils.Run(['tune2fs', '-i', '1', '-U', disk_uuid, image_mapping_path]) return disk_uuid
def SetupNetwork(): utils.LogStep('Setup Networking') utils.SecureDeleteFile('/etc/hostname') utils.WriteFile('/etc/hosts', ETC_HOSTS) utils.WriteFile('/etc/sysctl.d/70-disable-ipv6.conf', ETC_SYSCTL_D_70_DISABLE_IPV6_CONF) utils.EnableService('dhcpcd.service') utils.EnableService('systemd-networkd.service') utils.EnableService('systemd-networkd-wait-online.service') # Set Google Compute specific MTU # https://cloud.google.com/compute/docs/troubleshooting#packetfragmentation utils.WriteFile('/etc/systemd/system/[email protected]', ETC_SYSTEM_D_SET_MTU) utils.CreateDirectory('/etc/conf.d/') utils.WriteFile('/etc/conf.d/setmtu', ETC_CONF_D_SET_MTU) utils.EnableService('*****@*****.**')
def InstallBootloader(device, uuid, debugmode): utils.LogStep('Install Syslinux bootloader') ''' utils.Run(['syslinux-install_update', '-i', '-a', '-m']) ''' utils.Run(['blkid', '-s', 'PTTYPE', '-o', 'value', device]) utils.CreateDirectory('/boot/syslinux') utils.CopyFiles('/usr/lib/syslinux/bios/*.c32', '/boot/syslinux/') utils.Run(['extlinux', '--install', '/boot/syslinux']) utils.Replace('/boot/syslinux/syslinux.cfg', 'sda3', 'sda1') utils.Run(['fdisk', '-l', device]) utils.Run([ 'dd', 'bs=440', 'count=1', 'conv=notrunc', 'if=/usr/lib/syslinux/bios/mbr.bin', 'of=%s' % device ]) boot_params = [ 'console=ttyS0,38400', 'CONFIG_KVM_GUEST=y', 'CONFIG_KVM_CLOCK=y', 'CONFIG_VIRTIO_PCI=y', 'CONFIG_SCSI_VIRTIO=y', 'CONFIG_VIRTIO_NET=y', 'CONFIG_STRICT_DEVMEM=y', 'CONFIG_DEVKMEM=n', 'CONFIG_DEFAULT_MMAP_MIN_ADDR=65536', 'CONFIG_DEBUG_RODATA=y', 'CONFIG_DEBUG_SET_MODULE_RONX=y', 'CONFIG_CC_STACKPROTECTOR=y', 'CONFIG_COMPAT_VDSO=n', 'CONFIG_COMPAT_BRK=n', 'CONFIG_X86_PAE=y', 'CONFIG_SYN_COOKIES=y', 'CONFIG_SECURITY_YAMA=y', 'CONFIG_SECURITY_YAMA_STACKED=y', ] if debugmode: boot_params += [ 'systemd.log_level=debug', 'systemd.log_target=console', 'systemd.journald.forward_to_syslog=yes', 'systemd.journald.forward_to_kmsg=yes', 'systemd.journald.forward_to_console=yes', ] boot_params = ' '.join(boot_params) boot_spec = ' APPEND root=UUID=%s rw append %s' % (uuid, boot_params) utils.ReplaceLine('/boot/syslinux/syslinux.cfg', 'APPEND root=', boot_spec)
def PrepareBootstrap(workspace_dir, mirror_server, use_pacman_keys): utils.LogStep('Setting up Bootstrap Environment') arch_root = os.path.join(workspace_dir, os.listdir(workspace_dir)[0]) mirrorlist = 'Server = {MIRROR_SERVER}'.format(MIRROR_SERVER=mirror_server) utils.AppendFile(os.path.join(arch_root, 'etc/pacman.d/mirrorlist'), mirrorlist) utils.CreateDirectory(os.path.join(arch_root, 'run/shm')) if use_pacman_keys: utils.RunChroot(arch_root, 'pacman-key --init') utils.RunChroot(arch_root, 'pacman-key --populate archlinux') else: utils.ReplaceLine(os.path.join(arch_root, 'etc/pacman.conf'), 'SigLevel', 'SigLevel = Never') # Install the most basic utilities for the bootstrapper. utils.RunChroot(arch_root, 'pacman --noconfirm -Sy python3 sed') return arch_root
def DownloadArchBootstrap(bootstrap_tarball): utils.LogStep('Download Arch Linux Bootstrap') if bootstrap_tarball: url = bootstrap_tarball sha1sum = None else: url, sha1sum = GetLatestBootstrapUrl() logging.debug('Downloading %s', url) local_bootstrap = os.path.join(os.getcwd(), os.path.basename(url)) if os.path.isfile(local_bootstrap): logging.debug('Using local file instead.') if sha1sum and utils.Sha1Sum(local_bootstrap) == sha1sum: return local_bootstrap utils.DownloadFile(url, local_bootstrap) if not sha1sum or utils.Sha1Sum(local_bootstrap) != sha1sum: raise ValueError('Bad checksum') return local_bootstrap
def ConfigureArchInstall(args, mount_path, parent_path, disk_uuid): relative_builder_path = utils.CopyBuilder(mount_path) utils.LogStep('Download compute-image-packages') packages_dir = utils.CreateTempDirectory(mount_path) utils.Run(['git', 'clone', COMPUTE_IMAGE_PACKAGES_GIT_URL, packages_dir]) utils.CreateDirectory(os.path.join(mount_path, '')) packages_dir = os.path.relpath(packages_dir, mount_path) params = { 'packages_dir': '/%s' % packages_dir, 'device': parent_path, 'disk_uuid': disk_uuid, 'accounts': args['accounts'], 'debugmode': args['debugmode'], } params.update(args) config_arch_py = os.path.join('/', relative_builder_path, 'arch-image.py') utils.RunChroot(mount_path, '%s "%s"' % (config_arch_py, utils.EncodeArgs(params)), use_custom_path=False) utils.DeleteDirectory(os.path.join(mount_path, relative_builder_path))
def ConfigureSecurity(): utils.LogStep('Compute Engine Security Recommendations') utils.WriteFile('/etc/sysctl.d/70-gce-security-strongly-recommended.conf', ETC_SYSCTL_D_70_GCE_SECURITY_STRONGLY_RECOMMENDED_CONF) utils.WriteFile('/etc/sysctl.d/70-gce-security-recommended.conf', ETC_SYSCTL_D_70_GCE_SECURITY_RECOMMENDED_CONF) utils.LogStep('Lock Root User Account') utils.Run(['usermod', '-L', 'root']) utils.LogStep('PAM Security Settings') utils.WriteFile('/etc/pam.d/passwd', ETC_PAM_D_PASSWD) utils.LogStep('Disable CAP_SYS_MODULE') utils.WriteFile('/proc/sys/kernel/modules_disabled', '1') utils.LogStep('Remove the kernel symbol table') utils.SecureDeleteFile('/boot/System.map') utils.LogStep('Sudo Access') utils.WriteFile('/etc/sudoers.d/add-group-adm', ETC_SUDOERS_D_ADD_GROUP_ADM) utils.Run(['chown', 'root:root', '/etc/sudoers.d/add-group-adm']) utils.Run(['chmod', '0440', '/etc/sudoers.d/add-group-adm'])
def ConfigMessageOfTheDay(): utils.LogStep('Configure Message of the Day') utils.WriteFile('/etc/motd', ETC_MOTD)
def SaveImage(disk_image_file, image_filename): utils.LogStep('Save Arch Linux Image in GCE format') image_raw = os.path.join(os.getcwd(), IMAGE_FILE) gce_image_file = os.path.join(os.getcwd(), image_filename) utils.Run(['tar', '-Szcf', image_filename, IMAGE_FILE]) return gce_image_file
def ShrinkDisk(image_mapping_path): utils.LogStep('Shrink Disk') utils.Run(['zerofree', image_mapping_path])
def OptimizePackages(): utils.LogStep('Cleanup Cached Package Data') utils.Pacman(['-Syu']) utils.Pacman(['-Sc']) utils.Run(['pacman-optimize'])
def SetupLocale(): utils.LogStep('Set Locale to US English (UTF-8)') utils.SetupArchLocale()
def SetupNtpServer(): utils.LogStep('Configure NTP') utils.WriteFile('/etc/ntp.conf', 'server metadata.google.internal iburst')
def ForwardSystemdToConsole(): utils.LogStep('Installing syslinux bootloader') utils.AppendFile('/etc/systemd/journald.conf', 'ForwardToConsole=yes')
def ConfigureTimeZone(): utils.LogStep('Set Timezone to UTC') utils.Run(['ln', '-sf', '/usr/share/zoneinfo/UTC', '/etc/localtime'])