def main(): logs_path = utils.GetMetadataParam('daisy-logs-path', raise_on_not_found=True) outs_path = utils.GetMetadataParam('daisy-outs-path', raise_on_not_found=True) # Mount the installer disk. utils.Execute(['mount', '-t', 'ext4', '/dev/sdb1', '/mnt']) logging.info('Installer root: %s', os.listdir('/mnt')) logging.info('Build logs: %s', os.listdir('/mnt/build-logs')) # For some reason we need to remove the gsutil credentials. utils.Execute(['rm', '-Rf', '/root/.gsutil']) utils.Execute(['gsutil', 'cp', '/mnt/ks.cfg', '%s/' % logs_path], raise_errors=False) utils.Execute(['gsutil', 'cp', '/mnt/build-logs/*', '%s/' % logs_path], raise_errors=False) utils.Execute([ 'gsutil', 'cp', '/mnt/build-logs/synopsis.json', '%s/synopsis.json' % outs_path ], raise_errors=False) utils.Execute(['umount', '-l', '/mnt'])
def DistroSpecific(g): el_release = utils.GetMetadataParam('el_release') install_gce = utils.GetMetadataParam('install_gce_packages') rhel_license = utils.GetMetadataParam('use_rhel_gce_license') if rhel_license == 'true': if 'Red Hat' in g.cat('/etc/redhat-release'): g.command(['yum', 'remove', '-y', '*rhui*']) logging.info('Adding in GCE RHUI package.') g.write('/etc/yum.repos.d/google-cloud.repo', repo_compute % el_release) g.command([ 'yum', 'install', '-y', 'google-rhui-client-rhel%s' % el_release ]) if install_gce == 'true': logging.info('Installing GCE packages.') g.write('/etc/yum.repos.d/google-cloud.repo', repo_compute % el_release) if el_release == '7': g.write_append('/etc/yum.repos.d/google-cloud.repo', repo_sdk % el_release) g.command(['yum', '-y', 'install', 'google-cloud-sdk']) g.command([ 'yum', '-y', 'install', 'google-compute-engine', 'python-google-compute-engine' ]) logging.info('Updating initramfs') for kver in g.ls('/lib/modules'): if el_release == '6': # Version 6 doesn't have option --kver g.command(['dracut', '-v', '-f', kver]) else: g.command(['dracut', '-v', '-f', '--kver', kver]) logging.info('Update grub configuration') if el_release == '6': # Version 6 doesn't have grub2, file grub.conf needs to be updated by hand g.write('/tmp/grub_gce_generated', grub_cfg) g.sh(r'grep -P "^[\t ]*initrd|^[\t ]*root|^[\t ]*kernel|^[\t ]*title" ' r'/boot/grub/grub.conf >> /tmp/grub_gce_generated;' r'sed -i "s/console=ttyS0[^ ]*//g" /tmp/grub_gce_generated;' r'sed -i "/^[\t ]*kernel/s/$/ console=ttyS0,38400n8/" ' r'/tmp/grub_gce_generated;' r'mv /tmp/grub_gce_generated /boot/grub/grub.conf') else: g.write('/etc/default/grub', grub2_cfg) g.command(['grub2-mkconfig', '-o', '/boot/grub2/grub.cfg']) # Reset network for DHCP. logging.info('Resetting network to DHCP for eth0.') g.write('/etc/sysconfig/network-scripts/ifcfg-eth0', ifcfg_eth0)
def DistroSpecific(g): ubu_release = utils.GetMetadataParam('ubuntu_release') install_gce = utils.GetMetadataParam('install_gce_packages') if install_gce == 'true': g.command(['apt-get', 'update']) logging.info('Installing cloud-init.') g.sh('DEBIAN_FRONTEND=noninteractive apt-get install -y' ' --no-install-recommends cloud-init') # Try to remove azure or aws configs so cloud-init has a chance. g.sh('rm -f /etc/cloud/cloud.cfg.d/*azure*') g.sh('rm -f /etc/cloud/cloud.cfg.d/*waagent*') g.sh('rm -f /etc/cloud/cloud.cfg.d/*walinuxagent*') g.sh('rm -f /etc/cloud/cloud.cfg.d/*aws*') g.sh('rm -f /etc/cloud/cloud.cfg.d/*amazon*') # Remove Azure agent. try: g.command( ['apt-get', 'remove', '-y', '-f', 'waagent', 'walinuxagent']) except Exception as e: logging.debug(str(e)) logging.warn('Could not uninstall Azure agent. Continuing anyway.') g.write('/etc/apt/sources.list.d/partner.list', partner_list.format(ubu_release=ubu_release)) g.write('/etc/cloud/cloud.cfg.d/91-gce-system.cfg', gce_system) # Use host machine as http proxy so cloud-init can access GCE API with open('/etc/tinyproxy/tinyproxy.conf', 'w') as cfg: cfg.write(tinyproxy_cfg) utils.Execute(['/etc/init.d/tinyproxy', 'restart']) default_gw = g.sh("ip route | awk '/default/ { printf $3 }'") logging.debug( g.sh('http_proxy="http://%s:8888" cloud-init -d init' % default_gw)) logging.info('Installing GCE packages.') g.command(['apt-get', 'update']) g.sh( 'DEBIAN_FRONTEND=noninteractive apt-get install -y' ' --no-install-recommends gce-compute-image-packages google-cloud-sdk' ) # Update grub config to log to console. g.command([ 'sed', '-i', r's#^\(GRUB_CMDLINE_LINUX=".*\)"$#\1 console=ttyS0,38400n8"#', '/etc/default/grub' ]) g.command(['update-grub2'])
def DistroSpecific(g): install_gce = utils.GetMetadataParam('install_gce_packages') deb_release = utils.GetMetadataParam('debian_release') if install_gce == 'true': print 'Installing GCE packages.' g.command([ 'wget', 'https://packages.cloud.google.com/apt/doc/apt-key.gpg', '-O', '/tmp/gce_key' ]) g.command(['apt-key', 'add', '/tmp/gce_key']) g.rm('/tmp/gce_key') g.write('/etc/apt/sources.list.d/google-cloud.list', google_cloud.format(deb_release=deb_release)) # Remove Azure agent. try: g.command( ['apt-get', 'remove', '-y', '-f', 'waagent', 'walinuxagent']) except Exception as e: logging.debug(str(e)) logging.warn('Could not uninstall Azure agent. Continuing anyway.') g.command(['apt-get', 'update']) g.sh('DEBIAN_FRONTEND=noninteractive ' 'apt-get install --assume-yes --no-install-recommends ' 'google-cloud-packages-archive-keyring google-cloud-sdk ' 'google-compute-engine python-google-compute-engine ' 'python3-google-compute-engine') # Update grub config to log to console. g.command([ 'sed', '-i', r's#^\(GRUB_CMDLINE_LINUX=".*\)"$#\1 console=ttyS0,38400n8"#', '/etc/default/grub' ]) # Disable predictive network interface naming in Stretch. if deb_release == 'stretch': g.command([ 'sed', '-i', r's#^\(GRUB_CMDLINE_LINUX=".*\)"$#\1 net.ifnames=0 biosdevname=0"#', '/etc/default/grub' ]) g.command(['update-grub2']) # Reset network for DHCP. print 'Resetting network to DHCP for eth0.' g.write('/etc/network/interfaces', interfaces)
def main(): global COMPUTE global ZONE global PROJECT global TESTEE COMPUTE = utils.GetCompute(discovery, GoogleCredentials) ZONE = utils.GetMetadataParam('zone') PROJECT = utils.GetMetadataParam('project') TESTEE = utils.GetMetadataParam('testee') test_login_ssh_keys(INSTANCE_LEVEL) test_login_ssh_keys(PROJECT_LEVEL) test_ssh_keys_with_sshKeys(INSTANCE_LEVEL) test_ssh_keys_with_sshKeys(PROJECT_LEVEL) test_ssh_keys_mixed_project_instance_level() test_sshKeys_ignores_project_level_keys() test_block_project_ssh_keys_ignores_project_level_keys()
def DistroSpecific(g): el_release = utils.GetMetadataParam('el_release') install_gce = utils.GetMetadataParam('install_gce_packages') rhel_license = utils.GetMetadataParam('use_rhel_gce_license') if rhel_license == 'true': if 'Red Hat' in g.cat('/etc/redhat-release'): g.command(['yum', 'remove', '-y', '*rhui*']) logging.info('Adding in GCE RHUI package.') g.write('/etc/yum.repos.d/google-cloud.repo', repo_compute % el_release) g.command([ 'yum', 'install', '-y', 'google-rhui-client-rhel%s' % el_release ]) if install_gce == 'true': logging.info('Installing GCE packages.') g.write('/etc/yum.repos.d/google-cloud.repo', repo_compute % el_release) if el_release == '7': g.write_append('/etc/yum.repos.d/google-cloud.repo', repo_sdk % el_release) g.command(['yum', '-y', 'install', 'google-cloud-sdk']) if el_release == '6': if 'CentOS' in g.cat('/etc/redhat-release'): logging.info('Installing CentOS SCL.') g.command(['rm', '-f', '/etc/yum.repos.d/CentOS-SCL.repo']) g.command(['yum', '-y', 'install', 'centos-release-scl']) # Install Google Cloud SDK from the upstream tar and create links for the # python27 SCL environment. logging.info('Installing python27 from SCL.') g.command(['yum', '-y', 'install', 'python27']) g.command([ 'scl', 'enable', 'python27', 'pip2.7 install --upgrade google_compute_engine' ]) logging.info('Installing Google Cloud SDK from tar.') sdk_base_url = 'https://dl.google.com/dl/cloudsdk/channels/rapid' sdk_base_tar = '%s/google-cloud-sdk.tar.gz' % sdk_base_url tar = utils.HttpGet(sdk_base_tar) g.write('/tmp/google-cloud-sdk.tar.gz', tar) g.command( ['tar', 'xzf', '/tmp/google-cloud-sdk.tar.gz', '-C', '/tmp']) sdk_version = g.cat('/tmp/google-cloud-sdk/VERSION').strip() logging.info('Getting Cloud SDK Version %s', sdk_version) sdk_version_tar = 'google-cloud-sdk-%s-linux-x86_64.tar.gz' % sdk_version sdk_version_tar_url = '%s/downloads/%s' % (sdk_base_url, sdk_version_tar) logging.info('Getting versioned Cloud SDK tar file from %s', sdk_version_tar_url) tar = utils.HttpGet(sdk_version_tar_url) sdk_version_tar_file = os.path.join('/tmp', sdk_version_tar) g.write(sdk_version_tar_file, tar) g.mkdir_p('/usr/local/share/google') g.command([ 'tar', 'xzf', sdk_version_tar_file, '-C', '/usr/local/share/google', '--no-same-owner' ]) logging.info('Creating CloudSDK SCL symlinks.') sdk_bin_path = '/usr/local/share/google/google-cloud-sdk/bin' g.ln_s(os.path.join(sdk_bin_path, 'git-credential-gcloud.sh'), os.path.join('/usr/bin', 'git-credential-gcloud.sh')) for binary in ['bq', 'gcloud', 'gsutil']: binary_path = os.path.join(sdk_bin_path, binary) new_bin_path = os.path.join('/usr/bin', binary) bin_str = '#!/bin/bash\nsource /opt/rh/python27/enable\n%s $@' % binary_path g.write(new_bin_path, bin_str) g.chmod(0755, new_bin_path) g.command([ 'yum', '-y', 'install', 'google-compute-engine', 'python-google-compute-engine' ]) logging.info('Updating initramfs') for kver in g.ls('/lib/modules'): if el_release == '6': # Version 6 doesn't have option --kver g.command(['dracut', '-v', '-f', kver]) else: g.command(['dracut', '-v', '-f', '--kver', kver]) logging.info('Update grub configuration') if el_release == '6': # Version 6 doesn't have grub2, file grub.conf needs to be updated by hand g.write('/tmp/grub_gce_generated', grub_cfg) g.sh(r'grep -P "^[\t ]*initrd|^[\t ]*root|^[\t ]*kernel|^[\t ]*title" ' r'/boot/grub/grub.conf >> /tmp/grub_gce_generated;' r'sed -i "s/console=ttyS0[^ ]*//g" /tmp/grub_gce_generated;' r'sed -i "/^[\t ]*kernel/s/$/ console=ttyS0,38400n8/" ' r'/tmp/grub_gce_generated;' r'mv /tmp/grub_gce_generated /boot/grub/grub.conf') else: g.write('/etc/default/grub', grub2_cfg) g.command(['grub2-mkconfig', '-o', '/boot/grub2/grub.cfg']) # Reset network for DHCP. logging.info('Resetting network to DHCP for eth0.') g.write('/etc/sysconfig/network-scripts/ifcfg-eth0', ifcfg_eth0)
def main(): # Get Parameters. bvz_manifest = utils.GetMetadataParam( 'bootstrap_vz_manifest', raise_on_not_found=True) bvz_version = utils.GetMetadataParam( 'bootstrap_vz_version', raise_on_not_found=True) repo = utils.GetMetadataParam('google_cloud_repo', raise_on_not_found=True) image_dest = utils.GetMetadataParam('image_dest', raise_on_not_found=True) outs_path = utils.GetMetadataParam('daisy-outs-path', raise_on_not_found=True) if repo not in REPOS: raise ValueError( 'Metadata "google_cloud_repo" must be one of %s.' % REPOS) utils.Status('Bootstrap_vz manifest: %s' % bvz_manifest) utils.Status('Bootstrap_vz version: %s' % bvz_version) utils.Status('Google Cloud repo: %s' % repo) # Download and setup bootstrap_vz. bvz_url = 'https://github.com/andsens/bootstrap-vz/archive/%s.zip' bvz_url %= bvz_version bvz_zip_dir = 'bvz_zip' utils.Status('Downloading bootstrap-vz at commit %s' % bvz_version) urllib.urlretrieve(bvz_url, 'bvz.zip') with zipfile.ZipFile('bvz.zip', 'r') as z: z.extractall(bvz_zip_dir) utils.Status('Downloaded and extracted %s to bvz.zip.' % bvz_url) bvz_zip_contents = [d for d in os.listdir(bvz_zip_dir)] bvz_zip_subdir = os.path.join(bvz_zip_dir, bvz_zip_contents[0]) utils.Execute(['mv', bvz_zip_subdir, BVZ_DIR]) utils.Status('Moved bootstrap_vz from %s to %s.' % (bvz_zip_subdir, BVZ_DIR)) bvz_bin = os.path.join(BVZ_DIR, 'bootstrap-vz') utils.MakeExecutable(bvz_bin) utils.Status('Made %s executable.' % bvz_bin) bvz_manifest_file = os.path.join(BVZ_DIR, 'manifests', bvz_manifest) # Inject Google Cloud test repo plugin if using staging or unstable repos. # This is used to test new package releases in images. if repo is not 'stable': utils.Status('Adding Google Cloud test repos plugin for bootstrapvz.') repo_plugin_dir = '/build_files/google_cloud_test_repos' bvz_plugins = os.path.join(BVZ_DIR, 'bootstrapvz', 'plugins') shutil.move(repo_plugin_dir, bvz_plugins) with open(bvz_manifest_file, 'r+') as manifest_file: manifest_data = yaml.load(manifest_file) manifest_plugins = manifest_data['plugins'] manifest_plugins['google_cloud_test_repos'] = {repo: True} manifest_yaml = yaml.dump(manifest_data, default_flow_style=False) manifest_file.write(manifest_yaml) # Run bootstrap_vz build. cmd = [bvz_bin, '--debug', bvz_manifest_file] utils.Status('Starting build in %s with params: %s' % (BVZ_DIR, str(cmd))) utils.Execute(cmd, cwd=BVZ_DIR) # Upload tar. image_tar_gz = '/target/disk.tar.gz' if os.path.exists(image_tar_gz): image_tar_gz_dest = os.path.join(image_dest, 'root.tar.gz') utils.Status('Saving %s to %s' % (image_tar_gz, image_tar_gz_dest)) utils.Gsutil(['cp', image_tar_gz, image_tar_gz_dest]) # Create and upload the synopsis of the image. utils.Status('Creating image synopsis.') synopsis = {} packages = collections.OrderedDict() _, output, _ = utils.Execute(['dpkg-query', '-W'], capture_output=True) for line in output.split('\n')[:-1]: # Last line is an empty line. parts = line.split() packages[parts[0]] = parts[1] synopsis['installed_packages'] = packages with open('/tmp/synopsis.json', 'w') as f: f.write(json.dumps(synopsis)) utils.Status('Uploading image synopsis.') synopsis_dest = os.path.join(outs_path, 'synopsis.json') utils.Gsutil(['cp', '/tmp/synopsis.json', synopsis_dest])
def main(): # Get Parameters repo = utils.GetMetadataParam('google_cloud_repo', raise_on_not_found=True) release = utils.GetMetadataParam('el_release', raise_on_not_found=True) savelogs = utils.GetMetadataParam('el_savelogs', raise_on_not_found=False) savelogs = savelogs == 'true' byol = utils.GetMetadataParam('rhel_byol', raise_on_not_found=False) byol = byol == 'true' logging.info('EL Installer Builder') logging.info('==============') logging.info('Release: %s', release) logging.info('Google Cloud repo: %s', repo) logging.info('Build working directory: %s', os.getcwd()) iso_file = 'installer.iso' # Necessary libs and tools to build the installer disk. utils.AptGetInstall(['extlinux', 'rsync']) # Build the kickstart file. ks_content = ks_helpers.BuildKsConfig(release, repo, byol) ks_cfg = 'ks.cfg' utils.WriteFile(ks_cfg, ks_content) # Write the installer disk. Write extlinux MBR, create partition, # copy installer ISO and ISO boot files over. logging.info('Writing installer disk.') utils.Execute(['parted', '/dev/sdb', 'mklabel', 'msdos']) utils.Execute(['sync']) utils.Execute(['parted', '/dev/sdb', 'mkpart', 'primary', '1MB', '100%']) utils.Execute(['sync']) utils.Execute(['parted', '/dev/sdb', 'set', '1', 'boot', 'on']) utils.Execute(['sync']) utils.Execute(['dd', 'if=/usr/lib/EXTLINUX/mbr.bin', 'of=/dev/sdb']) utils.Execute(['sync']) utils.Execute(['mkfs.ext4', '-L', 'INSTALLER', '/dev/sdb1']) utils.Execute(['sync']) utils.Execute(['mkdir', 'iso', 'installer']) utils.Execute(['mount', '-o', 'ro,loop', '-t', 'iso9660', iso_file, 'iso']) utils.Execute(['mount', '-t', 'ext4', '/dev/sdb1', 'installer']) utils.Execute( ['rsync', '-Pav', 'iso/images', 'iso/isolinux', 'installer/']) utils.Execute(['cp', iso_file, 'installer/']) utils.Execute(['cp', ks_cfg, 'installer/']) # Modify boot files on installer disk. utils.Execute(['mv', 'installer/isolinux', 'installer/extlinux']) utils.Execute([ 'mv', 'installer/extlinux/isolinux.cfg', 'installer/extlinux/extlinux.conf' ]) # Modify boot config. with open('installer/extlinux/extlinux.conf', 'r+') as f: oldcfg = f.read() cfg = re.sub(r'^default.*', r'default linux', oldcfg, count=1) # Change boot args. args = ' '.join([ 'text', 'ks=hd:/dev/sda1:/%s' % ks_cfg, 'console=ttyS0,38400n8', 'sshd=1', 'loglevel=debug' ]) # Tell Anaconda not to store its logs in the installed image, # unless requested to keep them for debugging. if not savelogs: args += ' inst.nosave=all' cfg = re.sub(r'append initrd=initrd\.img.*', r'\g<0> %s' % args, cfg) # Change labels to explicit partitions. if release in ['centos7', 'rhel7', 'oraclelinux7']: cfg = re.sub(r'LABEL=[^ ]+', 'LABEL=INSTALLER', cfg) # Print out a the modifications. diff = difflib.Differ().compare(oldcfg.splitlines(1), cfg.splitlines(1)) logging.info('Modified extlinux.conf:\n%s', '\n'.join(diff)) f.seek(0) f.write(cfg) f.truncate() # Activate extlinux. utils.Execute(['extlinux', '--install', 'installer/extlinux'])
def main(): # Get Parameters. BVZ_MANIFEST = utils.GetMetadataParam('bootstrap_vz_manifest', raise_on_not_found=True) bvz_version = utils.GetMetadataParam('bootstrap_vz_version', raise_on_not_found=True) build_files_gcs_dir = utils.GetMetadataParam('build_files_gcs_dir', raise_on_not_found=True) repo = utils.GetMetadataParam('google_cloud_repo', raise_on_not_found=True) outs_path = utils.GetMetadataParam('daisy-outs-path', raise_on_not_found=True) license_id = utils.GetMetadataParam('license_id', raise_on_not_found=True) if repo not in REPOS: raise ValueError('Metadata "google_cloud_repo" must be one of %s.' % REPOS) release = utils.GetMetadataParam('release', raise_on_not_found=True) logging.info('Debian Builder') logging.info('==============') logging.info('Bootstrap_vz manifest: %s', BVZ_MANIFEST) logging.info('Bootstrap_vz version: %s', bvz_version) logging.info('Google Cloud repo: %s', repo) logging.info('Debian Builder Sources: %s', build_files_gcs_dir) # Download and setup bootstrap_vz. bvz_url = 'https://github.com/andsens/bootstrap-vz/archive/%s.zip' bvz_url %= bvz_version bvz_zip_dir = 'bvz_zip' logging.info('Downloading bootstrap-vz') urllib.urlretrieve(bvz_url, 'bvz.zip') with zipfile.ZipFile('bvz.zip', 'r') as z: z.extractall(bvz_zip_dir) logging.info('Downloaded and extracted %s to %s', bvz_url, 'bvz_zip') bvz_zip_contents = [d for d in os.listdir(bvz_zip_dir)] bvz_zip_subdir = os.path.join(bvz_zip_dir, bvz_zip_contents[0]) utils.Execute(['mv', bvz_zip_subdir, BVZ_DIR]) logging.info('Moved bootstrap_vz from %s to %s.', bvz_zip_subdir, BVZ_DIR) bvz_bin = os.path.join(BVZ_DIR, 'bootstrap-vz') utils.MakeExecutable(bvz_bin) logging.info('Made %s executable.', bvz_bin) # Run bootstrap_vz build. cmd = [ bvz_bin, '--debug', os.path.join(BVZ_DIR, 'manifests', BVZ_MANIFEST) ] logging.info('Starting build in %s with params: %s', BVZ_DIR, str(cmd)) utils.Execute(cmd, cwd=BVZ_DIR) # Setup tmpfs. tmpfs = '/mnt/tmpfs' os.makedirs(tmpfs) utils.Execute(['mount', '-t', 'tmpfs', '-o', 'size=20g', 'tmpfs', tmpfs]) # Create license manifest. license_manifest = os.path.join(tmpfs, 'manifest.json') logging.info('Creating license manifest for %s', license_id) manifest = '{"licenses": ["%s"]}' % license_id with open(license_manifest, 'w') as manifest_file: manifest_file.write(manifest) # Extract raw image. image = '/target/disk.tar.gz' logging.info('Creating licensed tar for %s', image) with tarfile.open(image, 'r:gz') as tar: tar.extractall(tmpfs) # Create tar with license manifest included. image_tar_gz = os.path.join(tmpfs, os.path.basename(image)) with tarfile.open(image_tar_gz, 'w:gz') as tar: tar_info = tarfile.TarInfo(name='disk.raw') tar_info.type = tarfile.GNUTYPE_SPARSE tar.add(license_manifest, arcname='manifest.json') tar.add(os.path.join(tmpfs, 'disk.raw'), arcname='disk.raw') # Upload tar. image_tar_gz_dest = os.path.join(outs_path, 'image.tar.gz') logging.info('Saving %s to %s', image_tar_gz, image_tar_gz_dest) utils.Gsutil(['cp', image_tar_gz, image_tar_gz_dest]) # Create and upload the synopsis of the image. logging.info('Creating image synopsis.') synopsis = {} packages = collections.OrderedDict() _, output, _ = utils.Execute(['dpkg-query', '-W'], capture_output=True) for line in output.split('\n')[:-1]: # Last line is an empty line. parts = line.split() packages[parts[0]] = parts[1] synopsis['installed_packages'] = packages with open('/tmp/synopsis.json', 'w') as f: f.write(json.dumps(synopsis)) logging.info('Uploading image synopsis.') synopsis_dest = os.path.join(outs_path, 'synopsis.json') utils.Gsutil(['cp', '/tmp/synopsis.json', synopsis_dest])