def AddImageToComputeEngineProject(image_name, gs_path, description):
utils.LogStep('Add image to project')
utils.Run(['gcloud', 'compute', 'images', 'delete', image_name, '-q'])
utils.Run([
'gcloud', 'compute', 'images', 'create', image_name, '-q',
'--source-uri', gs_path, '--description', description
])
def CreateBlankImage(image_path, size_gb=10, fs_type='ext4'):
utils.LogStep('Create Image')
utils.Run(['rm', '-f', image_path])
utils.Run(['truncate', image_path, '--size=%sG' % size_gb])
utils.Run(['parted', image_path, 'mklabel', 'msdos'])
utils.Run(['parted', image_path, 'mkpart', 'primary',
fs_type, '1', str(int(size_gb) * 1024)])
def UploadImage(image_path, gs_path, make_public=False):
utils.LogStep('Upload Image to Cloud Storage')
utils.SecureDeleteFile('~/.gsutil/*.url')
utils.Run(['gsutil', 'rm', gs_path])
utils.Run(['gsutil', 'cp', image_path, gs_path])
if make_public:
utils.Run(['gsutil', 'acl', 'set', 'public-read', gs_path])
def InstallGoogleCloudSdk():
# TODO: There's a google-cloud-sdk in AUR which should be used
# but it's not optimal for cloud use. The image is too large.
utils.LogStep('Install Google Cloud SDK')
usr_share_google = '/usr/share/google'
archive = os.path.join(usr_share_google, 'google-cloud-sdk.zip')
unzip_dir = os.path.join(usr_share_google, 'google-cloud-sdk')
utils.CreateDirectory(usr_share_google)
utils.DownloadFile(
'https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.zip',
archive)
utils.Run(['unzip', archive, '-d', usr_share_google])
utils.AppendFile('/etc/bash.bashrc',
'export CLOUDSDK_PYTHON=/usr/bin/python2')
utils.Run([
os.path.join(unzip_dir, 'install.sh'), '--usage-reporting', 'false',
'--bash-completion', 'true', '--disable-installation-options',
'--rc-path', '/etc/bash.bashrc', '--path-update', 'true'
],
cwd=unzip_dir,
env={'CLOUDSDK_PYTHON': '/usr/bin/python2'})
utils.Symlink(os.path.join(unzip_dir, 'bin/gcloud'), '/usr/bin/gcloud')
utils.Symlink(os.path.join(unzip_dir, 'bin/gcutil'), '/usr/bin/gcutil')
utils.Symlink(os.path.join(unzip_dir, 'bin/gsutil'), '/usr/bin/gsutil')
utils.SecureDeleteFile(archive)
def SaveImage(arch_root, image_filename):
utils.LogStep('Save Arch Linux Image in GCE format')
source_image_raw = os.path.join(arch_root, 'disk.raw')
image_raw = os.path.join(os.getcwd(), 'disk.raw')
image_file = os.path.join(os.getcwd(), image_filename)
utils.Run(['cp', '--sparse=always', source_image_raw, image_raw])
utils.Run(['tar', '-Szcf', image_file, 'disk.raw'])
return image_file
def AddImageToComputeEngineProject(image_name, gs_path, description):
utils.LogStep('Add image to project')
utils.Run(['gcloud', 'compute', 'images', 'delete', image_name, '-q'],
env={'CLOUDSDK_PYTHON': '/usr/bin/python2'})
utils.Run([
'gcloud', 'compute', 'images', 'create', image_name, '-q',
'--source-uri', gs_path, '--description', description
],
env={'CLOUDSDK_PYTHON': '/usr/bin/python2'})
def UploadImage(image_path, gs_path, make_public=False):
utils.LogStep('Upload Image to Cloud Storage')
utils.SecureDeleteFile('~/.gsutil/*.url')
utils.Run(['gsutil', 'rm', gs_path],
env={'CLOUDSDK_PYTHON': '/usr/bin/python2'})
utils.Run(['gsutil', 'cp', image_path, gs_path],
env={'CLOUDSDK_PYTHON': '/usr/bin/python2'})
if make_public:
utils.Run(['gsutil', 'acl', 'set', 'public-read', gs_path],
env={'CLOUDSDK_PYTHON': '/usr/bin/python2'})
def SetupAccounts(args):
accounts = args['accounts']
if accounts:
utils.LogStep('Add Accounts')
for account in accounts:
username, password = account.split(':')
logging.info(' - %s', username)
utils.Run(['useradd', username, '-m', '-s', '/bin/bash',
'-G', 'adm,video'])
utils.Run('echo %s:%s | chpasswd' % (username, password), shell=True)
def SetupFileSystem(base_dir, image_mapping_path):
utils.LogStep('File Systems')
_, fstab_contents, _ = utils.Run(['genfstab', '-p', base_dir],
capture_output=True)
utils.WriteFile(os.path.join(base_dir, 'etc', 'fstab'), fstab_contents)
_, disk_uuid, _ = utils.Run(
['blkid', '-s', 'UUID', '-o', 'value', image_mapping_path],
capture_output=True)
disk_uuid = disk_uuid.strip()
utils.WriteFile(os.path.join(base_dir, 'etc', 'fstab'),
'UUID=%s / ext4 defaults 0 1' % disk_uuid)
utils.Run(['tune2fs', '-i', '1', '-U', disk_uuid, image_mapping_path])
return disk_uuid
def InstallBootloader(device, uuid, debugmode):
utils.LogStep('Install Syslinux bootloader')
'''
utils.Run(['syslinux-install_update', '-i', '-a', '-m'])
'''
utils.Run(['blkid', '-s', 'PTTYPE', '-o', 'value', device])
utils.CreateDirectory('/boot/syslinux')
utils.CopyFiles('/usr/lib/syslinux/bios/*.c32', '/boot/syslinux/')
utils.Run(['extlinux', '--install', '/boot/syslinux'])
utils.Replace('/boot/syslinux/syslinux.cfg', 'sda3', 'sda1')
utils.Run(['fdisk', '-l', device])
utils.Run([
'dd', 'bs=440', 'count=1', 'conv=notrunc',
'if=/usr/lib/syslinux/bios/mbr.bin',
'of=%s' % device
])
boot_params = [
'console=ttyS0,38400',
'CONFIG_KVM_GUEST=y',
'CONFIG_KVM_CLOCK=y',
'CONFIG_VIRTIO_PCI=y',
'CONFIG_SCSI_VIRTIO=y',
'CONFIG_VIRTIO_NET=y',
'CONFIG_STRICT_DEVMEM=y',
'CONFIG_DEVKMEM=n',
'CONFIG_DEFAULT_MMAP_MIN_ADDR=65536',
'CONFIG_DEBUG_RODATA=y',
'CONFIG_DEBUG_SET_MODULE_RONX=y',
'CONFIG_CC_STACKPROTECTOR=y',
'CONFIG_COMPAT_VDSO=n',
'CONFIG_COMPAT_BRK=n',
'CONFIG_X86_PAE=y',
'CONFIG_SYN_COOKIES=y',
'CONFIG_SECURITY_YAMA=y',
'CONFIG_SECURITY_YAMA_STACKED=y',
]
if debugmode:
boot_params += [
'systemd.log_level=debug',
'systemd.log_target=console',
'systemd.journald.forward_to_syslog=yes',
'systemd.journald.forward_to_kmsg=yes',
'systemd.journald.forward_to_console=yes',
]
boot_params = ' '.join(boot_params)
boot_spec = ' APPEND root=UUID=%s rw append %s' % (uuid, boot_params)
utils.ReplaceLine('/boot/syslinux/syslinux.cfg', 'APPEND root=', boot_spec)
def CreateArchImage(args, aur_packages):
logging.info('Creating Arch Image')
logging.info('========================')
image_path = os.path.join(os.getcwd(), IMAGE_FILE)
CreateBlankImage(image_path, size_gb=int(args.size_gb), fs_type=args.fs_type)
mount_path = utils.CreateTempDirectory(base_dir='/')
image_mapping = utils.ImageMapper(image_path, mount_path)
try:
image_mapping.InstallLoopback()
image_mapping.Map()
primary_mapping = image_mapping.GetFirstMapping()
image_mapping_path = primary_mapping['path']
FormatImage(image_mapping_path)
try:
image_mapping.Mount()
utils.CreateDirectory('/run/shm')
utils.CreateDirectory(os.path.join(mount_path, 'run', 'shm'))
InstallArchLinux(mount_path)
disk_uuid = SetupFileSystem(mount_path, image_mapping_path, args.fs_type)
ConfigureArchInstall(
args, mount_path, primary_mapping['parent'], disk_uuid, aur_packages)
utils.DeleteDirectory(os.path.join(mount_path, 'run', 'shm'))
PurgeDisk(mount_path)
finally:
image_mapping.Unmount()
ShrinkDisk(image_mapping_path)
finally:
image_mapping.Unmap()
utils.Run(['parted', image_path, 'set', '1', 'boot', 'on'])
utils.Sync()
logging.info('========================')
return image_path
def ConfigureArchInstall(args, mount_path, parent_path, disk_uuid, aur_packages):
relative_builder_path = utils.CopyBuilder(mount_path)
packages_dir = utils.CreateTempDirectory(mount_path)
utils.Run(['git', 'clone', COMPUTE_IMAGE_PACKAGES_GIT_URL, packages_dir])
utils.CreateDirectory(os.path.join(mount_path, ''))
aur_packages_dir = os.path.join(packages_dir, 'aur')
for aur_package in aur_packages:
utils.CopyFiles(aur_package, aur_packages_dir + '/')
packages_dir = os.path.relpath(packages_dir, mount_path)
params = {
'packages_dir': '/%s' % packages_dir,
'device': parent_path,
'disk_uuid': disk_uuid,
'accounts': args.accounts,
'debugmode': args.debug,
'quiet': args.quiet,
'verbose': args.verbose,
'packages': args.packages,
'size_gb': args.size_gb
}
config_arch_py = os.path.join(
'/', relative_builder_path, 'arch-image.py')
utils.RunChroot(mount_path,
'%s "%s"' % (config_arch_py, utils.EncodeArgs(params)),
use_custom_path=False)
utils.DeleteDirectory(os.path.join(mount_path, relative_builder_path))
def InstallComputeImagePackages(packages_dir):
utils.LogStep('Install compute-image-packages')
utils.Run([
"egrep -lRZ 'python' %s | "
"xargs -0 -l sed -i -e '/#!.*python/c\#!/usr/bin/env python2'" %
packages_dir
],
shell=True)
utils.CopyFiles(os.path.join(packages_dir, 'google-daemon', '*'), '/')
utils.CopyFiles(os.path.join(packages_dir, 'google-startup-scripts', '*'),
'/')
utils.SecureDeleteFile('/README.md')
# TODO: Fix gcimagebundle does not work with Arch yet.
#InstallGcimagebundle(packages_dir)
# Patch Google services to run after the network is actually available.
PatchGoogleSystemdService(
'/usr/lib/systemd/system/google-startup-scripts.service')
PatchGoogleSystemdService(
'/usr/lib/systemd/system/google-accounts-manager.service')
PatchGoogleSystemdService(
'/usr/lib/systemd/system/google-address-manager.service')
PatchGoogleSystemdService('/usr/lib/systemd/system/google.service')
utils.EnableService('google-accounts-manager.service')
utils.EnableService('google-address-manager.service')
utils.EnableService('google.service')
utils.EnableService('google-startup-scripts.service')
utils.DeleteDirectory(packages_dir)
def main():
args = utils.DecodeArgs(sys.argv[1])
utils.SetupLogging(quiet=args['quiet'], verbose=args['verbose'])
logging.info('Setup Bootstrapper Environment')
utils.SetupArchLocale()
InstallPackagesForStagingEnvironment()
image_path = os.path.join(os.getcwd(), IMAGE_FILE)
CreateImage(image_path, size_gb=int(args['size_gb']))
mount_path = utils.CreateTempDirectory(base_dir='/')
image_mapping = utils.ImageMapper(image_path, mount_path)
try:
image_mapping.Map()
primary_mapping = image_mapping.GetFirstMapping()
image_mapping_path = primary_mapping['path']
FormatImage(image_mapping_path)
try:
image_mapping.Mount()
utils.CreateDirectory('/run/shm')
utils.CreateDirectory(os.path.join(mount_path, 'run', 'shm'))
InstallArchLinux(mount_path)
disk_uuid = SetupFileSystem(mount_path, image_mapping_path)
ConfigureArchInstall(args, mount_path, primary_mapping['parent'],
disk_uuid)
utils.DeleteDirectory(os.path.join(mount_path, 'run', 'shm'))
PurgeDisk(mount_path)
finally:
image_mapping.Unmount()
ShrinkDisk(image_mapping_path)
finally:
image_mapping.Unmap()
utils.Run(['parted', image_path, 'set', '1', 'boot', 'on'])
utils.Sync()
def pushRemote(self, file_loc, file_remote, follow=False):
rsync_flags = "-aP"
# if they asked us to follow symlinks, then add '-L' into the arguments.
if follow:
rsync_flags += "L"
utils.Run([
"rsync", rsync_flags, file_loc, self.HostName + ":" + file_remote
])
def extract(self, filename):
if "tar.bz2" in filename:
tar = tarfile.open(self.folder + filename)
tar.extractall(self.folder)
tar.close()
elif "zip" in filename:
with utils.chdir(self.folder):
utils.Run(["unzip", filename], silent=True)
def report_html():
"""Create the report."""
print("-", report_html.__name__)
with open("%sdata.json" % (utils.results), 'r') as json_file:
data_json = json.load(json_file)
print_html_doc(data_json)
print(" : index.html generated")
utils.Run("rm %sdata.json" % (utils.results))
def ConfigureKernel():
utils.LogStep('Configure Kernel')
utils.Replace(
'/etc/mkinitcpio.conf', 'MODULES=""',
'MODULES="virtio virtio_blk virtio_pci virtio_scsi virtio_net"')
utils.Replace('/etc/mkinitcpio.conf', 'autodetect ', '')
utils.Run([
'mkinitcpio', '-g', '/boot/initramfs-linux.img', '-k',
'/boot/vmlinuz-linux', '-c', '/etc/mkinitcpio.conf'
])
def phoronix_install(b):
"""Install benchamrks."""
print(" :", phoronix_install.__name__)
for i in b:
cmd = "phoronix-test-suite force-install " + i
print("\t", cmd)
rc, o, err = utils.Run(cmd, xenv=pxenv)
if err:
print(err, "(%s)" % rc)
def get_release_notes(regression):
"""Get the release notes."""
global release_notes
url = 'https://cdn.download.clearlinux.org/releases/'+ regression\
+ '/clear/RELEASENOTES'
r, release_notes, err = utils.Run("curl " + url)
if r != 0:
print("NO RELSEASENOTES FOUND!! for relase: " + regression)
return ""
class RemoteSlave(Slave):
def __init__(self, name):
super(RemoteSlave, self).__init__(name)
self.HostName = utils.config_get_default(name, 'hostname', name)
self.delayed = None
self.delayedCommand = None
def prepare(self, engines):
self.pushRemote(utils.DriverPath + os.path.sep, self.DriverPath)
self.pushRemote(utils.BenchmarkPath + os.path.sep, self.BenchmarkPath)
for engine in engines:
shell = os.path.join(utils.RepoPath, engine.source, engine.shell())
rshell = os.path.join(self.RepoPath, engine.source, engine.shell())
self.runRemote(["mkdir", "-p", os.path.dirname(rshell)])
self.pushRemote(shell, rshell, follow=True)
def benchmark(self, submit, native, modes):
fd = open("state.p", "wb")
# dump the global state gathered from the config file
pickle.dump(utils.config, fd)
# dump out all the arguments
pickle.dump(submit, fd)
pickle.dump(native, fd)
pickle.dump(modes, fd)
fd.close()
# send the pickled data over the wire so we can make a call
self.pushRemote(os.path.join(utils.DriverPath, "state.p"),
os.path.join(self.DriverPath, "state.p"))
# cd into the driver's directory, then start running the module.
self.runRemote([
"cd", self.DriverPath, ";", self.PythonName, 'slaves.py',
os.path.join(self.DriverPath, "state.p")
],
async=True)
def runRemote(self, cmds, async=False):
# no matter what, we don't want to start running a new command until the old one is gone.
self.synchronize()
fullcmd = ["ssh", self.HostName, "--"] + cmds
if async:
print("ASYNC: " + " ".join(fullcmd))
self.delayed = subprocess.Popen(fullcmd,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE)
subprocess.Popen(['sed', '-e', 's/^/' + self.name + ': /'],
stdin=self.delayed.stdout)
self.delayedCommand = str(fullcmd)
else:
utils.Run(fullcmd)
def phoronix_run(benchmark):
"""Run benchmark suite."""
print(" :", phoronix_run.__name__)
for b in benchmark:
blog = b.split("/")[1]
strace = "/usr/bin/strace -ff -o %s/%s -ttt " % (logs, blog)
pxenv.update({'EXECUTE_BINARY_PREPEND': strace})
cmd = "phoronix-test-suite batch-run " + b
print("\t", cmd)
rc, o, err = utils.Run(cmd, xenv=pxenv)
if err:
print(err, "(%s)" % rc)
else:
with open("%s%s.log" % (utils.results, blog), "wb", 0) as f:
cmd = "/usr/bin/strace-log-merge %s/%s" % (logs, blog)
rc, o, err = utils.Run(cmd, stdout=f)
if rc != 0:
print(err, "(%s)" % rc)
def ConfigureSecurity():
utils.LogStep('Compute Engine Security Recommendations')
utils.WriteFile('/etc/sysctl.d/70-gce-security-strongly-recommended.conf',
ETC_SYSCTL_D_70_GCE_SECURITY_STRONGLY_RECOMMENDED_CONF)
utils.WriteFile('/etc/sysctl.d/70-gce-security-recommended.conf',
ETC_SYSCTL_D_70_GCE_SECURITY_RECOMMENDED_CONF)
utils.LogStep('Lock Root User Account')
utils.Run(['usermod', '-L', 'root'])
utils.LogStep('PAM Security Settings')
utils.WriteFile('/etc/pam.d/passwd', ETC_PAM_D_PASSWD)
utils.LogStep('Disable CAP_SYS_MODULE')
utils.WriteFile('/proc/sys/kernel/modules_disabled', '1')
utils.LogStep('Remove the kernel symbol table')
utils.SecureDeleteFile('/boot/System.map')
utils.LogStep('Sudo Access')
utils.WriteFile('/etc/sudoers.d/add-group-adm', ETC_SUDOERS_D_ADD_GROUP_ADM)
utils.Run(['chown', 'root:root', '/etc/sudoers.d/add-group-adm'])
utils.Run(['chmod', '0440', '/etc/sudoers.d/add-group-adm'])
def whatprovides(file_name):
"""Get the bundle that provides a file name."""
pkgs = []
pkg = ""
cmd = "repoquery -c /etc/yum.conf --whatprovides " + file_name
r, o, err = utils.Run(cmd)
if r != 0:
cmd = "dnf-3 --releasever=clear --config=/etc/dnf.conf provides "
cmd += file_name
r, o, err = utils.Run(cmd)
if r != 0:
return pkgs
for line in iter(o.splitlines()):
if ".x86_64" in line:
pkg = line.split("-")[0]
if pkg not in pkgs:
pkgs.append(pkg)
# for pkg in pkgs:
# print("\tFile : " + file_name + " is provided by : " + pkg)
return pkg
def ConfigureArchInstall(args, mount_path, parent_path, disk_uuid):
relative_builder_path = utils.CopyBuilder(mount_path)
utils.LogStep('Download compute-image-packages')
packages_dir = utils.CreateTempDirectory(mount_path)
utils.Run(['git', 'clone', COMPUTE_IMAGE_PACKAGES_GIT_URL, packages_dir])
utils.CreateDirectory(os.path.join(mount_path, ''))
packages_dir = os.path.relpath(packages_dir, mount_path)
params = {
'packages_dir': '/%s' % packages_dir,
'device': parent_path,
'disk_uuid': disk_uuid,
'accounts': args['accounts'],
'debugmode': args['debugmode'],
}
params.update(args)
config_arch_py = os.path.join('/', relative_builder_path, 'arch-image.py')
utils.RunChroot(mount_path,
'%s "%s"' % (config_arch_py, utils.EncodeArgs(params)),
use_custom_path=False)
utils.DeleteDirectory(os.path.join(mount_path, relative_builder_path))
def runGaiaTest(test):
args = ['./run-benchmark.sh', b2gDir, test]
nb = 0
output = ""
m = None
# Retry because we might fail to connect to the wifi
while nb < 10:
try:
output = utils.Run(args)
m = filterOutput.search(output)
if m != None:
return m.group(1)
except subprocess.CalledProcessError, e:
print e.output
nb = nb + 1
if nb == 10:
raise Exception("Fail to execute")
def InstallGcimagebundle(packages_dir):
utils.WriteFile(
os.path.join(packages_dir, 'gcimagebundle/gcimagebundlelib/arch.py'),
GCIMAGEBUNDLE_ARCH_PY)
utils.Run(['python2', 'setup.py', 'install'],
cwd=os.path.join(packages_dir, 'gcimagebundle'))
def ConfigureTimeZone():
utils.LogStep('Set Timezone to UTC')
utils.Run(['ln', '-sf', '/usr/share/zoneinfo/UTC', '/etc/localtime'])
def OptimizePackages():
utils.LogStep('Cleanup Cached Package Data')
utils.Pacman(['-Syu'])
utils.Pacman(['-Sc'])
utils.Run(['pacman-optimize'])
def SaveImage(disk_image_file, image_filename):
utils.LogStep('Save Arch Linux Image in GCE format')
image_raw = os.path.join(os.getcwd(), IMAGE_FILE)
gce_image_file = os.path.join(os.getcwd(), image_filename)
utils.Run(['tar', '-Szcf', image_filename, IMAGE_FILE])
return gce_image_file
