def run(self, command): # Run a shell command in the VM. self.forward(22) host = "linux-vm" if self.system == 'windows': host = "windows-vm" sh("ssh -F %s %s \"%s\"" % (CTL_DIR + "/ssh_config", host, command)) self.unforward(22)
def put(self, src_filename, dst_filename): # Copy a file to the VM. self.forward(22) host = "linux-vm" if self.system == 'windows': host = "windows-vm" sh("scp -rF %s \"%s\" %s:\"%s\"" % (CTL_DIR + "/ssh_config", src_filename, host, dst_filename)) self.unforward(22)
def get(self, src_filename, dst_filename): # Read a file from VM. self.forward(22) host = "linux-vm" if self.system == 'windows': host = "windows-vm" sh("scp -rF %s %s:\"%s\" \"%s\"" % (CTL_DIR + "/ssh_config", host, src_filename, dst_filename)) self.unforward(22)
def compress(self, backing_name=None): # Run `qemu-img convert -c ...`. opts = "convert -c -f qcow2 -O qcow2" if backing_name: opts += " -o backing_file=%s.qcow2" % backing_name sh("qemu-img %s %s.qcow2 %s-compressed.qcow2" % (opts, self.name, self.name), cd=IMG_DIR) mv(IMG_DIR+"/%s-compressed.qcow2" % self.name, IMG_DIR+"/%s.qcow2" % self.name)
def PKG_DEB(): """create Debian packages This task creates Debian packages from source packages. """ if deb_vm.missing(): raise fail("VM is not built: {}", deb_vm.name) if deb_vm.running(): deb_vm.stop() if os.path.exists("./build/pkg/deb"): rmtree("./build/pkg/deb") if os.path.exists("./build/tmp"): rmtree("./build/tmp") version = get_version() debian_version = ".".join(version.split(".")[:3]) moves = load_moves(DATA_ROOT + "/pkg/debian/moves.yaml") deb_vm.start() pubkey = pipe("gpg --armour --export %s" % KEYSIG) seckey = pipe("gpg --armour --export-secret-key %s" % KEYSIG) deb_vm.write("/root/sign.key", pubkey + seckey) deb_vm.run("gpg --import /root/sign.key") try: for move in moves: package = "%s-%s" % (move.code.upper(), version) debian_package = "%s_%s" % (move.code, debian_version) archive = "./build/pkg/src/%s.tar.gz" % package if not os.path.exists(archive): raise fail("cannot find a source package;" " run `cogs pkg-src` first") changelog = open(DATA_ROOT + "/pkg/debian/changelog").read() if ('htsql (%s-1)' % debian_version) not in changelog: raise fatal("run `job pkg-deb-changelog`" " to update the changelog file") changelog = changelog.replace('htsql (', '%s (' % move.code) mktree("./build/tmp") cp(archive, "./build/tmp/%s.orig.tar.gz" % debian_package) sh("tar -xzf %s -C ./build/tmp" % archive) move(DATA_ROOT + "/pkg/debian", "./build/tmp/%s" % package) open("./build/tmp/%s/debian/changelog" % package, 'w') \ .write(changelog) deb_vm.put("./build/tmp", "./build") deb_vm.run("cd ./build/%s && dpkg-buildpackage -k%s" % (package, KEYSIG)) if not os.path.exists("./build/pkg/deb"): mktree("./build/pkg/deb") deb_vm.get("./build/*.deb", "./build/pkg/deb") deb_vm.run("rm -rf build") rmtree("./build/tmp") finally: deb_vm.stop() log() log("The generated Debian packages are placed in:") for filename in glob.glob("./build/pkg/deb/*"): log(" `{}`", filename) log()
def kvm(self, opts=None): # Run `kvm`. net_model = "virtio" if self.system == 'windows': net_model = "rtl8139" sh("kvm -name %s -monitor unix:%s,server,nowait" " -drive file=%s,cache=writeback" " -net nic,model=%s -net user -vga cirrus" " -rtc clock=vm -m %s" % (self.name, self.ctl_path, self.img_path, net_model, MEM_SIZE) + ((" " + opts) if opts else "") + ("" if env.debug else " -vnc none"))
def build(self): # Generate a VM image. if not self.missing(): raise fail("VM is already built") for path in [IMG_DIR, CTL_DIR, TMP_DIR]: if not os.path.exists(path): mktree(path) identity_path = CTL_DIR + "/identity" if not os.path.exists(identity_path): sh("ssh-keygen -q -N \"\" -f %s" % identity_path) config_path = CTL_DIR + "/ssh_config" if not os.path.exists(config_path): config_template_path = DATA_ROOT + "/vm/ssh_config" config_template = open(config_template_path).read() config = config_template.replace("$VM_ROOT", VM_ROOT) assert config != config_template debug("translating: {} => {}", config_template_path, config_path) open(config_path, 'w').write(config)
def PKG_DEB_CHANGELOG(message=None): """update the Debian changelog This task updates the Debian changelog to the current HTSQL version. """ if message is None: message = "new upstream release" version = get_version() debian_version = ".".join(version.split(".")[:3]) debian_version += "-1" changelog = open(DATA_ROOT + "/pkg/debian/changelog").read() if ('htsql (%s)' % debian_version) in changelog: raise fail("changelog is already up-to-date") sh("dch --check-dirname-level 0 -D unstable -v %s %s" % (debian_version, message), cd=DATA_ROOT + "/pkg/debian") log("The Debian changelog is updated to version:") log(" `{}`", debian_version) log()
def build(self): super(DebianTemplateVM, self).build() log("building VM: `{}`...", self.name) start_time = datetime.datetime.now() src_iso_path = getattr(env, self.iso_env) if not (src_iso_path and os.path.isfile(src_iso_path)): src_iso_path = self.download(self.iso_urls) unpack_path = TMP_DIR + "/" + self.name if os.path.exists(unpack_path): rmtree(unpack_path) self.unpack_iso(src_iso_path, unpack_path) cp(DATA_ROOT + "/vm/%s-isolinux.cfg" % self.name, unpack_path + "/isolinux/isolinux.cfg") cp(DATA_ROOT + "/vm/%s-preseed.cfg" % self.name, unpack_path + "/preseed.cfg") cp(DATA_ROOT + "/vm/%s-install.sh" % self.name, unpack_path + "/install.sh") cp(CTL_DIR + "/identity.pub", unpack_path + "/identity.pub") sh( "md5sum" " `find ! -name \"md5sum.txt\"" " ! -path \"./isolinux/*\" -follow -type f` > md5sum.txt", cd=unpack_path) iso_path = TMP_DIR + "/%s.iso" % self.name if os.path.exists(iso_path): rm(iso_path) sh("mkisofs -o %s" " -q -r -J -no-emul-boot -boot-load-size 4 -boot-info-table" " -b isolinux/isolinux.bin -c isolinux/boot.cat %s" % (iso_path, unpack_path)) rmtree(unpack_path) try: self.kvm_img() self.kvm("-cdrom %s -boot d" % iso_path) rm(iso_path) self.compress() except: if os.path.exists(self.img_path): rm(self.img_path) raise stop_time = datetime.datetime.now() log("VM is built successfully: `{}` ({})", self.name, stop_time - start_time)
def build(self): super(LinuxBenchVM, self).build() parent_vm = VM.find(self.parent) if parent_vm.missing(): parent_vm.build() if parent_vm.running(): raise fail("unable to copy VM while it is running: {}", parent_vm.name) log("building VM: `{}`...", self.name) start_time = datetime.datetime.now() try: sh("qemu-img create -b %s.qcow2 -f qcow2 %s.qcow2" % (parent_vm.name, self.name), cd=IMG_DIR) self.kvm("-daemonize") time.sleep(60.0) self.put(DATA_ROOT + "/vm/%s-update.sh" % self.name, "/root/update.sh") self.run("/root/update.sh") self.run("rm /root/update.sh") self.run("shutdown") self.wait() #self.compress(parent_vm.name) self.kvm("-daemonize") time.sleep(60.0) self.ctl("savevm %s" % self.state) self.ctl("quit") self.wait() except: if self.running(): self.ctl("quit") self.wait() if os.path.exists(self.img_path): rm(self.img_path) raise stop_time = datetime.datetime.now() log("VM is built successfully: `{}` ({})", self.name, stop_time - start_time)
def build(self): super(CentOSTemplateVM, self).build() log("building VM: `{}`...", self.name) start_time = datetime.datetime.now() src_iso_path = env.centos_iso if not (src_iso_path and os.path.isfile(src_iso_path)): src_iso_path = self.download(CENTOS_ISO_URLS) unpack_path = TMP_DIR + "/" + self.name if os.path.exists(unpack_path): rmtree(unpack_path) self.unpack_iso(src_iso_path, unpack_path) cp(DATA_ROOT + "/vm/%s-isolinux.cfg" % self.name, unpack_path + "/isolinux/isolinux.cfg") cp(DATA_ROOT + "/vm/%s-ks.cfg" % self.name, unpack_path + "/ks.cfg") cp(DATA_ROOT + "/vm/%s-install.sh" % self.name, unpack_path + "/install.sh") cp(CTL_DIR + "/identity.pub", unpack_path + "/identity.pub") iso_path = TMP_DIR + "/%s.iso" % self.name if os.path.exists(iso_path): rm(iso_path) sh("mkisofs -o %s" " -q -r -J -T -no-emul-boot -boot-load-size 4 -boot-info-table" " -b isolinux/isolinux.bin -c isolinux/boot.cat %s" % (iso_path, unpack_path)) rmtree(unpack_path) try: self.kvm_img() self.kvm("-cdrom %s -boot d" % iso_path) rm(iso_path) self.compress() except: if os.path.exists(self.img_path): rm(self.img_path) raise stop_time = datetime.datetime.now() log("VM is built successfully: `{}` ({})", (self.name, stop_time - start_time))
def python(command, cd=None): # Run `python <command>`. with env(debug=True): sh(env.python_path + " " + command, cd=cd)
def CHECK_ALL(): """run regression tests for all supported backends This task runs HTSQL regression tests on all combinations of client and server platforms. """ vms = [ py26_vm, py27_vm, pgsql84_vm, pgsql90_vm, pgsql91_vm, mysql51_vm, oracle10g_vm, mssql2005_vm, mssql2008_vm ] for vm in vms: if vm.missing(): warn("VM is not built: {}", vm.name) for vm in vms: if vm.running(): vm.stop() errors = 0 try: for client_vm in [py26_vm, py27_vm]: if client_vm.missing(): continue client_vm.start() client_vm.run("~/bin/pip -q install" " hg+http://bitbucket.org/prometheus/pbbt") sh("hg clone --ssh='ssh -F %s' . ssh://linux-vm/src/htsql" % (CTL_DIR + "/ssh_config")) errors += trial("hg update && python setup.py install", "installing HTSQL under %s" % client_vm.name) errors += trial( "pbbt test/regress.yaml -E test/regress.py" " -q -S /all/sqlite", "testing sqlite backend") for server_vm, suite in [(pgsql84_vm, 'pgsql'), (pgsql90_vm, 'pgsql'), (pgsql91_vm, 'pgsql'), (mysql51_vm, 'mysql'), (oracle10g_vm, 'oracle'), (mssql2005_vm, 'mssql'), (mssql2008_vm, 'mssql')]: if server_vm.missing(): continue server_vm.start() username_key = "%s_USERNAME" % suite.upper() password_key = "%s_PASSWORD" % suite.upper() host_key = "%s_HOST" % suite.upper() port_key = "%s_PORT" % suite.upper() username_value = { 'pgsql': "postgres", 'mysql': "root", 'oracle': "system", 'mssql': "sa" }[suite] password_value = "admin" host_value = "10.0.2.2" port_value = 10000 + server_vm.port command = "pbbt test/regress.yaml -E test/regress.py" \ " -q -S /all/%s" \ " -D %s=%s -D %s=%s -D %s=%s -D %s=%s" \ % (suite, username_key, username_value, password_key, password_value, host_key, host_value, port_key, port_value) message = "testing %s backend against %s" \ % (suite, server_vm.name) errors += trial(command, message) server_vm.stop() errors += trial( "pbbt test/regress.yaml -E test/regress.py" " -q -S /all/routine", "testing htsql-ctl routines") client_vm.stop() except: for vm in vms: if vm.running(): vm.stop() raise log() if errors: if errors == 1: warn("1 failed test") else: warn("{} failed tests", errors) else: log("`All tests passed`")
def PKG_SRC(): """create a source package This task creates Python source distribution. """ if src_vm.missing(): raise fail("VM is not built: {}", src_vm.name) if src_vm.running(): src_vm.stop() if os.path.exists("./build/pkg/src"): rmtree("./build/pkg/src") if os.path.exists("./build/tmp"): rmtree("./build/tmp") version = get_version() all_routines = get_routines() all_addons = get_addons() moves = load_moves(DATA_ROOT + "/pkg/source/moves.yaml") src_vm.start() src_vm.run("pip install wheel") try: for move in moves: with_doc = move.variables['with-doc'] packages = move.variables['packages'].strip().splitlines() routines = "".join( routine + "\n" for routine in all_routines if routine.split('=', 1)[1].strip().split('.')[0] in packages) addons = "".join( addon + "\n" for addon in all_addons if addon.split('=', 1)[1].strip().split('.')[0] in packages) move.variables['version'] = version move.variables['htsql-routines'] = routines move.variables['htsql-addons'] = addons mktree("./build/tmp") sh("hg archive ./build/tmp/htsql") if with_doc: setup_py("-q download_vendor", cd="./build/tmp/htsql") for dirname in glob.glob("./build/tmp/htsql/src/*"): if os.path.basename(dirname) not in packages: rmtree(dirname) packages = setuptools.find_packages("./build/tmp/htsql/src") move.variables['packages'] = "".join(package + "\n" for package in packages) if not with_doc: rmtree("./build/tmp/htsql/doc") move(DATA_ROOT + "/pkg/source", "./build/tmp/htsql") src_vm.put("./build/tmp/htsql", ".") if with_doc: src_vm.run("cd htsql &&" " PYTHONPATH=src sphinx-build -d doc doc doc/html") for filename in glob.glob("./build/tmp/htsql/doc/man/*.?.rst"): basename = os.path.basename(filename) target = basename[:-4] src_vm.run("rst2man htsql/doc/man/%s htsql/doc/man/%s" % (basename, target)) src_vm.run("cd htsql && python setup.py sdist --formats=zip,gztar") src_vm.run("cd htsql && python setup.py bdist_wheel") if not os.path.exists("./build/pkg/src"): mktree("./build/pkg/src") src_vm.get("./htsql/dist/*", "./build/pkg/src") src_vm.run("rm -rf htsql") rmtree("./build/tmp") finally: src_vm.stop() log() log("The generated source packages are placed in:") for filename in glob.glob("./build/pkg/src/*"): log(" `{}`", filename) log()
def build(self): super(WindowsTemplateVM, self).build() log("building VM: `{}`...", self.name) start_time = datetime.datetime.now() src_iso_path = env.windows_iso if not (src_iso_path and os.path.isfile(src_iso_path)): src_iso_path = None output = pipe("locate %s || true" % " ".join(WINDOWS_ISO_FILES)) for line in output.splitlines(): if os.path.exists(line): src_iso_path = line break if src_iso_path is None: log("unable to find an ISO image for Windows XP or Windows 2003") src_iso_path = prompt("enter path to an ISO image:") if not (src_iso_path and os.path.isfile(src_iso_path)): raise fail("invalid path: %s" % src_iso_path) key_regexp = re.compile(r'^\w{5}-\w{5}-\w{5}-\w{5}-\w{5}$') key = env.windows_key if not (key and key_regexp.match(key)): key = None key_path = os.path.splitext(src_iso_path)[0] + ".key" if os.path.isfile(key_path): key = open(key_path).readline().strip() if not key_regexp.match(key): key = None if key is None: log("unable to find a Windows product key") key = prompt("enter product key:") if not key_regexp.match(key): raise fail("invalid product key: {}", key) wget_path = self.download(WGET_EXE_URLS) unpack_path = TMP_DIR + "/" + self.name boot_path = unpack_path + "/eltorito.img" if os.path.exists(unpack_path): rmtree(unpack_path) self.unpack_iso(src_iso_path, unpack_path) self.unpack_iso_boot(src_iso_path, boot_path) sif_template_path = DATA_ROOT + "/vm/%s-winnt.sif" % self.name sif_path = unpack_path + "/I386/WINNT.SIF" debug("translating: {} => {}", sif_template_path, sif_path) sif_template = open(sif_template_path).read() sif = sif_template.replace("#####-#####-#####-#####-#####", key) assert sif != sif_template open(sif_path, 'w').write(sif) install_path = unpack_path + "/$OEM$/$1/INSTALL" mktree(install_path) cp(wget_path, install_path) cp(CTL_DIR + "/identity.pub", install_path) cp(DATA_ROOT + "/vm/%s-install.cmd" % self.name, install_path + "/INSTALL.CMD") iso_path = TMP_DIR + "/%s.iso" % self.name if os.path.exists(iso_path): rm(iso_path) sh("mkisofs -o %s -q -iso-level 2 -J -l -D -N" " -joliet-long -relaxed-filenames -no-emul-boot" " -boot-load-size 4 -b eltorito.img %s" % (iso_path, unpack_path)) rmtree(unpack_path) try: self.kvm_img() self.kvm("-cdrom %s -boot d" % iso_path) rm(iso_path) self.compress() except: if os.path.exists(self.img_path): rm(self.img_path) raise stop_time = datetime.datetime.now() log("VM is built successfully: `{}` ({})", self.name, stop_time - start_time)
def regress(command): # Run `pbbt test/regress.yaml <command>`. variables = make_variables() with env(debug=True): sh(env.pbbt_path + " test/regress.yaml -E test/regress.py " + variables + command)
def htsql_ctl(command, environ=None): # Run `htsql-ctl <command>`. with env(debug=True): sh(env.ctl_path + " " + command)
def pyflakes(command): # Run `pyflakes <command>`. with env(debug=True): sh(env.pyflakes_path + " " + command)
def sphinx(command, cd=None): # Run `sphinx-build <command>`. with env(debug=True): sh(env.sphinx_path + " " + command, cd=cd)
def kvm_img(self): # Run `qemu-img create -f qcow2 <img_path> <DISK_SIZE>`. sh("qemu-img create -f qcow2 %s %s" % (self.img_path, DISK_SIZE))
def unpack_iso_boot(self, iso_path, boot_path): # Unpack El Torito boot image from and ISO. assert os.path.isfile(iso_path) debug("unpacking boot image: {} => {}", iso_path, boot_path) sh("geteltorito -o %s %s" % (boot_path, iso_path))
def coverage_py(command, environ=None): # Run `coverage <command>`. path = env.coverage_path with env(debug=True): sh(path + " " + command, environ=environ)