def forward(self, port): # Forward a port from a local host to the VM. local_port = port + 10000 debug("forwarding: localhost:{} -> {}:{}", local_port, self.name, port) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: sock.connect(('127.0.0.1', local_port)) sock.close() is_taken = True except socket.error: is_taken = False port_path = CTL_DIR + "/port.%s" % port if is_taken: if os.path.exists(port_path): name = open(port_path).read().strip() if name != self.name: vm = VM.find(name) if not vm.running(): raise fail("unable to forward port: {} -> {}:{}", local_port, self.name, port) raise fail( "unable to forward port: {} -> {}:{}" " (already forwarded by VM {})", local_port, self.name, port, name) else: raise fail("unable to forward port: {} -> {}:{}", local_port, self.name, port) else: self.ctl("hostfwd_add tcp:127.0.0.1:%s-:%s" % (local_port, port)) open(port_path, 'w').write("%s\n" % self.name)
def delete(self): # Delete the VM image. if self.missing(): raise fail("VM is not built: {}", self.name) if self.running(): raise fail("VM is running: {}", self.name) log("deleting VM: {}", self.name) rm(self.img_path) if os.path.exists(self.ctl_path): rm(self.ctl_path)
def PKG_DEB(): """create Debian packages This task creates Debian packages from source packages. """ if deb_vm.missing(): raise fail("VM is not built: {}", deb_vm.name) if deb_vm.running(): deb_vm.stop() if os.path.exists("./build/pkg/deb"): rmtree("./build/pkg/deb") if os.path.exists("./build/tmp"): rmtree("./build/tmp") version = get_version() debian_version = ".".join(version.split(".")[:3]) moves = load_moves(DATA_ROOT + "/pkg/debian/moves.yaml") deb_vm.start() pubkey = pipe("gpg --armour --export %s" % KEYSIG) seckey = pipe("gpg --armour --export-secret-key %s" % KEYSIG) deb_vm.write("/root/sign.key", pubkey + seckey) deb_vm.run("gpg --import /root/sign.key") try: for move in moves: package = "%s-%s" % (move.code.upper(), version) debian_package = "%s_%s" % (move.code, debian_version) archive = "./build/pkg/src/%s.tar.gz" % package if not os.path.exists(archive): raise fail("cannot find a source package;" " run `cogs pkg-src` first") changelog = open(DATA_ROOT + "/pkg/debian/changelog").read() if ('htsql (%s-1)' % debian_version) not in changelog: raise fatal("run `job pkg-deb-changelog`" " to update the changelog file") changelog = changelog.replace('htsql (', '%s (' % move.code) mktree("./build/tmp") cp(archive, "./build/tmp/%s.orig.tar.gz" % debian_package) sh("tar -xzf %s -C ./build/tmp" % archive) move(DATA_ROOT + "/pkg/debian", "./build/tmp/%s" % package) open("./build/tmp/%s/debian/changelog" % package, 'w') \ .write(changelog) deb_vm.put("./build/tmp", "./build") deb_vm.run("cd ./build/%s && dpkg-buildpackage -k%s" % (package, KEYSIG)) if not os.path.exists("./build/pkg/deb"): mktree("./build/pkg/deb") deb_vm.get("./build/*.deb", "./build/pkg/deb") deb_vm.run("rm -rf build") rmtree("./build/tmp") finally: deb_vm.stop() log() log("The generated Debian packages are placed in:") for filename in glob.glob("./build/pkg/deb/*"): log(" `{}`", filename) log()
def stop(self): # Stop a VM. if self.missing(): raise fail("VM is not built: {}", self.name) if not self.running(): raise fail("VM is not running: {}", self.name) log("stopping VM: {}", self.name) for port in self.ports(): self.unforward(port) self.ctl("quit") self.wait()
def PKG_RPM(): """create RedHat/CentOS packages This task creates RedHat/CentOS packages from source packages. """ if rpm_vm.missing(): raise fail("VM is not built: {}", rpm_vm.name) if rpm_vm.running(): rpm_vm.stop() if os.path.exists("./build/pkg/rpm"): rmtree("./build/pkg/rpm") if os.path.exists("./build/tmp"): rmtree("./build/tmp") version = get_version() redhat_version = ".".join(version.split(".")[:3]) moves = load_moves(DATA_ROOT + "/pkg/redhat/moves.yaml") rpm_vm.start() pubkey = pipe("gpg --armour --export %s" % KEYSIG) seckey = pipe("gpg --armour --export-secret-key %s" % KEYSIG) rpm_vm.write("/root/sign.key", pubkey + seckey) rpm_vm.run("gpg --import /root/sign.key") rpm_vm.put(DATA_ROOT + "/pkg/redhat/.rpmmacros", ".") try: for move in moves: name = move.variables['name'] move.variables['version'] = redhat_version move.variables['package'] = "%s-%s" % (name, version) package = "%s-%s" % (name, version) archive = "./build/pkg/src/%s.tar.gz" % package if not os.path.exists(archive): raise fail("cannot find a source package;" " run `cogs pkg-src` first") mktree("./build/tmp") move(DATA_ROOT + "/pkg/redhat", "./build/tmp") cp(archive, "./build/tmp/SOURCES") rpm_vm.put("./build/tmp", "./rpmbuild") rpm_vm.run("rpmbuild -bb rpmbuild/SPECS/%s.spec" % name) if not os.path.exists("./build/pkg/rpm"): mktree("./build/pkg/rpm") #rpm_vm.run("rpmsign --addsign ./rpmbuild/RPMS/noarch/*.rpm") rpm_vm.get("./rpmbuild/RPMS/noarch/*.rpm", "./build/pkg/rpm") rpm_vm.run("rm -rf rpmbuild") rmtree("./build/tmp") finally: rpm_vm.stop() log() log("The generated RedHat/CentOS packages are placed in:") for filename in glob.glob("./build/pkg/rpm/*"): log(" `{}`", filename) log()
def start(self): # Start a VM. if self.missing(): raise fail("VM is not built: {}", self.name) if self.running(): raise fail("VM is already running: {}", self.name) log("starting VM: {}", self.name) for filename in glob.glob(CTL_DIR + "/port.*"): name = open(filename).read().strip() if name == self.name: rm(filename) if self.state: self.kvm("-daemonize -loadvm %s" % self.state) else: self.kvm("-daemonize -snapshot")
def PYPI(): """upload the source distribution to PyPI This task uploads `zip` and `tar.gz` source distributions to PyPI. The distributions must be already built with `cogs pkg-src`. """ if not (glob.glob("./build/pkg/src/HTSQL-*.tar.gz") and glob.glob("./build/pkg/src/HTSQL-*.zip")): raise fail("cannot find source packages; run `cogs pkg-src` first") if os.path.exists("./build/tmp"): rmtree("./build/tmp") mktree("./build/tmp") archives = [] for tgzname in glob.glob("./build/pkg/src/*.tar.gz"): dirname = tgzname[:-7] zipname = dirname + ".zip" dirname = os.path.basename(dirname) project, version = dirname.rsplit('-', 1) dirname = "./build/tmp/" + dirname run("tar -xzf %s -C ./build/tmp" % tgzname) mktree(dirname + "/dist") run("tar -xzf %s -C %s" % (tgzname, dirname)) cp(tgzname, dirname + "/dist") cp(zipname, dirname + "/dist") setup_py("sdist --formats=zip,gztar --dry-run" " register upload --sign --identity=" + KEYSIG, cd=dirname) archives.append((project, version)) rmtree("./build/tmp") log() log("Source distribution archives are uploaded to:") for project, version in archives: log(" `http://pypi.python.org/pypi/{}/{}/`", project, version) log()
def Factorial(n): """calculate n! This task calculates the value of the factorial of the given positive number `n`. Factorial of n, also known as n!, is defined by the formula: n! = 1*2*...*(n-1)*n """ try: n = int(n) except ValueError: raise fail("n must be an integer") if n < 1: raise fail("n must be positive") f = 1 for k in range(2, n + 1): f *= k log("{}! = `{}`", n, f)
def VM_CTL(name, cmd): """send a command to a virtual machine This task sends a low-level command to the virtual machine monitor. For the list of commands, see: `http://en.wikibooks.org/wiki/QEMU/Monitor` """ vm = VM.find(name) if not vm.running(): raise fail("VM is not running: {}", vm.name) vm.ctl(cmd)
def VM_START(*names): """start a virtual machine This task starts a virtual machine. """ if not names: raise fail("VM is not specified") vms = [VM.find(name) for name in names] for vm in vms: if vm.running(): warn("VM is already running: {}", vm.name) continue vm.start()
def VM_SSH(name): """open a shell to a virtual machine Open an SSH session to a running virtual machine. """ vm = VM.find(name) if not vm.running(): raise fail("VM is not running: {}", vm.name) vm.forward(22) host = "linux-vm" if vm.system == 'windows': host = "windows-vm" with env(debug=True): exe("ssh -F %s %s" % (CTL_DIR + "/ssh_config", host))
def VM_DELETE(*names): """delete a virtual machine This task deletes an existing virtual machine image. """ if not names: raise fail("VM is not specified") vms = [VM.find(name) for name in names] for vm in vms: if vm.missing(): warn("VM is not built: {}", vm.name) continue if vm.running(): warn("VM is running: {}", vm.name) continue vm.delete()
def build(self): # Generate a VM image. if not self.missing(): raise fail("VM is already built") for path in [IMG_DIR, CTL_DIR, TMP_DIR]: if not os.path.exists(path): mktree(path) identity_path = CTL_DIR + "/identity" if not os.path.exists(identity_path): sh("ssh-keygen -q -N \"\" -f %s" % identity_path) config_path = CTL_DIR + "/ssh_config" if not os.path.exists(config_path): config_template_path = DATA_ROOT + "/vm/ssh_config" config_template = open(config_template_path).read() config = config_template.replace("$VM_ROOT", VM_ROOT) assert config != config_template debug("translating: {} => {}", config_template_path, config_path) open(config_path, 'w').write(config)
def download(self, urls): # Download a file from a list of URLs and save it to `./vm/tmp`. for url in urls: path = TMP_DIR + "/" + os.path.basename(url) if os.path.exists(path): return path data = None debug("downloading: {} => {}", url, path) try: data = urllib2.urlopen(url).read() except urllib2.HTTPError: pass if data is not None: stream = open(path, 'w') stream.write(data) stream.close() return path raise fail("failed to download: {}", ", ".join(urls))
def PKG_DEB_CHANGELOG(message=None): """update the Debian changelog This task updates the Debian changelog to the current HTSQL version. """ if message is None: message = "new upstream release" version = get_version() debian_version = ".".join(version.split(".")[:3]) debian_version += "-1" changelog = open(DATA_ROOT + "/pkg/debian/changelog").read() if ('htsql (%s)' % debian_version) in changelog: raise fail("changelog is already up-to-date") sh("dch --check-dirname-level 0 -D unstable -v %s %s" % (debian_version, message), cd=DATA_ROOT + "/pkg/debian") log("The Debian changelog is updated to version:") log(" `{}`", debian_version) log()
def build(self): super(LinuxBenchVM, self).build() parent_vm = VM.find(self.parent) if parent_vm.missing(): parent_vm.build() if parent_vm.running(): raise fail("unable to copy VM while it is running: {}", parent_vm.name) log("building VM: `{}`...", self.name) start_time = datetime.datetime.now() try: sh("qemu-img create -b %s.qcow2 -f qcow2 %s.qcow2" % (parent_vm.name, self.name), cd=IMG_DIR) self.kvm("-daemonize") time.sleep(60.0) self.put(DATA_ROOT + "/vm/%s-update.sh" % self.name, "/root/update.sh") self.run("/root/update.sh") self.run("rm /root/update.sh") self.run("shutdown") self.wait() #self.compress(parent_vm.name) self.kvm("-daemonize") time.sleep(60.0) self.ctl("savevm %s" % self.state) self.ctl("quit") self.wait() except: if self.running(): self.ctl("quit") self.wait() if os.path.exists(self.img_path): rm(self.img_path) raise stop_time = datetime.datetime.now() log("VM is built successfully: `{}` ({})", self.name, stop_time - start_time)
def build(self): super(WindowsBenchVM, self).build() parent_vm = VM.find(self.parent) if parent_vm.missing(): parent_vm.build() if parent_vm.running(): raise fail("unable to copy VM while it is running: {}", parent_vm.name) log("building VM: `{}`...", self.name) start_time = datetime.datetime.now() try: cp(parent_vm.img_path, self.img_path) self.kvm("-daemonize") time.sleep(120.0) self.put(DATA_ROOT + "/vm/%s-update.cmd" % self.name, "/cygdrive/c/INSTALL/UPDATE.CMD") self.run( "reg add 'HKLM\Software\Microsoft\Windows\CurrentVersion\RunOnce'" " /v %s /t REG_SZ /d 'C:\INSTALL\UPDATE.CMD' /f" % self.name) self.run("shutdown /r /t 0 /f") self.wait() #self.compress(parent_vm.name) self.kvm("-daemonize") time.sleep(120.0) self.ctl("savevm %s" % self.state) self.ctl("quit") self.wait() except: if self.running(): self.ctl("quit") self.wait() if os.path.exists(self.img_path): rm(self.img_path) raise stop_time = datetime.datetime.now() log("VM is built successfully: `{}` ({})", self.name, stop_time - start_time)
def find(cls, name): # Get an instance by name. for vm in env.vms: if vm.name == name: return vm raise fail("unknown VM: {}", name)
def validate_database(database): # Check if `database` parameter is valid. valid_databases = ['demo', 'edge', 'etl', 'sandbox'] if database not in valid_databases: raise fail("invalid database name: expected one of {}; got {}", ", ".join(valid_databases), database)
def PKG_SRC(): """create a source package This task creates Python source distribution. """ if src_vm.missing(): raise fail("VM is not built: {}", src_vm.name) if src_vm.running(): src_vm.stop() if os.path.exists("./build/pkg/src"): rmtree("./build/pkg/src") if os.path.exists("./build/tmp"): rmtree("./build/tmp") version = get_version() all_routines = get_routines() all_addons = get_addons() moves = load_moves(DATA_ROOT + "/pkg/source/moves.yaml") src_vm.start() src_vm.run("pip install wheel") try: for move in moves: with_doc = move.variables['with-doc'] packages = move.variables['packages'].strip().splitlines() routines = "".join( routine + "\n" for routine in all_routines if routine.split('=', 1)[1].strip().split('.')[0] in packages) addons = "".join( addon + "\n" for addon in all_addons if addon.split('=', 1)[1].strip().split('.')[0] in packages) move.variables['version'] = version move.variables['htsql-routines'] = routines move.variables['htsql-addons'] = addons mktree("./build/tmp") sh("hg archive ./build/tmp/htsql") if with_doc: setup_py("-q download_vendor", cd="./build/tmp/htsql") for dirname in glob.glob("./build/tmp/htsql/src/*"): if os.path.basename(dirname) not in packages: rmtree(dirname) packages = setuptools.find_packages("./build/tmp/htsql/src") move.variables['packages'] = "".join(package + "\n" for package in packages) if not with_doc: rmtree("./build/tmp/htsql/doc") move(DATA_ROOT + "/pkg/source", "./build/tmp/htsql") src_vm.put("./build/tmp/htsql", ".") if with_doc: src_vm.run("cd htsql &&" " PYTHONPATH=src sphinx-build -d doc doc doc/html") for filename in glob.glob("./build/tmp/htsql/doc/man/*.?.rst"): basename = os.path.basename(filename) target = basename[:-4] src_vm.run("rst2man htsql/doc/man/%s htsql/doc/man/%s" % (basename, target)) src_vm.run("cd htsql && python setup.py sdist --formats=zip,gztar") src_vm.run("cd htsql && python setup.py bdist_wheel") if not os.path.exists("./build/pkg/src"): mktree("./build/pkg/src") src_vm.get("./htsql/dist/*", "./build/pkg/src") src_vm.run("rm -rf htsql") rmtree("./build/tmp") finally: src_vm.stop() log() log("The generated source packages are placed in:") for filename in glob.glob("./build/pkg/src/*"): log(" `{}`", filename) log()
def validate_engine(engine): # Check if `engine` parameter is valid. valid_engines = ['sqlite', 'pgsql', 'mysql', 'oracle', 'mssql'] if engine not in valid_engines: raise fail("invalid engine: expected one of {}; got {}", ", ".join(valid_engines), engine)
def build(self): super(WindowsTemplateVM, self).build() log("building VM: `{}`...", self.name) start_time = datetime.datetime.now() src_iso_path = env.windows_iso if not (src_iso_path and os.path.isfile(src_iso_path)): src_iso_path = None output = pipe("locate %s || true" % " ".join(WINDOWS_ISO_FILES)) for line in output.splitlines(): if os.path.exists(line): src_iso_path = line break if src_iso_path is None: log("unable to find an ISO image for Windows XP or Windows 2003") src_iso_path = prompt("enter path to an ISO image:") if not (src_iso_path and os.path.isfile(src_iso_path)): raise fail("invalid path: %s" % src_iso_path) key_regexp = re.compile(r'^\w{5}-\w{5}-\w{5}-\w{5}-\w{5}$') key = env.windows_key if not (key and key_regexp.match(key)): key = None key_path = os.path.splitext(src_iso_path)[0] + ".key" if os.path.isfile(key_path): key = open(key_path).readline().strip() if not key_regexp.match(key): key = None if key is None: log("unable to find a Windows product key") key = prompt("enter product key:") if not key_regexp.match(key): raise fail("invalid product key: {}", key) wget_path = self.download(WGET_EXE_URLS) unpack_path = TMP_DIR + "/" + self.name boot_path = unpack_path + "/eltorito.img" if os.path.exists(unpack_path): rmtree(unpack_path) self.unpack_iso(src_iso_path, unpack_path) self.unpack_iso_boot(src_iso_path, boot_path) sif_template_path = DATA_ROOT + "/vm/%s-winnt.sif" % self.name sif_path = unpack_path + "/I386/WINNT.SIF" debug("translating: {} => {}", sif_template_path, sif_path) sif_template = open(sif_template_path).read() sif = sif_template.replace("#####-#####-#####-#####-#####", key) assert sif != sif_template open(sif_path, 'w').write(sif) install_path = unpack_path + "/$OEM$/$1/INSTALL" mktree(install_path) cp(wget_path, install_path) cp(CTL_DIR + "/identity.pub", install_path) cp(DATA_ROOT + "/vm/%s-install.cmd" % self.name, install_path + "/INSTALL.CMD") iso_path = TMP_DIR + "/%s.iso" % self.name if os.path.exists(iso_path): rm(iso_path) sh("mkisofs -o %s -q -iso-level 2 -J -l -D -N" " -joliet-long -relaxed-filenames -no-emul-boot" " -boot-load-size 4 -b eltorito.img %s" % (iso_path, unpack_path)) rmtree(unpack_path) try: self.kvm_img() self.kvm("-cdrom %s -boot d" % iso_path) rm(iso_path) self.compress() except: if os.path.exists(self.img_path): rm(self.img_path) raise stop_time = datetime.datetime.now() log("VM is built successfully: `{}` ({})", self.name, stop_time - start_time)