def run(self, command):
# Run a shell command in the VM.
self.forward(22)
host = "linux-vm"
if self.system == 'windows':
host = "windows-vm"
sh("ssh -F %s %s \"%s\"" % (CTL_DIR + "/ssh_config", host, command))
self.unforward(22)
def put(self, src_filename, dst_filename):
# Copy a file to the VM.
self.forward(22)
host = "linux-vm"
if self.system == 'windows':
host = "windows-vm"
sh("scp -rF %s \"%s\" %s:\"%s\"" %
(CTL_DIR + "/ssh_config", src_filename, host, dst_filename))
self.unforward(22)
def get(self, src_filename, dst_filename):
# Read a file from VM.
self.forward(22)
host = "linux-vm"
if self.system == 'windows':
host = "windows-vm"
sh("scp -rF %s %s:\"%s\" \"%s\"" %
(CTL_DIR + "/ssh_config", host, src_filename, dst_filename))
self.unforward(22)
def compress(self, backing_name=None):
# Run `qemu-img convert -c ...`.
opts = "convert -c -f qcow2 -O qcow2"
if backing_name:
opts += " -o backing_file=%s.qcow2" % backing_name
sh("qemu-img %s %s.qcow2 %s-compressed.qcow2"
% (opts, self.name, self.name), cd=IMG_DIR)
mv(IMG_DIR+"/%s-compressed.qcow2" % self.name,
IMG_DIR+"/%s.qcow2" % self.name)
def PKG_DEB():
"""create Debian packages
This task creates Debian packages from source packages.
"""
if deb_vm.missing():
raise fail("VM is not built: {}", deb_vm.name)
if deb_vm.running():
deb_vm.stop()
if os.path.exists("./build/pkg/deb"):
rmtree("./build/pkg/deb")
if os.path.exists("./build/tmp"):
rmtree("./build/tmp")
version = get_version()
debian_version = ".".join(version.split(".")[:3])
moves = load_moves(DATA_ROOT + "/pkg/debian/moves.yaml")
deb_vm.start()
pubkey = pipe("gpg --armour --export %s" % KEYSIG)
seckey = pipe("gpg --armour --export-secret-key %s" % KEYSIG)
deb_vm.write("/root/sign.key", pubkey + seckey)
deb_vm.run("gpg --import /root/sign.key")
try:
for move in moves:
package = "%s-%s" % (move.code.upper(), version)
debian_package = "%s_%s" % (move.code, debian_version)
archive = "./build/pkg/src/%s.tar.gz" % package
if not os.path.exists(archive):
raise fail("cannot find a source package;"
" run `cogs pkg-src` first")
changelog = open(DATA_ROOT + "/pkg/debian/changelog").read()
if ('htsql (%s-1)' % debian_version) not in changelog:
raise fatal("run `job pkg-deb-changelog`"
" to update the changelog file")
changelog = changelog.replace('htsql (', '%s (' % move.code)
mktree("./build/tmp")
cp(archive, "./build/tmp/%s.orig.tar.gz" % debian_package)
sh("tar -xzf %s -C ./build/tmp" % archive)
move(DATA_ROOT + "/pkg/debian", "./build/tmp/%s" % package)
open("./build/tmp/%s/debian/changelog" % package, 'w') \
.write(changelog)
deb_vm.put("./build/tmp", "./build")
deb_vm.run("cd ./build/%s && dpkg-buildpackage -k%s" %
(package, KEYSIG))
if not os.path.exists("./build/pkg/deb"):
mktree("./build/pkg/deb")
deb_vm.get("./build/*.deb", "./build/pkg/deb")
deb_vm.run("rm -rf build")
rmtree("./build/tmp")
finally:
deb_vm.stop()
log()
log("The generated Debian packages are placed in:")
for filename in glob.glob("./build/pkg/deb/*"):
log(" `{}`", filename)
log()
def kvm(self, opts=None):
# Run `kvm`.
net_model = "virtio"
if self.system == 'windows':
net_model = "rtl8139"
sh("kvm -name %s -monitor unix:%s,server,nowait"
" -drive file=%s,cache=writeback"
" -net nic,model=%s -net user -vga cirrus"
" -rtc clock=vm -m %s" %
(self.name, self.ctl_path, self.img_path, net_model, MEM_SIZE) +
((" " + opts) if opts else "") +
("" if env.debug else " -vnc none"))
def build(self):
# Generate a VM image.
if not self.missing():
raise fail("VM is already built")
for path in [IMG_DIR, CTL_DIR, TMP_DIR]:
if not os.path.exists(path):
mktree(path)
identity_path = CTL_DIR + "/identity"
if not os.path.exists(identity_path):
sh("ssh-keygen -q -N \"\" -f %s" % identity_path)
config_path = CTL_DIR + "/ssh_config"
if not os.path.exists(config_path):
config_template_path = DATA_ROOT + "/vm/ssh_config"
config_template = open(config_template_path).read()
config = config_template.replace("$VM_ROOT", VM_ROOT)
assert config != config_template
debug("translating: {} => {}", config_template_path, config_path)
open(config_path, 'w').write(config)
def PKG_DEB_CHANGELOG(message=None):
"""update the Debian changelog
This task updates the Debian changelog to the current HTSQL version.
"""
if message is None:
message = "new upstream release"
version = get_version()
debian_version = ".".join(version.split(".")[:3])
debian_version += "-1"
changelog = open(DATA_ROOT + "/pkg/debian/changelog").read()
if ('htsql (%s)' % debian_version) in changelog:
raise fail("changelog is already up-to-date")
sh("dch --check-dirname-level 0 -D unstable -v %s %s" %
(debian_version, message),
cd=DATA_ROOT + "/pkg/debian")
log("The Debian changelog is updated to version:")
log(" `{}`", debian_version)
log()
def build(self):
super(DebianTemplateVM, self).build()
log("building VM: `{}`...", self.name)
start_time = datetime.datetime.now()
src_iso_path = getattr(env, self.iso_env)
if not (src_iso_path and os.path.isfile(src_iso_path)):
src_iso_path = self.download(self.iso_urls)
unpack_path = TMP_DIR + "/" + self.name
if os.path.exists(unpack_path):
rmtree(unpack_path)
self.unpack_iso(src_iso_path, unpack_path)
cp(DATA_ROOT + "/vm/%s-isolinux.cfg" % self.name,
unpack_path + "/isolinux/isolinux.cfg")
cp(DATA_ROOT + "/vm/%s-preseed.cfg" % self.name,
unpack_path + "/preseed.cfg")
cp(DATA_ROOT + "/vm/%s-install.sh" % self.name,
unpack_path + "/install.sh")
cp(CTL_DIR + "/identity.pub", unpack_path + "/identity.pub")
sh(
"md5sum"
" `find ! -name \"md5sum.txt\""
" ! -path \"./isolinux/*\" -follow -type f` > md5sum.txt",
cd=unpack_path)
iso_path = TMP_DIR + "/%s.iso" % self.name
if os.path.exists(iso_path):
rm(iso_path)
sh("mkisofs -o %s"
" -q -r -J -no-emul-boot -boot-load-size 4 -boot-info-table"
" -b isolinux/isolinux.bin -c isolinux/boot.cat %s" %
(iso_path, unpack_path))
rmtree(unpack_path)
try:
self.kvm_img()
self.kvm("-cdrom %s -boot d" % iso_path)
rm(iso_path)
self.compress()
except:
if os.path.exists(self.img_path):
rm(self.img_path)
raise
stop_time = datetime.datetime.now()
log("VM is built successfully: `{}` ({})", self.name,
stop_time - start_time)
def build(self):
super(LinuxBenchVM, self).build()
parent_vm = VM.find(self.parent)
if parent_vm.missing():
parent_vm.build()
if parent_vm.running():
raise fail("unable to copy VM while it is running: {}",
parent_vm.name)
log("building VM: `{}`...", self.name)
start_time = datetime.datetime.now()
try:
sh("qemu-img create -b %s.qcow2 -f qcow2 %s.qcow2" %
(parent_vm.name, self.name),
cd=IMG_DIR)
self.kvm("-daemonize")
time.sleep(60.0)
self.put(DATA_ROOT + "/vm/%s-update.sh" % self.name,
"/root/update.sh")
self.run("/root/update.sh")
self.run("rm /root/update.sh")
self.run("shutdown")
self.wait()
#self.compress(parent_vm.name)
self.kvm("-daemonize")
time.sleep(60.0)
self.ctl("savevm %s" % self.state)
self.ctl("quit")
self.wait()
except:
if self.running():
self.ctl("quit")
self.wait()
if os.path.exists(self.img_path):
rm(self.img_path)
raise
stop_time = datetime.datetime.now()
log("VM is built successfully: `{}` ({})", self.name,
stop_time - start_time)
def build(self):
super(CentOSTemplateVM, self).build()
log("building VM: `{}`...", self.name)
start_time = datetime.datetime.now()
src_iso_path = env.centos_iso
if not (src_iso_path and os.path.isfile(src_iso_path)):
src_iso_path = self.download(CENTOS_ISO_URLS)
unpack_path = TMP_DIR + "/" + self.name
if os.path.exists(unpack_path):
rmtree(unpack_path)
self.unpack_iso(src_iso_path, unpack_path)
cp(DATA_ROOT + "/vm/%s-isolinux.cfg" % self.name,
unpack_path + "/isolinux/isolinux.cfg")
cp(DATA_ROOT + "/vm/%s-ks.cfg" % self.name, unpack_path + "/ks.cfg")
cp(DATA_ROOT + "/vm/%s-install.sh" % self.name,
unpack_path + "/install.sh")
cp(CTL_DIR + "/identity.pub", unpack_path + "/identity.pub")
iso_path = TMP_DIR + "/%s.iso" % self.name
if os.path.exists(iso_path):
rm(iso_path)
sh("mkisofs -o %s"
" -q -r -J -T -no-emul-boot -boot-load-size 4 -boot-info-table"
" -b isolinux/isolinux.bin -c isolinux/boot.cat %s" %
(iso_path, unpack_path))
rmtree(unpack_path)
try:
self.kvm_img()
self.kvm("-cdrom %s -boot d" % iso_path)
rm(iso_path)
self.compress()
except:
if os.path.exists(self.img_path):
rm(self.img_path)
raise
stop_time = datetime.datetime.now()
log("VM is built successfully: `{}` ({})",
(self.name, stop_time - start_time))
def python(command, cd=None):
# Run `python <command>`.
with env(debug=True):
sh(env.python_path + " " + command, cd=cd)
def CHECK_ALL():
"""run regression tests for all supported backends
This task runs HTSQL regression tests on all combinations of client
and server platforms.
"""
vms = [
py26_vm, py27_vm, pgsql84_vm, pgsql90_vm, pgsql91_vm, mysql51_vm,
oracle10g_vm, mssql2005_vm, mssql2008_vm
]
for vm in vms:
if vm.missing():
warn("VM is not built: {}", vm.name)
for vm in vms:
if vm.running():
vm.stop()
errors = 0
try:
for client_vm in [py26_vm, py27_vm]:
if client_vm.missing():
continue
client_vm.start()
client_vm.run("~/bin/pip -q install"
" hg+http://bitbucket.org/prometheus/pbbt")
sh("hg clone --ssh='ssh -F %s' . ssh://linux-vm/src/htsql" %
(CTL_DIR + "/ssh_config"))
errors += trial("hg update && python setup.py install",
"installing HTSQL under %s" % client_vm.name)
errors += trial(
"pbbt test/regress.yaml -E test/regress.py"
" -q -S /all/sqlite", "testing sqlite backend")
for server_vm, suite in [(pgsql84_vm, 'pgsql'),
(pgsql90_vm, 'pgsql'),
(pgsql91_vm, 'pgsql'),
(mysql51_vm, 'mysql'),
(oracle10g_vm, 'oracle'),
(mssql2005_vm, 'mssql'),
(mssql2008_vm, 'mssql')]:
if server_vm.missing():
continue
server_vm.start()
username_key = "%s_USERNAME" % suite.upper()
password_key = "%s_PASSWORD" % suite.upper()
host_key = "%s_HOST" % suite.upper()
port_key = "%s_PORT" % suite.upper()
username_value = {
'pgsql': "postgres",
'mysql': "root",
'oracle': "system",
'mssql': "sa"
}[suite]
password_value = "admin"
host_value = "10.0.2.2"
port_value = 10000 + server_vm.port
command = "pbbt test/regress.yaml -E test/regress.py" \
" -q -S /all/%s" \
" -D %s=%s -D %s=%s -D %s=%s -D %s=%s" \
% (suite, username_key, username_value,
password_key, password_value,
host_key, host_value, port_key, port_value)
message = "testing %s backend against %s" \
% (suite, server_vm.name)
errors += trial(command, message)
server_vm.stop()
errors += trial(
"pbbt test/regress.yaml -E test/regress.py"
" -q -S /all/routine", "testing htsql-ctl routines")
client_vm.stop()
except:
for vm in vms:
if vm.running():
vm.stop()
raise
log()
if errors:
if errors == 1:
warn("1 failed test")
else:
warn("{} failed tests", errors)
else:
log("`All tests passed`")
def PKG_SRC():
"""create a source package
This task creates Python source distribution.
"""
if src_vm.missing():
raise fail("VM is not built: {}", src_vm.name)
if src_vm.running():
src_vm.stop()
if os.path.exists("./build/pkg/src"):
rmtree("./build/pkg/src")
if os.path.exists("./build/tmp"):
rmtree("./build/tmp")
version = get_version()
all_routines = get_routines()
all_addons = get_addons()
moves = load_moves(DATA_ROOT + "/pkg/source/moves.yaml")
src_vm.start()
src_vm.run("pip install wheel")
try:
for move in moves:
with_doc = move.variables['with-doc']
packages = move.variables['packages'].strip().splitlines()
routines = "".join(
routine + "\n" for routine in all_routines
if routine.split('=', 1)[1].strip().split('.')[0] in packages)
addons = "".join(
addon + "\n" for addon in all_addons
if addon.split('=', 1)[1].strip().split('.')[0] in packages)
move.variables['version'] = version
move.variables['htsql-routines'] = routines
move.variables['htsql-addons'] = addons
mktree("./build/tmp")
sh("hg archive ./build/tmp/htsql")
if with_doc:
setup_py("-q download_vendor", cd="./build/tmp/htsql")
for dirname in glob.glob("./build/tmp/htsql/src/*"):
if os.path.basename(dirname) not in packages:
rmtree(dirname)
packages = setuptools.find_packages("./build/tmp/htsql/src")
move.variables['packages'] = "".join(package + "\n"
for package in packages)
if not with_doc:
rmtree("./build/tmp/htsql/doc")
move(DATA_ROOT + "/pkg/source", "./build/tmp/htsql")
src_vm.put("./build/tmp/htsql", ".")
if with_doc:
src_vm.run("cd htsql &&"
" PYTHONPATH=src sphinx-build -d doc doc doc/html")
for filename in glob.glob("./build/tmp/htsql/doc/man/*.?.rst"):
basename = os.path.basename(filename)
target = basename[:-4]
src_vm.run("rst2man htsql/doc/man/%s htsql/doc/man/%s" %
(basename, target))
src_vm.run("cd htsql && python setup.py sdist --formats=zip,gztar")
src_vm.run("cd htsql && python setup.py bdist_wheel")
if not os.path.exists("./build/pkg/src"):
mktree("./build/pkg/src")
src_vm.get("./htsql/dist/*", "./build/pkg/src")
src_vm.run("rm -rf htsql")
rmtree("./build/tmp")
finally:
src_vm.stop()
log()
log("The generated source packages are placed in:")
for filename in glob.glob("./build/pkg/src/*"):
log(" `{}`", filename)
log()
def build(self):
super(WindowsTemplateVM, self).build()
log("building VM: `{}`...", self.name)
start_time = datetime.datetime.now()
src_iso_path = env.windows_iso
if not (src_iso_path and os.path.isfile(src_iso_path)):
src_iso_path = None
output = pipe("locate %s || true" % " ".join(WINDOWS_ISO_FILES))
for line in output.splitlines():
if os.path.exists(line):
src_iso_path = line
break
if src_iso_path is None:
log("unable to find an ISO image for Windows XP or Windows 2003")
src_iso_path = prompt("enter path to an ISO image:")
if not (src_iso_path and os.path.isfile(src_iso_path)):
raise fail("invalid path: %s" % src_iso_path)
key_regexp = re.compile(r'^\w{5}-\w{5}-\w{5}-\w{5}-\w{5}$')
key = env.windows_key
if not (key and key_regexp.match(key)):
key = None
key_path = os.path.splitext(src_iso_path)[0] + ".key"
if os.path.isfile(key_path):
key = open(key_path).readline().strip()
if not key_regexp.match(key):
key = None
if key is None:
log("unable to find a Windows product key")
key = prompt("enter product key:")
if not key_regexp.match(key):
raise fail("invalid product key: {}", key)
wget_path = self.download(WGET_EXE_URLS)
unpack_path = TMP_DIR + "/" + self.name
boot_path = unpack_path + "/eltorito.img"
if os.path.exists(unpack_path):
rmtree(unpack_path)
self.unpack_iso(src_iso_path, unpack_path)
self.unpack_iso_boot(src_iso_path, boot_path)
sif_template_path = DATA_ROOT + "/vm/%s-winnt.sif" % self.name
sif_path = unpack_path + "/I386/WINNT.SIF"
debug("translating: {} => {}", sif_template_path, sif_path)
sif_template = open(sif_template_path).read()
sif = sif_template.replace("#####-#####-#####-#####-#####", key)
assert sif != sif_template
open(sif_path, 'w').write(sif)
install_path = unpack_path + "/$OEM$/$1/INSTALL"
mktree(install_path)
cp(wget_path, install_path)
cp(CTL_DIR + "/identity.pub", install_path)
cp(DATA_ROOT + "/vm/%s-install.cmd" % self.name,
install_path + "/INSTALL.CMD")
iso_path = TMP_DIR + "/%s.iso" % self.name
if os.path.exists(iso_path):
rm(iso_path)
sh("mkisofs -o %s -q -iso-level 2 -J -l -D -N"
" -joliet-long -relaxed-filenames -no-emul-boot"
" -boot-load-size 4 -b eltorito.img %s" % (iso_path, unpack_path))
rmtree(unpack_path)
try:
self.kvm_img()
self.kvm("-cdrom %s -boot d" % iso_path)
rm(iso_path)
self.compress()
except:
if os.path.exists(self.img_path):
rm(self.img_path)
raise
stop_time = datetime.datetime.now()
log("VM is built successfully: `{}` ({})", self.name,
stop_time - start_time)
def regress(command):
# Run `pbbt test/regress.yaml <command>`.
variables = make_variables()
with env(debug=True):
sh(env.pbbt_path + " test/regress.yaml -E test/regress.py " +
variables + command)
def htsql_ctl(command, environ=None):
# Run `htsql-ctl <command>`.
with env(debug=True):
sh(env.ctl_path + " " + command)
def pyflakes(command):
# Run `pyflakes <command>`.
with env(debug=True):
sh(env.pyflakes_path + " " + command)
def sphinx(command, cd=None):
# Run `sphinx-build <command>`.
with env(debug=True):
sh(env.sphinx_path + " " + command, cd=cd)
def kvm_img(self):
# Run `qemu-img create -f qcow2 <img_path> <DISK_SIZE>`.
sh("qemu-img create -f qcow2 %s %s" % (self.img_path, DISK_SIZE))
def unpack_iso_boot(self, iso_path, boot_path):
# Unpack El Torito boot image from and ISO.
assert os.path.isfile(iso_path)
debug("unpacking boot image: {} => {}", iso_path, boot_path)
sh("geteltorito -o %s %s" % (boot_path, iso_path))
def coverage_py(command, environ=None):
# Run `coverage <command>`.
path = env.coverage_path
with env(debug=True):
sh(path + " " + command, environ=environ)
