def forward(self, port):
# Forward a port from a local host to the VM.
local_port = port + 10000
debug("forwarding: localhost:{} -> {}:{}", local_port, self.name, port)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect(('127.0.0.1', local_port))
sock.close()
is_taken = True
except socket.error:
is_taken = False
port_path = CTL_DIR + "/port.%s" % port
if is_taken:
if os.path.exists(port_path):
name = open(port_path).read().strip()
if name != self.name:
vm = VM.find(name)
if not vm.running():
raise fail("unable to forward port: {} -> {}:{}",
local_port, self.name, port)
raise fail(
"unable to forward port: {} -> {}:{}"
" (already forwarded by VM {})", local_port, self.name,
port, name)
else:
raise fail("unable to forward port: {} -> {}:{}", local_port,
self.name, port)
else:
self.ctl("hostfwd_add tcp:127.0.0.1:%s-:%s" % (local_port, port))
open(port_path, 'w').write("%s\n" % self.name)
def delete(self):
# Delete the VM image.
if self.missing():
raise fail("VM is not built: {}", self.name)
if self.running():
raise fail("VM is running: {}", self.name)
log("deleting VM: {}", self.name)
rm(self.img_path)
if os.path.exists(self.ctl_path):
rm(self.ctl_path)
def PKG_DEB():
"""create Debian packages
This task creates Debian packages from source packages.
"""
if deb_vm.missing():
raise fail("VM is not built: {}", deb_vm.name)
if deb_vm.running():
deb_vm.stop()
if os.path.exists("./build/pkg/deb"):
rmtree("./build/pkg/deb")
if os.path.exists("./build/tmp"):
rmtree("./build/tmp")
version = get_version()
debian_version = ".".join(version.split(".")[:3])
moves = load_moves(DATA_ROOT + "/pkg/debian/moves.yaml")
deb_vm.start()
pubkey = pipe("gpg --armour --export %s" % KEYSIG)
seckey = pipe("gpg --armour --export-secret-key %s" % KEYSIG)
deb_vm.write("/root/sign.key", pubkey + seckey)
deb_vm.run("gpg --import /root/sign.key")
try:
for move in moves:
package = "%s-%s" % (move.code.upper(), version)
debian_package = "%s_%s" % (move.code, debian_version)
archive = "./build/pkg/src/%s.tar.gz" % package
if not os.path.exists(archive):
raise fail("cannot find a source package;"
" run `cogs pkg-src` first")
changelog = open(DATA_ROOT + "/pkg/debian/changelog").read()
if ('htsql (%s-1)' % debian_version) not in changelog:
raise fatal("run `job pkg-deb-changelog`"
" to update the changelog file")
changelog = changelog.replace('htsql (', '%s (' % move.code)
mktree("./build/tmp")
cp(archive, "./build/tmp/%s.orig.tar.gz" % debian_package)
sh("tar -xzf %s -C ./build/tmp" % archive)
move(DATA_ROOT + "/pkg/debian", "./build/tmp/%s" % package)
open("./build/tmp/%s/debian/changelog" % package, 'w') \
.write(changelog)
deb_vm.put("./build/tmp", "./build")
deb_vm.run("cd ./build/%s && dpkg-buildpackage -k%s" %
(package, KEYSIG))
if not os.path.exists("./build/pkg/deb"):
mktree("./build/pkg/deb")
deb_vm.get("./build/*.deb", "./build/pkg/deb")
deb_vm.run("rm -rf build")
rmtree("./build/tmp")
finally:
deb_vm.stop()
log()
log("The generated Debian packages are placed in:")
for filename in glob.glob("./build/pkg/deb/*"):
log(" `{}`", filename)
log()
def stop(self):
# Stop a VM.
if self.missing():
raise fail("VM is not built: {}", self.name)
if not self.running():
raise fail("VM is not running: {}", self.name)
log("stopping VM: {}", self.name)
for port in self.ports():
self.unforward(port)
self.ctl("quit")
self.wait()
def PKG_RPM():
"""create RedHat/CentOS packages
This task creates RedHat/CentOS packages from source packages.
"""
if rpm_vm.missing():
raise fail("VM is not built: {}", rpm_vm.name)
if rpm_vm.running():
rpm_vm.stop()
if os.path.exists("./build/pkg/rpm"):
rmtree("./build/pkg/rpm")
if os.path.exists("./build/tmp"):
rmtree("./build/tmp")
version = get_version()
redhat_version = ".".join(version.split(".")[:3])
moves = load_moves(DATA_ROOT + "/pkg/redhat/moves.yaml")
rpm_vm.start()
pubkey = pipe("gpg --armour --export %s" % KEYSIG)
seckey = pipe("gpg --armour --export-secret-key %s" % KEYSIG)
rpm_vm.write("/root/sign.key", pubkey + seckey)
rpm_vm.run("gpg --import /root/sign.key")
rpm_vm.put(DATA_ROOT + "/pkg/redhat/.rpmmacros", ".")
try:
for move in moves:
name = move.variables['name']
move.variables['version'] = redhat_version
move.variables['package'] = "%s-%s" % (name, version)
package = "%s-%s" % (name, version)
archive = "./build/pkg/src/%s.tar.gz" % package
if not os.path.exists(archive):
raise fail("cannot find a source package;"
" run `cogs pkg-src` first")
mktree("./build/tmp")
move(DATA_ROOT + "/pkg/redhat", "./build/tmp")
cp(archive, "./build/tmp/SOURCES")
rpm_vm.put("./build/tmp", "./rpmbuild")
rpm_vm.run("rpmbuild -bb rpmbuild/SPECS/%s.spec" % name)
if not os.path.exists("./build/pkg/rpm"):
mktree("./build/pkg/rpm")
#rpm_vm.run("rpmsign --addsign ./rpmbuild/RPMS/noarch/*.rpm")
rpm_vm.get("./rpmbuild/RPMS/noarch/*.rpm", "./build/pkg/rpm")
rpm_vm.run("rm -rf rpmbuild")
rmtree("./build/tmp")
finally:
rpm_vm.stop()
log()
log("The generated RedHat/CentOS packages are placed in:")
for filename in glob.glob("./build/pkg/rpm/*"):
log(" `{}`", filename)
log()
def start(self):
# Start a VM.
if self.missing():
raise fail("VM is not built: {}", self.name)
if self.running():
raise fail("VM is already running: {}", self.name)
log("starting VM: {}", self.name)
for filename in glob.glob(CTL_DIR + "/port.*"):
name = open(filename).read().strip()
if name == self.name:
rm(filename)
if self.state:
self.kvm("-daemonize -loadvm %s" % self.state)
else:
self.kvm("-daemonize -snapshot")
def PYPI():
"""upload the source distribution to PyPI
This task uploads `zip` and `tar.gz` source distributions
to PyPI. The distributions must be already built with
`cogs pkg-src`.
"""
if not (glob.glob("./build/pkg/src/HTSQL-*.tar.gz")
and glob.glob("./build/pkg/src/HTSQL-*.zip")):
raise fail("cannot find source packages; run `cogs pkg-src` first")
if os.path.exists("./build/tmp"):
rmtree("./build/tmp")
mktree("./build/tmp")
archives = []
for tgzname in glob.glob("./build/pkg/src/*.tar.gz"):
dirname = tgzname[:-7]
zipname = dirname + ".zip"
dirname = os.path.basename(dirname)
project, version = dirname.rsplit('-', 1)
dirname = "./build/tmp/" + dirname
run("tar -xzf %s -C ./build/tmp" % tgzname)
mktree(dirname + "/dist")
run("tar -xzf %s -C %s" % (tgzname, dirname))
cp(tgzname, dirname + "/dist")
cp(zipname, dirname + "/dist")
setup_py("sdist --formats=zip,gztar --dry-run"
" register upload --sign --identity=" + KEYSIG,
cd=dirname)
archives.append((project, version))
rmtree("./build/tmp")
log()
log("Source distribution archives are uploaded to:")
for project, version in archives:
log(" `http://pypi.python.org/pypi/{}/{}/`", project, version)
log()
def Factorial(n):
"""calculate n!
This task calculates the value of the factorial of the given
positive number `n`. Factorial of n, also known as n!, is
defined by the formula:
n! = 1*2*...*(n-1)*n
"""
try:
n = int(n)
except ValueError:
raise fail("n must be an integer")
if n < 1:
raise fail("n must be positive")
f = 1
for k in range(2, n + 1):
f *= k
log("{}! = `{}`", n, f)
def VM_CTL(name, cmd):
"""send a command to a virtual machine
This task sends a low-level command to the virtual machine monitor.
For the list of commands, see:
`http://en.wikibooks.org/wiki/QEMU/Monitor`
"""
vm = VM.find(name)
if not vm.running():
raise fail("VM is not running: {}", vm.name)
vm.ctl(cmd)
def VM_START(*names):
"""start a virtual machine
This task starts a virtual machine.
"""
if not names:
raise fail("VM is not specified")
vms = [VM.find(name) for name in names]
for vm in vms:
if vm.running():
warn("VM is already running: {}", vm.name)
continue
vm.start()
def VM_SSH(name):
"""open a shell to a virtual machine
Open an SSH session to a running virtual machine.
"""
vm = VM.find(name)
if not vm.running():
raise fail("VM is not running: {}", vm.name)
vm.forward(22)
host = "linux-vm"
if vm.system == 'windows':
host = "windows-vm"
with env(debug=True):
exe("ssh -F %s %s" % (CTL_DIR + "/ssh_config", host))
def VM_DELETE(*names):
"""delete a virtual machine
This task deletes an existing virtual machine image.
"""
if not names:
raise fail("VM is not specified")
vms = [VM.find(name) for name in names]
for vm in vms:
if vm.missing():
warn("VM is not built: {}", vm.name)
continue
if vm.running():
warn("VM is running: {}", vm.name)
continue
vm.delete()
def build(self):
# Generate a VM image.
if not self.missing():
raise fail("VM is already built")
for path in [IMG_DIR, CTL_DIR, TMP_DIR]:
if not os.path.exists(path):
mktree(path)
identity_path = CTL_DIR + "/identity"
if not os.path.exists(identity_path):
sh("ssh-keygen -q -N \"\" -f %s" % identity_path)
config_path = CTL_DIR + "/ssh_config"
if not os.path.exists(config_path):
config_template_path = DATA_ROOT + "/vm/ssh_config"
config_template = open(config_template_path).read()
config = config_template.replace("$VM_ROOT", VM_ROOT)
assert config != config_template
debug("translating: {} => {}", config_template_path, config_path)
open(config_path, 'w').write(config)
def download(self, urls):
# Download a file from a list of URLs and save it to `./vm/tmp`.
for url in urls:
path = TMP_DIR + "/" + os.path.basename(url)
if os.path.exists(path):
return path
data = None
debug("downloading: {} => {}", url, path)
try:
data = urllib2.urlopen(url).read()
except urllib2.HTTPError:
pass
if data is not None:
stream = open(path, 'w')
stream.write(data)
stream.close()
return path
raise fail("failed to download: {}", ", ".join(urls))
def PKG_DEB_CHANGELOG(message=None):
"""update the Debian changelog
This task updates the Debian changelog to the current HTSQL version.
"""
if message is None:
message = "new upstream release"
version = get_version()
debian_version = ".".join(version.split(".")[:3])
debian_version += "-1"
changelog = open(DATA_ROOT + "/pkg/debian/changelog").read()
if ('htsql (%s)' % debian_version) in changelog:
raise fail("changelog is already up-to-date")
sh("dch --check-dirname-level 0 -D unstable -v %s %s" %
(debian_version, message),
cd=DATA_ROOT + "/pkg/debian")
log("The Debian changelog is updated to version:")
log(" `{}`", debian_version)
log()
def build(self):
super(LinuxBenchVM, self).build()
parent_vm = VM.find(self.parent)
if parent_vm.missing():
parent_vm.build()
if parent_vm.running():
raise fail("unable to copy VM while it is running: {}",
parent_vm.name)
log("building VM: `{}`...", self.name)
start_time = datetime.datetime.now()
try:
sh("qemu-img create -b %s.qcow2 -f qcow2 %s.qcow2" %
(parent_vm.name, self.name),
cd=IMG_DIR)
self.kvm("-daemonize")
time.sleep(60.0)
self.put(DATA_ROOT + "/vm/%s-update.sh" % self.name,
"/root/update.sh")
self.run("/root/update.sh")
self.run("rm /root/update.sh")
self.run("shutdown")
self.wait()
#self.compress(parent_vm.name)
self.kvm("-daemonize")
time.sleep(60.0)
self.ctl("savevm %s" % self.state)
self.ctl("quit")
self.wait()
except:
if self.running():
self.ctl("quit")
self.wait()
if os.path.exists(self.img_path):
rm(self.img_path)
raise
stop_time = datetime.datetime.now()
log("VM is built successfully: `{}` ({})", self.name,
stop_time - start_time)
def build(self):
super(WindowsBenchVM, self).build()
parent_vm = VM.find(self.parent)
if parent_vm.missing():
parent_vm.build()
if parent_vm.running():
raise fail("unable to copy VM while it is running: {}",
parent_vm.name)
log("building VM: `{}`...", self.name)
start_time = datetime.datetime.now()
try:
cp(parent_vm.img_path, self.img_path)
self.kvm("-daemonize")
time.sleep(120.0)
self.put(DATA_ROOT + "/vm/%s-update.cmd" % self.name,
"/cygdrive/c/INSTALL/UPDATE.CMD")
self.run(
"reg add 'HKLM\Software\Microsoft\Windows\CurrentVersion\RunOnce'"
" /v %s /t REG_SZ /d 'C:\INSTALL\UPDATE.CMD' /f" % self.name)
self.run("shutdown /r /t 0 /f")
self.wait()
#self.compress(parent_vm.name)
self.kvm("-daemonize")
time.sleep(120.0)
self.ctl("savevm %s" % self.state)
self.ctl("quit")
self.wait()
except:
if self.running():
self.ctl("quit")
self.wait()
if os.path.exists(self.img_path):
rm(self.img_path)
raise
stop_time = datetime.datetime.now()
log("VM is built successfully: `{}` ({})", self.name,
stop_time - start_time)
def find(cls, name):
# Get an instance by name.
for vm in env.vms:
if vm.name == name:
return vm
raise fail("unknown VM: {}", name)
def validate_database(database):
# Check if `database` parameter is valid.
valid_databases = ['demo', 'edge', 'etl', 'sandbox']
if database not in valid_databases:
raise fail("invalid database name: expected one of {}; got {}",
", ".join(valid_databases), database)
def PKG_SRC():
"""create a source package
This task creates Python source distribution.
"""
if src_vm.missing():
raise fail("VM is not built: {}", src_vm.name)
if src_vm.running():
src_vm.stop()
if os.path.exists("./build/pkg/src"):
rmtree("./build/pkg/src")
if os.path.exists("./build/tmp"):
rmtree("./build/tmp")
version = get_version()
all_routines = get_routines()
all_addons = get_addons()
moves = load_moves(DATA_ROOT + "/pkg/source/moves.yaml")
src_vm.start()
src_vm.run("pip install wheel")
try:
for move in moves:
with_doc = move.variables['with-doc']
packages = move.variables['packages'].strip().splitlines()
routines = "".join(
routine + "\n" for routine in all_routines
if routine.split('=', 1)[1].strip().split('.')[0] in packages)
addons = "".join(
addon + "\n" for addon in all_addons
if addon.split('=', 1)[1].strip().split('.')[0] in packages)
move.variables['version'] = version
move.variables['htsql-routines'] = routines
move.variables['htsql-addons'] = addons
mktree("./build/tmp")
sh("hg archive ./build/tmp/htsql")
if with_doc:
setup_py("-q download_vendor", cd="./build/tmp/htsql")
for dirname in glob.glob("./build/tmp/htsql/src/*"):
if os.path.basename(dirname) not in packages:
rmtree(dirname)
packages = setuptools.find_packages("./build/tmp/htsql/src")
move.variables['packages'] = "".join(package + "\n"
for package in packages)
if not with_doc:
rmtree("./build/tmp/htsql/doc")
move(DATA_ROOT + "/pkg/source", "./build/tmp/htsql")
src_vm.put("./build/tmp/htsql", ".")
if with_doc:
src_vm.run("cd htsql &&"
" PYTHONPATH=src sphinx-build -d doc doc doc/html")
for filename in glob.glob("./build/tmp/htsql/doc/man/*.?.rst"):
basename = os.path.basename(filename)
target = basename[:-4]
src_vm.run("rst2man htsql/doc/man/%s htsql/doc/man/%s" %
(basename, target))
src_vm.run("cd htsql && python setup.py sdist --formats=zip,gztar")
src_vm.run("cd htsql && python setup.py bdist_wheel")
if not os.path.exists("./build/pkg/src"):
mktree("./build/pkg/src")
src_vm.get("./htsql/dist/*", "./build/pkg/src")
src_vm.run("rm -rf htsql")
rmtree("./build/tmp")
finally:
src_vm.stop()
log()
log("The generated source packages are placed in:")
for filename in glob.glob("./build/pkg/src/*"):
log(" `{}`", filename)
log()
def validate_engine(engine):
# Check if `engine` parameter is valid.
valid_engines = ['sqlite', 'pgsql', 'mysql', 'oracle', 'mssql']
if engine not in valid_engines:
raise fail("invalid engine: expected one of {}; got {}",
", ".join(valid_engines), engine)
def build(self):
super(WindowsTemplateVM, self).build()
log("building VM: `{}`...", self.name)
start_time = datetime.datetime.now()
src_iso_path = env.windows_iso
if not (src_iso_path and os.path.isfile(src_iso_path)):
src_iso_path = None
output = pipe("locate %s || true" % " ".join(WINDOWS_ISO_FILES))
for line in output.splitlines():
if os.path.exists(line):
src_iso_path = line
break
if src_iso_path is None:
log("unable to find an ISO image for Windows XP or Windows 2003")
src_iso_path = prompt("enter path to an ISO image:")
if not (src_iso_path and os.path.isfile(src_iso_path)):
raise fail("invalid path: %s" % src_iso_path)
key_regexp = re.compile(r'^\w{5}-\w{5}-\w{5}-\w{5}-\w{5}$')
key = env.windows_key
if not (key and key_regexp.match(key)):
key = None
key_path = os.path.splitext(src_iso_path)[0] + ".key"
if os.path.isfile(key_path):
key = open(key_path).readline().strip()
if not key_regexp.match(key):
key = None
if key is None:
log("unable to find a Windows product key")
key = prompt("enter product key:")
if not key_regexp.match(key):
raise fail("invalid product key: {}", key)
wget_path = self.download(WGET_EXE_URLS)
unpack_path = TMP_DIR + "/" + self.name
boot_path = unpack_path + "/eltorito.img"
if os.path.exists(unpack_path):
rmtree(unpack_path)
self.unpack_iso(src_iso_path, unpack_path)
self.unpack_iso_boot(src_iso_path, boot_path)
sif_template_path = DATA_ROOT + "/vm/%s-winnt.sif" % self.name
sif_path = unpack_path + "/I386/WINNT.SIF"
debug("translating: {} => {}", sif_template_path, sif_path)
sif_template = open(sif_template_path).read()
sif = sif_template.replace("#####-#####-#####-#####-#####", key)
assert sif != sif_template
open(sif_path, 'w').write(sif)
install_path = unpack_path + "/$OEM$/$1/INSTALL"
mktree(install_path)
cp(wget_path, install_path)
cp(CTL_DIR + "/identity.pub", install_path)
cp(DATA_ROOT + "/vm/%s-install.cmd" % self.name,
install_path + "/INSTALL.CMD")
iso_path = TMP_DIR + "/%s.iso" % self.name
if os.path.exists(iso_path):
rm(iso_path)
sh("mkisofs -o %s -q -iso-level 2 -J -l -D -N"
" -joliet-long -relaxed-filenames -no-emul-boot"
" -boot-load-size 4 -b eltorito.img %s" % (iso_path, unpack_path))
rmtree(unpack_path)
try:
self.kvm_img()
self.kvm("-cdrom %s -boot d" % iso_path)
rm(iso_path)
self.compress()
except:
if os.path.exists(self.img_path):
rm(self.img_path)
raise
stop_time = datetime.datetime.now()
log("VM is built successfully: `{}` ({})", self.name,
stop_time - start_time)
