def PYPI(): """upload the source distribution to PyPI This task uploads `zip` and `tar.gz` source distributions to PyPI. The distributions must be already built with `cogs pkg-src`. """ if not (glob.glob("./build/pkg/src/HTSQL-*.tar.gz") and glob.glob("./build/pkg/src/HTSQL-*.zip")): raise fail("cannot find source packages; run `cogs pkg-src` first") if os.path.exists("./build/tmp"): rmtree("./build/tmp") mktree("./build/tmp") archives = [] for tgzname in glob.glob("./build/pkg/src/*.tar.gz"): dirname = tgzname[:-7] zipname = dirname + ".zip" dirname = os.path.basename(dirname) project, version = dirname.rsplit('-', 1) dirname = "./build/tmp/" + dirname run("tar -xzf %s -C ./build/tmp" % tgzname) mktree(dirname + "/dist") run("tar -xzf %s -C %s" % (tgzname, dirname)) cp(tgzname, dirname + "/dist") cp(zipname, dirname + "/dist") setup_py("sdist --formats=zip,gztar --dry-run" " register upload --sign --identity=" + KEYSIG, cd=dirname) archives.append((project, version)) rmtree("./build/tmp") log() log("Source distribution archives are uploaded to:") for project, version in archives: log(" `http://pypi.python.org/pypi/{}/{}/`", project, version) log()
def PKG_DEB(): """create Debian packages This task creates Debian packages from source packages. """ if deb_vm.missing(): raise fail("VM is not built: {}", deb_vm.name) if deb_vm.running(): deb_vm.stop() if os.path.exists("./build/pkg/deb"): rmtree("./build/pkg/deb") if os.path.exists("./build/tmp"): rmtree("./build/tmp") version = get_version() debian_version = ".".join(version.split(".")[:3]) moves = load_moves(DATA_ROOT + "/pkg/debian/moves.yaml") deb_vm.start() pubkey = pipe("gpg --armour --export %s" % KEYSIG) seckey = pipe("gpg --armour --export-secret-key %s" % KEYSIG) deb_vm.write("/root/sign.key", pubkey + seckey) deb_vm.run("gpg --import /root/sign.key") try: for move in moves: package = "%s-%s" % (move.code.upper(), version) debian_package = "%s_%s" % (move.code, debian_version) archive = "./build/pkg/src/%s.tar.gz" % package if not os.path.exists(archive): raise fail("cannot find a source package;" " run `cogs pkg-src` first") changelog = open(DATA_ROOT + "/pkg/debian/changelog").read() if ('htsql (%s-1)' % debian_version) not in changelog: raise fatal("run `job pkg-deb-changelog`" " to update the changelog file") changelog = changelog.replace('htsql (', '%s (' % move.code) mktree("./build/tmp") cp(archive, "./build/tmp/%s.orig.tar.gz" % debian_package) sh("tar -xzf %s -C ./build/tmp" % archive) move(DATA_ROOT + "/pkg/debian", "./build/tmp/%s" % package) open("./build/tmp/%s/debian/changelog" % package, 'w') \ .write(changelog) deb_vm.put("./build/tmp", "./build") deb_vm.run("cd ./build/%s && dpkg-buildpackage -k%s" % (package, KEYSIG)) if not os.path.exists("./build/pkg/deb"): mktree("./build/pkg/deb") deb_vm.get("./build/*.deb", "./build/pkg/deb") deb_vm.run("rm -rf build") rmtree("./build/tmp") finally: deb_vm.stop() log() log("The generated Debian packages are placed in:") for filename in glob.glob("./build/pkg/deb/*"): log(" `{}`", filename) log()
def PKG_RPM(): """create RedHat/CentOS packages This task creates RedHat/CentOS packages from source packages. """ if rpm_vm.missing(): raise fail("VM is not built: {}", rpm_vm.name) if rpm_vm.running(): rpm_vm.stop() if os.path.exists("./build/pkg/rpm"): rmtree("./build/pkg/rpm") if os.path.exists("./build/tmp"): rmtree("./build/tmp") version = get_version() redhat_version = ".".join(version.split(".")[:3]) moves = load_moves(DATA_ROOT + "/pkg/redhat/moves.yaml") rpm_vm.start() pubkey = pipe("gpg --armour --export %s" % KEYSIG) seckey = pipe("gpg --armour --export-secret-key %s" % KEYSIG) rpm_vm.write("/root/sign.key", pubkey + seckey) rpm_vm.run("gpg --import /root/sign.key") rpm_vm.put(DATA_ROOT + "/pkg/redhat/.rpmmacros", ".") try: for move in moves: name = move.variables['name'] move.variables['version'] = redhat_version move.variables['package'] = "%s-%s" % (name, version) package = "%s-%s" % (name, version) archive = "./build/pkg/src/%s.tar.gz" % package if not os.path.exists(archive): raise fail("cannot find a source package;" " run `cogs pkg-src` first") mktree("./build/tmp") move(DATA_ROOT + "/pkg/redhat", "./build/tmp") cp(archive, "./build/tmp/SOURCES") rpm_vm.put("./build/tmp", "./rpmbuild") rpm_vm.run("rpmbuild -bb rpmbuild/SPECS/%s.spec" % name) if not os.path.exists("./build/pkg/rpm"): mktree("./build/pkg/rpm") #rpm_vm.run("rpmsign --addsign ./rpmbuild/RPMS/noarch/*.rpm") rpm_vm.get("./rpmbuild/RPMS/noarch/*.rpm", "./build/pkg/rpm") rpm_vm.run("rm -rf rpmbuild") rmtree("./build/tmp") finally: rpm_vm.stop() log() log("The generated RedHat/CentOS packages are placed in:") for filename in glob.glob("./build/pkg/rpm/*"): log(" `{}`", filename) log()
def build(self): # Generate a VM image. if not self.missing(): raise fail("VM is already built") for path in [IMG_DIR, CTL_DIR, TMP_DIR]: if not os.path.exists(path): mktree(path) identity_path = CTL_DIR + "/identity" if not os.path.exists(identity_path): sh("ssh-keygen -q -N \"\" -f %s" % identity_path) config_path = CTL_DIR + "/ssh_config" if not os.path.exists(config_path): config_template_path = DATA_ROOT + "/vm/ssh_config" config_template = open(config_template_path).read() config = config_template.replace("$VM_ROOT", VM_ROOT) assert config != config_template debug("translating: {} => {}", config_template_path, config_path) open(config_path, 'w').write(config)
def COVERAGE(): """measure code coverage by regression tests (coverage.py)""" if os.path.exists("./build/coverage"): rmtree("./build/coverage") mktree("./build/coverage") environ = {} environ['COVERAGE_FILE'] = "./build/coverage/coverage.dat" variables = make_variables() coverage_py("run --branch" " --source=htsql,htsql_sqlite,htsql_pgsql,htsql_oracle," "htsql_mssql,htsql_django" + " `which \"%s\"` test/regress.yaml -E test/regress.py -q " % env.pbbt_path + variables, environ=environ) coverage_py("html --directory=build/coverage", environ) log() log("To see the coverage report, open:") log(" `./build/coverage/index.html`") log()
def unpack_iso(self, iso_path, target_path): # Unpack an ISO image. assert os.path.isfile(iso_path) if not os.path.exists(target_path): mktree(target_path) debug("unpacking: {} => {}", iso_path, target_path) listing = pipe("isoinfo -i %s -R -f" % iso_path) for entry in listing.splitlines(): filename = target_path + entry dirname = os.path.dirname(filename) if not os.path.exists(dirname): mktree(dirname) with env(debug=False): content = pipe("isoinfo -i %s -R -x '%s'" % (iso_path, entry)) if not content: continue #debug("extracting: {} => {}", entry, filename) stream = open(filename, 'w') stream.write(content) stream.close()
def PKG_SRC(): """create a source package This task creates Python source distribution. """ if src_vm.missing(): raise fail("VM is not built: {}", src_vm.name) if src_vm.running(): src_vm.stop() if os.path.exists("./build/pkg/src"): rmtree("./build/pkg/src") if os.path.exists("./build/tmp"): rmtree("./build/tmp") version = get_version() all_routines = get_routines() all_addons = get_addons() moves = load_moves(DATA_ROOT + "/pkg/source/moves.yaml") src_vm.start() src_vm.run("pip install wheel") try: for move in moves: with_doc = move.variables['with-doc'] packages = move.variables['packages'].strip().splitlines() routines = "".join( routine + "\n" for routine in all_routines if routine.split('=', 1)[1].strip().split('.')[0] in packages) addons = "".join( addon + "\n" for addon in all_addons if addon.split('=', 1)[1].strip().split('.')[0] in packages) move.variables['version'] = version move.variables['htsql-routines'] = routines move.variables['htsql-addons'] = addons mktree("./build/tmp") sh("hg archive ./build/tmp/htsql") if with_doc: setup_py("-q download_vendor", cd="./build/tmp/htsql") for dirname in glob.glob("./build/tmp/htsql/src/*"): if os.path.basename(dirname) not in packages: rmtree(dirname) packages = setuptools.find_packages("./build/tmp/htsql/src") move.variables['packages'] = "".join(package + "\n" for package in packages) if not with_doc: rmtree("./build/tmp/htsql/doc") move(DATA_ROOT + "/pkg/source", "./build/tmp/htsql") src_vm.put("./build/tmp/htsql", ".") if with_doc: src_vm.run("cd htsql &&" " PYTHONPATH=src sphinx-build -d doc doc doc/html") for filename in glob.glob("./build/tmp/htsql/doc/man/*.?.rst"): basename = os.path.basename(filename) target = basename[:-4] src_vm.run("rst2man htsql/doc/man/%s htsql/doc/man/%s" % (basename, target)) src_vm.run("cd htsql && python setup.py sdist --formats=zip,gztar") src_vm.run("cd htsql && python setup.py bdist_wheel") if not os.path.exists("./build/pkg/src"): mktree("./build/pkg/src") src_vm.get("./htsql/dist/*", "./build/pkg/src") src_vm.run("rm -rf htsql") rmtree("./build/tmp") finally: src_vm.stop() log() log("The generated source packages are placed in:") for filename in glob.glob("./build/pkg/src/*"): log(" `{}`", filename) log()
def build(self): super(WindowsTemplateVM, self).build() log("building VM: `{}`...", self.name) start_time = datetime.datetime.now() src_iso_path = env.windows_iso if not (src_iso_path and os.path.isfile(src_iso_path)): src_iso_path = None output = pipe("locate %s || true" % " ".join(WINDOWS_ISO_FILES)) for line in output.splitlines(): if os.path.exists(line): src_iso_path = line break if src_iso_path is None: log("unable to find an ISO image for Windows XP or Windows 2003") src_iso_path = prompt("enter path to an ISO image:") if not (src_iso_path and os.path.isfile(src_iso_path)): raise fail("invalid path: %s" % src_iso_path) key_regexp = re.compile(r'^\w{5}-\w{5}-\w{5}-\w{5}-\w{5}$') key = env.windows_key if not (key and key_regexp.match(key)): key = None key_path = os.path.splitext(src_iso_path)[0] + ".key" if os.path.isfile(key_path): key = open(key_path).readline().strip() if not key_regexp.match(key): key = None if key is None: log("unable to find a Windows product key") key = prompt("enter product key:") if not key_regexp.match(key): raise fail("invalid product key: {}", key) wget_path = self.download(WGET_EXE_URLS) unpack_path = TMP_DIR + "/" + self.name boot_path = unpack_path + "/eltorito.img" if os.path.exists(unpack_path): rmtree(unpack_path) self.unpack_iso(src_iso_path, unpack_path) self.unpack_iso_boot(src_iso_path, boot_path) sif_template_path = DATA_ROOT + "/vm/%s-winnt.sif" % self.name sif_path = unpack_path + "/I386/WINNT.SIF" debug("translating: {} => {}", sif_template_path, sif_path) sif_template = open(sif_template_path).read() sif = sif_template.replace("#####-#####-#####-#####-#####", key) assert sif != sif_template open(sif_path, 'w').write(sif) install_path = unpack_path + "/$OEM$/$1/INSTALL" mktree(install_path) cp(wget_path, install_path) cp(CTL_DIR + "/identity.pub", install_path) cp(DATA_ROOT + "/vm/%s-install.cmd" % self.name, install_path + "/INSTALL.CMD") iso_path = TMP_DIR + "/%s.iso" % self.name if os.path.exists(iso_path): rm(iso_path) sh("mkisofs -o %s -q -iso-level 2 -J -l -D -N" " -joliet-long -relaxed-filenames -no-emul-boot" " -boot-load-size 4 -b eltorito.img %s" % (iso_path, unpack_path)) rmtree(unpack_path) try: self.kvm_img() self.kvm("-cdrom %s -boot d" % iso_path) rm(iso_path) self.compress() except: if os.path.exists(self.img_path): rm(self.img_path) raise stop_time = datetime.datetime.now() log("VM is built successfully: `{}` ({})", self.name, stop_time - start_time)