def _get_mk(self): Blog.debug("get mkpath for: %s" % self.name) mkpath = self._do_get_mk(Bos.distrodir) if not mkpath: mkpath = self._do_get_mk(Bos.topdir) Blog.debug("mk for: %s as: %s" % (self.name, mkpath)) if mkpath: return(mkpath[len(Bos.topdir):]) return mkpath
def compile(self): if self.compile_yes: Blog.info("compiling %s" % self.name) Bos.get_env(self._native) return bos_run(['make', '-C', os.path.join(Bos.topdir, self._src), '-f', os.path.join(Bos.mkdir, os.path.basename(self.mk)), '--no-print-directory', 'compile'], self._get_logdir() + '-compile') return (0, None)
def _get_gitdir(self): gitdir = None if self._src: from subprocess import Popen, PIPE out, err = Popen('cd %s; git rev-parse --show-toplevel' % os.path.join(Bos.topdir, self._src), shell = True, stdout = PIPE, stderr = PIPE ).communicate() if err: Blog.warn('%s: not a git repository.' % os.path.join(Bos.topdir, self._src)) else: gitdir = os.path.join(out.strip(), '.git')[len(Bos.topdir):] return gitdir
def _print_pkg_info(name): from bomb.package import BosPackage Blog.debug('print package info: %s' % name) try: pkg = BosPackage.open(name) pkg.dump() except: Blog.fatal('invalid package name: %s' % name) sys.exit(0)
def prepare(self): Blog.info("preparing %s" % self.name) self._apply_patch() if self.prepare_yes: Bos.get_env(self._native) return bos_run(['make', '-C', os.path.join(Bos.topdir, self._src), '-f', os.path.join(Bos.mkdir, os.path.basename(self.mk)), '--no-print-directory', 'MK=%s' % os.path.dirname(os.path.join(Bos.topdir, self.mk)), 'prepare'], self._get_logdir() + '-prepare') return (0, None)
def open(cls, name): try: db = shelve.open(_get_shelf_name(name)) pkg = db['obj'] Blog.debug('package: %s already on shelve' % name) if os.path.getmtime(os.path.join(Bos.topdir, pkg.mk)) != pkg._mtime: pkg._uninstall() pkg = None except KeyError: pkg = None if not pkg: return BosPackage(name) return pkg
def _revert_patch(self): """ revert package patches if available. """ if self._patch and os.path.exists(self._patched): for p in reversed(self._patch): Blog.debug("reverting %s: %s" % (self.name, p)) ret,logname = bos_run( ['patch', '-Rp1', '-d', os.path.join(Bos.topdir, self._src), '-i', os.path.join(Bos.topdir, os.path.dirname(self.mk), p)]) if 0 != ret: Blog.fatal('%s unable to revert patch: %s' % (self.name, p)) os.unlink(self._patched)
def _do_get_mk(self, root): Blog.debug('search mk at: %s for %s' % (root, self.name)) mk = None path = None for r,d,f in os.walk(os.path.join(root, 'bm')): for fn in f: if fn.endswith('.mk'): Blog.debug('mk found as: %s at %s' % (fn, r)) if fn == '%s.mk' % self.name: return os.path.join(r, fn) if True == self._native: if fn == '%s.mk' % self._basename: mk = fn path = r return None if not mk else os.path.join(path, mk)
def bos_rm_empty_path(path, base): """ recursively check and remove given path from base if path is empty. path: directory relative to base directory base: base directory """ base = os.path.realpath(base) if path[0] == '/': path = path[1:] path = os.path.join(base, path) Blog.debug('path: %s base: %s' % (path, base)) while os.path.realpath(path) != base and path != '/': if _rm_empty_dirs(path): path = os.path.dirname(path) else: break
def _apply_patch(self): """ apply package patches if available. """ ret = 0 if self._patch and not os.path.exists(self._patched): for p in self._patch: Blog.debug("patching %s: %s" % (self.name, p)) ret,logname = bos_run( ['patch', '-p1', '-d', os.path.join(Bos.topdir, self._src), '-i', os.path.join(Bos.topdir, os.path.dirname(self.mk), p)]) if 0 != ret: Blog.fatal('%s unable to apply patch: %s' % (self.name, p)) if 0 == ret and self._patch: Bos.touch(self._patched)
def _rm_empty_dirs(root): """ recursively check and remove all empty directories. return True if 'root' directory is empty and removed """ if not os.path.isdir(root): return False files = os.listdir(root) for df in files: path = os.path.join(root, df) if os.path.isdir(path): _rm_empty_dirs(path) if not os.listdir(root): Blog.debug('deleting: %s' % root) os.rmdir(root) return True else: return False
def install(self): ret = 0 logname = None self._uninstall() if self.install_yes: Blog.info("installing %s" % self.name) if not os.path.exists(self._get_stagingdir()): os.makedirs(self._get_stagingdir()) ret,logname = bos_run(['make', '-C', os.path.join(Bos.topdir, self._src), '-f', os.path.join(Bos.mkdir, os.path.basename(self.mk)), '--no-print-directory', 'DESTDIR=%s' % self._get_stagingdir(), 'install'], self._get_logdir() + '-install') if 0 == ret: ret = self._install() shutil.rmtree(self._get_stagingdir()) if 0 == ret: ## record package version self._version = self._get_version() self._flush() return (ret, logname)
def clean(self): if self.clean_yes: Blog.info("cleaning %s" % self.name) Bos.get_env(self._native) ret,logname = bos_run(['make', '-C', os.path.join(Bos.topdir, self._src), '-f', os.path.join(Bos.mkdir, os.path.basename(self.mk)), '--no-print-directory', 'clean']) if 0 != ret: Blog.warn('%s unable to clean' % self.name) self._revert_patch() if self._gitdir: with BosLockFile(os.path.join(Bos.topdir, self._gitdir, '.bos.lock')) as lock: bos_run(['git', '--git-dir=%s' % os.path.join(Bos.topdir, self._gitdir), 'clean', '-Xfd']) self._uninstall() try: for fn in glob.glob('%s.?' % (Bos.statesdir + self.name)): os.unlink(fn) Bos.touch(Bos.statesdir + self.name + '.v') except OSError as e: Blog.warn(e.strerror + ': ' + e.filename) try: shutil.rmtree(os.path.dirname(self._get_logdir())) except: pass return (0, None)
def _check_package_version(): from bomb.package import BosPackage names = _all_pkgs() for name in names: pkg = BosPackage.open(name) dot_v = os.path.join(Bos.statesdir, name + '.v') dot_d = os.path.join(Bos.statesdir, name + '.d') if not os.path.exists(dot_v): Bos.touch(dot_v) if True == pkg.is_version_diff(): if os.path.exists(dot_d): Blog.info("%s: rebuild required" % name) Bos.touch(dot_v) else: if not os.path.exists(dot_d): Bos.touch(os.path.join(Bos.statesdir, name + '.p')) Bos.touch(os.path.join(Bos.statesdir, name + '.f')) Bos.touch(os.path.join(Bos.statesdir, name + '.b')) Bos.touch(dot_d)
def _uninstall(self): """ uninstall package both from output and index DB area uninstall also checks to remove any path that becomes empty due to this package's uninstallation. """ try: ## must acquire the global lock lockdir = Bos.nativedirlock if self._native else Bos.targetdirlock with BosLockFile(lockdir) as lock: ## clean up output and index area based on cached package info for pn in self._contents: for lst in self._contents[pn]: fn = lst[3] Blog.debug('%s removing %s' % (self.name, fn[1:])) if self._native: os.unlink(os.path.join(Bos.nativedir, fn[1:])) os.unlink(os.path.join(Bos.nativeindexdir, fn[1:])) else: os.unlink(os.path.join(Bos.targetdir, fn[1:])) os.unlink(os.path.join(Bos.targetindexdir, fn[1:])) ## check output area to remove left-over empty paths for kn in self._files: for itm in self._files[kn].split('\n'): if not itm.strip(): continue dn = os.path.dirname(itm) Blog.debug('package %s removing item: %s' % (self.name, dn)) if dn and '/' != dn: if self._native: bos_rm_empty_path(dn, Bos.nativedir) bos_rm_empty_path(dn, Bos.nativeindexdir) else: bos_rm_empty_path(dn, Bos.targetdir) bos_rm_empty_path(dn, Bos.targetindexdir) except: ## all uninstall errors are ignored Blog.debug('%s unable to uninstall.' % self.name) self._contents = {} self._version = None self._flush()
def _install_files(src, ownership, context): if os.path.isdir(src): for ff in os.listdir(src): Blog.debug('installing: %s' % ff) _install_files(os.path.join(src, ff), ownership, context) else: rel_src = src[context.baselen:] if rel_src[0] == '/': rel_src = rel_src[1:] mode, size = bos_fileinfo(src) ## actual install must acquire the global lock lockdir = Bos.nativedirlock if context.pkg._native else Bos.targetdirlock with BosLockFile(lockdir) as lock: path = os.path.join(context.destdir, os.path.dirname(rel_src)) if not os.path.exists(path): os.makedirs(path) Blog.debug('installing from: %s to %s' % (src, path)) try: shutil.move(src, path + '/') except shutil.Error: owner = _who_has(rel_src, context.pkg._native) if owner == context.name: os.unlink(path, os.path.basename(src)) shutil.move(src, path + '/') else: Blog.fatal('package %s conflicts with: %s\n%s' % (context.name, owner, rel_src)) info = [] ## if file is no longer in src, it must be in destdir already if '----------' == mode: mode, size = bos_fileinfo(os.path.join(context.destdir, rel_src)) info.append(mode) info.append(ownership if ownership else 'root:root') info.append(size) info.append('/' + rel_src) context.contents.append(info)
def purge(self): if self.clean_yes: Blog.info("purging %s" % self.name) Bos.get_env(self._native) ret,logname = bos_run(['make', '-C', os.path.join(Bos.topdir, self._src), '-f', os.path.join(Bos.mkdir, os.path.basename(self.mk)), '--no-print-directory', 'clean']) if 0 != ret: Blog.warn('%s unable to clean' % self.name) self._revert_patch() if self._gitdir: with BosLockFile(os.path.join(Bos.topdir, self._gitdir, '.bos.lock')) as lock: from subprocess import Popen, PIPE Popen('rm -fr %s/*' % os.path.join(Bos.topdir, self._src), shell = True, stdout = PIPE, stderr = PIPE ).communicate() Popen('cd %s/.. && git reset --hard' % os.path.join(Bos.topdir, self._gitdir), shell = True, stdout = PIPE, stderr = PIPE ).communicate() self._purge() try: for fn in glob.glob('%s.?' % (Bos.statesdir + self.name)): os.unlink(fn) Bos.touch(Bos.statesdir + self.name + '.v') except OSError as e: Blog.warn(e.strerror + ': ' + e.filename) try: shutil.rmtree(os.path.dirname(self._get_logdir())) except: pass return (0, None)
def __init__(self, name): self.name = name if name[-7:] == '-native': self._basename = name[:-7] self._native = True else: self._basename = name self._native = False self.mk = self._get_mk() if not self.mk: Blog.fatal("unable to find mk for package: %s" % name) mk_full_path = os.path.join(Bos.topdir, self.mk) Blog.debug('start to parse .mk: %s' % self.mk) config = ConfigParser() meta = self._preprocess_mk() try: config.read(meta) except ParsingError as e: Blog.error(e.message) Blog.fatal("failed to parse .mk: %s <%s>" % (name, self.mk)) os.remove(meta) Blog.debug('parsing package .mk: %s' % self.mk) ## package description is required try: self._description = config.get('BOSMK', 'DESCRIPTION') except NoOptionError: Blog.fatal("package description missing: %s <%s>" % (name, self.mk)) ## everything else if optional Blog.debug('parsing package .mk: %s optional fields.' % self.mk) self.require = [] self._patch = [] self._patched = None self._src = None self._files = {} for fld in dict(config.items('BOSMK')): if 'require' == fld: require = dict(config.items('BOSMK'))[ 'require'].replace('\n', ' ').split(' ') for dep in require: if self._native: if dep[-7:] != '-native': self.require.append(dep + '-native') continue self.require.append(dep) elif 'source' == fld: sources = dict(config.items('BOSMK'))[ 'source'].replace('\n', ' ').split(' ') for s in sources: if os.path.splitext(s)[1] == '.patch': self._patch.append(s) self._src = sources[0] elif fld[:5] == 'files': self._files.update({fld: dict(config.items('BOSMK'))[fld]}) ## package misc if self._patch: self._patched = os.path.join(Bos.topdir, self._src, '.bos-patch-applied') self._mtime = os.path.getmtime(mk_full_path) self._gitdir = self._get_gitdir() ## installed contents directory: ## {package-name: [[mode ownership size path]]} self._contents = {} ## version info is available only after a successful install. self._version = None ## put it on shelf db = shelve.open(_get_shelf_name(name)) db['obj'] = self db.close()
def bosm(args): args.target = _fuzzy_target(args.target) ## cleanup all existing logs if 'clean' in args.target: try: shutil.rmtree(Bos.logdir) except: pass ## BOS internal environments required by logging system. os.environ['_BOS_DEBUG_'] = 'yes' if args.debug == True else 'no' os.environ['_BOS_TRACE_'] = 'yes' if args.trace == True else 'no' os.environ['_BOS_VERBOSE_'] = 'yes' if args.verbose == True else 'no' ### bootstrap build system Bos.setup() Blog.debug("entering to build system, topdir: %s" % Bos.topdir) if _bootstrapcheck(): Blog.debug("bootstrap required.") Bos.touch(os.path.join(Bos.cachedir, '.rebootstrap')) Bos.touch(os.path.join(Bos.cachedir, '.rebuild')) ret = 0 if not 'bootstrap' in args.target: Blog.debug("checking to bootstrap ...") ret = call(['make', '-C', Bos.cachedir, '-f', Bos.topdir + 'bos/mk/bootstrap.mk', '--no-print-directory']) if 0 != ret: Blog.fatal('unable to bootstrap') ## from this point on, build system is bootstraped and ready: ## - all envorinments are in place and ready to consume build target if 'info' in args.target: _print_info() if 'all' in args.target: _check_package_version() if 0 == ret: for target in args.target: if target[-5:] == '-info': _print_pkg_info(target[:-5]) Blog.debug("package %s top-level make" % target) call(['make', '-C', Bos.cachedir, '-f', Bos.topdir + 'bos/mk/main.mk', '-j' + str(args.jobs) if args.jobs else '-j', '--no-print-directory', target]) print '' print 'build summary at: {0}'.format(Blog.name())
def _install(self): """ install package from staging area to output area and populate DB examine contents in staging area to make sure that, - all package specified contents must exist, unless optional - all installed contents must associate with given package return: 0 if successful, error code otherwise """ ## walk through package and sub-package definitions if any for kn in self._files: if kn == 'files': pn = self._basename else: pn = self._basename + kn[5:] Blog.debug('processing package: %s' % pn) ctx = BosInstallContext(pn, self) try: for itm in self._files[kn].split('\n'): if '' == itm.strip(): continue ownership, pattern, optional = _parse_install_item(itm) Blog.debug('processing pattern: %s' % pattern) flist = glob.glob(os.path.join(self._get_stagingdir(), pattern[1:])) if (not flist) and (not optional): Blog.fatal('<%s> unable to find: %s' % (self.name, pattern)) for ff in flist: _install_files(ff, ownership, ctx) for ctnt in ctx.contents: ff = ctnt[3] path = os.path.join(ctx.indexdir, ff[1:]) if not os.path.exists(os.path.dirname(path)): os.makedirs(os.path.dirname(path)) #with open(path, 'w') as f: f.write(ctx.name) os.symlink(ctx.name, path) except: Blog.error("%s unable to install." % self.name) self._put_info({ctx.name:ctx.contents}) self._uninstall() return -1 Blog.debug('%s writing package info' % ctx.name) self._put_info({ctx.name:ctx.contents}) ## post process: walk the stagingdir to make sure there's no files left try: for r, d, f in os.walk(self._get_stagingdir()): if f: Blog.fatal('installed but unpackaged contents found: %s\n%s' % (self.name, _list_dir(self._get_stagingdir()))) except: Blog.error('%s unable to walk staging dir: %s' % (self.name, self._get_stagingdir())) self._uninstall() return -2 return 0