def rmtree(path, *args, **kargs): """version os shutil.rmtree that ignores no-such-file-or-directory errors, and tries harder if it finds immutable files""" do_selinux_ops = False if kargs.has_key('selinux'): do_selinux_ops = kargs['selinux'] del kargs['selinux'] tryAgain = 1 retries = 0 failedFilename = None getLog().debug("remove tree: %s" % path) while tryAgain: tryAgain = 0 try: shutil.rmtree(path, *args, **kargs) except OSError, e: if e.errno == errno.ENOENT: # no such file or directory pass elif do_selinux_ops and (e.errno==errno.EPERM or e.errno==errno.EACCES): tryAgain = 1 if failedFilename == e.filename: raise failedFilename = e.filename os.system("chattr -R -i %s" % path) elif e.errno == errno.EBUSY: retries += 1 if retries > 1: raise tryAgain = 1 getLog().debug("retrying failed tree remove after sleeping a bit") time.sleep(2) else: raise
def _selinuxAtExit(self): if os.path.exists(self.filesystems): try: os.unlink(self.filesystems) except OSError as e: getLog().warning("unable to delete selinux filesystems (%s): %s" % (self.filesystems, e)) pass
def __init__(self, rootObj, conf): self.rootObj = rootObj self.root_cache_opts = conf self.rootSharedCachePath = self.root_cache_opts['dir'] % self.root_cache_opts self.rootCacheFile = os.path.join(self.rootSharedCachePath, "cache.tar") self.rootCacheLock = None self.compressProgram = self.root_cache_opts['compress_program'] if self.compressProgram == 'pigz' and not os.path.exists('/usr/bin/pigz'): getLog().warning("specified 'pigz' as the root cache compress program but not available; using gzip") self.compressProgram = 'gzip' if self.compressProgram: self.compressArgs = ['--use-compress-program', self.compressProgram] self.rootCacheFile = self.rootCacheFile + self.root_cache_opts['extension'] else: self.compressArgs = [] rootObj.rootCacheObj = self rootObj.addHook("preinit", self._rootCachePreInitHook) rootObj.addHook("preshell", self._rootCachePreShellHook) rootObj.addHook("prechroot", self._rootCachePreShellHook) rootObj.addHook("preyum", self._rootCachePreYumHook) rootObj.addHook("postinit", self._rootCachePostInitHook) rootObj.addHook("postshell", self._rootCachePostShellHook) rootObj.addHook("postchroot", self._rootCachePostShellHook) rootObj.addHook("postyum", self._rootCachePostShellHook) self.exclude_dirs = self.root_cache_opts['exclude_dirs'] self.exclude_tar_cmds = [ "--exclude=" + dir for dir in self.exclude_dirs]
def condEnvironment(env=None): if not env: return getLog().debug("child environment: %s" % env) os.environ.clear() for k in env.keys(): os.putenv(k, env[k])
def _ccachePreInitHook(self): getLog().info("enabled ccache") mockbuild.util.mkdirIfAbsent(self.rootObj.makeChrootPath('/tmp/ccache')) os.environ['CCACHE_DIR'] = "/tmp/ccache" os.environ['CCACHE_UMASK'] = "002" mockbuild.util.mkdirIfAbsent(self.ccachePath) self.rootObj.uidManager.changeOwner(self.ccachePath)
def scrub(self, scrub_opts): """clean out chroot and/or cache dirs with extreme prejudice :)""" statestr = "scrub %s" % scrub_opts self.state.start(statestr) try: try: self.plugins.call_hooks('clean') for scrub in scrub_opts: #FIXME hooks for all plugins self.plugins.call_hooks('scrub', scrub) if scrub == 'all': self.buildroot.root_log.info("scrubbing everything for %s" % self.config_name) self.buildroot.delete() util.rmtree(self.buildroot.cachedir, selinux=self.buildroot.selinux) elif scrub == 'chroot': self.buildroot.root_log.info("scrubbing chroot for %s" % self.config_name) self.buildroot.delete() elif scrub == 'cache': self.buildroot.root_log.info("scrubbing cache for %s" % self.config_name) util.rmtree(self.buildroot.cachedir, selinux=self.buildroot.selinux) elif scrub == 'c-cache': self.buildroot.root_log.info("scrubbing c-cache for %s" % self.config_name) util.rmtree(os.path.join(self.buildroot.cachedir, 'ccache'), selinux=self.buildroot.selinux) elif scrub == 'root-cache': self.buildroot.root_log.info("scrubbing root-cache for %s" % self.config_name) util.rmtree(os.path.join(self.buildroot.cachedir, 'root_cache'), selinux=self.buildroot.selinux) elif scrub == 'yum-cache': self.buildroot.root_log.info("scrubbing yum-cache for %s" % self.config_name) util.rmtree(os.path.join(self.buildroot.cachedir, 'yum_cache'), selinux=self.buildroot.selinux) except IOError as e: getLog().warn("parts of chroot do not exist: %s" % e) raise finally: print("finishing: %s" % statestr) self.state.finish(statestr)
def __init__(self, rootObj, conf): self.rootObj = rootObj self.scan_opts = conf self.regexes = self.rootObj.pluginConf['chroot_scan_opts']['regexes'] self.resultdir = os.path.join(rootObj.resultdir, "chroot_scan") rootObj.scanObj = self rootObj.addHook("postbuild", self._scanChroot) getLog().info("chroot_scan: initialized")
def unshare(flags): getLog().debug("Unsharing. Flags: %s" % flags) try: res = _libc.unshare(flags) if res: raise mockbuild.exception.UnshareFailed(os.strerror(_errno.value)) except AttributeError, e: pass
def __init__(self, plugins, conf, buildroot): self.buildroot = buildroot self.config = buildroot.config self.state = buildroot.state self.scan_opts = conf self.resultdir = os.path.join(buildroot.resultdir, "chroot_scan") plugins.add_hook("postbuild", self._scanChroot) getLog().info("chroot_scan: initialized")
def __init__(self, plugins, conf, buildroot): self.buildroot = buildroot self.config = buildroot.config self.state = buildroot.state self.conf = conf self.command = self.conf['command'] plugins.add_hook("postbuild", self._compress_logs) getLog().info("compress_logs: initialized")
def __init__(self, plugins, conf, buildroot): self.plugins = plugins self.conf = conf self.buildroot = buildroot self.plugins.add_hook('postbuild', self.sign_results) getLog().info(conf) getLog().info("enabled package signing")
def _init(self, prebuild, do_log): # If previous run didn't finish properly self._umount_residual() self.state.start("chroot init") util.mkdirIfAbsent(self.basedir) mockgid = grp.getgrnam('mock').gr_gid os.chown(self.basedir, os.getuid(), mockgid) os.chmod(self.basedir, 0o2775) util.mkdirIfAbsent(self.make_chroot_path()) self.plugins.call_hooks('mount_root') self.chroot_was_initialized = self.chroot_is_initialized() self._setup_result_dir() getLog().info("calling preinit hooks") self.plugins.call_hooks('preinit') self.chroot_was_initialized = self.chroot_is_initialized() self._setup_dirs() if not util.USE_NSPAWN: self._setup_devices() self._setup_files() self._setup_nosync() self.mounts.mountall() if do_log: self._resetLogging() # write out config details self.root_log.debug('rootdir = %s' % self.make_chroot_path()) self.root_log.debug('resultdir = %s' % self.resultdir) self.pkg_manager.initialize() if not self.chroot_was_initialized: self._setup_resolver_config() self._setup_dbus_uuid() self._init_aux_files() if not util.USE_NSPAWN: self._setup_timezone() self._init_pkg_management() self._make_build_user() self._setup_build_dirs() elif prebuild: # Recreates build user to ensure the uid/gid are up to date with config # and there's no garbage left by previous build self._make_build_user() self._setup_build_dirs() if self.config['online'] and self.config['update_before_build']: update_state = "{0} update".format(self.pkg_manager.name) self.state.start(update_state) self.pkg_manager.update() self.state.finish(update_state) # mark the buildroot as initialized util.touch(self.make_chroot_path('.initialized')) # done with init self.plugins.call_hooks('postinit') self.state.finish("chroot init")
def _tmpfsUmount(self): getLog().info("unmounting tmpfs.") mountCmd = ["umount", "-n", self.rootObj.makeChrootPath()] # since we're in a separate namespace, the mount will be cleaned up # on exit, so just warn if it fails here try: mockbuild.util.do(mountCmd, shell=False) except: getLog().warning("tmpfs-plugin: exception while umounting tmpfs! (cwd: %s)" % os.getcwd())
def sign_results(self): rpms = glob.glob('%s/*.rpm' % self.buildroot.resultdir) if rpms: getLog().info("Signing %s", ', '.join(rpms)) opts = self.conf['opts'] % {'rpms': ' '.join(rpms), 'resultdir': self.buildroot.resultdir} cmd = "{0} {1}".format(self.conf['cmd'], opts) getLog().info("Executing %s", cmd) with self.buildroot.uid_manager: subprocess.call(cmd, shell=True, env=os.environ)
def _ccachePreInitHook(self): getLog().info("enabled ccache") envupd = {"CCACHE_DIR": "/tmp/ccache", "CCACHE_UMASK": "002"} if self.ccache_opts.get("compress") is not None: envupd["CCACHE_COMPRESS"] = str(self.ccache_opts["compress"]) self.rootObj.env.update(envupd) mockbuild.util.mkdirIfAbsent(self.rootObj.makeChrootPath("/tmp/ccache")) mockbuild.util.mkdirIfAbsent(self.ccachePath) self.rootObj.uidManager.changeOwner(self.ccachePath)
def _ccachePreInitHook(self): getLog().info("enabled ccache") envupd = {"CCACHE_DIR": "/var/tmp/ccache", "CCACHE_UMASK": "002"} if self.ccache_opts.get('compress') is not None: envupd["CCACHE_COMPRESS"] = str(self.ccache_opts['compress']) self.buildroot.env.update(envupd) mockbuild.util.mkdirIfAbsent(self.buildroot.make_chroot_path('/var/tmp/ccache')) mockbuild.util.mkdirIfAbsent(self.ccachePath) self.buildroot.uid_manager.changeOwner(self.ccachePath, recursive=True)
def _tmpfsMount(self): getLog().info("mounting tmpfs at %s.", self.buildroot.make_chroot_path()) if not self.mounted: mountCmd = ["mount", "-n", "-t", "tmpfs"] + self.optArgs + \ ["mock_chroot_tmpfs", self.buildroot.make_chroot_path()] mockbuild.util.do(mountCmd, shell=False) else: getLog().info("reusing tmpfs at %s.", self.buildroot.make_chroot_path()) self.mounted = True
def _setupDev(self, interactive=False): if not self.internal_dev_setup: self._state_log.info("Skipping device setup due to config") return self.start("device setup") try: # files in /dev mockbuild.util.rmtree(self.makeChrootPath("dev"), selinux=self.selinux) mockbuild.util.mkdirIfAbsent(self.makeChrootPath("dev", "pts")) mockbuild.util.mkdirIfAbsent(self.makeChrootPath("dev", "shm")) prevMask = os.umask(0000) devFiles = [ (stat.S_IFCHR | 0666, os.makedev(1, 3), "dev/null"), (stat.S_IFCHR | 0666, os.makedev(1, 7), "dev/full"), (stat.S_IFCHR | 0666, os.makedev(1, 5), "dev/zero"), (stat.S_IFCHR | 0666, os.makedev(1, 8), "dev/random"), (stat.S_IFCHR | 0444, os.makedev(1, 9), "dev/urandom"), (stat.S_IFCHR | 0666, os.makedev(5, 0), "dev/tty"), (stat.S_IFCHR | 0600, os.makedev(5, 1), "dev/console"), (stat.S_IFCHR | 0666, os.makedev(5, 2), "dev/ptmx"), ] kver = os.uname()[2] getLog().debug("kernel version == %s" % kver) for i in devFiles: # create node os.mknod( self.makeChrootPath(i[2]), i[0], i[1]) # set context. (only necessary if host running selinux enabled.) # fails gracefully if chcon not installed. if self.selinux: mockbuild.util.do( ["chcon", "--reference=/%s"% i[2], self.makeChrootPath(i[2])] , raiseExc=0, shell=False, env=self.env) os.symlink("/proc/self/fd/0", self.makeChrootPath("dev/stdin")) os.symlink("/proc/self/fd/1", self.makeChrootPath("dev/stdout")) os.symlink("/proc/self/fd/2", self.makeChrootPath("dev/stderr")) if os.path.isfile(self.makeChrootPath('etc', 'mtab')): os.remove(self.makeChrootPath('etc', 'mtab')) os.symlink("/proc/self/mounts", self.makeChrootPath('etc', 'mtab')) os.chown(self.makeChrootPath('dev/tty'), pwd.getpwnam('root')[2], grp.getgrnam('tty')[2]) os.chown(self.makeChrootPath('dev/ptmx'), pwd.getpwnam('root')[2], grp.getgrnam('tty')[2]) # symlink /dev/fd in the chroot for everything except RHEL4 if mockbuild.util.cmpKernelVer(kver, '2.6.9') > 0: os.symlink("/proc/self/fd", self.makeChrootPath("dev/fd")) os.umask(prevMask) if mockbuild.util.cmpKernelVer(kver, '2.6.18') >= 0: os.unlink(self.makeChrootPath('/dev/ptmx')) os.symlink("pts/ptmx", self.makeChrootPath('/dev/ptmx')) finally: self.finish("device setup")
def init(rootObj, conf): system_ram_bytes = os.sysconf(os.sysconf_names['SC_PAGE_SIZE']) * os.sysconf(os.sysconf_names['SC_PHYS_PAGES']) system_ram_mb = system_ram_bytes / (1024 * 1024) if system_ram_mb > conf['required_ram_mb']: Tmpfs(rootObj, conf) else: getLog().warning("Tmpfs plugin disabled. " "System does not have the required amount of RAM to enable the tmpfs plugin. " "System has %sMB RAM, but the config specifies the minimum required is %sMB RAM. " % (system_ram_mb, conf['required_ram_mb']))
def orphansKill(rootToKill, killsig=SIGTERM): """kill off anything that is still chrooted.""" getLog().debug("kill orphans") for fn in [ d for d in os.listdir("/proc") if d.isdigit() ]: try: root = os.readlink("/proc/%s/root" % fn) if os.path.realpath(root) == os.path.realpath(rootToKill): getLog().warning("Process ID %s still running in chroot. Killing..." % fn) pid = int(fn, 10) os.kill(pid, killsig) os.waitpid(pid, 0) except OSError, e: pass
def __init__(self, plugins, conf, buildroot): self.buildroot = buildroot self.main_config = buildroot.config self.state = buildroot.state self.conf = conf self.maxSize = self.conf['max_fs_size'] self.mode = self.conf['mode'] self.optArgs = ['-o', 'mode=%s' % self.mode] if self.maxSize: self.optArgs += ['-o', 'size=' + self.maxSize] plugins.add_hook("mount_root", self._tmpfsMount) plugins.add_hook("postumount", self._tmpfsUmount) plugins.add_hook("umount_root", self._tmpfsUmount) self.mounted = False getLog().info("tmpfs initialized")
def __init__(self, rootObj, conf): self.rootObj = rootObj self.conf = conf self.maxSize = self.conf['max_fs_size'] self.mode = self.conf['mode'] self.optArgs = ['-o', 'mode=%s' % self.mode] if self.maxSize: self.optArgs += ['-o', 'size=' + self.maxSize] rootObj.addHook("preinit", self._tmpfsMount) rootObj.addHook("preshell", self._tmpfsMount) rootObj.addHook("prechroot", self._tmpfsMount) rootObj.addHook("postshell", self._tmpfsUmount) rootObj.addHook("postbuild", self._tmpfsUmount) rootObj.addHook("postchroot", self._tmpfsUmount) rootObj.addHook("initfailed", self._tmpfsUmount) getLog().info("tmpfs initialized")
def shell(self, options, cmd=None): log = getLog() self.tryLockBuildRoot() log.debug("shell: calling preshell hooks") self._callHooks("preshell") if options.unpriv or self.no_root_shells: uid = self.chrootuid gid = self.chrootgid else: uid = 0 gid = 0 try: log.debug("shell: setting up root files") self._setupDirs() self._setupDev() self._setupFiles() log.debug("shell: mounting all filesystems") self._mountall() self.start("shell") ret = mockbuild.util.doshell(chrootPath=self.makeChrootPath(), environ=self.env, uid=uid, gid=gid, cmd=cmd) finally: log.debug("shell: unmounting all filesystems") self._umountall() log.debug("shell: calling postshell hooks") self._callHooks('postshell') self.unlockBuildRoot() self.finish("shell") return ret
def chroot(self, args, options): log = getLog() shell = False if len(args) == 1: args = args[0] shell = True log.info("Running in chroot: %s" % args) self.tryLockBuildRoot() self._resetLogging() self._callHooks("prechroot") try: self._setupDirs() self._setupDev() self._setupFiles() self._mountall() chrootstate = "chroot %s" % args self.start(chrootstate) if options.unpriv: self.doChroot(args, shell=shell, printOutput=True, uid=self.chrootuid, gid=self.chrootgid, cwd=options.cwd) else: self.doChroot(args, shell=shell, cwd=options.cwd, printOutput=True) finally: self._umountall() self._callHooks("postchroot") self.unlockBuildRoot() self.finish(chrootstate)
def chroot(self, args, options): log = getLog() shell=False if len(args) == 1: args = args[0] shell=True log.info("Running in chroot: %s" % args) self.tryLockBuildRoot() self._resetLogging() self._callHooks("prechroot") try: self._setupDirs() self._setupDev() self._setupFiles() self._mountall() chrootstate = "chroot %s" % args self.start(chrootstate) if options.unpriv: self.doChroot(args, shell=shell, printOutput=True, uid=self.chrootuid, gid=self.chrootgid, cwd=options.cwd) else: self.doChroot(args, shell=shell, cwd=options.cwd, printOutput=True) finally: self._umountall() self.finish(chrootstate) self._callHooks("postchroot") self.unlockBuildRoot()
def shell(self, options, cmd=None): log = getLog() self.tryLockBuildRoot() log.debug("shell: calling preshell hooks") self._callHooks("preshell") if options.unpriv or self.no_root_shells: uid=self.chrootuid gid=self.chrootgid else: uid=0 gid=0 try: log.debug("shell: setting up root files") self._setupDirs() self._setupDev() self._setupFiles() log.debug("shell: mounting all filesystems") self._mountall() self.state("shell") ret = mockbuild.util.doshell(chrootPath=self.makeChrootPath(), uid=uid, gid=gid, cmd=cmd) finally: log.debug("shell: unmounting all filesystems") self._umountall() log.debug("shell: calling postshell hooks") self._callHooks('postshell') self.unlockBuildRoot() return ret
def __init__(self, rootObj, conf): self.rootObj = rootObj self.conf = conf self.maxSize = self.conf['max_fs_size'] if self.maxSize: self.optArgs = ['-o', 'size=' + self.maxSize] else: self.optArgs = [] rootObj.addHook("preinit", self._tmpfsMount) rootObj.addHook("preshell", self._tmpfsMount) rootObj.addHook("prechroot", self._tmpfsMount) rootObj.addHook("postshell", self._tmpfsUmount) rootObj.addHook("postbuild", self._tmpfsUmount) rootObj.addHook("postchroot", self._tmpfsUmount) rootObj.addHook("initfailed", self._tmpfsUmount) getLog().info("tmpfs initialized")
def __scanChroot(self): regexstr = "|".join(self.scan_opts['regexes']) regex = re.compile(regexstr) chroot = self.buildroot.make_chroot_path() mockbuild.util.mkdirIfAbsent(self.resultdir) count = 0 logger = getLog() logger.debug("chroot_scan: Starting scan of %s", chroot) copied = [] for root, _, files in os.walk(chroot): for f in files: m = regex.search(f) if m: srcpath = os.path.join(root, f) subprocess.call("cp --preserve=mode --parents %s %s" % (srcpath, self.resultdir), shell=True) count += 1 copied.append(srcpath) logger.debug("chroot_scan: finished with %d files found", count) if count: logger.info("chroot_scan: %d files copied to %s", count, self.resultdir) logger.info("\n".join(copied)) self.buildroot.uid_manager.changeOwner(self.resultdir, recursive=True) # some packages installs 555 perms on dirs, # so user can't delete/move chroot_scan's results subprocess.call(['chmod', '-R', 'u+w', self.resultdir])
def do(command, shell=False, chrootPath=None, cwd=None, timeout=0, raiseExc=True, returnOutput=0, uid=None, gid=None, personality=None, printOutput=False, env=None, *args, **kargs): logger = kargs.get("logger", getLog()) output = "" start = time.time() preexec = ChildPreExec(personality, chrootPath, cwd, uid, gid) if env is None: env = clean_env() try: child = None logger.debug("Executing command: %s with env %s" % (command, env)) child = subprocess.Popen( command, shell=shell, env=env, bufsize=0, close_fds=True, stdin=open("/dev/null", "r"), stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn = preexec, ) # use select() to poll for output so we dont block output = logOutput([child.stdout, child.stderr], logger, returnOutput, start, timeout, printOutput=printOutput, child=child, chrootPath=chrootPath) except: # kill children if they arent done if child is not None and child.returncode is None: os.killpg(child.pid, 9) try: if child is not None: os.waitpid(child.pid, 0) except: pass raise # wait until child is done, kill it if it passes timeout niceExit=1 while child.poll() is None: if (time.time() - start)>timeout and timeout!=0: niceExit=0 os.killpg(child.pid, 15) if (time.time() - start)>(timeout+1) and timeout!=0: niceExit=0 os.killpg(child.pid, 9) if not niceExit: raise commandTimeoutExpired, ("Timeout(%s) expired for command:\n # %s\n%s" % (timeout, command, output)) logger.debug("Child return code was: %s" % str(child.returncode)) if raiseExc and child.returncode: if returnOutput: raise mockbuild.exception.Error, ("Command failed: \n # %s\n%s" % (command, output), child.returncode) else: raise mockbuild.exception.Error, ("Command failed. See logs for output.\n # %s" % (command,), child.returncode) return output
def __init__(self, rootObj, conf): self.rootObj = rootObj self.conf = conf self.filesystems = self._selinuxCreateFauxFilesystems() self.chrootFilesystems = rootObj.makeChrootPath("/proc/filesystems") atexit.register(self._selinuxAtExit) self.rootObj.mounts.add(BindMountPoint(srcpath=self.filesystems, bindpath=self.chrootFilesystems)) if self._selinuxYumIsSetoptSupported(): rootObj.addHook("preyum", self._selinuxPreYumHook) rootObj.addHook("postyum", self._selinuxPostYumHook) else: getLog().warning("selinux: 'yum' does not support '--setopt' option")
def sign_results(self): if self.buildroot.final_rpm_list: rpms = [ os.path.join(self.buildroot.resultdir, rpm) for rpm in self.buildroot.final_rpm_list ] if rpms: getLog().info("Signing %s", ', '.join(rpms)) opts = self.conf['opts'] % { 'rpms': ' '.join(rpms), 'resultdir': self.buildroot.resultdir } cmd = "{0} {1}".format(self.conf['cmd'], opts) getLog().info("Executing %s", cmd) with self.buildroot.uid_manager: subprocess.check_call(cmd, shell=True, env=os.environ)
def shell(self, options, cmd=None): log = getLog() log.debug("shell: calling preshell hooks") self.plugins.call_hooks("preshell") if options.unpriv or self.no_root_shells: uid = self.buildroot.chrootuid gid = self.buildroot.chrootgid else: uid = 0 gid = 0 try: self.state.start("shell") ret = util.doshell(chrootPath=self.buildroot.make_chroot_path(), environ=self.buildroot.env, uid=uid, gid=gid, user=self.buildroot.chrootuser, cmd=cmd) finally: log.debug("shell: unmounting all filesystems") self.state.finish("shell") log.debug("shell: calling postshell hooks") self.plugins.call_hooks('postshell') return ret
def __init__(self, plugins, conf, buildroot): self.buildroot = buildroot self.config = buildroot.config self.state = buildroot.state self.opts = conf self.log = getLog() plugins.add_hook("pre_srpm_build", self._preprocess_proxy) self.log.info("rpkg_preprocessor: initialized")
def _compress_logs(self): logger = getLog() for f_name in ('root.log', 'build.log', 'state.log', 'available_pkgs.log', 'installed_pkgs.log', 'hw_info.log'): f_path = os.path.join(self.buildroot.resultdir, f_name) if os.path.exists(f_path): command = "{0} {1}".format(self.command, f_path) logger.debug("Running %s", command) util.do(command, shell=True)
def __init__(self, plugins, conf, buildroot): self.buildroot = buildroot self.root_cache_opts = conf self.config = buildroot.config self.state = buildroot.state self.rootSharedCachePath = self.root_cache_opts[ 'dir'] % self.root_cache_opts self.rootCacheFile = os.path.join(self.rootSharedCachePath, "cache.tar") self.rootCacheLock = None self.compressProgram = self.root_cache_opts['compress_program'] if self.compressProgram == 'pigz' and not os.path.exists('/bin/pigz'): getLog().warning( "specified 'pigz' as the root cache compress program but not available; using gzip" ) self.compressProgram = 'gzip' # bsdtar use different decompress program self.decompressProgram = self.root_cache_opts[ 'decompress_program'] or self.compressProgram if self.compressProgram: self.compressArgs = [ '--use-compress-program', self.compressProgram ] self.rootCacheFile = self.rootCacheFile + self.root_cache_opts[ 'extension'] else: self.compressArgs = [] if self.decompressProgram: self.decompressArgs = [ '--use-compress-program', self.decompressProgram ] else: self.decompressArgs = [] plugins.add_hook("preinit", self._rootCachePreInitHook) plugins.add_hook("preshell", self._rootCachePreShellHook) plugins.add_hook("prechroot", self._rootCachePreShellHook) plugins.add_hook("preyum", self._rootCachePreYumHook) plugins.add_hook("postinit", self._rootCachePostInitHook) plugins.add_hook("postshell", self._rootCachePostShellHook) plugins.add_hook("postchroot", self._rootCachePostShellHook) plugins.add_hook("postyum", self._rootCachePostShellHook) plugins.add_hook("postupdate", self._rootCachePostUpdateHook) self.exclude_dirs = self.root_cache_opts['exclude_dirs'] self.exclude_tar_cmds = [] for ex_dir in self.exclude_dirs: self._tarExcludeOption(ex_dir)
def _compress_logs(self): logger = getLog() for f_name in ('root.log', 'build.log', 'state.log', 'available_pkgs', 'installed_pkgs'): f_path = os.path.join(self.buildroot.resultdir, f_name) if os.path.exists(f_path): command = "{0} {1}".format(self.command, f_path) logger.debug("Running {0}".format(command)) util.do(command, shell=True)
def _compress_logs(self): logger = getLog() for f_name in ('root.log', 'build.log', 'state.log'): f_path = os.path.join(self.buildroot.resultdir, f_name) if os.path.exists(f_path): command = "{0} {1}".format(self.command, f_path) logger.debug("Running {0}".format(command)) util.do(command, shell=True)
def __init__(self, plugins, conf, buildroot): self.buildroot = buildroot self.main_config = buildroot.config self.state = buildroot.state self.conf = conf self.maxSize = self.conf['max_fs_size'] self.mode = self.conf['mode'] self.optArgs = ['-o', 'mode=%s' % self.mode] if self.maxSize: self.optArgs += ['-o', 'size=' + self.maxSize] plugins.add_hook("mount_root", self._tmpfsMount) plugins.add_hook("postumount", self._tmpfsPostUmount) plugins.add_hook("umount_root", self._tmpfsUmount) if not os.path.ismount(self.buildroot.make_chroot_path()): self.mounted = False else: self.mounted = True getLog().info("tmpfs initialized")
def test_get_log(self): global __name__ oldname = __name__ __name__ = 'test_name' try: logger = trace_decorator.getLog() self.assertEqual('test_name', logger.name) finally: __name__ = oldname
def _yumCachePreInitHook(self): getLog().info(self._format_pm("enabled {pm} cache")) mockbuild.util.mkdirIfAbsent( self.buildroot.make_chroot_path(self.target_path)) # lock so others dont accidentally use yum cache while we operate on it. self._yumCachePreYumHook() if self.online: state = self._format_pm("cleaning {pm} metadata") self.state.start(state) for (dirpath, _, filenames) in os.walk(self.yumSharedCachePath): for filename in filenames: fullPath = os.path.join(dirpath, filename) statinfo = os.stat(fullPath) file_age_days = (time.time() - statinfo.st_ctime) / (60 * 60 * 24) # prune repodata so yum redownloads. # prevents certain errors where yum gets stuck due to bad metadata for ext in self.METADATA_EXTS: if filename.endswith( ext) and file_age_days > self.yum_cache_opts[ 'max_metadata_age_days']: os.unlink(fullPath) fullPath = None break if fullPath is None: continue if file_age_days > self.yum_cache_opts['max_age_days']: os.unlink(fullPath) continue self.state.finish(state) # yum made an rpmdb cache dir in $cachedir/installed for a while; # things can go wrong in a specific mock case if this happened. # So - just nuke the dir and all that's in it. if os.path.exists(self.yumSharedCachePath + '/installed'): for fn in glob.glob(self.yumSharedCachePath + '/installed/*'): os.unlink(fn) os.rmdir(self.yumSharedCachePath + '/installed') self._yumCachePostYumHook()
def __init__(self, rootObj, conf): self.rootObj = rootObj self.conf = conf self.filesystems = self._selinuxCreateFauxFilesystems() self.chrootFilesystems = rootObj.makeChrootPath("/proc/filesystems") atexit.register(self._selinuxAtExit) self.rootObj.mounts.add( BindMountPoint(srcpath=self.filesystems, bindpath=self.chrootFilesystems)) if self._selinuxYumIsSetoptSupported(): rootObj.addHook("preyum", self._selinuxPreYumHook) rootObj.addHook("postyum", self._selinuxPostYumHook) else: getLog().warning( "selinux: 'yum' does not support '--setopt' option")
def __init__(self, rootObj, conf): self.rootObj = rootObj self.root_cache_opts = conf self.rootSharedCachePath = self.root_cache_opts['dir'] % self.root_cache_opts self.rootCacheFile = os.path.join(self.rootSharedCachePath, "cache.tar") self.rootCacheLock = None self.compressProgram = self.root_cache_opts['compress_program'] if self.compressProgram == 'pigz' and not os.path.exists('/usr/bin/pigz'): getLog().warning("specified 'pigz' as the root cache compress program but not available; using gzip") self.compressProgram = 'gzip' if self.compressProgram: self.compressArgs = ['--use-compress-program', self.compressProgram] self.rootCacheFile = self.rootCacheFile + self.root_cache_opts['extension'] else: self.compressArgs = [] rootObj.rootCacheObj = self rootObj.addHook("preinit", self._rootCachePreInitHook) rootObj.addHook("postinit", self._rootCachePostInitHook) self.exclude_dirs = self.root_cache_opts['exclude_dirs'] self.exclude_tar_cmds = [ "--exclude=" + dir for dir in self.exclude_dirs]
def __init__(self, plugins, conf, buildroot): self._originalUtilDo = mockbuild.util.do self.buildroot = buildroot self.config = buildroot.config self.state = buildroot.state self.conf = conf self.filesystems = self._selinuxCreateFauxFilesystems() self.chrootFilesystems = buildroot.make_chroot_path("/proc/filesystems") atexit.register(self._selinuxAtExit) self.buildroot.mounts.add(BindMountPoint(srcpath=self.filesystems, bindpath=self.chrootFilesystems)) if self._selinuxYumIsSetoptSupported(): plugins.add_hook("preyum", self._selinuxPreYumHook) plugins.add_hook("postyum", self._selinuxPostYumHook) else: getLog().warning("selinux: 'yum' does not support '--setopt' option")
def _nuke_rpm_db(self): """remove rpm DB lock files from the chroot""" dbfiles = glob.glob(self.makeChrootPath('var/lib/rpm/__db*')) if not dbfiles: return self.root_log.debug("removing %d rpm db files" % len(dbfiles)) # become root self.uidManager.becomeUser(0, 0) try: for tmp in dbfiles: self.root_log.debug("_nuke_rpm_db: removing %s" % tmp) try: os.unlink(tmp) except OSError, e: getLog().error("%s" % e) raise finally: #restore previous privs self.uidManager.restorePrivs()
def _tmpfsUmount(self): if not self.mounted: return force = False getLog().info("unmounting tmpfs.") umountCmd = ["umount", "-n", self.buildroot.make_chroot_path()] # since we're in a separate namespace, the mount will be cleaned up # on exit, so just warn if it fails here try: mockbuild.util.do(umountCmd, shell=False) except: getLog().warning( "tmpfs-plugin: exception while umounting tmpfs! (cwd: %s)", mockbuild.util.pretty_getcwd()) force = True if force: # try umounting with force option umountCmd = [ "umount", "-n", "-f", self.buildroot.make_chroot_path() ] try: mockbuild.util.do(umountCmd, shell=False) except: getLog().warning( "tmpfs-plugin: exception while force umounting tmpfs! (cwd: %s)", mockbuild.util.pretty_getcwd()) self.mounted = False
def _PreInitHook(self): getLog().info("enabled HW Info plugin") out_file = self.buildroot.resultdir + '/hw_info.log' out = codecs.open(out_file, 'w', 'utf-8', 'replace') cmd = ["/usr/bin/lscpu"] output = mockbuild.util.do(cmd, shell=False, returnOutput=True, raiseExc=False) out.write("CPU info:\n") out.write(output) cmd = ["/bin/free"] output = mockbuild.util.do(cmd, shell=False, returnOutput=True, raiseExc=False) out.write("\n\nMemory:\n") out.write(output) cmd = ["/bin/df", "-H", self.buildroot.make_chroot_path()] output = mockbuild.util.do(cmd, shell=False, returnOutput=True, raiseExc=False) out.write("\n\nStorage:\n") out.write(output) out.close() self.buildroot.uid_manager.changeOwner(out_file, gid=self.config['chrootgid'])
def modifySpec(self): getLog().info("Modify the spec file") root = self.root specs = glob.glob(root.makeChrootPath(root.builddir, "SPECS", "*.spec")) spec = specs[0] chrootspec = spec.replace(root.makeChrootPath(), '') # get rid of rootdir prefix root.doChroot( [ "sed", "-i", "-e", 's/^Release: .*/\\0.%s/' % (self.suffix), "-e", "1i %define debug_package %{nil}", "-e", "1i %define debug_packages %{nil}", "-e", "1i %define __strip :", chrootspec ], shell=False, logger=root.build_log, timeout=0, uid=root.chrootuid, gid=root.chrootgid, )
def _unpack_root_cache(self): # check cache status try: if self.root_cache_opts['age_check']: # see if it aged out statinfo = os.stat(self.rootCacheFile) file_age_days = (time.time() - statinfo.st_ctime) / (60 * 60 * 24) if file_age_days > self.root_cache_opts['max_age_days']: getLog().info("root cache aged out! cache will be rebuilt") os.unlink(self.rootCacheFile) else: # make sure no config file is newer than the cache file for cfg in self.config['config_paths']: if os.stat(cfg).st_mtime > statinfo.st_mtime: getLog().info( "%s newer than root cache; cache will be rebuilt", cfg) os.unlink(self.rootCacheFile) break else: getLog().info("skipping root_cache aging check") except OSError: pass mockbuild.util.mkdirIfAbsent(self.rootSharedCachePath) # lock so others dont accidentally use root cache while we operate on it. if self.rootCacheLock is None: self.rootCacheLock = open( os.path.join(self.rootSharedCachePath, "rootcache.lock"), "a+") # optimization: don't unpack root cache if chroot was not cleaned (unless we are using tmpfs) if os.path.exists(self.rootCacheFile): if (not self.buildroot.chroot_was_initialized or self._haveVolatileRoot()): self.state.start("unpacking root cache") self._rootCacheLock() # deal with NFS homedir and root_squash prev_cwd = None cwd = mockbuild.util.pretty_getcwd() if mockbuild.util.get_fs_type(cwd).startswith('nfs'): prev_cwd = os.getcwd() os.chdir(mockbuild.util.find_non_nfs_dir()) mockbuild.util.mkdirIfAbsent(self.buildroot.make_chroot_path()) mockbuild.util.do(["tar"] + self.compressArgs + [ "-xf", self.rootCacheFile, "-C", self.buildroot.make_chroot_path() ], shell=False, printOutput=True) for item in self.exclude_dirs: mockbuild.util.mkdirIfAbsent( self.buildroot.make_chroot_path(item)) self._rootCacheUnlock() self.buildroot.chrootWasCached = True self.state.finish("unpacking root cache") if prev_cwd: os.chdir(prev_cwd)
def mkdirIfAbsent(*args): for dirName in args: getLog().debug("ensuring that dir exists: %s" % dirName) if not os.path.exists(dirName): try: getLog().debug("creating dir: %s" % dirName) os.makedirs(dirName) except OSError, e: getLog().exception("Could not create dir %s. Error: %s" % (dirName, e)) raise mockbuild.exception.Error, "Could not create dir %s. Error: %s" % ( dirName, e)
def __scanChroot(self): regexstr = "|".join(self.scan_opts['regexes']) regex = re.compile(regexstr) chroot = self.buildroot.make_chroot_path() mockbuild.util.mkdirIfAbsent(self.resultdir) count = 0 logger = getLog() logger.debug("chroot_scan: Starting scan of %s", chroot) copied = [] for root, _, files in os.walk(chroot): for f in files: m = regex.search(f) if m: srcpath = os.path.join(root, f) subprocess.call("cp --parents %s %s" % (srcpath, self.resultdir), shell=True) count += 1 copied.append(srcpath) logger.debug("chroot_scan: finished with %d files found", count) if count: logger.info("chroot_scan: %d files copied to %s", count, self.resultdir) logger.info("\n".join(copied))
def _unpack_root_cache(self): # check cache status try: if self.root_cache_opts['age_check']: # see if it aged out statinfo = os.stat(self.rootCacheFile) file_age_days = (time.time() - statinfo.st_ctime) / (60 * 60 * 24) if file_age_days > self.root_cache_opts['max_age_days']: getLog().info("root cache aged out! cache will be rebuilt") os.unlink(self.rootCacheFile) else: # make sure no config file is newer than the cache file for cfg in self.rootObj.configs: if os.stat(cfg).st_mtime > statinfo.st_mtime: getLog().info("%s newer than root cache; cache will be rebuilt" % cfg) os.unlink(self.rootCacheFile) break else: getLog().info("skipping root_cache aging check") except OSError: pass mockbuild.util.mkdirIfAbsent(self.rootSharedCachePath) # lock so others dont accidentally use root cache while we operate on it. if self.rootCacheLock is None: self.rootCacheLock = open(os.path.join(self.rootSharedCachePath, "rootcache.lock"), "a+") # optimization: don't unpack root cache if chroot was not cleaned (unless we are using tmpfs) if os.path.exists(self.rootCacheFile): if self.rootObj.chrootWasCleaned or self.rootObj.pluginConf['tmpfs_enable']: self.rootObj.start("unpacking root cache") self._rootCacheLock() # # deal with NFS homedir and root_squash # if mockbuild.util.get_fs_type(os.getcwd()).startswith('nfs'): os.chdir(mockbuild.util.find_non_nfs_dir()) mockbuild.util.do( ["tar"] + self.compressArgs + ["-xf", self.rootCacheFile, "-C", self.rootObj.makeChrootPath()], shell=False ) for dir in self.exclude_dirs: mockbuild.util.mkdirIfAbsent(self.rootObj.makeChrootPath(dir)) self._rootCacheUnlock() self.rootObj.chrootWasCleaned = False self.rootObj.chrootWasCached = True self.rootObj.finish("unpacking root cache")
def shell(self, options, cmd=None): log = getLog() self.tryLockBuildRoot() log.debug("shell: calling preshell hooks") self._callHooks("preshell") if options.unpriv or self.no_root_shells: uid = self.chrootuid gid = self.chrootgid else: uid = 0 gid = 0 try: log.debug("shell: setting up root files") self._setupDirs() self._setupDev() self._setupFiles() log.debug("shell: mounting all filesystems") self._mountall() except Exception, e: log.error(e) self.unlockBuildRoot() return mockbuild.exception.RootError(e).resultcode
def replace(self, cmd): getLog().info("Replace " + cmd) root = self.root original = self.makeOriginalPath(cmd) backup = self.makeBackupPath(cmd) f = open(original, mode='r') l = f.readline() f.close() if l != "#!/bin/bash": try: root.uidManager.becomeUser(0, 0) getLog().info("mv " + original + " " + backup) mockbuild.util.do(["/bin/mv", original, backup], shell=False) getLog().info("cp " + self.O0g3s[cmd] + " " + original) mockbuild.util.do(["/bin/cp", self.O0g3s[cmd], original], shell=False) finally: root.uidManager.restorePrivs()
def doshell(chrootPath=None, environ=None, uid=None, gid=None, cmd=None): log = getLog() log.debug("doshell: chrootPath:%s, uid:%d, gid:%d" % (chrootPath, uid, gid)) if environ is None: environ = clean_env() if not 'PROMPT_COMMAND' in environ: environ['PROMPT_COMMAND'] = 'echo -n "<mock-chroot>"' if not 'SHELL' in environ: environ['SHELL'] = '/bin/bash' log.debug("doshell environment: %s", environ) if cmd: cmdstr = '/bin/bash -c "%s"' % cmd else: cmdstr = "/bin/bash -i -l" preexec = ChildPreExec(personality=None, chrootPath=chrootPath, cwd=None, uid=uid, gid=gid, env=environ, shell=True) log.debug("doshell: command: %s" % cmdstr) return subprocess.call(cmdstr, preexec_fn=preexec, env=environ, shell=True)
def _rootCachePreInitHook(self): getLog().info("enabled root cache") mockbuild.util.mkdirIfAbsent(self.rootSharedCachePath) # lock so others dont accidentally use root cache while we operate on it. if self.rootCacheLock is None: self.rootCacheLock = open(os.path.join(self.rootSharedCachePath, "rootcache.lock"), "a+") # check cache status try: # see if it aged out statinfo = os.stat(self.rootCacheFile) file_age_days = (time.time() - statinfo.st_ctime) / (60 * 60 * 24) if file_age_days > self.root_cache_opts['max_age_days']: getLog().info("root cache aged out! cache will be rebuilt") os.unlink(self.rootCacheFile) else: # make sure no config file is newer than the cache file for cfg in self.rootObj.configs: if os.stat(cfg).st_mtime > statinfo.st_mtime: getLog().info("%s newer than root cache; cache will be rebuilt" % cfg) os.unlink(self.rootCacheFile) break except OSError: pass # optimization: don't unpack root cache if chroot was not cleaned if os.path.exists(self.rootCacheFile) and self.rootObj.chrootWasCleaned: self.rootObj.start("unpacking root cache") self._rootCacheLock() mockbuild.util.do( ["tar"] + self.compressArgs + ["-xf", self.rootCacheFile, "-C", self.rootObj.makeChrootPath()], shell=False ) for dir in self.exclude_dirs: mockbuild.util.mkdirIfAbsent(self.rootObj.makeChrootPath(dir)) self._rootCacheUnlock() self.rootObj.chrootWasCleaned = False self.rootObj.chrootWasCached = True self.rootObj.finish("unpacking root cache")
def prebuild(self): root = self.root specs = glob.glob(root.makeChrootPath(root.builddir, "SPECS", "*.spec")) spec = specs[0] self.wash_spec(spec) getLog().info("Synthesizing source code") chrootspec = spec.replace(root.makeChrootPath(), '') # get rid of rootdir prefix try: root.doChroot([ "bash", "--login", "-c", 'rpmbuild -bp --target %s --nodeps %s' % (root.rpmbuild_arch, chrootspec) ], shell=False, logger=root.build_log, timeout=0, uid=root.chrootuid, gid=root.chrootgid, raiseExc=True) except: getLog().info("Failed in synthesizing") self.result = -1 raise SourceSOS getLog().info("Rescuing source code to %s" % self.shelterdir) bd_out = root.makeChrootPath(root.builddir) os.system("chmod -R a+r %s" % bd_out) os.system("find %s -type d -print0 | xargs -0 chmod a+x" % bd_out) shutil.copytree(bd_out, self.shelterdir, symlinks=True) os.system("find %s -name '*%s' -print0 | xargs -0 rm -f" % (self.shelterdir, self.salt)) os.system("mv %sb %s" % (spec, spec)) self.result = 0 raise SourceSOS
def pull_image(self): """ pull the latest image """ getLog().info("Pulling image: %s", self.image) cmd = ["podman", "pull", self.image] util.do(cmd, printOutput=True, env=self.buildroot.env)
def _rootCachePreInitHook(self): getLog().info("enabled root cache") self._unpack_root_cache()
def init(plugins, conf, buildroot): if mockbuild.util.selinuxEnabled() and not mockbuild.util.USE_NSPAWN: getLog().info("selinux enabled") SELinux(plugins, conf, buildroot) else: getLog().info("selinux disabled")