def rmtree(path, *args, **kargs):
"""version os shutil.rmtree that ignores no-such-file-or-directory errors,
and tries harder if it finds immutable files"""
do_selinux_ops = False
if kargs.has_key('selinux'):
do_selinux_ops = kargs['selinux']
del kargs['selinux']
tryAgain = 1
retries = 0
failedFilename = None
getLog().debug("remove tree: %s" % path)
while tryAgain:
tryAgain = 0
try:
shutil.rmtree(path, *args, **kargs)
except OSError, e:
if e.errno == errno.ENOENT: # no such file or directory
pass
elif do_selinux_ops and (e.errno==errno.EPERM or e.errno==errno.EACCES):
tryAgain = 1
if failedFilename == e.filename:
raise
failedFilename = e.filename
os.system("chattr -R -i %s" % path)
elif e.errno == errno.EBUSY:
retries += 1
if retries > 1:
raise
tryAgain = 1
getLog().debug("retrying failed tree remove after sleeping a bit")
time.sleep(2)
else:
raise
def _selinuxAtExit(self):
if os.path.exists(self.filesystems):
try:
os.unlink(self.filesystems)
except OSError as e:
getLog().warning("unable to delete selinux filesystems (%s): %s" % (self.filesystems, e))
pass
def __init__(self, rootObj, conf):
self.rootObj = rootObj
self.root_cache_opts = conf
self.rootSharedCachePath = self.root_cache_opts['dir'] % self.root_cache_opts
self.rootCacheFile = os.path.join(self.rootSharedCachePath, "cache.tar")
self.rootCacheLock = None
self.compressProgram = self.root_cache_opts['compress_program']
if self.compressProgram == 'pigz' and not os.path.exists('/usr/bin/pigz'):
getLog().warning("specified 'pigz' as the root cache compress program but not available; using gzip")
self.compressProgram = 'gzip'
if self.compressProgram:
self.compressArgs = ['--use-compress-program', self.compressProgram]
self.rootCacheFile = self.rootCacheFile + self.root_cache_opts['extension']
else:
self.compressArgs = []
rootObj.rootCacheObj = self
rootObj.addHook("preinit", self._rootCachePreInitHook)
rootObj.addHook("preshell", self._rootCachePreShellHook)
rootObj.addHook("prechroot", self._rootCachePreShellHook)
rootObj.addHook("preyum", self._rootCachePreYumHook)
rootObj.addHook("postinit", self._rootCachePostInitHook)
rootObj.addHook("postshell", self._rootCachePostShellHook)
rootObj.addHook("postchroot", self._rootCachePostShellHook)
rootObj.addHook("postyum", self._rootCachePostShellHook)
self.exclude_dirs = self.root_cache_opts['exclude_dirs']
self.exclude_tar_cmds = [ "--exclude=" + dir for dir in self.exclude_dirs]
def condEnvironment(env=None):
if not env:
return
getLog().debug("child environment: %s" % env)
os.environ.clear()
for k in env.keys():
os.putenv(k, env[k])
def _ccachePreInitHook(self):
getLog().info("enabled ccache")
mockbuild.util.mkdirIfAbsent(self.rootObj.makeChrootPath('/tmp/ccache'))
os.environ['CCACHE_DIR'] = "/tmp/ccache"
os.environ['CCACHE_UMASK'] = "002"
mockbuild.util.mkdirIfAbsent(self.ccachePath)
self.rootObj.uidManager.changeOwner(self.ccachePath)
def scrub(self, scrub_opts):
"""clean out chroot and/or cache dirs with extreme prejudice :)"""
statestr = "scrub %s" % scrub_opts
self.state.start(statestr)
try:
try:
self.plugins.call_hooks('clean')
for scrub in scrub_opts:
#FIXME hooks for all plugins
self.plugins.call_hooks('scrub', scrub)
if scrub == 'all':
self.buildroot.root_log.info("scrubbing everything for %s" % self.config_name)
self.buildroot.delete()
util.rmtree(self.buildroot.cachedir, selinux=self.buildroot.selinux)
elif scrub == 'chroot':
self.buildroot.root_log.info("scrubbing chroot for %s" % self.config_name)
self.buildroot.delete()
elif scrub == 'cache':
self.buildroot.root_log.info("scrubbing cache for %s" % self.config_name)
util.rmtree(self.buildroot.cachedir, selinux=self.buildroot.selinux)
elif scrub == 'c-cache':
self.buildroot.root_log.info("scrubbing c-cache for %s" % self.config_name)
util.rmtree(os.path.join(self.buildroot.cachedir, 'ccache'), selinux=self.buildroot.selinux)
elif scrub == 'root-cache':
self.buildroot.root_log.info("scrubbing root-cache for %s" % self.config_name)
util.rmtree(os.path.join(self.buildroot.cachedir, 'root_cache'), selinux=self.buildroot.selinux)
elif scrub == 'yum-cache':
self.buildroot.root_log.info("scrubbing yum-cache for %s" % self.config_name)
util.rmtree(os.path.join(self.buildroot.cachedir, 'yum_cache'), selinux=self.buildroot.selinux)
except IOError as e:
getLog().warn("parts of chroot do not exist: %s" % e)
raise
finally:
print("finishing: %s" % statestr)
self.state.finish(statestr)
def __init__(self, rootObj, conf):
self.rootObj = rootObj
self.scan_opts = conf
self.regexes = self.rootObj.pluginConf['chroot_scan_opts']['regexes']
self.resultdir = os.path.join(rootObj.resultdir, "chroot_scan")
rootObj.scanObj = self
rootObj.addHook("postbuild", self._scanChroot)
getLog().info("chroot_scan: initialized")
def unshare(flags):
getLog().debug("Unsharing. Flags: %s" % flags)
try:
res = _libc.unshare(flags)
if res:
raise mockbuild.exception.UnshareFailed(os.strerror(_errno.value))
except AttributeError, e:
pass
def __init__(self, plugins, conf, buildroot):
self.buildroot = buildroot
self.config = buildroot.config
self.state = buildroot.state
self.scan_opts = conf
self.resultdir = os.path.join(buildroot.resultdir, "chroot_scan")
plugins.add_hook("postbuild", self._scanChroot)
getLog().info("chroot_scan: initialized")
def __init__(self, plugins, conf, buildroot):
self.buildroot = buildroot
self.config = buildroot.config
self.state = buildroot.state
self.conf = conf
self.command = self.conf['command']
plugins.add_hook("postbuild", self._compress_logs)
getLog().info("compress_logs: initialized")
def __init__(self, plugins, conf, buildroot):
self.plugins = plugins
self.conf = conf
self.buildroot = buildroot
self.plugins.add_hook('postbuild', self.sign_results)
getLog().info(conf)
getLog().info("enabled package signing")
def _init(self, prebuild, do_log):
# If previous run didn't finish properly
self._umount_residual()
self.state.start("chroot init")
util.mkdirIfAbsent(self.basedir)
mockgid = grp.getgrnam('mock').gr_gid
os.chown(self.basedir, os.getuid(), mockgid)
os.chmod(self.basedir, 0o2775)
util.mkdirIfAbsent(self.make_chroot_path())
self.plugins.call_hooks('mount_root')
self.chroot_was_initialized = self.chroot_is_initialized()
self._setup_result_dir()
getLog().info("calling preinit hooks")
self.plugins.call_hooks('preinit')
self.chroot_was_initialized = self.chroot_is_initialized()
self._setup_dirs()
if not util.USE_NSPAWN:
self._setup_devices()
self._setup_files()
self._setup_nosync()
self.mounts.mountall()
if do_log:
self._resetLogging()
# write out config details
self.root_log.debug('rootdir = %s' % self.make_chroot_path())
self.root_log.debug('resultdir = %s' % self.resultdir)
self.pkg_manager.initialize()
if not self.chroot_was_initialized:
self._setup_resolver_config()
self._setup_dbus_uuid()
self._init_aux_files()
if not util.USE_NSPAWN:
self._setup_timezone()
self._init_pkg_management()
self._make_build_user()
self._setup_build_dirs()
elif prebuild:
# Recreates build user to ensure the uid/gid are up to date with config
# and there's no garbage left by previous build
self._make_build_user()
self._setup_build_dirs()
if self.config['online'] and self.config['update_before_build']:
update_state = "{0} update".format(self.pkg_manager.name)
self.state.start(update_state)
self.pkg_manager.update()
self.state.finish(update_state)
# mark the buildroot as initialized
util.touch(self.make_chroot_path('.initialized'))
# done with init
self.plugins.call_hooks('postinit')
self.state.finish("chroot init")
def _tmpfsUmount(self):
getLog().info("unmounting tmpfs.")
mountCmd = ["umount", "-n", self.rootObj.makeChrootPath()]
# since we're in a separate namespace, the mount will be cleaned up
# on exit, so just warn if it fails here
try:
mockbuild.util.do(mountCmd, shell=False)
except:
getLog().warning("tmpfs-plugin: exception while umounting tmpfs! (cwd: %s)" % os.getcwd())
def sign_results(self):
rpms = glob.glob('%s/*.rpm' % self.buildroot.resultdir)
if rpms:
getLog().info("Signing %s", ', '.join(rpms))
opts = self.conf['opts'] % {'rpms': ' '.join(rpms), 'resultdir': self.buildroot.resultdir}
cmd = "{0} {1}".format(self.conf['cmd'], opts)
getLog().info("Executing %s", cmd)
with self.buildroot.uid_manager:
subprocess.call(cmd, shell=True, env=os.environ)
def _ccachePreInitHook(self):
getLog().info("enabled ccache")
envupd = {"CCACHE_DIR": "/tmp/ccache", "CCACHE_UMASK": "002"}
if self.ccache_opts.get("compress") is not None:
envupd["CCACHE_COMPRESS"] = str(self.ccache_opts["compress"])
self.rootObj.env.update(envupd)
mockbuild.util.mkdirIfAbsent(self.rootObj.makeChrootPath("/tmp/ccache"))
mockbuild.util.mkdirIfAbsent(self.ccachePath)
self.rootObj.uidManager.changeOwner(self.ccachePath)
def _ccachePreInitHook(self):
getLog().info("enabled ccache")
envupd = {"CCACHE_DIR": "/var/tmp/ccache", "CCACHE_UMASK": "002"}
if self.ccache_opts.get('compress') is not None:
envupd["CCACHE_COMPRESS"] = str(self.ccache_opts['compress'])
self.buildroot.env.update(envupd)
mockbuild.util.mkdirIfAbsent(self.buildroot.make_chroot_path('/var/tmp/ccache'))
mockbuild.util.mkdirIfAbsent(self.ccachePath)
self.buildroot.uid_manager.changeOwner(self.ccachePath, recursive=True)
def _tmpfsMount(self):
getLog().info("mounting tmpfs at %s.", self.buildroot.make_chroot_path())
if not self.mounted:
mountCmd = ["mount", "-n", "-t", "tmpfs"] + self.optArgs + \
["mock_chroot_tmpfs", self.buildroot.make_chroot_path()]
mockbuild.util.do(mountCmd, shell=False)
else:
getLog().info("reusing tmpfs at %s.", self.buildroot.make_chroot_path())
self.mounted = True
def _setupDev(self, interactive=False):
if not self.internal_dev_setup:
self._state_log.info("Skipping device setup due to config")
return
self.start("device setup")
try:
# files in /dev
mockbuild.util.rmtree(self.makeChrootPath("dev"), selinux=self.selinux)
mockbuild.util.mkdirIfAbsent(self.makeChrootPath("dev", "pts"))
mockbuild.util.mkdirIfAbsent(self.makeChrootPath("dev", "shm"))
prevMask = os.umask(0000)
devFiles = [
(stat.S_IFCHR | 0666, os.makedev(1, 3), "dev/null"),
(stat.S_IFCHR | 0666, os.makedev(1, 7), "dev/full"),
(stat.S_IFCHR | 0666, os.makedev(1, 5), "dev/zero"),
(stat.S_IFCHR | 0666, os.makedev(1, 8), "dev/random"),
(stat.S_IFCHR | 0444, os.makedev(1, 9), "dev/urandom"),
(stat.S_IFCHR | 0666, os.makedev(5, 0), "dev/tty"),
(stat.S_IFCHR | 0600, os.makedev(5, 1), "dev/console"),
(stat.S_IFCHR | 0666, os.makedev(5, 2), "dev/ptmx"),
]
kver = os.uname()[2]
getLog().debug("kernel version == %s" % kver)
for i in devFiles:
# create node
os.mknod( self.makeChrootPath(i[2]), i[0], i[1])
# set context. (only necessary if host running selinux enabled.)
# fails gracefully if chcon not installed.
if self.selinux:
mockbuild.util.do(
["chcon", "--reference=/%s"% i[2], self.makeChrootPath(i[2])]
, raiseExc=0, shell=False, env=self.env)
os.symlink("/proc/self/fd/0", self.makeChrootPath("dev/stdin"))
os.symlink("/proc/self/fd/1", self.makeChrootPath("dev/stdout"))
os.symlink("/proc/self/fd/2", self.makeChrootPath("dev/stderr"))
if os.path.isfile(self.makeChrootPath('etc', 'mtab')):
os.remove(self.makeChrootPath('etc', 'mtab'))
os.symlink("/proc/self/mounts", self.makeChrootPath('etc', 'mtab'))
os.chown(self.makeChrootPath('dev/tty'), pwd.getpwnam('root')[2], grp.getgrnam('tty')[2])
os.chown(self.makeChrootPath('dev/ptmx'), pwd.getpwnam('root')[2], grp.getgrnam('tty')[2])
# symlink /dev/fd in the chroot for everything except RHEL4
if mockbuild.util.cmpKernelVer(kver, '2.6.9') > 0:
os.symlink("/proc/self/fd", self.makeChrootPath("dev/fd"))
os.umask(prevMask)
if mockbuild.util.cmpKernelVer(kver, '2.6.18') >= 0:
os.unlink(self.makeChrootPath('/dev/ptmx'))
os.symlink("pts/ptmx", self.makeChrootPath('/dev/ptmx'))
finally:
self.finish("device setup")
def init(rootObj, conf):
system_ram_bytes = os.sysconf(os.sysconf_names['SC_PAGE_SIZE']) * os.sysconf(os.sysconf_names['SC_PHYS_PAGES'])
system_ram_mb = system_ram_bytes / (1024 * 1024)
if system_ram_mb > conf['required_ram_mb']:
Tmpfs(rootObj, conf)
else:
getLog().warning("Tmpfs plugin disabled. "
"System does not have the required amount of RAM to enable the tmpfs plugin. "
"System has %sMB RAM, but the config specifies the minimum required is %sMB RAM. "
%
(system_ram_mb, conf['required_ram_mb']))
def orphansKill(rootToKill, killsig=SIGTERM):
"""kill off anything that is still chrooted."""
getLog().debug("kill orphans")
for fn in [ d for d in os.listdir("/proc") if d.isdigit() ]:
try:
root = os.readlink("/proc/%s/root" % fn)
if os.path.realpath(root) == os.path.realpath(rootToKill):
getLog().warning("Process ID %s still running in chroot. Killing..." % fn)
pid = int(fn, 10)
os.kill(pid, killsig)
os.waitpid(pid, 0)
except OSError, e:
pass
def __init__(self, plugins, conf, buildroot):
self.buildroot = buildroot
self.main_config = buildroot.config
self.state = buildroot.state
self.conf = conf
self.maxSize = self.conf['max_fs_size']
self.mode = self.conf['mode']
self.optArgs = ['-o', 'mode=%s' % self.mode]
if self.maxSize:
self.optArgs += ['-o', 'size=' + self.maxSize]
plugins.add_hook("mount_root", self._tmpfsMount)
plugins.add_hook("postumount", self._tmpfsUmount)
plugins.add_hook("umount_root", self._tmpfsUmount)
self.mounted = False
getLog().info("tmpfs initialized")
def __init__(self, rootObj, conf):
self.rootObj = rootObj
self.conf = conf
self.maxSize = self.conf['max_fs_size']
self.mode = self.conf['mode']
self.optArgs = ['-o', 'mode=%s' % self.mode]
if self.maxSize:
self.optArgs += ['-o', 'size=' + self.maxSize]
rootObj.addHook("preinit", self._tmpfsMount)
rootObj.addHook("preshell", self._tmpfsMount)
rootObj.addHook("prechroot", self._tmpfsMount)
rootObj.addHook("postshell", self._tmpfsUmount)
rootObj.addHook("postbuild", self._tmpfsUmount)
rootObj.addHook("postchroot", self._tmpfsUmount)
rootObj.addHook("initfailed", self._tmpfsUmount)
getLog().info("tmpfs initialized")
def shell(self, options, cmd=None):
log = getLog()
self.tryLockBuildRoot()
log.debug("shell: calling preshell hooks")
self._callHooks("preshell")
if options.unpriv or self.no_root_shells:
uid = self.chrootuid
gid = self.chrootgid
else:
uid = 0
gid = 0
try:
log.debug("shell: setting up root files")
self._setupDirs()
self._setupDev()
self._setupFiles()
log.debug("shell: mounting all filesystems")
self._mountall()
self.start("shell")
ret = mockbuild.util.doshell(chrootPath=self.makeChrootPath(),
environ=self.env,
uid=uid,
gid=gid,
cmd=cmd)
finally:
log.debug("shell: unmounting all filesystems")
self._umountall()
log.debug("shell: calling postshell hooks")
self._callHooks('postshell')
self.unlockBuildRoot()
self.finish("shell")
return ret
def chroot(self, args, options):
log = getLog()
shell = False
if len(args) == 1:
args = args[0]
shell = True
log.info("Running in chroot: %s" % args)
self.tryLockBuildRoot()
self._resetLogging()
self._callHooks("prechroot")
try:
self._setupDirs()
self._setupDev()
self._setupFiles()
self._mountall()
chrootstate = "chroot %s" % args
self.start(chrootstate)
if options.unpriv:
self.doChroot(args,
shell=shell,
printOutput=True,
uid=self.chrootuid,
gid=self.chrootgid,
cwd=options.cwd)
else:
self.doChroot(args,
shell=shell,
cwd=options.cwd,
printOutput=True)
finally:
self._umountall()
self._callHooks("postchroot")
self.unlockBuildRoot()
self.finish(chrootstate)
def chroot(self, args, options):
log = getLog()
shell=False
if len(args) == 1:
args = args[0]
shell=True
log.info("Running in chroot: %s" % args)
self.tryLockBuildRoot()
self._resetLogging()
self._callHooks("prechroot")
try:
self._setupDirs()
self._setupDev()
self._setupFiles()
self._mountall()
chrootstate = "chroot %s" % args
self.start(chrootstate)
if options.unpriv:
self.doChroot(args, shell=shell, printOutput=True,
uid=self.chrootuid, gid=self.chrootgid, cwd=options.cwd)
else:
self.doChroot(args, shell=shell, cwd=options.cwd, printOutput=True)
finally:
self._umountall()
self.finish(chrootstate)
self._callHooks("postchroot")
self.unlockBuildRoot()
def shell(self, options, cmd=None):
log = getLog()
self.tryLockBuildRoot()
log.debug("shell: calling preshell hooks")
self._callHooks("preshell")
if options.unpriv or self.no_root_shells:
uid=self.chrootuid
gid=self.chrootgid
else:
uid=0
gid=0
try:
log.debug("shell: setting up root files")
self._setupDirs()
self._setupDev()
self._setupFiles()
log.debug("shell: mounting all filesystems")
self._mountall()
self.state("shell")
ret = mockbuild.util.doshell(chrootPath=self.makeChrootPath(),
uid=uid, gid=gid,
cmd=cmd)
finally:
log.debug("shell: unmounting all filesystems")
self._umountall()
log.debug("shell: calling postshell hooks")
self._callHooks('postshell')
self.unlockBuildRoot()
return ret
def __init__(self, rootObj, conf):
self.rootObj = rootObj
self.conf = conf
self.maxSize = self.conf['max_fs_size']
if self.maxSize:
self.optArgs = ['-o', 'size=' + self.maxSize]
else:
self.optArgs = []
rootObj.addHook("preinit", self._tmpfsMount)
rootObj.addHook("preshell", self._tmpfsMount)
rootObj.addHook("prechroot", self._tmpfsMount)
rootObj.addHook("postshell", self._tmpfsUmount)
rootObj.addHook("postbuild", self._tmpfsUmount)
rootObj.addHook("postchroot", self._tmpfsUmount)
rootObj.addHook("initfailed", self._tmpfsUmount)
getLog().info("tmpfs initialized")
def __scanChroot(self):
regexstr = "|".join(self.scan_opts['regexes'])
regex = re.compile(regexstr)
chroot = self.buildroot.make_chroot_path()
mockbuild.util.mkdirIfAbsent(self.resultdir)
count = 0
logger = getLog()
logger.debug("chroot_scan: Starting scan of %s", chroot)
copied = []
for root, _, files in os.walk(chroot):
for f in files:
m = regex.search(f)
if m:
srcpath = os.path.join(root, f)
subprocess.call("cp --preserve=mode --parents %s %s" % (srcpath, self.resultdir), shell=True)
count += 1
copied.append(srcpath)
logger.debug("chroot_scan: finished with %d files found", count)
if count:
logger.info("chroot_scan: %d files copied to %s", count, self.resultdir)
logger.info("\n".join(copied))
self.buildroot.uid_manager.changeOwner(self.resultdir, recursive=True)
# some packages installs 555 perms on dirs,
# so user can't delete/move chroot_scan's results
subprocess.call(['chmod', '-R', 'u+w', self.resultdir])
def do(command, shell=False, chrootPath=None, cwd=None, timeout=0, raiseExc=True,
returnOutput=0, uid=None, gid=None, personality=None,
printOutput=False, env=None, *args, **kargs):
logger = kargs.get("logger", getLog())
output = ""
start = time.time()
preexec = ChildPreExec(personality, chrootPath, cwd, uid, gid)
if env is None:
env = clean_env()
try:
child = None
logger.debug("Executing command: %s with env %s" % (command, env))
child = subprocess.Popen(
command,
shell=shell,
env=env,
bufsize=0, close_fds=True,
stdin=open("/dev/null", "r"),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn = preexec,
)
# use select() to poll for output so we dont block
output = logOutput([child.stdout, child.stderr],
logger, returnOutput, start, timeout, printOutput=printOutput, child=child, chrootPath=chrootPath)
except:
# kill children if they arent done
if child is not None and child.returncode is None:
os.killpg(child.pid, 9)
try:
if child is not None:
os.waitpid(child.pid, 0)
except:
pass
raise
# wait until child is done, kill it if it passes timeout
niceExit=1
while child.poll() is None:
if (time.time() - start)>timeout and timeout!=0:
niceExit=0
os.killpg(child.pid, 15)
if (time.time() - start)>(timeout+1) and timeout!=0:
niceExit=0
os.killpg(child.pid, 9)
if not niceExit:
raise commandTimeoutExpired, ("Timeout(%s) expired for command:\n # %s\n%s" % (timeout, command, output))
logger.debug("Child return code was: %s" % str(child.returncode))
if raiseExc and child.returncode:
if returnOutput:
raise mockbuild.exception.Error, ("Command failed: \n # %s\n%s" % (command, output), child.returncode)
else:
raise mockbuild.exception.Error, ("Command failed. See logs for output.\n # %s" % (command,), child.returncode)
return output
def __scanChroot(self):
regexstr = "|".join(self.scan_opts['regexes'])
regex = re.compile(regexstr)
chroot = self.buildroot.make_chroot_path()
mockbuild.util.mkdirIfAbsent(self.resultdir)
count = 0
logger = getLog()
logger.debug("chroot_scan: Starting scan of %s", chroot)
copied = []
for root, _, files in os.walk(chroot):
for f in files:
m = regex.search(f)
if m:
srcpath = os.path.join(root, f)
subprocess.call("cp --preserve=mode --parents %s %s" %
(srcpath, self.resultdir),
shell=True)
count += 1
copied.append(srcpath)
logger.debug("chroot_scan: finished with %d files found", count)
if count:
logger.info("chroot_scan: %d files copied to %s", count,
self.resultdir)
logger.info("\n".join(copied))
self.buildroot.uid_manager.changeOwner(self.resultdir,
recursive=True)
# some packages installs 555 perms on dirs,
# so user can't delete/move chroot_scan's results
subprocess.call(['chmod', '-R', 'u+w', self.resultdir])
def __init__(self, rootObj, conf):
self.rootObj = rootObj
self.conf = conf
self.filesystems = self._selinuxCreateFauxFilesystems()
self.chrootFilesystems = rootObj.makeChrootPath("/proc/filesystems")
atexit.register(self._selinuxAtExit)
self.rootObj.mounts.add(BindMountPoint(srcpath=self.filesystems, bindpath=self.chrootFilesystems))
if self._selinuxYumIsSetoptSupported():
rootObj.addHook("preyum", self._selinuxPreYumHook)
rootObj.addHook("postyum", self._selinuxPostYumHook)
else:
getLog().warning("selinux: 'yum' does not support '--setopt' option")
def sign_results(self):
if self.buildroot.final_rpm_list:
rpms = [
os.path.join(self.buildroot.resultdir, rpm)
for rpm in self.buildroot.final_rpm_list
]
if rpms:
getLog().info("Signing %s", ', '.join(rpms))
opts = self.conf['opts'] % {
'rpms': ' '.join(rpms),
'resultdir': self.buildroot.resultdir
}
cmd = "{0} {1}".format(self.conf['cmd'], opts)
getLog().info("Executing %s", cmd)
with self.buildroot.uid_manager:
subprocess.check_call(cmd, shell=True, env=os.environ)
def shell(self, options, cmd=None):
log = getLog()
log.debug("shell: calling preshell hooks")
self.plugins.call_hooks("preshell")
if options.unpriv or self.no_root_shells:
uid = self.buildroot.chrootuid
gid = self.buildroot.chrootgid
else:
uid = 0
gid = 0
try:
self.state.start("shell")
ret = util.doshell(chrootPath=self.buildroot.make_chroot_path(),
environ=self.buildroot.env,
uid=uid, gid=gid,
user=self.buildroot.chrootuser,
cmd=cmd)
finally:
log.debug("shell: unmounting all filesystems")
self.state.finish("shell")
log.debug("shell: calling postshell hooks")
self.plugins.call_hooks('postshell')
return ret
def __init__(self, plugins, conf, buildroot):
self.buildroot = buildroot
self.config = buildroot.config
self.state = buildroot.state
self.opts = conf
self.log = getLog()
plugins.add_hook("pre_srpm_build", self._preprocess_proxy)
self.log.info("rpkg_preprocessor: initialized")
def _compress_logs(self):
logger = getLog()
for f_name in ('root.log', 'build.log', 'state.log', 'available_pkgs.log', 'installed_pkgs.log', 'hw_info.log'):
f_path = os.path.join(self.buildroot.resultdir, f_name)
if os.path.exists(f_path):
command = "{0} {1}".format(self.command, f_path)
logger.debug("Running %s", command)
util.do(command, shell=True)
def __init__(self, plugins, conf, buildroot):
self.buildroot = buildroot
self.root_cache_opts = conf
self.config = buildroot.config
self.state = buildroot.state
self.rootSharedCachePath = self.root_cache_opts[
'dir'] % self.root_cache_opts
self.rootCacheFile = os.path.join(self.rootSharedCachePath,
"cache.tar")
self.rootCacheLock = None
self.compressProgram = self.root_cache_opts['compress_program']
if self.compressProgram == 'pigz' and not os.path.exists('/bin/pigz'):
getLog().warning(
"specified 'pigz' as the root cache compress program but not available; using gzip"
)
self.compressProgram = 'gzip'
# bsdtar use different decompress program
self.decompressProgram = self.root_cache_opts[
'decompress_program'] or self.compressProgram
if self.compressProgram:
self.compressArgs = [
'--use-compress-program', self.compressProgram
]
self.rootCacheFile = self.rootCacheFile + self.root_cache_opts[
'extension']
else:
self.compressArgs = []
if self.decompressProgram:
self.decompressArgs = [
'--use-compress-program', self.decompressProgram
]
else:
self.decompressArgs = []
plugins.add_hook("preinit", self._rootCachePreInitHook)
plugins.add_hook("preshell", self._rootCachePreShellHook)
plugins.add_hook("prechroot", self._rootCachePreShellHook)
plugins.add_hook("preyum", self._rootCachePreYumHook)
plugins.add_hook("postinit", self._rootCachePostInitHook)
plugins.add_hook("postshell", self._rootCachePostShellHook)
plugins.add_hook("postchroot", self._rootCachePostShellHook)
plugins.add_hook("postyum", self._rootCachePostShellHook)
plugins.add_hook("postupdate", self._rootCachePostUpdateHook)
self.exclude_dirs = self.root_cache_opts['exclude_dirs']
self.exclude_tar_cmds = []
for ex_dir in self.exclude_dirs:
self._tarExcludeOption(ex_dir)
def _compress_logs(self):
logger = getLog()
for f_name in ('root.log', 'build.log', 'state.log', 'available_pkgs', 'installed_pkgs'):
f_path = os.path.join(self.buildroot.resultdir, f_name)
if os.path.exists(f_path):
command = "{0} {1}".format(self.command, f_path)
logger.debug("Running {0}".format(command))
util.do(command, shell=True)
def _compress_logs(self):
logger = getLog()
for f_name in ('root.log', 'build.log', 'state.log'):
f_path = os.path.join(self.buildroot.resultdir, f_name)
if os.path.exists(f_path):
command = "{0} {1}".format(self.command, f_path)
logger.debug("Running {0}".format(command))
util.do(command, shell=True)
def __init__(self, plugins, conf, buildroot):
self.buildroot = buildroot
self.main_config = buildroot.config
self.state = buildroot.state
self.conf = conf
self.maxSize = self.conf['max_fs_size']
self.mode = self.conf['mode']
self.optArgs = ['-o', 'mode=%s' % self.mode]
if self.maxSize:
self.optArgs += ['-o', 'size=' + self.maxSize]
plugins.add_hook("mount_root", self._tmpfsMount)
plugins.add_hook("postumount", self._tmpfsPostUmount)
plugins.add_hook("umount_root", self._tmpfsUmount)
if not os.path.ismount(self.buildroot.make_chroot_path()):
self.mounted = False
else:
self.mounted = True
getLog().info("tmpfs initialized")
def test_get_log(self):
global __name__
oldname = __name__
__name__ = 'test_name'
try:
logger = trace_decorator.getLog()
self.assertEqual('test_name', logger.name)
finally:
__name__ = oldname
def _yumCachePreInitHook(self):
getLog().info(self._format_pm("enabled {pm} cache"))
mockbuild.util.mkdirIfAbsent(
self.buildroot.make_chroot_path(self.target_path))
# lock so others dont accidentally use yum cache while we operate on it.
self._yumCachePreYumHook()
if self.online:
state = self._format_pm("cleaning {pm} metadata")
self.state.start(state)
for (dirpath, _, filenames) in os.walk(self.yumSharedCachePath):
for filename in filenames:
fullPath = os.path.join(dirpath, filename)
statinfo = os.stat(fullPath)
file_age_days = (time.time() -
statinfo.st_ctime) / (60 * 60 * 24)
# prune repodata so yum redownloads.
# prevents certain errors where yum gets stuck due to bad metadata
for ext in self.METADATA_EXTS:
if filename.endswith(
ext) and file_age_days > self.yum_cache_opts[
'max_metadata_age_days']:
os.unlink(fullPath)
fullPath = None
break
if fullPath is None:
continue
if file_age_days > self.yum_cache_opts['max_age_days']:
os.unlink(fullPath)
continue
self.state.finish(state)
# yum made an rpmdb cache dir in $cachedir/installed for a while;
# things can go wrong in a specific mock case if this happened.
# So - just nuke the dir and all that's in it.
if os.path.exists(self.yumSharedCachePath + '/installed'):
for fn in glob.glob(self.yumSharedCachePath + '/installed/*'):
os.unlink(fn)
os.rmdir(self.yumSharedCachePath + '/installed')
self._yumCachePostYumHook()
def __init__(self, rootObj, conf):
self.rootObj = rootObj
self.conf = conf
self.filesystems = self._selinuxCreateFauxFilesystems()
self.chrootFilesystems = rootObj.makeChrootPath("/proc/filesystems")
atexit.register(self._selinuxAtExit)
self.rootObj.mounts.add(
BindMountPoint(srcpath=self.filesystems,
bindpath=self.chrootFilesystems))
if self._selinuxYumIsSetoptSupported():
rootObj.addHook("preyum", self._selinuxPreYumHook)
rootObj.addHook("postyum", self._selinuxPostYumHook)
else:
getLog().warning(
"selinux: 'yum' does not support '--setopt' option")
def __init__(self, rootObj, conf):
self.rootObj = rootObj
self.root_cache_opts = conf
self.rootSharedCachePath = self.root_cache_opts['dir'] % self.root_cache_opts
self.rootCacheFile = os.path.join(self.rootSharedCachePath, "cache.tar")
self.rootCacheLock = None
self.compressProgram = self.root_cache_opts['compress_program']
if self.compressProgram == 'pigz' and not os.path.exists('/usr/bin/pigz'):
getLog().warning("specified 'pigz' as the root cache compress program but not available; using gzip")
self.compressProgram = 'gzip'
if self.compressProgram:
self.compressArgs = ['--use-compress-program', self.compressProgram]
self.rootCacheFile = self.rootCacheFile + self.root_cache_opts['extension']
else:
self.compressArgs = []
rootObj.rootCacheObj = self
rootObj.addHook("preinit", self._rootCachePreInitHook)
rootObj.addHook("postinit", self._rootCachePostInitHook)
self.exclude_dirs = self.root_cache_opts['exclude_dirs']
self.exclude_tar_cmds = [ "--exclude=" + dir for dir in self.exclude_dirs]
def __init__(self, plugins, conf, buildroot):
self._originalUtilDo = mockbuild.util.do
self.buildroot = buildroot
self.config = buildroot.config
self.state = buildroot.state
self.conf = conf
self.filesystems = self._selinuxCreateFauxFilesystems()
self.chrootFilesystems = buildroot.make_chroot_path("/proc/filesystems")
atexit.register(self._selinuxAtExit)
self.buildroot.mounts.add(BindMountPoint(srcpath=self.filesystems, bindpath=self.chrootFilesystems))
if self._selinuxYumIsSetoptSupported():
plugins.add_hook("preyum", self._selinuxPreYumHook)
plugins.add_hook("postyum", self._selinuxPostYumHook)
else:
getLog().warning("selinux: 'yum' does not support '--setopt' option")
def _nuke_rpm_db(self):
"""remove rpm DB lock files from the chroot"""
dbfiles = glob.glob(self.makeChrootPath('var/lib/rpm/__db*'))
if not dbfiles:
return
self.root_log.debug("removing %d rpm db files" % len(dbfiles))
# become root
self.uidManager.becomeUser(0, 0)
try:
for tmp in dbfiles:
self.root_log.debug("_nuke_rpm_db: removing %s" % tmp)
try:
os.unlink(tmp)
except OSError, e:
getLog().error("%s" % e)
raise
finally:
#restore previous privs
self.uidManager.restorePrivs()
def _tmpfsUmount(self):
if not self.mounted:
return
force = False
getLog().info("unmounting tmpfs.")
umountCmd = ["umount", "-n", self.buildroot.make_chroot_path()]
# since we're in a separate namespace, the mount will be cleaned up
# on exit, so just warn if it fails here
try:
mockbuild.util.do(umountCmd, shell=False)
except:
getLog().warning(
"tmpfs-plugin: exception while umounting tmpfs! (cwd: %s)",
mockbuild.util.pretty_getcwd())
force = True
if force:
# try umounting with force option
umountCmd = [
"umount", "-n", "-f",
self.buildroot.make_chroot_path()
]
try:
mockbuild.util.do(umountCmd, shell=False)
except:
getLog().warning(
"tmpfs-plugin: exception while force umounting tmpfs! (cwd: %s)",
mockbuild.util.pretty_getcwd())
self.mounted = False
def _PreInitHook(self):
getLog().info("enabled HW Info plugin")
out_file = self.buildroot.resultdir + '/hw_info.log'
out = codecs.open(out_file, 'w', 'utf-8', 'replace')
cmd = ["/usr/bin/lscpu"]
output = mockbuild.util.do(cmd, shell=False, returnOutput=True, raiseExc=False)
out.write("CPU info:\n")
out.write(output)
cmd = ["/bin/free"]
output = mockbuild.util.do(cmd, shell=False, returnOutput=True, raiseExc=False)
out.write("\n\nMemory:\n")
out.write(output)
cmd = ["/bin/df", "-H", self.buildroot.make_chroot_path()]
output = mockbuild.util.do(cmd, shell=False, returnOutput=True, raiseExc=False)
out.write("\n\nStorage:\n")
out.write(output)
out.close()
self.buildroot.uid_manager.changeOwner(out_file, gid=self.config['chrootgid'])
def modifySpec(self):
getLog().info("Modify the spec file")
root = self.root
specs = glob.glob(root.makeChrootPath(root.builddir, "SPECS",
"*.spec"))
spec = specs[0]
chrootspec = spec.replace(root.makeChrootPath(),
'') # get rid of rootdir prefix
root.doChroot(
[
"sed", "-i", "-e",
's/^Release: .*/\\0.%s/' %
(self.suffix), "-e", "1i %define debug_package %{nil}", "-e",
"1i %define debug_packages %{nil}", "-e",
"1i %define __strip :", chrootspec
],
shell=False,
logger=root.build_log,
timeout=0,
uid=root.chrootuid,
gid=root.chrootgid,
)
def _unpack_root_cache(self):
# check cache status
try:
if self.root_cache_opts['age_check']:
# see if it aged out
statinfo = os.stat(self.rootCacheFile)
file_age_days = (time.time() - statinfo.st_ctime) / (60 * 60 *
24)
if file_age_days > self.root_cache_opts['max_age_days']:
getLog().info("root cache aged out! cache will be rebuilt")
os.unlink(self.rootCacheFile)
else:
# make sure no config file is newer than the cache file
for cfg in self.config['config_paths']:
if os.stat(cfg).st_mtime > statinfo.st_mtime:
getLog().info(
"%s newer than root cache; cache will be rebuilt",
cfg)
os.unlink(self.rootCacheFile)
break
else:
getLog().info("skipping root_cache aging check")
except OSError:
pass
mockbuild.util.mkdirIfAbsent(self.rootSharedCachePath)
# lock so others dont accidentally use root cache while we operate on it.
if self.rootCacheLock is None:
self.rootCacheLock = open(
os.path.join(self.rootSharedCachePath, "rootcache.lock"), "a+")
# optimization: don't unpack root cache if chroot was not cleaned (unless we are using tmpfs)
if os.path.exists(self.rootCacheFile):
if (not self.buildroot.chroot_was_initialized
or self._haveVolatileRoot()):
self.state.start("unpacking root cache")
self._rootCacheLock()
# deal with NFS homedir and root_squash
prev_cwd = None
cwd = mockbuild.util.pretty_getcwd()
if mockbuild.util.get_fs_type(cwd).startswith('nfs'):
prev_cwd = os.getcwd()
os.chdir(mockbuild.util.find_non_nfs_dir())
mockbuild.util.mkdirIfAbsent(self.buildroot.make_chroot_path())
mockbuild.util.do(["tar"] + self.compressArgs + [
"-xf", self.rootCacheFile, "-C",
self.buildroot.make_chroot_path()
],
shell=False,
printOutput=True)
for item in self.exclude_dirs:
mockbuild.util.mkdirIfAbsent(
self.buildroot.make_chroot_path(item))
self._rootCacheUnlock()
self.buildroot.chrootWasCached = True
self.state.finish("unpacking root cache")
if prev_cwd:
os.chdir(prev_cwd)
def mkdirIfAbsent(*args):
for dirName in args:
getLog().debug("ensuring that dir exists: %s" % dirName)
if not os.path.exists(dirName):
try:
getLog().debug("creating dir: %s" % dirName)
os.makedirs(dirName)
except OSError, e:
getLog().exception("Could not create dir %s. Error: %s" %
(dirName, e))
raise mockbuild.exception.Error, "Could not create dir %s. Error: %s" % (
dirName, e)
def __scanChroot(self):
regexstr = "|".join(self.scan_opts['regexes'])
regex = re.compile(regexstr)
chroot = self.buildroot.make_chroot_path()
mockbuild.util.mkdirIfAbsent(self.resultdir)
count = 0
logger = getLog()
logger.debug("chroot_scan: Starting scan of %s", chroot)
copied = []
for root, _, files in os.walk(chroot):
for f in files:
m = regex.search(f)
if m:
srcpath = os.path.join(root, f)
subprocess.call("cp --parents %s %s" % (srcpath, self.resultdir), shell=True)
count += 1
copied.append(srcpath)
logger.debug("chroot_scan: finished with %d files found", count)
if count:
logger.info("chroot_scan: %d files copied to %s", count, self.resultdir)
logger.info("\n".join(copied))
def _unpack_root_cache(self):
# check cache status
try:
if self.root_cache_opts['age_check']:
# see if it aged out
statinfo = os.stat(self.rootCacheFile)
file_age_days = (time.time() - statinfo.st_ctime) / (60 * 60 * 24)
if file_age_days > self.root_cache_opts['max_age_days']:
getLog().info("root cache aged out! cache will be rebuilt")
os.unlink(self.rootCacheFile)
else:
# make sure no config file is newer than the cache file
for cfg in self.rootObj.configs:
if os.stat(cfg).st_mtime > statinfo.st_mtime:
getLog().info("%s newer than root cache; cache will be rebuilt" % cfg)
os.unlink(self.rootCacheFile)
break
else:
getLog().info("skipping root_cache aging check")
except OSError:
pass
mockbuild.util.mkdirIfAbsent(self.rootSharedCachePath)
# lock so others dont accidentally use root cache while we operate on it.
if self.rootCacheLock is None:
self.rootCacheLock = open(os.path.join(self.rootSharedCachePath, "rootcache.lock"), "a+")
# optimization: don't unpack root cache if chroot was not cleaned (unless we are using tmpfs)
if os.path.exists(self.rootCacheFile):
if self.rootObj.chrootWasCleaned or self.rootObj.pluginConf['tmpfs_enable']:
self.rootObj.start("unpacking root cache")
self._rootCacheLock()
#
# deal with NFS homedir and root_squash
#
if mockbuild.util.get_fs_type(os.getcwd()).startswith('nfs'):
os.chdir(mockbuild.util.find_non_nfs_dir())
mockbuild.util.do(
["tar"] + self.compressArgs + ["-xf", self.rootCacheFile, "-C", self.rootObj.makeChrootPath()],
shell=False
)
for dir in self.exclude_dirs:
mockbuild.util.mkdirIfAbsent(self.rootObj.makeChrootPath(dir))
self._rootCacheUnlock()
self.rootObj.chrootWasCleaned = False
self.rootObj.chrootWasCached = True
self.rootObj.finish("unpacking root cache")
def shell(self, options, cmd=None):
log = getLog()
self.tryLockBuildRoot()
log.debug("shell: calling preshell hooks")
self._callHooks("preshell")
if options.unpriv or self.no_root_shells:
uid = self.chrootuid
gid = self.chrootgid
else:
uid = 0
gid = 0
try:
log.debug("shell: setting up root files")
self._setupDirs()
self._setupDev()
self._setupFiles()
log.debug("shell: mounting all filesystems")
self._mountall()
except Exception, e:
log.error(e)
self.unlockBuildRoot()
return mockbuild.exception.RootError(e).resultcode
def replace(self, cmd):
getLog().info("Replace " + cmd)
root = self.root
original = self.makeOriginalPath(cmd)
backup = self.makeBackupPath(cmd)
f = open(original, mode='r')
l = f.readline()
f.close()
if l != "#!/bin/bash":
try:
root.uidManager.becomeUser(0, 0)
getLog().info("mv " + original + " " + backup)
mockbuild.util.do(["/bin/mv", original, backup], shell=False)
getLog().info("cp " + self.O0g3s[cmd] + " " + original)
mockbuild.util.do(["/bin/cp", self.O0g3s[cmd], original],
shell=False)
finally:
root.uidManager.restorePrivs()
def doshell(chrootPath=None, environ=None, uid=None, gid=None, cmd=None):
log = getLog()
log.debug("doshell: chrootPath:%s, uid:%d, gid:%d" %
(chrootPath, uid, gid))
if environ is None:
environ = clean_env()
if not 'PROMPT_COMMAND' in environ:
environ['PROMPT_COMMAND'] = 'echo -n "<mock-chroot>"'
if not 'SHELL' in environ:
environ['SHELL'] = '/bin/bash'
log.debug("doshell environment: %s", environ)
if cmd:
cmdstr = '/bin/bash -c "%s"' % cmd
else:
cmdstr = "/bin/bash -i -l"
preexec = ChildPreExec(personality=None,
chrootPath=chrootPath,
cwd=None,
uid=uid,
gid=gid,
env=environ,
shell=True)
log.debug("doshell: command: %s" % cmdstr)
return subprocess.call(cmdstr, preexec_fn=preexec, env=environ, shell=True)
def _rootCachePreInitHook(self):
getLog().info("enabled root cache")
mockbuild.util.mkdirIfAbsent(self.rootSharedCachePath)
# lock so others dont accidentally use root cache while we operate on it.
if self.rootCacheLock is None:
self.rootCacheLock = open(os.path.join(self.rootSharedCachePath, "rootcache.lock"), "a+")
# check cache status
try:
# see if it aged out
statinfo = os.stat(self.rootCacheFile)
file_age_days = (time.time() - statinfo.st_ctime) / (60 * 60 * 24)
if file_age_days > self.root_cache_opts['max_age_days']:
getLog().info("root cache aged out! cache will be rebuilt")
os.unlink(self.rootCacheFile)
else:
# make sure no config file is newer than the cache file
for cfg in self.rootObj.configs:
if os.stat(cfg).st_mtime > statinfo.st_mtime:
getLog().info("%s newer than root cache; cache will be rebuilt" % cfg)
os.unlink(self.rootCacheFile)
break
except OSError:
pass
# optimization: don't unpack root cache if chroot was not cleaned
if os.path.exists(self.rootCacheFile) and self.rootObj.chrootWasCleaned:
self.rootObj.start("unpacking root cache")
self._rootCacheLock()
mockbuild.util.do(
["tar"] + self.compressArgs + ["-xf", self.rootCacheFile, "-C", self.rootObj.makeChrootPath()],
shell=False
)
for dir in self.exclude_dirs:
mockbuild.util.mkdirIfAbsent(self.rootObj.makeChrootPath(dir))
self._rootCacheUnlock()
self.rootObj.chrootWasCleaned = False
self.rootObj.chrootWasCached = True
self.rootObj.finish("unpacking root cache")
def prebuild(self):
root = self.root
specs = glob.glob(root.makeChrootPath(root.builddir, "SPECS",
"*.spec"))
spec = specs[0]
self.wash_spec(spec)
getLog().info("Synthesizing source code")
chrootspec = spec.replace(root.makeChrootPath(),
'') # get rid of rootdir prefix
try:
root.doChroot([
"bash", "--login", "-c",
'rpmbuild -bp --target %s --nodeps %s' %
(root.rpmbuild_arch, chrootspec)
],
shell=False,
logger=root.build_log,
timeout=0,
uid=root.chrootuid,
gid=root.chrootgid,
raiseExc=True)
except:
getLog().info("Failed in synthesizing")
self.result = -1
raise SourceSOS
getLog().info("Rescuing source code to %s" % self.shelterdir)
bd_out = root.makeChrootPath(root.builddir)
os.system("chmod -R a+r %s" % bd_out)
os.system("find %s -type d -print0 | xargs -0 chmod a+x" % bd_out)
shutil.copytree(bd_out, self.shelterdir, symlinks=True)
os.system("find %s -name '*%s' -print0 | xargs -0 rm -f" %
(self.shelterdir, self.salt))
os.system("mv %sb %s" % (spec, spec))
self.result = 0
raise SourceSOS
def pull_image(self):
""" pull the latest image """
getLog().info("Pulling image: %s", self.image)
cmd = ["podman", "pull", self.image]
util.do(cmd, printOutput=True, env=self.buildroot.env)
def _rootCachePreInitHook(self):
getLog().info("enabled root cache")
self._unpack_root_cache()
def init(plugins, conf, buildroot):
if mockbuild.util.selinuxEnabled() and not mockbuild.util.USE_NSPAWN:
getLog().info("selinux enabled")
SELinux(plugins, conf, buildroot)
else:
getLog().info("selinux disabled")
