def _unlock(self, root, cleanup=False):
if not self.lockFile:
return
if cleanup:
util.removeIfExists(self.lockFile.name)
self.lockFile.close()
self.lockFile = None
def loadSchemaDump(self, name, db, dumpPath):
BaseSQLServer.loadSchemaDump(self, name, db, dumpPath)
tfile = util.AtomicFile(name)
cmd = "sqlite3 '%s' < '%s'" % (tfile.name, dumpPath)
execute(cmd)
util.removeIfExists(name + '.journal')
tfile.commit()
def restore(self, fileContents, root, target, journal=None, nameLookup=True,
**kwargs):
util.removeIfExists(target)
util.mkdirChain(os.path.dirname(target))
os.mkfifo(target)
return File.restore(self, root, target, journal=journal,
nameLookup=nameLookup, **kwargs)
def restore(self,
fileContents,
root,
target,
journal=None,
nameLookup=True,
**kwargs):
util.removeIfExists(target)
if not journal and os.getuid(): return target
util.mkdirChain(os.path.dirname(target))
if journal:
journal.mknod(root, target, self.lsTag, self.devt.major(),
self.devt.minor(), self.inode.perms(),
self.inode.owner(), self.inode.group())
else:
if self.lsTag == 'c':
flags = stat.S_IFCHR
else:
flags = stat.S_IFBLK
os.mknod(target, flags,
os.makedev(self.devt.major(), self.devt.minor()))
return File.restore(self,
root,
target,
journal=journal,
nameLookup=nameLookup,
**kwargs)
return target
def _mount_dev(self):
# Temporarily bind-mount the jobslave /dev into the chroot so
# grub2-install can see the loop device it's targeting.
logCall("mount -o bind /dev %s/dev" % self.image_root)
# /etc/grub.d/10_linux tries to find the backing device for loop
# devices, on the assumption that it's a block device with cryptoloop
# on top. Replace losetup with a stub while running mkconfig so it
# keeps the loop device name and all the right UUIDs get emitted.
losetup = util.joinPaths(self.image_root, '/sbin/losetup')
os.rename(losetup, losetup + '.bak')
with open(losetup, 'w') as f_losetup:
print >> f_losetup, '#!/bin/sh'
print >> f_losetup, 'echo "$1"'
os.chmod(losetup, 0755)
# In order for the root device to be detected as a FS UUID and not
# /dev/loop0 there needs to be a link in /dev/disk/by-uuid, which
# doesn't happen with the jobmaster's containerized environment.
link_path = None
if self.root_device.uuid:
link_path = util.joinPaths(self.image_root, '/dev/disk/by-uuid',
self.root_device.uuid)
util.mkdirChain(os.path.dirname(link_path))
util.removeIfExists(link_path)
os.symlink(self.root_device.devPath, link_path)
try:
yield
finally:
try:
if link_path:
os.unlink(link_path)
os.rename(losetup + '.bak', losetup)
logCall("umount %s/dev" % self.image_root)
except:
pass
def _unlock(self, root, cleanup=False):
if not self.lockFile:
return
if cleanup:
util.removeIfExists(self.lockFile.name)
self.lockFile.close()
self.lockFile = None
def restore(self, fileContents, root, target, journal=None, nameLookup=True, **kwargs):
util.removeIfExists(target)
util.mkdirChain(os.path.dirname(target))
os.symlink(self.target(), target)
# utime() follows symlinks and Linux currently does not implement
# lutimes()
return File.restore(self, root, target, skipMtime=True, journal=journal,
nameLookup=nameLookup, **kwargs)
def restore(self, chrootFingerprint, root):
path = self._fingerPrintToPath(chrootFingerprint)
if os.path.isdir(root):
self._remove(root)
with locking.LockFile(path + '.lock', share=True):
open(os.path.join(path, '.used'), 'w').close()
self._copy(path, root)
util.removeIfExists(os.path.join(root, '.used'))
def restore(self, chrootFingerprint, root):
path = self._fingerPrintToPath(chrootFingerprint)
if os.path.isdir(root):
self._remove(root)
with locking.LockFile(path + '.lock', share=True):
open(os.path.join(path, '.used'), 'w').close()
self._copy(path, root)
util.removeIfExists(os.path.join(root, '.used'))
def removeIssuerFromStore(self, crt, store):
certHash = crt.hash
issuerHash = crt.hash_issuer
destPath = os.path.join(store, "%s.%d" % (issuerHash, 0))
if certHash == issuerHash:
# Self-signed cert
return False
util.removeIfExists(destPath)
return True
def restore(self, fileContents, root, target, journal=None, nameLookup=True,
**kwargs):
util.removeIfExists(target)
util.mkdirChain(os.path.dirname(target))
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0);
sock.bind(target)
sock.close()
return File.restore(self, root, target, journal=journal,
nameLookup=nameLookup, **kwargs)
def runCommand(self, thisCommand, cfg, *args, **kw):
cfg.socketPath = cfg.root + cfg.socketPath
cfg.logDir = cfg.root + cfg.logDir
cfg.lockDir = cfg.root + cfg.lockDir
util.removeIfExists(cfg.socketPath)
util.mkdirChain(os.path.dirname(cfg.socketPath))
util.mkdirChain(cfg.lockDir)
util.mkdirChain(cfg.logDir)
return daemon.Daemon.runCommand(self, thisCommand, cfg, *args, **kw)
def runCommand(self, thisCommand, cfg, *args, **kw):
basename = os.path.basename(cfg.socketPath)
cfg.socketPath = cfg.root + cfg.socketPath
cfg.logDir = '%s%s.%s' % (cfg.root, cfg.logDir, basename)
cfg.lockDir = '%s%s.%s' % (cfg.root, cfg.lockDir, basename)
util.removeIfExists(cfg.socketPath)
util.mkdirChain(os.path.dirname(cfg.socketPath))
util.mkdirChain(cfg.lockDir)
util.mkdirChain(cfg.logDir)
return daemon.Daemon.runCommand(self, thisCommand, cfg, *args, **kw)
def runCommand(self, thisCommand, cfg, *args, **kw):
basename = os.path.basename(cfg.socketPath)
cfg.socketPath = cfg.root + cfg.socketPath
cfg.logDir = '%s%s.%s' % (cfg.root, cfg.logDir, basename)
cfg.lockDir = '%s%s.%s' % (cfg.root, cfg.lockDir, basename)
util.removeIfExists(cfg.socketPath)
util.mkdirChain(os.path.dirname(cfg.socketPath))
util.mkdirChain(cfg.lockDir)
util.mkdirChain(cfg.logDir)
return daemon.Daemon.runCommand(self, thisCommand, cfg, *args, **kw)
def store(self, chrootFingerprint, root):
path = self._fingerPrintToPath(chrootFingerprint)
prefix = sha1ToString(chrootFingerprint) + '.'
util.mkdirChain(self.cacheDir)
fd, fn = tempfile.mkstemp('.tar.gz', prefix, self.cacheDir)
os.close(fd)
try:
subprocess.call('tar cSpf - -C %s . | gzip -1 - > %s' % (root, fn),
shell=True)
os.rename(fn, path)
finally:
util.removeIfExists(fn)
def store(self, chrootFingerprint, root):
path = self._fingerPrintToPath(chrootFingerprint)
prefix = sha1ToString(chrootFingerprint) + '.'
util.mkdirChain(self.cacheDir)
fd, fn = tempfile.mkstemp('.tar.gz', prefix, self.cacheDir)
os.close(fd)
try:
subprocess.call('tar cSpf - -C %s . | gzip -1 - > %s' %(root, fn),
shell=True)
os.rename(fn, path)
finally:
util.removeIfExists(fn)
def _extractLayer(self, unpackDir, tarFile):
util.mkdirChain(unpackDir)
# Walk the files in the tar file, looking for .wh.*
tf = tarfile.open(tarFile)
toDeleteAfter = set()
for tinfo in tf:
bname = os.path.basename(tinfo.name)
if bname.startswith('.wh.') and tinfo.mode == 0:
util.rmtree(util.joinPaths(unpackDir,
os.path.dirname(tinfo.name), bname[4:]),
ignore_errors=True)
toDeleteAfter.add(util.joinPaths(unpackDir, tinfo.name))
logCall(["tar", "-C", unpackDir, "-xf", tarFile])
for fname in toDeleteAfter:
util.removeIfExists(fname)
def testHasSystemModel(self):
modelPath = util.joinPaths(self.cfg.root, self.cfg.modelPath)
util.removeIfExists(modelPath)
client = conaryclient.ConaryClient(self.cfg)
self.assertEquals(client.hasSystemModel(), False)
self.assertEquals(client.getSystemModel(), None)
# Create file now
util.mkdirChain(os.path.dirname(modelPath))
file(modelPath, "w").write("install group-me\n")
self.assertEquals(client.hasSystemModel(), True)
sysmodel = client.getSystemModel()
self.assertEquals(sysmodel.model.filedata, ["install group-me\n"])
self.assertEquals(sysmodel.fileFullName, modelPath)
self.assertEquals(sysmodel.mtime, os.stat(modelPath).st_mtime)
class rMakeDaemon(daemon.Daemon):
name = 'rmake-server'
commandName = 'rmake-server'
version = constants.version
configClass = servercfg.rMakeConfiguration
loggerClass = server.ServerLogger
user = constants.rmakeUser
groups = [constants.chrootUser]
capabilities = 'cap_sys_chroot+ep'
commandList = list(daemon.Daemon.commandList) + [ResetCommand, HelpCommand]
def getConfigFile(self, argv):
p = plugins.getPluginManager(argv, servercfg.rMakeConfiguration)
p.callServerHook('server_preInit', self, argv)
self.plugins = p
cfg = daemon.Daemon.getConfigFile(self, argv)
cfg.sanityCheck()
return cfg
def doWork(self):
cfg = self.cfg
try:
cfg.sanityCheckForStart()
except Exception, e:
self.logger.error(e)
sys.exit(1)
reposPid = None
proxyPid = None
rMakeServer = None
try:
if not cfg.isExternalRepos():
reposPid = repos.startRepository(cfg,
fork=True,
logger=self.logger)
if cfg.proxyUrl and not cfg.isExternalProxy():
proxyPid = repos.startProxy(cfg, fork=True, logger=self.logger)
if cfg.getSocketPath():
util.removeIfExists(cfg.getSocketPath())
rMakeServer = server.rMakeServer(cfg.getServerUri(),
cfg,
repositoryPid=reposPid,
proxyPid=proxyPid,
pluginMgr=self.plugins)
rMakeServer._installSignalHandlers()
rMakeServer.serve_forever()
finally:
if rMakeServer:
if rMakeServer.repositoryPid:
self.killRepos(reposPid)
if rMakeServer.proxyPid:
self.killRepos(proxyPid, 'proxy')
else:
# rmake server failed to start
if reposPid:
self.killRepos(reposPid)
if proxyPid:
self.killRepos(proxyPid, 'proxy')
def store(self, chrootFingerprint, root):
path = self._fingerPrintToPath(chrootFingerprint)
prefix = sha1ToString(chrootFingerprint) + '.'
util.mkdirChain(self.cacheDir)
lock = locking.LockFile(path + '.lock')
if not lock.acquire(wait=False):
# Busy, just do nothing
return
fd, fn = tempfile.mkstemp(self.suffix, prefix, self.cacheDir)
os.close(fd)
try:
subprocess.call('tar -cC %s . | %s > %s' % (root, self.compress,
fn), shell=True)
os.rename(fn, path)
finally:
util.removeIfExists(fn)
lock.release()
ChrootManifest.store(root, path)
self.prune()
def store(self, chrootFingerprint, root):
path = self._fingerPrintToPath(chrootFingerprint)
prefix = sha1ToString(chrootFingerprint) + '.'
util.mkdirChain(self.cacheDir)
lock = locking.LockFile(path + '.lock')
if not lock.acquire(wait=False):
# Busy, just do nothing
return
fd, fn = tempfile.mkstemp(self.suffix, prefix, self.cacheDir)
os.close(fd)
try:
subprocess.call('tar -cC %s . | %s > %s' %
(root, self.compress, fn),
shell=True)
os.rename(fn, path)
finally:
util.removeIfExists(fn)
lock.release()
ChrootManifest.store(root, path)
self.prune()
def __init__(self,
listener,
uri=None,
client=None,
showTroveDetails=False):
self.uri = uri
self.client = client
self.showTroveDetails = showTroveDetails
self.listener = listener
serverObj = None
if uri:
if isinstance(uri, str):
type, url = urllib.splittype(uri)
if type == 'unix':
util.removeIfExists(url)
serverObj = rpclib.UnixDomainDelayableXMLRPCServer(
url, logRequests=False)
elif type in ('http', 'https'):
# path is ignored with simple server.
host, path = urllib.splithost(url)
if ':' in host:
host, port = urllib.splitport(host)
port = int(port)
else:
port = 0
serverObj = rpclib.DelayableXMLRPCServer(('', port))
if not port:
uri = '%s://%s:%s' % (type, host, serverObj.getPort())
else:
raise NotImplmentedError
else:
serverObj = uri
self.uri = uri
self.server = serverObj
if serverObj:
serverObj.register_instance(self)
def loadMacros(paths):
"""
Load default macros from a series of I{paths}.
@rtype: dict
@return: A dictionary of default macros
"""
baseMacros = {}
loadPaths = []
for path in paths:
globPaths = sorted(list(glob.glob(path)))
loadPaths.extend(globPaths)
for path in loadPaths:
compiledPath = path + "c"
deleteCompiled = not util.exists(compiledPath)
macroModule = imp.load_source("tmpmodule", path)
if deleteCompiled:
util.removeIfExists(compiledPath)
baseMacros.update(x for x in macroModule.__dict__.iteritems() if not x[0].startswith("__"))
return baseMacros
def restore(self, fileContents, root, target, journal=None, nameLookup=True,
**kwargs):
util.removeIfExists(target)
if not journal and os.getuid(): return target
util.mkdirChain(os.path.dirname(target))
if journal:
journal.mknod(root, target, self.lsTag, self.devt.major(),
self.devt.minor(), self.inode.perms(),
self.inode.owner(), self.inode.group())
else:
if self.lsTag == 'c':
flags = stat.S_IFCHR
else:
flags = stat.S_IFBLK
os.mknod(target, flags, os.makedev(self.devt.major(),
self.devt.minor()))
return File.restore(self, root, target, journal=journal,
nameLookup=nameLookup, **kwargs)
return target
def __init__(self, listener, uri=None, client=None,
showTroveDetails=False):
self.uri = uri
self.client = client
self.showTroveDetails = showTroveDetails
self.listener = listener
serverObj = None
if uri:
if isinstance(uri, str):
type, url = urllib.splittype(uri)
if type == 'unix':
util.removeIfExists(url)
serverObj = rpclib.UnixDomainDelayableXMLRPCServer(url,
logRequests=False)
elif type in ('http', 'https'):
# path is ignored with simple server.
host, path = urllib.splithost(url)
if ':' in host:
host, port = urllib.splitport(host)
port = int(port)
else:
port = 0
serverObj = rpclib.DelayableXMLRPCServer(('', port))
if not port:
uri = '%s://%s:%s' % (type, host,
serverObj.getPort())
else:
raise NotImplmentedError
else:
serverObj = uri
self.uri = uri
self.server = serverObj
if serverObj:
serverObj.register_instance(self)
def loadMacros(paths):
'''
Load default macros from a series of I{paths}.
@rtype: dict
@return: A dictionary of default macros
'''
baseMacros = {}
loadPaths = []
for path in paths:
globPaths = sorted(list(glob.glob(path)))
loadPaths.extend(globPaths)
for path in loadPaths:
compiledPath = path + 'c'
deleteCompiled = not util.exists(compiledPath)
macroModule = imp.load_source('tmpmodule', path)
if deleteCompiled:
util.removeIfExists(compiledPath)
baseMacros.update(x for x in macroModule.__dict__.iteritems()
if not x[0].startswith('__'))
return baseMacros
def apply(self, fileDict, justDatabase = False, noScripts = False):
if not justDatabase and self.preserveSet:
capsuleJournal = ConaryOwnedJournal(self.root)
for path, unlink in self.preserveSet.iteritems():
fullPath = self.root + path
capsuleJournal.backup(fullPath, skipDirs = True)
if unlink and not util.removeIfExists(fullPath):
capsuleJournal.create(fullPath)
else:
capsuleJournal = None
try:
self.doApply(fileDict, justDatabase = justDatabase, noScripts = noScripts)
finally:
if capsuleJournal:
capsuleJournal.revert()
def apply(self, fileDict, justDatabase = False, noScripts = False):
if not justDatabase and self.preserveSet:
capsuleJournal = ConaryOwnedJournal(self.root)
for path in self.preserveSet:
fullPath = self.root + path
capsuleJournal.backup(fullPath, skipDirs = True)
if not util.removeIfExists(fullPath):
capsuleJournal.create(fullPath)
else:
capsuleJournal = None
try:
self.doApply(fileDict, justDatabase = justDatabase, noScripts = noScripts)
finally:
if capsuleJournal:
capsuleJournal.revert()
def close(self):
"""Flush the gzipped log and unlink the plaintext log"""
self.f_plain.close()
self.f_gz.close()
util.removeIfExists(self.path)
def remove(self, chrootFingerprint):
path = self._fingerPrintToPath(chrootFingerprint)
util.removeIfExists(path + ChrootManifest.AR_SUFFIX)
util.removeIfExists(path)
def deleteLogs(self, logHashes):
for logHash in logHashes:
path = self._hashToPath(logHash)
util.removeIfExists(path)
util.removeIfExists(path + '.gz')
def remove(self, chrootFingerprint):
path = self._fingerPrintToPath(chrootFingerprint)
util.removeIfExists(path + ChrootManifest.AR_SUFFIX)
util.removeIfExists(path)
def deleteFile(self, hash):
path = self.hashToPath(hash)
util.removeIfExists(path)
def close(self):
"""Flush the gzipped log and unlink the plaintext log"""
self.f_plain.close()
self.f_gz.close()
util.removeIfExists(self.path)
def deleteLogs(self, logHashes):
for logHash in logHashes:
path = self._hashToPath(logHash)
util.removeIfExists(path)
util.removeIfExists(path + '.gz')
def removeSurveyFromStore(self, uuid, store):
self.surveyPath = os.path.join(store, 'survey-%s.xml' % uuid)
if self.surveyPath is None:
return False
util.removeIfExists(self.surveyPath)
return True
def deleteFile(self, hash):
path = self.hashToPath(hash)
util.removeIfExists(path)
