def _checkDir(self, name, path, requiredOwner=None,
requiredMode=None):
if not os.path.exists(path):
raise errors.RmakeError('%s does not exist, expected at %s - cannot start server' % (name, path))
sys.exit(1)
if not (requiredOwner or requiredMode):
return
statInfo = os.stat(path)
if requiredMode and statInfo.st_mode & 0777 != requiredMode:
raise errors.RmakeError('%s (%s) must have mode %o' % (path, name, requiredMode))
def SubscriberFactory(name, protocol, uri):
if not _builtInsLoaded:
_loadBuiltIns()
try:
return _registeredProtocols[protocol](name, uri)
except KeyError:
raise errors.RmakeError('cannot get subscriber for %s: Unknown protocol' % uri)
def _getChroot(self, chroot):
params = chroot.split(':', 1)
if len(params) != 2:
self.usage()
raise errors.RmakeError(
'Chroot name needs to be <host>:<chroot> format')
return params
def _getPathList(repos, cfg, recipePath, relative=False):
loader, recipeClass, sourceVersion = cook.getRecipeInfoFromPath(repos, cfg,
recipePath)
log.info("Getting relevant path information from %s..." % recipeClass.name)
recipeDir = os.path.dirname(recipePath)
srcdirs = [ recipeDir ]
recipeObj = None
buildLabel = sourceVersion.trailingLabel()
macros = {'buildlabel' : buildLabel.asString(),
'buildbranch' : sourceVersion.branch().asString()}
if recipe.isPackageRecipe(recipeClass):
recipeObj = recipeClass(cfg, None,
srcdirs, macros, lightInstance=True)
elif recipe.isGroupRecipe(recipeClass):
recipeObj = recipeClass(repos, cfg, buildLabel, None, None,
srcdirs=srcdirs,
extraMacros=macros)
else:
# no included files for the rest of the recipe types
pathList = [recipePath]
recipeObj = None
if recipeObj:
try:
if hasattr(recipeObj, 'loadPolicy'):
recipeObj.loadPolicy()
cook._callSetup(cfg, recipeObj)
except (conaryerrors.ConaryError, conaryerrors.CvcError), msg:
raise errors.RmakeError("could not initialize recipe: %s" % (msg))
pathList = recipeObj.fetchLocalSources() + [recipePath ]
def _getLocalCook(conaryclient, cfg, recipePath, message):
if not hasattr(cook, 'getRecipeInfoFromPath'):
raise errors.RmakeError('Local cooks require at least conary 1.0.19')
recipeDir = os.path.dirname(recipePath)
# We do not want to sign commits to the local repository, doing so
# would require that we manage keys in this repository as well.
oldKey = cfg.signatureKey
oldMap = cfg.signatureKeyMap
oldInteractive = cfg.interactive
oldWorkDir = os.getcwd()
try:
cfg.signatureKey = None
cfg.signatureKeyMap = {}
cfg.interactive = False
if os.access(recipeDir + '/CONARY', os.R_OK):
os.chdir(recipeDir)
stateFile = state.ConaryStateFromFile(recipeDir + '/CONARY')
if stateFile.hasSourceState():
stateFile = stateFile.getSourceState()
if stateFile.getVersion() != versions.NewVersion():
return _shadowAndCommit(conaryclient, cfg, recipeDir,
stateFile, message)
else:
return _commitRecipe(conaryclient, cfg, recipePath, message,
branch=stateFile.getBranch())
return _commitRecipe(conaryclient, cfg, recipePath, message)
finally:
cfg.signatureKey = oldKey
cfg.signatureKeyMap = oldMap
cfg.interactive = oldInteractive
os.chdir(oldWorkDir)
def runCommand(self, client, cfg, argSet, args):
command, subCommand = self.requireParameters(args, 'command')
commandFn = getattr(self, 'list%s' % subCommand.title(), None)
if not commandFn:
self.usage()
raise errors.RmakeError('No such list command %s' % subCommand)
commandFn(client, cfg, argSet)
def openTroveBuildLog(self, trove):
try:
return self.logStore.openTroveLog(trove.logPath)
except KeyError:
raise errors.RmakeError('Log for %s=%s[%s] from %s missing' % \
(trove.getNameVersionFlavor() +
(trove.jobId,)))
def openTroveBuildLog(self, trove):
if trove.logPath:
try:
return open(trove.logPath, 'r')
except (IOError, OSError), err:
raise errors.RmakeError(
'Could not open log for %s=%s[%s] from %s: %s' %
(trove.getNameVersionFlavor() + (trove.jobId, err)))
def deleteJobs(self, callData, jobIdList):
jobIdList = self.db.convertToJobIds(jobIdList)
jobs = self.db.getJobs(jobIdList, withTroves=False)
for job in jobs:
if job.isBuilding():
raise errors.RmakeError('cannot delete active job %s' % job.jobId)
deletedJobIds = self.db.deleteJobs(jobIdList)
return deletedJobIds
def startChrootServer(self, callData, jobId, troveTuple, command,
superUser, chrootHost, chrootPath):
jobId = self.db.convertToJobId(jobId)
trove = self.db.getTrove(jobId, *troveTuple)
if not chrootHost:
if not trove.getChrootPath():
raise errors.RmakeError('Chroot does not exist')
chrootHost = trove.getChrootHost()
chrootPath = trove.getChrootPath()
success, data = self.worker.startSession(chrootHost,
chrootPath,
command,
superUser=superUser,
buildTrove=trove)
if not success:
raise errors.RmakeError('Chroot failed: %s' % data)
return data
def getChrootCache(self):
if not self.chrootCache:
return None
elif self.chrootCache[0] == 'local':
return chrootcache.LocalChrootCache(self.chrootCache[1])
else:
raise errors.RmakeError(
'unknown chroot cache type of "%s" specified' %
self.chrootCache[0])
def _doCommit(recipePath, repos, cfg, message):
try:
kw = {}
if compat.ConaryVersion().supportsForceCommit():
kw.update(force=True)
rv = checkin.commit(repos, cfg, message, **kw)
except (conaryerrors.CvcError, conaryerrors.ConaryError), msg:
raise errors.RmakeError("Could not commit changes to build"
" recipe %s: %s" % (recipePath, msg))
def commitJobs(self, jobIds, message=None, commitOutdatedSources=False,
commitWithFailures=True, waitForJob=False,
sourceOnly=False, updateRecipes=True, excludeSpecs=None,
writeToFile=None):
"""
Commits a set of jobs.
Committing in rMake is slightly different from committing in
conary. rMake uses the conary "clone" command to move the binary
stored in its internal repository out into the repository the
source component came from.
@param jobId: jobId or uuid for a given job.
@type jobId: int or uuid
@param message: Message to use for source commits.
@type message: str
@param commitOutdatedSources: if True, allow commit of sources
even if someone else has changed the source component outside
of rMake before you.
@param commitWithFailures: if True, allow commit of this job
even if parts of the job have failed.
@param waitForJob: if True, wait for the job to finish if necessary
before committing.
@param sourceOnly: if True, only commit the source component.
@param writeToFile: if set to a path, the changeset is written to
that path instead of committed to the repository (Advanced)
@return: False if job failed to commit, True if it succeeded.
@raise: JobNotFound: If job does not exist
"""
if not isinstance(jobIds, (list, tuple)):
jobIds = [jobIds]
jobs = self.client.getJobs(jobIds, withConfigs=True)
finalJobs = []
for job in jobs:
jobId = job.jobId
if job.isCommitting():
raise errors.RmakeError("Job %s is already committing" % job.jobId)
if not job.isFinished() and waitForJob:
print "Waiting for job %s to complete before committing" % jobId
try:
self.waitForJob(jobId)
except Exception, err:
print "Wait interrupted, not committing"
print "You can restart commit by running 'rmake commit %s'" % jobId
raise
job = self.client.getJob(jobId)
if not job.isFinished():
log.error('Job %s is not yet finished' % jobId)
return False
if job.isFailed() and not commitWithFailures:
log.error('Job %s has failures, not committing' % jobId)
return False
if not list(job.iterBuiltTroves()):
log.error('Job %s has no built troves to commit' % jobId)
return True
finalJobs.append(job)
def stopJob(self, callData, jobId):
callData.logger.logRPCDetails('stopJob', jobId=jobId)
jobId = self.db.convertToJobId(jobId)
job = self.db.getJob(jobId, withTroves=True)
if job.isCommitting():
return
elif not job.isQueued() and not job.isRunning():
raise errors.RmakeError('Cannot stop job %s - it is'
' already stopped' % job.jobId)
self.nodeClient.stopJob(job.jobId)
def release(self, callData):
delayed = self._delayed
self._delayed = []
for callData, delayedVal in delayed:
if delayedVal < 0:
callData.respondWithException(
errors.RmakeError('Cannot accept %s' % delayedVal))
else:
callData.respond(delayedVal)
return len(delayed)
def requireFactoryRecipeGeneration(self):
'''
Checks to see if the FactoryRecipe generator exists, added pre conary
2.0.26
'''
if not self.checkVersion(minVer='2.0.26'):
raise errors.RmakeError('rMake requires a conary version 2.0.26 or '
'greater to build factories'
% (version, msg))
return True
def _getResolveTroveTups(cfg, repos):
# get resolve troves - use installLabelPath and install flavor
# for these since they're used for dep resolution
try:
allResolveTroves = itertools.chain(*cfg.resolveTroves)
results = repos.findTroves(cfg.installLabelPath,
list(allResolveTroves), cfg.flavor)
except Exception, err:
context = cfg.context
if not context:
context = 'default'
raise errors.RmakeError("Could not find resolve troves for [%s] context: %s\n" % (context, err))
def getChrootCache(self):
if not self.chrootCache:
return None
cls = chrootcache.CACHE_TYPES.get(self.chrootCache[0])
if not cls:
raise errors.RmakeError("Unknown chroot cache type of '%s' "
"specified. Valid types are: " % (self.chrootCache[0],)
+ " ".join(chrootcache.CACHE_TYPES))
cacheDir = self.chrootCache[1]
if len(self.chrootCache) > 2:
sizeLimit = self.chrootCache[2]
else:
sizeLimit = None
return cls(cacheDir, sizeLimit, self.getChrootHelper())
def watchImage(self, buildId):
curStatus = None
while True:
error, buildStatus = self.client.server.getBuildStatus(buildId)
if error:
raise errors.RmakeError(buildStatus)
if curStatus != buildStatus:
curStatus = buildStatus
print '%s: %s' % (buildId, curStatus['message'])
sys.stdout.flush()
if curStatus['status'] > 200:
break
time.sleep(2)
return curStatus['status'], curStatus['message']
def _setContext(self, buildConfig, conaryConfig, argSet):
context = self._getContext(buildConfig, conaryConfig, argSet)
usedContext = False
if conaryConfig and context:
if conaryConfig.hasSection(context):
usedContext = True
conaryConfig.setContext(context)
buildConfig.useConaryConfig(conaryConfig)
if context and buildConfig.hasSection(context):
buildConfig.setContext(context)
usedContext = True
if not usedContext and context:
raise errors.RmakeError('No such context "%s"' % context)
def startChrootSession(self, jobId, troveSpec, command,
superUser=False, chrootHost=None, chrootPath=None):
job = self.client.getJob(jobId, withTroves=False)
if not troveSpec:
troveTups = list(job.iterTroveList(True))
if len(troveTups) > 1:
raise errors.RmakeError('job has more than one trove in it, must specify trovespec to chroot into')
else:
newTroveSpec = cmdutil.parseTroveSpec(troveSpec)
newTroveSpec = (newTroveSpec[0].split(':')[0] + ':source',) + newTroveSpec[1:]
troveTups = job.findTrovesWithContext(None, [newTroveSpec])[newTroveSpec]
if len(troveTups) > 1:
err = ['%s matches more than one trove:' % troveSpec]
for troveTup in troveTups:
err.append(' %s=%s[%s]{%s}' % troveTup)
raise errors.RmakeError('\n'.join(err))
troveTup = troveTups[0]
chrootConnection = self.client.connectToChroot(jobId, troveTup,
command,
superUser=superUser,
chrootHost=chrootHost,
chrootPath=chrootPath)
chrootConnection.interact()
def startSession(self,
host,
chrootPath,
commandLine,
superUser=False,
buildTrove=None):
if host != '_local_':
raise errors.RmakeError('Unknown host %s!' % host)
try:
chrootFactory = self.chrootManager.useExistingChroot(
chrootPath, useChrootUser=not superUser, buildTrove=buildTrove)
commandId = self.idgen.getSessionCommandId(chrootPath)
cmd = self.runCommand(self.commandClasses['session'], self.cfg,
commandId, chrootFactory, commandLine)
except errors.RmakeError, err:
f = failure.ChrootFailed('%s: %s' % (chrootPath, err))
return False, f
def sanityCheckForStart(self):
if self.proxyUrl is None and self.rbuilderUrl:
self.proxyUrl = self.rbuilderUrl
if self.hostName == 'localhost':
self.hostName = procutil.getNetName()
currUser = pwd.getpwuid(os.getuid()).pw_name
cfgPaths = ['logDir', 'lockDir', 'serverDir']
for uri in self.getServerUris():
if not uri.startswith('unix://'):
continue
socketPath = uri[7:]
if not os.access(os.path.dirname(socketPath), os.W_OK):
log.error('cannot write to socketPath directory at %s - cannot start server' % os.path.dirname(socketPath))
sys.exit(1)
ret = self._sanityCheckForSSL()
if ret:
sys.exit(ret)
cfgPaths = ['buildDir', 'logDir', 'lockDir', 'serverDir']
for path in cfgPaths:
if not os.path.exists(self[path]):
log.error('%s does not exist, expected at %s - cannot start server' % (path, self[path]))
sys.exit(1)
if not os.access(self[path], os.W_OK):
log.error('user "%s" cannot write to %s at %s - cannot start server' % (currUser, path, self[path]))
sys.exit(1)
if self.useResolverCache:
util.mkdirChain(self.getResolverCachePath())
if self.rbuilderUrl:
try:
try:
urllib2.urlopen(self.rbuilderUrl).read(1024)
except urllib2.HTTPError, err:
if 200 <= err.code < 400:
# Something benign like a redirect
pass
else:
raise
except Exception, err:
raise errors.RmakeError('Could not access rbuilder at %s. '
'Please ensure you have a line "rbuilderUrl '
'https://<yourRbuilder>" set correctly in your serverrc '
'file. Error: %s' % (self.rbuilderUrl, err))
def _stopJob(self, job):
if job.isQueued(): # job isn't started yet, just stop it.
# FIXME: when we make this multiprocess,
# there will be a race condition here.
# We'll have to hold the queue lock
# while we make this check.
return
elif job.isCommitting():
return
elif not job.isRunning():
raise errors.RmakeError('Cannot stop job %s - it is'
' already stopped' % job.jobId)
if job.pid not in self._buildPids:
self.warning('job %s is not in active job list', job.jobId)
return
else:
self._killPid(job.pid,
killGroup=True,
hook=self.handleRequestIfReady)
def _promptPassword(self, cfg):
# Try to share descriptor with rbuild so only one prompt is seen
user = cfg.rmakeUser[0]
url = cfg.rmakeUrl.replace(':9999', '')
keyDesc = 'rbuild:user:%s:%s' % (user, url)
passwd = keystore.getPassword(keyDesc)
if passwd and self._setAndCheckPassword(cfg, passwd):
return
for x in range(3):
print "Please enter the password for user %r on %s" % (user,
cfg.rmakeUrl)
passwd = getpass.getpass("Password: "******"The specified credentials were not valid."
print
raise errors.RmakeError("Could not authenticate to remote rMake server")
def sanityCheckForStart(self):
if self.proxyUrl is None:
self.proxyUrl = self.rbuilderUrl
if self.hostName == 'localhost':
self.hostName = procutil.getNetName()
self.oldSanityCheck()
try:
try:
urllib2.urlopen(self.rbuilderUrl).read(1024)
except urllib2.HTTPError, err:
if 200 <= err.code < 400:
# Something benign like a redirect
pass
else:
raise
except Exception, err:
raise errors.RmakeError(
'Could not access rbuilder at %s. '
'Please ensure you have a line "rbuilderUrl '
'https://<yourRbuilder>" set correctly in your serverrc '
'file. Error: %s' % (self.rbuilderUrl, err))
class rMakeBuilderConfiguration(daemon.DaemonConfig):
buildDir = (CfgPath, '/var/rmake')
helperDir = (CfgPath, "/usr/libexec/rmake")
slots = (CfgInt, 1)
useCache = (CfgBool, False)
useTmpfs = (CfgBool, False)
pluginDirs = (CfgPathList, ['/usr/share/rmake/plugins'])
usePlugins = (CfgBool, True)
usePlugin = CfgDict(CfgBool)
chrootLimit = (CfgInt, 4)
chrootCache = CfgChrootCache
chrootCaps = (CfgBool, False,
"Set capability masks as directed by chroot contents. "
"This has the potential to be unsafe.")
chrootServerPorts = (CfgPortRange, (63000, 64000),
"Port range to be used for 'rmake chroot' sessions.")
hostName = (CfgString, 'localhost')
verbose = False
def getAuthUrl(self):
return None
def getCommandSocketDir(self):
return self.buildDir + '/tmp/'
def getName(self):
return '_local_'
def getCacheDir(self):
return self.buildDir + '/cscache'
def getChrootDir(self):
return self.buildDir + '/chroots'
def getChrootArchiveDir(self):
return self.buildDir + '/archive'
def getBuildLogDir(self, jobId=None):
if jobId:
return self.logDir + '/buildlogs/%d/' % jobId
return self.logDir + '/buildlogs/'
def getBuildLogPath(self, jobId):
return self.logDir + '/buildlogs/%d.log' % jobId
def getChrootHelper(self):
return self.helperDir + '/chroothelper'
def getChrootCache(self):
if not self.chrootCache:
return None
elif self.chrootCache[0] == 'local':
return chrootcache.LocalChrootCache(self.chrootCache[1])
else:
raise errors.RmakeError(
'unknown chroot cache type of "%s" specified' %
self.chrootCache[0])
def _getChrootCacheDir(self):
if not self.chrootCache:
return None
elif self.chrootCache[0] == 'local':
return self.chrootCache[1]
return None
def _checkDir(self, name, path, requiredOwner=None, requiredMode=None):
if not os.path.exists(path):
raise errors.RmakeError(
'%s does not exist, expected at %s - cannot start server' %
(name, path))
sys.exit(1)
if not (requiredOwner or requiredMode):
return
statInfo = os.stat(path)
if requiredMode and statInfo.st_mode & 0777 != requiredMode:
raise errors.RmakeError('%s (%s) must have mode %o' %
(path, name, requiredMode))
if requiredOwner:
ownerName = pwd.getpwuid(statInfo.st_uid).pw_name
if ownerName != requiredOwner:
raise errors.RmakeError('%s (%s) must have owner %s' %
(path, name, requiredOwner))
def startRepository(cfg, fork=True, logger=None):
global conaryDir
baseDir = cfg.serverDir
if logger is None:
logger = log
reposDir = '%s/repos' % baseDir
util.mkdirChain(reposDir)
if not cfg.reposUser:
passwordFile = reposDir + '/password'
if os.path.exists(passwordFile):
password = open(passwordFile).readline()[:-1]
else:
password = ''.join(
[chr(random.randrange(ord('a'), ord('z'))) for x in range(10)])
open(passwordFile, 'w').write(password + '\n')
os.chmod(reposDir + '/password', 0700)
cfg.reposUser.addServerGlob(cfg.reposName, 'rmake', password)
serverConfig = os.path.join(cfg.getReposDir(), 'serverrc')
if os.path.exists(serverConfig):
os.unlink(serverConfig)
serverCfg = server.ServerConfig(os.path.join(cfg.getReposDir(),
'serverrc'))
serverCfg.repositoryDB = ('sqlite', cfg.getReposDbPath())
serverCfg.contentsDir = cfg.getContentsPath()
serverCfg.port = cfg.getReposInfo()[1]
serverCfg.configKey('serverName', cfg.reposName) # this works with either
# 1.0.16 or 1.0.17+
serverCfg.logFile = cfg.getReposDir() + '/repos.log'
serverCfg.logFile = None
# Transfer SSL settings from rMake config object
if hasattr(server, 'SSL') and server.SSL:
# The server supports starting in SSL mode
serverCfg.useSSL = cfg.reposRequiresSsl()
serverCfg.sslCert = cfg.sslCertPath
serverCfg.sslKey = cfg.sslCertPath
elif cfg.reposRequiresSsl():
raise errors.RmakeError(
'Tried to start repository at %s, but missing ssl server library: Please install m2crypto'
% (cfg.getRepositoryUrl(), ))
(driver, database) = serverCfg.repositoryDB
db = dbstore.connect(database, driver)
# Note - this will automatically migrate this repository!
# Since this is a throwaway repos anyway, I think that's
# acceptable.
compat.ConaryVersion().loadServerSchema(db)
db.commit()
db.close()
user, password = cfg.reposUser.find(cfg.reposName)
addUser(serverCfg, user, password, write=True)
if not serverCfg.useSSL:
# allow anonymous access if we're not securing this repos
# by using SSL - no use securing access if it's all going to be
# viewable via tcpdump.
addUser(serverCfg, 'anonymous', 'anonymous')
if fork:
pid = os.fork()
if pid:
try:
pingServer(cfg)
except:
killServer(pid)
raise
logger.info('Started repository "%s" on port %s (pid %s)' %
(cfg.reposName, serverCfg.port, pid))
return pid
elif hasattr(logger, 'close'):
logger.close()
try:
os.chdir(cfg.getReposDir())
serverrc = open(cfg.getReposConfigPath(), 'w')
serverCfg.store(serverrc, includeDocs=False)
util.mkdirChain(os.path.dirname(cfg.getReposLogPath()))
logFile = logfile.LogFile(cfg.getReposLogPath())
logFile.redirectOutput(close=True)
serverrc.close()
os.execv('%s/server/server.py' % conaryDir, [
'%s/server/server.py' % conaryDir, '--config-file',
cfg.getReposConfigPath()
])
except Exception, err:
print >> sys.stderr, 'Could not start repository server: %s' % err
os._exit(1)
def deleteChroot(self, callData, host, chrootPath):
if self.db.chrootIsActive(host, chrootPath):
raise errors.RmakeError('Chroot is in use!')
self.worker.deleteChroot(host, chrootPath)
self.db.removeChroot(host, chrootPath)
def archiveChroot(self, callData, host, chrootPath, newPath):
if self.db.chrootIsActive(host, chrootPath):
raise errors.RmakeError('Chroot is in use!')
newPath = self.worker.archiveChroot(host, chrootPath, newPath)
self.db.moveChroot(host, chrootPath, newPath)