def testCook(self):
repos = self.openRepository()
self.makeSourceTrove('test1', test1Recipe)
cookFlavor = deps.parseFlavor('readline,ssl,X')
troveTup = repos.findTrove(self.cfg.buildLabel,
('test1:source', None, None), None)[0]
troveTup = (troveTup[0], troveTup[1], cookFlavor)
targetLabel = versions.Label('localhost@LOCAL:linux')
newTargetLabel = versions.Label('localhost@LOCAL:linux')
# adding an unknown flavor shouldn't matter with latest conary.
self.cfg.buildFlavor = deps.overrideFlavor(self.cfg.buildFlavor,
deps.parseFlavor('foobar'))
logger_ = logger.Logger()
logPath, pid, buildInfo = self.discardOutput(cook.cookTrove,
self.cfg,
repos,
logger_,
targetLabel=targetLabel,
*troveTup)
result = self._wait(buildInfo)
assert (result.isBuildSuccess())
repos.commitChangeSetFile(result.getChangeSetFile())
troveTup = repos.findTrove(
newTargetLabel, ('test1', None, deps.parseFlavor('readline,ssl')),
None)[0]
assert (troveTup[1].branch().label() == newTargetLabel)
assert (str(troveTup[2]) == 'readline,ssl')
self.updatePkg('test1=%s[readline,ssl]' % newTargetLabel,
raiseError=True)
self.verifyFile(self.rootDir + '/foo/bar',
str(self.cfg.buildLabel) + '\n')
def __init__(self,
readConfigFiles=False,
root='',
conaryConfig=None,
serverConfig=None,
ignoreErrors=False,
log=None,
strictMode=None):
# we default the value of these items to whatever they
# are set to on the local system's conaryrc.
if log is None:
log = logger.Logger('buildcfg')
conarycfg.ConaryConfiguration.__init__(self, readConfigFiles=False)
if hasattr(self, 'setIgnoreErrors'):
self.setIgnoreErrors(ignoreErrors)
if hasattr(self, 'addConfigOption'):
# Conary < 2.5
for info in RmakeBuildContext._getConfigOptions():
if info[0] not in self:
self.addConfigOption(*info)
if strictMode is not None:
self.strictMode = strictMode
if not hasattr(self, 'rmakeUrl'):
self.rmakeUrl = None
if not hasattr(self, 'clientCert'):
self.clientCert = None
if readConfigFiles:
if os.path.exists(root + '/etc/rmake/clientrc'):
log.warning(root + '/etc/rmake/clientrc should be renamed'
' to /etc/rmake/rmakerc')
self.read(root + '/etc/rmake/clientrc', exception=False)
self.read(root + '/etc/rmake/rmakerc', exception=False)
if os.environ.has_key("HOME"):
self.read(root + os.environ["HOME"] + "/" + ".rmakerc",
exception=False)
self.read('rmakerc', exception=False)
if self.strictMode:
self.enforceManagedPolicy = True
self.copyInConary = False
self.copyInConfig = False
# these values are not set based on
# config file values - we don't want to touch the system database,
# and we don't want to use conary's logging mechanism.
self.root = ':memory:'
self.dbPath = ':memory:'
self.logFile = []
if hasattr(self, '_lowerCaseMap'):
# Conary < 2.5
for option in self._cfg_hidden:
del self._lowerCaseMap[option.lower()]
self.useConaryConfig(conaryConfig)
if serverConfig:
self.reposName = serverConfig.reposName
self.repositoryMap.update(serverConfig.getRepositoryMap())
self.user.extend(serverConfig.reposUser)
def testChangedRecipeCook(self):
repos = self.openRepository()
self.openRmakeRepository()
trv = self.addComponent(
'test1:source=/localhost@rpl:linux//rmakehost@LOCAL:linux/1.2-0.1',
[('test1.recipe', test1Recipe)])
cookFlavor = deps.parseFlavor('readline,ssl,X')
troveTup = trv.getNameVersionFlavor()
troveTup = (troveTup[0], troveTup[1], cookFlavor)
targetLabel = versions.Label('rmakehost@LOCAL:linux')
newTargetLabel = versions.Label('rmakehost@LOCAL:linux')
# adding an unknown flavor shouldn't matter with latest conary.
self.cfg.buildFlavor = deps.overrideFlavor(self.cfg.buildFlavor,
deps.parseFlavor('foobar'))
logger_ = logger.Logger()
logPath, pid, buildInfo = self.discardOutput(cook.cookTrove,
self.cfg,
repos,
logger_,
targetLabel=targetLabel,
*troveTup)
result = self._wait(buildInfo)
assert (result.isBuildSuccess())
def testCookTwoGroups(self):
self.addComponent('test:run', '1', '!ssl,~test.foo')
self.addComponent('test:run', '1', 'ssl,~test.foo')
trv = self.addComponent('group-foo:source', '1', '',
[('group-foo.recipe', groupRecipeWithFlags)])
flavorList = [deps.parseFlavor('!ssl'), deps.parseFlavor('ssl')]
repos = self.openRepository()
logger_ = logger.Logger()
targetLabel = versions.Label('localhost@rpl:TARGET')
self.cfg.shortenGroupFlavors = True
logPath, pid, buildInfo = self.discardOutput(cook.cookTrove,
self.cfg,
repos,
logger_,
trv.getName(),
trv.getVersion(),
flavorList,
targetLabel=targetLabel)
result = self._wait(buildInfo)
assert (result.isBuildSuccess())
cs = changeset.ChangeSetFromFile(result.getChangeSetFile())
newTroves = [
x.getNewNameVersionFlavor() for x in cs.iterNewTroveList()
]
assert (len(set([x[1] for x in newTroves])))
flavorList = set([x[2] for x in newTroves])
self.assertEquals(sorted(str(x) for x in flavorList),
['readline,!ssl', 'ssl'])
def testDefaultBuildReqs(self):
# add a defautl build req on foo:run and make sure that it shows
# up in the build requirements for the finishehd trove.
repos = self.openRepository()
self.cfg.defaultBuildReqs = ['foo:run']
foo = self.addComponent('foo:run', '1')
self.updatePkg('foo:run')
test1 = self.addComponent('test1:source', '1',
[('test1.recipe', test1Recipe)])
troveTup = test1.getNameVersionFlavor()
targetLabel = versions.Label('localhost@LOCAL:linux')
logger_ = logger.Logger()
logPath, pid, buildInfo = self.discardOutput(
cook.cookTrove,
self.cfg,
repos,
logger_,
targetLabel=self.cfg.buildLabel,
*troveTup)
result = self._wait(buildInfo)
assert (result.isBuildSuccess())
repos = self.openRepository()
repos.commitChangeSetFile(result.getChangeSetFile())
troveTup = repos.findTrove(self.cfg.buildLabel, ('test1', None, None),
None)[0]
test1 = repos.getTrove(*troveTup)
assert (test1.getBuildRequirements() == [foo.getNameVersionFlavor()])
def __init__(self, logger=None):
if logger is None:
logger = logger_.Logger()
self._logger = logger
self._halt = False
self._haltSignal = None
self._pids = {}
self._serving = False
def __init__(self, cfg, db):
self.client = DispatcherNodeClient(cfg.getMessageBusHost(),
cfg.messageBusPort, cfg, self)
server.Server.__init__(self, self.client.getLogger())
subscriberLog = logger.Logger('subscriber', cfg.getSubscriberLogPath())
# detaile data about the nodes is stored in the NodeList.
self._nodes = NodeList(db, self._logger)
# commands that haven't been assigned to a node.
self._queuedCommands = []
self._queuedCommandsById = {}
self.db = db
def __init__(self, cfg, db, map=None, fork=None, logger=None):
self.logStore = db.logStore
if not fork:
fork = lambda name: os.fork()
self.fork = fork
if logger is None:
logger = logger_.Logger()
self._logger = logger
asyncore.dispatcher.__init__(self, map=map)
self.create_socket(socket.AF_INET6, socket.SOCK_STREAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.bind(('', cfg.logServerPort))
# in case the port was set to 0
cfg.logServerPort = self.socket.getsockname()[1]
self.listen(5)
def __init__(self, serverCfg, logger=None):
self.serverCfg = serverCfg
self.baseDir = os.path.realpath(serverCfg.getChrootDir())
self.archiveDir = os.path.realpath(serverCfg.getChrootArchiveDir())
self.chrootHelperPath = serverCfg.getChrootHelper()
cacheDir = serverCfg.getCacheDir()
util.mkdirChain(cacheDir)
if self.serverCfg.useCache:
self.csCache = repocache.RepositoryCache(cacheDir)
else:
self.csCache = None
self.chrootCache = serverCfg.getChrootCache()
if logger is None:
logger = logger_.Logger()
self.logger = logger
self.queue = ChrootQueue(self.baseDir, self.serverCfg.chrootLimit)
def __init__(self, cfg, messageBusInfo=None):
serverLogger = logger.Logger('rmake-node',
logPath=cfg.logDir + '/rmake-node.log')
try:
serverLogger.info('Starting rMake Node (pid %s)' % os.getpid())
worker.Worker.__init__(self, cfg, serverLogger, slots=cfg.slots)
#calculates current state of the rmake chroot directory.
chroots = self.listChroots()
self.client = WorkerNodeClient(cfg,
self,
procutil.MachineInformation(),
chroots=chroots,
messageBusInfo=messageBusInfo)
self.lastStatusSent = 0
self.statusPeriod = 60
except Exception, err:
self.error('Error initializing Node Server:\n %s\n%s', err,
traceback.format_exc())
raise
def __init__(self, cfg, db):
self.client = DispatcherNodeClient(cfg.getMessageBusHost(),
cfg.messageBusPort, cfg, self)
server.Server.__init__(self, self.client.getLogger())
subscriberLog = logger.Logger('subscriber', cfg.getSubscriberLogPath())
# In multinode rMake, publishing events to external subscribers
# is done by a process forked from the dispatcher (since the
# dispatcher gets all events anyway) instead of passing them
# back to the rMake XMLRPC front end.
self._publisher = publish._RmakeServerPublisher(
subscriberLog, db, self._fork)
# detaile data about the nodes is stored in the NodeList.
self._nodes = NodeList(db, self._logger)
# commands that haven't been assigned to a node.
self._queuedCommands = []
self._queuedCommandsById = {}
self.db = db
def _testCookGroupDoesntPickBuilt(self, builtTroves):
repos = self.openRepository()
repos = self.openRepository(1)
logger_ = logger.Logger()
troveTup = repos.findTrove(self.cfg.buildLabel,
('group-foo:source', None, None), None)[0]
targetLabel = versions.Label('localhost1@rpl:TARGET')
logPath, pid, buildInfo = self.discardOutput(cook.cookTrove,
self.cfg,
repos,
logger_,
troveTup[0],
troveTup[1],
troveTup[2],
targetLabel=targetLabel,
builtTroves=builtTroves)
result = self._wait(buildInfo)
if not result.csFile:
print result.failureReason
raise RuntimeError
repos.commitChangeSetFile(result.csFile)
group = self.findAndGetTrove('group-foo=localhost1@rpl:TARGET')
childTroves = [
x for x in group.iterTroveList(strongRefs=True)
if x[0] != 'test:run'
]
self.assertEquals(len(childTroves), 3)
def _getHost(name):
return str([
x[1].trailingLabel().getHost() for x in childTroves
if x[0] == name
][0])
assert (_getHost('foo:run') == 'localhost')
assert (_getHost('bar:run') == 'localhost')
assert (_getHost('bam:run') == 'localhost1')
def testFailedTrove(self):
repos = self.openRepository()
self.makeSourceTrove('failed', failedRecipe)
troveTup = repos.findTrove(self.cfg.buildLabel,
('failed:source', None, None), None)[0]
self.logFilter.add()
logger_ = logger.Logger()
logPath, pid, buildInfo = self.discardOutput(cook.cookTrove,
self.cfg,
repos,
logger_,
targetLabel=None,
*troveTup)
result = self._wait(buildInfo)
assert (not result.isBuildSuccess())
assert (not result.getChangeSetFile())
assert (not result.getExitSignal())
assert (result.getExitStatus() == 1)
# test again, with signal
self.updateSourceTrove('failed', failedSignalRecipe)
troveTup = repos.findTrove(self.cfg.buildLabel,
('failed:source', None, None), None)[0]
logPath, pid, buildInfo = self.discardOutput(cook.cookTrove,
self.cfg,
repos,
logger_,
targetLabel=None,
*troveTup)
result = self._wait(buildInfo)
assert (result.getExitSignal() == 11)
assert (not result.isBuildSuccess())
assert (not result.getChangeSetFile())
def __init__(self,
uri,
cfg,
repositoryPid=None,
proxyPid=None,
pluginMgr=None,
quiet=False):
util.mkdirChain(cfg.logDir)
logPath = cfg.logDir + '/rmake.log'
rpcPath = cfg.logDir + '/xmlrpc.log'
serverLogger = ServerLogger()
serverLogger.disableRPCConsole()
serverLogger.logToFile(logPath)
serverLogger.logRPCToFile(rpcPath)
if quiet:
serverLogger.setQuietMode()
else:
if cfg.verbose:
logLevel = logging.DEBUG
else:
logLevel = logging.INFO
serverLogger.enableConsole(logLevel)
serverLogger.info(
'*** Started rMake Server at pid %s (serving at %s)' %
(os.getpid(), uri))
try:
self._initialized = False
self.db = None
# forked jobs that are currently active
self._buildPids = {}
self.uri = uri
self.cfg = cfg
self.repositoryPid = repositoryPid
self.proxyPid = proxyPid
apirpc.XMLApiServer.__init__(
self,
uri,
logger=serverLogger,
forkByDefault=True,
sslCertificate=cfg.getSslCertificatePath(),
caCertificate=cfg.getCACertificatePath())
self._setUpInternalUser()
self.db = database.Database(cfg.getDbPath(),
cfg.getDbContentsPath())
self.auth = auth.AuthenticationManager(cfg.getAuthUrl(), self.db)
if pluginMgr is None:
pluginMgr = plugins.PluginManager([])
self.plugins = pluginMgr
# any jobs that were running before are not running now
subscriberLog = logger.Logger('susbscriber',
self.cfg.getSubscriberLogPath())
self._publisher = publish._RmakeServerPublisher(
subscriberLog, self.db, self._fork)
self.worker = worker.Worker(self.cfg, self._logger)
dbLogger = subscriber._JobDbLogger(self.db)
# note - it's important that the db logger
# comes first, before the general publisher,
# so that whatever published is actually
# recorded in the DB.
self._subscribers = [dbLogger]
if self.uri:
s = subscriber._RmakeServerPublisherProxy(self.uri)
else:
# testsuite path - external subscribers also go through
# internal interface when the server is not run as a separate
# process.
s = subscriber._RmakeServerPublisherProxy(ShimAddress(self))
self._subscribers.append(s)
self._internalSubscribers = [dbLogger]
s = subscriber._RmakeServerPublisherProxy(ShimAddress(self))
self._internalSubscribers.append(s)
self.plugins.callServerHook('server_postInit', self)
except errors.uncatchableExceptions:
raise
except Exception, err:
self.error('Error initializing rMake Server:\n %s\n%s', err,
traceback.format_exc())
self._try('halt', self._shutDown)
raise
def testWorker(self):
# RMK-367 - test that we handle alrdeady stopped commands
repos = self.openRepository()
w = worker.Worker(self.rmakeCfg, logger.Logger())
rc, txt = self.captureOutput(w.stopCommand, 'foo')
assert(txt.endswith('warning: Asked to stop unknown command foo\n'))