def testResolutionLogging(self): myDep= ('soname: ELF32/a.so(x86)' ' soname: ELF32/b.so(x86)' ' soname: ELF32/c.so(x86)' ' soname: ELF32/d.so(x86)' ' soname: ELF32/e.so(x86)' ' soname: ELF32/f.so(x86)') log.setVerbosity(log.DEBUG) self.addComponent('prov:runtime', '1.0', provides=myDep, filePrimer=1) self.addComponent('req:runtime', '1.0', requires=myDep, filePrimer=2) self.logFilter.add() self.checkUpdate(['req:runtime'], ['req:runtime', 'prov:runtime'], resolve=True) self.logFilter.remove() self.logFilter.compare('''\ + Resolved: req:runtime=localhost@rpl:linux/1.0-1-1[] Required: soname: ELF32/a.so(x86) soname: ELF32/b.so(x86) soname: ELF32/c.so(x86) soname: ELF32/d.so(x86) soname: ELF32/e.so(x86) ... Adding: prov:runtime=localhost@rpl:linux/1.0-1-1[]''')
def testResolutionUpdateExistingLogging(self): self.addComponent('foo:runtime', '1.0', provides='trove:foo(1.0)', filePrimer=7) self.addComponent('foo-build:runtime', '1.0', requires='trove:foo(1.0)', filePrimer=8) self.addComponent('foo:runtime', '2.0', provides='trove:foo(2.0)', filePrimer=9) self.addComponent('foo-build:runtime', '2.0', requires='trove:foo(2.0)', filePrimer=10) self.addCollection('foo-build', '2.0', [':runtime']) self.updatePkg(['foo:runtime=1.0', 'foo-build:runtime=1.0']) log.setVerbosity(log.DEBUG) self.logFilter.add() self.checkUpdate('foo:runtime', ['foo:runtime=1.0--2.0', 'foo-build=2.0', 'foo-build:runtime=1.0--2.0']) self.logFilter.compare([ '''+ Update breaks dependencies!''', '''+ Broken dependency: (dep needed by the system but being removed): foo-build:runtime=1.0-1-1 Requires: trove: foo(1.0) Provided by removed or updated packages: foo:runtime''', '''+ Attempting to update the following packages in order to remove their dependency on something being erased: foo-build''', '''+ updated 2 troves: foo-build:runtime=localhost@rpl:linux/2.0-1-1[] foo-build=localhost@rpl:linux/2.0-1-1[]'''])
def testResolutionKeepExistingLogging2(self): self.addComponent('keepprov:lib', '1.0', provides='trove:keepreq(1.0)', filePrimer=3) self.addCollection('keepprov', '1.0', [':lib']) self.addComponent('keepprov:data', '2.0', filePrimer=4) # they conflict! self.addCollection('keepprov', '2.0', [':data']) self.addComponent('keepreq:runtime', '1.0', requires='trove:keepreq(1.0)', filePrimer=5) self.updatePkg(['keepprov=1.0', 'keepreq:runtime']) log.setVerbosity(log.DEBUG) self.logFilter.add() self.checkUpdate('keepprov', ['keepprov=--2.0', 'keepprov:data=--2.0'], keepRequired=True) self.logFilter.compare([ '''+ Update breaks dependencies!''', '''+ Broken dependency: (dep needed by the system but being removed): keepreq:runtime=1.0-1-1 Requires: trove: keepreq(1.0) Provided by removed or updated packages: keepprov:lib''', '''+ Resolved (undoing erasure) keepreq:runtime=1.0-1-1[] Required: trove: keepreq(1.0) Keeping: keepprov:lib=1.0-1-1[]''', '''warning: keeping keepprov:lib - required by at least keepreq:runtime''' ])
def runCommand(self, client, cfg, argSet, args): if self.verbose: log.setVerbosity(log.DEBUG) else: log.setVerbosity(log.INFO) command, jobId, troveSpecs = self.requireParameters(args, 'jobId', allowExtra=True) jobId = _getJobIdOrUUId(jobId) noUpdate = argSet.pop('no-update', False) clearBuildList = argSet.pop('clear-build-list', False) clearPrebuiltList = argSet.pop('clear-prebuilt-list', False) updateConfigKeys = argSet.pop('update-config', None) if noUpdate: updateSpecs = ['-*'] else: updateSpecs = [] updateSpecs.extend(argSet.pop('update', [])) excludeSpecs = argSet.pop('exclude', []) self._prep(client, argSet) job = client.createRestartJob(jobId, troveSpecs, updateSpecs=updateSpecs, excludeSpecs=excludeSpecs, updateConfigKeys=updateConfigKeys, clearBuildList=clearBuildList, clearPrebuiltList=clearPrebuiltList) return self._build(client, job, argSet)
def processConfigOptions(self, rbuildConfig, cfgMap, argSet): """ Add any configuration files mentioned on the command line to the contents of the config object, overriding any configuration data already found. Then add any configuration items specified explicitly on the command line, overriding all configuration files, including any specified with C{--config-file}. @param rbuildConfig: rbuild configuration @param cfgMap: legacy option - ignore @param argSet: dictionary of option : value mapping based on options passed in at the command line. """ configFileList = argSet.pop('config-file', []) for path in configFileList: rbuildConfig.read(path, exception=True) command.AbstractCommand.processConfigOptions(self, rbuildConfig, cfgMap, argSet) rbuildConfig.quiet = argSet.get('quiet', False) if argSet.get('verbose', False): log.setVerbosity(log.DEBUG) self.processLocalConfigOptions(rbuildConfig, argSet)
def testResolutionLogging(self): myDep = ('soname: ELF32/a.so(x86)' ' soname: ELF32/b.so(x86)' ' soname: ELF32/c.so(x86)' ' soname: ELF32/d.so(x86)' ' soname: ELF32/e.so(x86)' ' soname: ELF32/f.so(x86)') log.setVerbosity(log.DEBUG) self.addComponent('prov:runtime', '1.0', provides=myDep, filePrimer=1) self.addComponent('req:runtime', '1.0', requires=myDep, filePrimer=2) self.logFilter.add() self.checkUpdate(['req:runtime'], ['req:runtime', 'prov:runtime'], resolve=True) self.logFilter.remove() self.logFilter.compare('''\ + Resolved: req:runtime=localhost@rpl:linux/1.0-1-1[] Required: soname: ELF32/a.so(x86) soname: ELF32/b.so(x86) soname: ELF32/c.so(x86) soname: ELF32/d.so(x86) soname: ELF32/e.so(x86) ... Adding: prov:runtime=localhost@rpl:linux/1.0-1-1[]''')
def testSourceGoesAway(self): self.resetRepository() cfg = self.cfg self.repos = self.openRepository() self.resetWork() self.resetCache() origDir = os.getcwd() os.chdir(self.workDir) self.newpkg('test') os.chdir('test') testdir = '/'.join((self.workDir, 'test')) log.setVerbosity(log.INFO) self.logFilter.add() recipeStr = """ class SourceGoesAwayTest(PackageRecipe): name = 'test' version = '0' clearBuildReqs() def setup(r): r.addSource('%s1', dir='/foo/') """ try: contentServer = rephelp.HTTPServerController(getRequester()) contentURL = contentServer.url() curl = contentURL + "file1.txt" self.writeFile('test.recipe', recipeStr % (curl, )) self.addfile('test.recipe') # Test all sorts of cooks, to make sure cooking from the recipe # does not re-download self.cookItem(self.repos, cfg, 'test.recipe') self.commit() self.cookItem(self.repos, cfg, 'test.recipe') self.cookItem(self.repos, cfg, 'test') finally: contentServer.kill() os.chdir(origDir) # Get rid of the cache self.resetWork() self.resetCache() # Server for source file is shut down now try: # Get rid of this directory os.chdir(self.workDir) self.checkout('test') os.chdir('test') # Cook the recipe, should work self.cookItem(self.repos, cfg, 'test.recipe') finally: os.chdir(origDir)
def testSourceGoesAway(self): self.resetRepository() cfg = self.cfg self.repos = self.openRepository() self.resetWork() self.resetCache() origDir = os.getcwd() os.chdir(self.workDir) self.newpkg('test') os.chdir('test') testdir = '/'.join((self.workDir, 'test')) log.setVerbosity(log.INFO) self.logFilter.add() recipeStr = """ class SourceGoesAwayTest(PackageRecipe): name = 'test' version = '0' clearBuildReqs() def setup(r): r.addSource('%s1', dir='/foo/') """ try: contentServer = rephelp.HTTPServerController(getRequester()) contentURL = contentServer.url() curl = contentURL + "file1.txt" self.writeFile('test.recipe', recipeStr %(curl, )) self.addfile('test.recipe') # Test all sorts of cooks, to make sure cooking from the recipe # does not re-download self.cookItem(self.repos, cfg, 'test.recipe') self.commit() self.cookItem(self.repos, cfg, 'test.recipe') self.cookItem(self.repos, cfg, 'test') finally: contentServer.kill() os.chdir(origDir) # Get rid of the cache self.resetWork() self.resetCache() # Server for source file is shut down now try: # Get rid of this directory os.chdir(self.workDir) self.checkout('test') os.chdir('test') # Cook the recipe, should work self.cookItem(self.repos, cfg, 'test.recipe') finally: os.chdir(origDir)
def startLogging(): import logging # set a format which is simpler for console use formatter = logging.Formatter('%(asctime)s %(message)s', datefmt = "%m-%d %H:%M") # tell the handler to use this format log.logger.handlers[0].setFormatter(formatter) log.setVerbosity(log.DEBUG) log.info("Logging system started")
def runCommand(self, client, cfg, argSet, args): if self.verbose: log.setVerbosity(log.DEBUG) else: log.setVerbosity(log.INFO) command, loadPath = self.requireParameters(args, 'path') self._prep(client, argSet) job = client.loadJobFromFile(loadPath) return self._build(client, job, argSet)
def _runMirrorCfg(self, src, dst, cfg, verbose=False): if verbose: from conary.lib import log log.setVerbosity(log.DEBUG) while 1: ret = mirror.mirrorRepository(src, dst, cfg) if not ret: break return
def startLogging(): import logging # set a format which is simpler for console use formatter = logging.Formatter('%(asctime)s %(message)s', datefmt="%m-%d %H:%M") # tell the handler to use this format log.logger.handlers[0].setFormatter(formatter) log.setVerbosity(log.DEBUG) log.info("Logging system started")
def testLocalFactoryWithLocalRecipe(self): os.chdir(self.workDir) self.newpkg("foo", factory = "test") os.chdir("foo") self.writeFile("factory-test.recipe", """ class TestFactory(Factory): name = "factory-test" version = "1.0" def getRecipeClass(self): class TestSubclass(PackageRecipe): name = "testsubclass" version = "1.0" internalAbstractBaseClass = True clearBuildReqs() return TestSubclass """) self.writeFile("foo.recipe", """ # CNY-2813. importing log inside a recipe used to reset the loglevel from conary.lib import log class FooRecipe(FactoryRecipeClass): name = "foo" version = "1.1" def setup(self): self.Create("/etc/foo", "foo") """) self.addfile("foo.recipe") repos = self.openRepository() cstate = state.ConaryStateFromFile('CONARY') level = log.getVerbosity() try: log.setVerbosity(log.INFO) klass = logging.getLoggerClass() self.discardOutput(cook.cookCommand, self.cfg, [cstate], False, {}) self.assertEquals(klass, logging.getLoggerClass()) finally: log.setVerbosity(level) ccs = changeset.ChangeSetFromFile(os.path.join(self.workDir, 'foo', 'foo-1.1.ccs')) trvs = [trove.Trove(x) for x in ccs.iterNewTroveList()] trv = [x for x in trvs if x.getName() == 'foo:debuginfo'][0] files = [x for x in trv.iterFileList() if \ x[1] == '/usr/src/debug/buildlogs/foo-1.1-log.bz2'] fileId, path, pathId, ver = files[0] fileInfo, fileObj = ccs.getFileContents(fileId, pathId) decomp = bz2.BZ2Decompressor() data = decomp.decompress(fileObj.f.read()) self.assertFalse("+ Processing" not in data, "build log data appears to be incomplete")
def testLocalFactoryWithLocalRecipe(self): os.chdir(self.workDir) self.newpkg("foo", factory="test") os.chdir("foo") self.writeFile( "factory-test.recipe", """ class TestFactory(Factory): name = "factory-test" version = "1.0" def getRecipeClass(self): class TestSubclass(PackageRecipe): name = "testsubclass" version = "1.0" internalAbstractBaseClass = True clearBuildReqs() return TestSubclass """) self.writeFile( "foo.recipe", """ # CNY-2813. importing log inside a recipe used to reset the loglevel from conary.lib import log class FooRecipe(FactoryRecipeClass): name = "foo" version = "1.1" def setup(self): self.Create("/etc/foo", "foo") """) self.addfile("foo.recipe") repos = self.openRepository() cstate = state.ConaryStateFromFile('CONARY') level = log.getVerbosity() try: log.setVerbosity(log.INFO) klass = logging.getLoggerClass() self.discardOutput(cook.cookCommand, self.cfg, [cstate], False, {}) self.assertEquals(klass, logging.getLoggerClass()) finally: log.setVerbosity(level) ccs = changeset.ChangeSetFromFile( os.path.join(self.workDir, 'foo', 'foo-1.1.ccs')) trvs = [trove.Trove(x) for x in ccs.iterNewTroveList()] trv = [x for x in trvs if x.getName() == 'foo:debuginfo'][0] files = [x for x in trv.iterFileList() if \ x[1] == '/usr/src/debug/buildlogs/foo-1.1-log.bz2'] fileId, path, pathId, ver = files[0] fileInfo, fileObj = ccs.getFileContents(fileId, pathId) decomp = bz2.BZ2Decompressor() data = decomp.decompress(fileObj.f.read()) self.assertFalse("+ Processing" not in data, "build log data appears to be incomplete")
def testLogMinVerbosity(self): log.resetErrorOccurred() log.setVerbosity(log.ERROR) self.logCheck(log.warning, ("a warning", ), []) assert (not log.setMinVerbosity(log.ERROR)) self.logCheck(log.warning, ("a warning", ), []) assert (log.setMinVerbosity(log.WARNING) == log.ERROR) self.logCheck(log.warning, ("a warning", ), ["warning: a warning"]) assert (not log.setMinVerbosity(log.ERROR)) self.logCheck(log.warning, ("a warning", ), ["warning: a warning"])
def testLogMinVerbosity(self): log.resetErrorOccurred() log.setVerbosity(log.ERROR) self.logCheck(log.warning, ("a warning",), []) assert(not log.setMinVerbosity(log.ERROR)) self.logCheck(log.warning, ("a warning",), []) assert(log.setMinVerbosity(log.WARNING) == log.ERROR) self.logCheck(log.warning, ("a warning",), ["warning: a warning"]) assert(not log.setMinVerbosity(log.ERROR)) self.logCheck(log.warning, ("a warning",), ["warning: a warning"])
def testLogWithObject(self): log.resetErrorOccurred() log.setVerbosity(2) foo = object() fooString = str(foo) self.logCheck(log.error, (foo,), "error: %s" % fooString) assert(log.errorOccurred()) self.logCheck(log.warning, (foo,), "warning: %s" % fooString) self.logCheck(log.debug, (foo,), "+ %s" % fooString) self.logCheck(log.info, (foo,), "+ %s" % fooString)
def testLogWithObject(self): log.resetErrorOccurred() log.setVerbosity(2) foo = object() fooString = str(foo) self.logCheck(log.error, (foo, ), "error: %s" % fooString) assert (log.errorOccurred()) self.logCheck(log.warning, (foo, ), "warning: %s" % fooString) self.logCheck(log.debug, (foo, ), "+ %s" % fooString) self.logCheck(log.info, (foo, ), "+ %s" % fooString)
def checkSyncRepos(config, sourceRepos, targetRepos): checkConfig(config) targets = _makeTargets(config, targetRepos) log.setVerbosity(log.DEBUG) # retrieve the set of troves from a give repository def _getTroveSet(config, repo): def _flatten(troveSpec): l = [] for name, versionD in troveSpec.iteritems(): for version, flavorList in versionD.iteritems(): l += [(name, version, flavor) for flavor in flavorList] return set(l) troveSpecs = {} if config.labels: d = troveSpecs.setdefault(None, {}) for l in config.labels: d[l] = '' t = repo.getTroveVersionsByLabel( troveSpecs, troveTypes=netclient.TROVE_QUERY_ALL) else: troveSpecs = {None: None} t = repo.getTroveVersionList(config.host, troveSpecs, troveTypes=netclient.TROVE_QUERY_ALL) return _flatten(t) # compare source with each target def _compare(src, dst): srcName, srcSet = src dstName, dstSet = dst counter = 0 for x in srcSet.difference(dstSet): log.debug(" - %s %s " % (srcName, x)) counter += 1 for x in dstSet.difference(srcSet): log.debug(" + %s %s" % (dstName, x)) counter += 1 return counter log.debug("Retrieving list of troves from source %s" % str(sourceRepos.c.map)) sourceSet = _getTroveSet(config, sourceRepos) hasDiff = 0 for target in targets: log.debug("Retrieving list of troves from %s %s" % (target.name, str(target.repo.c.map))) targetSet = _getTroveSet(config, target.repo) log.debug("Diffing source and %s" % target.name) hasDiff += _compare(("source", sourceSet), (target.name, targetSet)) log.debug("Done") return hasDiff
def cookTrove(cfg, repos, logger, name, version, flavorList, targetLabel, loadSpecsList=None, logData=None, buildReqs=None, crossReqs=None): if not isinstance(flavorList, (tuple, list)): flavorList = [flavorList] util.mkdirChain(cfg.root + '/tmp') fd, csFile = tempfile.mkstemp(dir=cfg.root + '/tmp', prefix='rmake-%s-' % name, suffix='.ccs') os.chmod(csFile, 0644) os.close(fd) logPath = cfg.root + '/tmp/rmake/%s-%s.log' % (name, version.trailingRevision()) logFile = logfile.LogFile(logPath) os.chmod(logPath, 0664) os.chmod(cfg.root + '/tmp/rmake', 0775) results = CookResults(name, version, flavorList) # ignore child output problems signal.signal(signal.SIGTTOU, signal.SIG_IGN) inF, outF = pipereader.makePipes() pid = os.fork() if not pid: try: try: signal.signal(signal.SIGTERM, signal.SIG_DFL) os.close(inF) os.setpgid(0, 0) # don't accidentally make world writable files os.umask(0022) # don't allow us to create core dumps resource.setrlimit(resource.RLIMIT_CORE, (0,0)) log.setVerbosity(log.DEBUG) log.info("Cook process started (pid %s)" % os.getpid()) _cookTrove(cfg, repos, name, version, flavorList, targetLabel, loadSpecsList, csFile, buildReqs=buildReqs, crossReqs=crossReqs, failureFd=outF, logger=logger) except Exception, msg: if len(flavorList) > 1: errMsg = 'Error cooking %s=%s with flavors %s: %s' % \ (name, version, ', '.join([str(x) for x in flavorList]), str(msg)) else: errMsg = str(msg) _buildFailed(outF, errMsg, traceback.format_exc()) logFile.close() os._exit(1) else: logFile.close() os._exit(0)
def runCommand(self, client, cfg, argSet, args): command, jobId = self.requireParameters(args, 'jobId') log.setVerbosity(log.INFO) quiet = argSet.pop('quiet', False) commit = argSet.pop('commit', False) jobId = _getJobIdOrUUId(jobId) success = client.watch(jobId, showBuildLogs = not quiet, showTroveLogs = not quiet, commit = commit) if success: return 0 else: return 1
def tearDown(self): from conary.lib import log log.setVerbosity(self._logLevel) # reattached the original stdio fds after the test has completed try: sys.stdout.flush() sys.stderr.flush() except: pass sys.stdin = self.savedStdin sys.stdout = self.savedStdout sys.stderr = self.savedStderr
def testUpdateRecipes(self): self.logFilter.add() log.setVerbosity(log.INFO) simpleRecipe = recipes.simpleRecipe os.chdir(self.workDir) self.newpkg('simple') os.chdir('simple') self.writeFile('simple.recipe', simpleRecipe) self.addfile('simple.recipe') self.writeFile('foo', 'bar\n') self.buildCfg.configLine('[foo]') self.buildCfg.configLine('[bar]') conaryStateFile = state.ConaryStateFromFile(os.getcwd() + '/CONARY') conaryStateFile.setContext('foo') conaryStateFile.write(os.getcwd() + '/CONARY') # here we pretend we've done an rmake commit of the same job. repos = self.openRepository() trv = self.addComponent('simple:source', '1.0', [('simple.recipe', simpleRecipe)]) commit.updateRecipes(repos, self.buildCfg, [os.getcwd() + '/simple.recipe'], [trv.getNameVersionFlavor()]) assert (os.path.exists('foo')) conaryStateFile = state.ConaryStateFromFile(os.getcwd() + '/CONARY') stateFile = conaryStateFile.getSourceState() assert (stateFile.getVersion() == trv.getVersion()) assert (conaryStateFile.getContext() == 'foo') conaryStateFile.setContext('bar') conaryStateFile.write(os.getcwd() + '/CONARY') assert (checkin.diff(repos) == 0) trv = self.addComponent( 'simple:source', '2.0', [('simple.recipe', simpleRecipe + '\n\t#change\n')]) assert (checkin.diff(repos) == 0) commit.updateRecipes(repos, self.buildCfg, [os.getcwd() + '/simple.recipe'], [trv.getNameVersionFlavor()]) assert (checkin.diff(repos) == 0) assert (os.path.exists('foo')) conaryStateFile = state.ConaryStateFromFile(os.getcwd() + '/CONARY') stateFile = conaryStateFile.getSourceState() assert (stateFile.getVersion() == trv.getVersion()) assert (conaryStateFile.getContext() == 'bar') self.logFilter.compare([ '+ Replacing CONARY file %s/simple after initial commit' % self.workDir, '+ Updating %s/simple after commit' % self.workDir, '+ patching %s/simple/simple.recipe' % self.workDir, '+ patch: applying hunk 1 of 1' ])
def testLog(self): log.resetErrorOccurred() self.logCheck(log.warning, ("a warning",), "warning: a warning") self.logCheck(log.debug, ("a debug",), []) assert(not log.errorOccurred()) log.setVerbosity(2) self.logCheck(log.warning, ("a warning",), "warning: a warning") self.logCheck(log.debug, ("a debug",), "+ a debug") log.setVerbosity(0) self.logCheck(log.warning, ("a warning",), "warning: a warning") self.logCheck(log.debug, ("a debug",), []) assert(not log.errorOccurred()) self.logCheck(log.error, ("an error",), "error: an error") assert(log.errorOccurred())
def runCommand(self, client, cfg, argSet, args): if self.verbose: log.setVerbosity(log.DEBUG) else: log.setVerbosity(log.INFO) command, troveSpecs = self.requireParameters(args, 'troveSpec', appendExtra=True) if command == 'buildgroup': log.warning( '"buildgroup" is deprecated and will be removed in a future release - use "build --recurse" instead' ) rebuild = (command == 'rebuild') flavorSpec = argSet.pop('flavor', None) if flavorSpec: flavor = deps.parseFlavor(flavorSpec) if flavor is None: raise errors.ParseError("Invalid flavor: '%s'" % flavorSpec) newFlavor = deps.overrideFlavor(client.buildConfig.buildFlavor, flavor) client.buildConfig.buildFlavor = newFlavor newFlavors = [] for oldFlavor in client.buildConfig.flavor: newFlavors.append(deps.overrideFlavor(oldFlavor, flavor)) client.buildConfig.flavor = newFlavors matchSpecs = argSet.pop('match', []) hosts = argSet.pop('host', []) labels = argSet.pop('label', []) recurseGroups = argSet.pop('recurse', False) or command == 'buildgroup' if recurseGroups: if argSet.pop('binary-search', False): recurseGroups = client.BUILD_RECURSE_GROUPS_BINARY elif not compat.ConaryVersion().supportsFindGroupSources(): log.warning('Your conary does not support recursing a group' ' source component, defaulting to searching the' ' binary version') recurseGroups = client.BUILD_RECURSE_GROUPS_BINARY else: recurseGroups = client.BUILD_RECURSE_GROUPS_SOURCE self._prep(client, argSet) job = client.createBuildJob(troveSpecs, limitToHosts=hosts, limitToLabels=labels, recurseGroups=recurseGroups, matchSpecs=matchSpecs, rebuild=rebuild) return self._build(client, job, argSet)
def runCommand(self, client, cfg, argSet, args): command, jobId = self.requireParameters(args, 'jobId') log.setVerbosity(log.INFO) quiet = argSet.pop('quiet', False) commit = argSet.pop('commit', False) jobId = _getJobIdOrUUId(jobId) success = client.watch(jobId, showBuildLogs=not quiet, showTroveLogs=not quiet, commit=commit) if success: return 0 else: return 1
def testLog(self): log.resetErrorOccurred() self.logCheck(log.warning, ("a warning", ), "warning: a warning") self.logCheck(log.debug, ("a debug", ), []) assert (not log.errorOccurred()) log.setVerbosity(2) self.logCheck(log.warning, ("a warning", ), "warning: a warning") self.logCheck(log.debug, ("a debug", ), "+ a debug") log.setVerbosity(0) self.logCheck(log.warning, ("a warning", ), "warning: a warning") self.logCheck(log.debug, ("a debug", ), []) assert (not log.errorOccurred()) self.logCheck(log.error, ("an error", ), "error: an error") assert (log.errorOccurred())
def testUpdateRecipes(self): self.logFilter.add() log.setVerbosity(log.INFO) simpleRecipe = recipes.simpleRecipe os.chdir(self.workDir) self.newpkg('simple') os.chdir('simple') self.writeFile('simple.recipe', simpleRecipe) self.addfile('simple.recipe') self.writeFile('foo', 'bar\n') self.buildCfg.configLine('[foo]') self.buildCfg.configLine('[bar]') conaryStateFile = state.ConaryStateFromFile(os.getcwd() + '/CONARY') conaryStateFile.setContext('foo') conaryStateFile.write(os.getcwd() + '/CONARY') # here we pretend we've done an rmake commit of the same job. repos = self.openRepository() trv = self.addComponent('simple:source', '1.0', [('simple.recipe', simpleRecipe)]) commit.updateRecipes(repos, self.buildCfg, [os.getcwd() + '/simple.recipe'], [trv.getNameVersionFlavor()]) assert(os.path.exists('foo')) conaryStateFile = state.ConaryStateFromFile(os.getcwd() + '/CONARY') stateFile = conaryStateFile.getSourceState() assert(stateFile.getVersion() == trv.getVersion()) assert(conaryStateFile.getContext() == 'foo') conaryStateFile.setContext('bar') conaryStateFile.write(os.getcwd() + '/CONARY') assert(checkin.diff(repos) == 0) trv = self.addComponent('simple:source', '2.0', [('simple.recipe', simpleRecipe + '\n\t#change\n')]) assert(checkin.diff(repos) == 0) commit.updateRecipes(repos, self.buildCfg, [os.getcwd() + '/simple.recipe'], [trv.getNameVersionFlavor()]) assert(checkin.diff(repos) == 0) assert(os.path.exists('foo')) conaryStateFile = state.ConaryStateFromFile(os.getcwd() + '/CONARY') stateFile = conaryStateFile.getSourceState() assert(stateFile.getVersion() == trv.getVersion()) assert(conaryStateFile.getContext() == 'bar') self.logFilter.compare([ '+ Replacing CONARY file %s/simple after initial commit' % self.workDir, '+ Updating %s/simple after commit' % self.workDir, '+ patching %s/simple/simple.recipe' % self.workDir, '+ patch: applying hunk 1 of 1'])
def logCheck(self, fn, args, records, kwargs={}, regExp = False, verbosity = None): from conary.lib import log self.logFilter.add() if verbosity != None: log.setVerbosity(verbosity) rc = fn(*args, **kwargs) try: if regExp: self.logFilter.regexpCompare(records) else: self.logFilter.compare(records) finally: self.logFilter.remove() return rc
def main(argv): log.setVerbosity(log.WARNING) try: argv = list(argv) debugAll = "--debug-all" in argv if debugAll: debuggerException = Exception argv.remove("--debug-all") else: debuggerException = errors.RmakeInternalError sys.excepthook = errors.genExcepthook(debug=debugAll, debugCtrlC=debugAll) compat.ConaryVersion().checkRequiredVersion() return RmakeMain().main(argv, debuggerException=debuggerException) except debuggerException, err: raise
def main(argv): log.setVerbosity(log.WARNING) try: argv = list(argv) debugAll = '--debug-all' in argv if debugAll: debuggerException = Exception argv.remove('--debug-all') else: debuggerException = errors.RmakeInternalError sys.excepthook = errors.genExcepthook(debug=debugAll, debugCtrlC=debugAll) compat.ConaryVersion().checkRequiredVersion() return RmakeMain().main(argv, debuggerException=debuggerException) except debuggerException, err: raise
def checkSyncRepos(config, sourceRepos, targetRepos): checkConfig(config) targets = _makeTargets(config, targetRepos) log.setVerbosity(log.DEBUG) # retrieve the set of troves from a give repository def _getTroveSet(config, repo): def _flatten(troveSpec): l = [] for name, versionD in troveSpec.iteritems(): for version, flavorList in versionD.iteritems(): l += [ (name, version, flavor) for flavor in flavorList ] return set(l) troveSpecs = {} if config.labels: d = troveSpecs.setdefault(None, {}) for l in config.labels: d[l] = '' t = repo.getTroveVersionsByLabel(troveSpecs, troveTypes = netclient.TROVE_QUERY_ALL) else: troveSpecs = {None : None} t = repo.getTroveVersionList(config.host, troveSpecs, troveTypes = netclient.TROVE_QUERY_ALL) return _flatten(t) # compare source with each target def _compare(src, dst): srcName, srcSet = src dstName, dstSet = dst counter = 0 for x in srcSet.difference(dstSet): log.debug(" - %s %s " % (srcName, x)) counter += 1 for x in dstSet.difference(srcSet): log.debug(" + %s %s" % (dstName, x)) counter += 1 return counter log.debug("Retrieving list of troves from source %s" % str(sourceRepos.c.map)) sourceSet = _getTroveSet(config, sourceRepos) hasDiff = 0 for target in targets: log.debug("Retrieving list of troves from %s %s" % (target.name, str(target.repo.c.map))) targetSet = _getTroveSet(config, target.repo) log.debug("Diffing source and %s" % target.name) hasDiff += _compare( ("source", sourceSet), (target.name, targetSet) ) log.debug("Done") return hasDiff
def runCommand(self, client, cfg, argSet, args): if self.verbose: log.setVerbosity(log.DEBUG) else: log.setVerbosity(log.INFO) command, troveSpecs = self.requireParameters(args, 'troveSpec', appendExtra=True) if command == 'buildgroup': log.warning('"buildgroup" is deprecated and will be removed in a future release - use "build --recurse" instead') rebuild = (command == 'rebuild') flavorSpec = argSet.pop('flavor', None) if flavorSpec: flavor = deps.parseFlavor(flavorSpec) if flavor is None: raise errors.ParseError("Invalid flavor: '%s'" % flavorSpec) newFlavor = deps.overrideFlavor(client.buildConfig.buildFlavor, flavor) client.buildConfig.buildFlavor = newFlavor newFlavors = [] for oldFlavor in client.buildConfig.flavor: newFlavors.append(deps.overrideFlavor(oldFlavor, flavor)) client.buildConfig.flavor = newFlavors matchSpecs = argSet.pop('match', []) hosts = argSet.pop('host', []) labels = argSet.pop('label', []) recurseGroups = argSet.pop('recurse', False) or command == 'buildgroup' if recurseGroups: if argSet.pop('binary-search', False): recurseGroups = client.BUILD_RECURSE_GROUPS_BINARY elif not compat.ConaryVersion().supportsFindGroupSources(): log.warning('Your conary does not support recursing a group' ' source component, defaulting to searching the' ' binary version') recurseGroups = client.BUILD_RECURSE_GROUPS_BINARY else: recurseGroups = client.BUILD_RECURSE_GROUPS_SOURCE self._prep(client, argSet) job = client.createBuildJob(troveSpecs, limitToHosts=hosts, limitToLabels=labels, recurseGroups=recurseGroups, matchSpecs=matchSpecs, rebuild=rebuild) return self._build(client, job, argSet)
def setUp(self): TestCase.setUp(self) self.slaveCfg = slave.SlaveConfig() self.slaveCfg.configLine('debugMode True') self.slaveCfg.configLine('masterUrl http://no.master/api/') self.finishedDir = os.path.join(self.tmpDir, "finished-images") self.entDir = os.path.join(self.tmpDir, "entitlements") generators.constants.tmpDir = os.path.join(self.tmpDir, 'tmp') for d in [ self.finishedDir, self.entDir, generators.constants.tmpDir ]: util.mkdirChain(d) generators.constants.finishedDir = self.finishedDir generators.constants.entDir = self.entDir generators.constants.skelDir = os.path.abspath(os.path.dirname(__file__) + '/../skel') generators.constants.templateDir = os.path.abspath(os.path.dirname(__file__) + '/../templates') self.constants = generators.constants self.testDir = os.path.join(os.path.dirname(os.path.abspath(__file__))) # Suppress warning messages logged during tests since they are # supposed to be exercising these paths. conary_log.setVerbosity(conary_log.ERROR) # Neuter a few functions that might result in unwanted output # during the testsuite run: # - don't let conary set the loglevel when cooking conary_log.setVerbosity = lambda x: None # - add a dummy handler so that logging doesn't open a # default one log.addHandler(DummyHandler()) # make sure we always delete mkdtemp directories self.mkdCreated = [] self.realMkdtemp = tempfile.mkdtemp def fakeMkdtemp(*args, **kwargs): kwargs['prefix'] = 'test' d = self.realMkdtemp(*args, **kwargs) self.mkdCreated.append(d) return d tempfile.mkdtemp = fakeMkdtemp data = self.data = copy.deepcopy(self.Data) data['project']['conaryCfg'] = 'root %s/_ROOT_' % self.cfg.root
class CommitCommand(CvcCommand): commands = ['commit', 'ci'] help = 'Commit changes to a source component' commandGroup = 'Repository Access' docs = { 'message': 'Use MESSAGE to describe why the commit was performed', 'test': ('Runs through all the steps of committing but does not ' 'modify the repository'), 'log-file': 'Read the commit message from file LOGFILE (use - for ' 'standard input)', } # allow "cvc commit -m'foo bar'" to work hobbleShortOpts = False def addParameters(self, argDef): CvcCommand.addParameters(self, argDef) argDef["message"] = '-m', ONE_PARAM argDef["test"] = NO_PARAM argDef["log-file"] = '-l', ONE_PARAM def runCommand(self, cfg, argSet, args, profile=False, callback=None, repos=None): args = args[1:] level = log.getVerbosity() message = argSet.pop("message", None) test = argSet.pop("test", False) logfile = argSet.pop("log-file", None) if argSet or len(args) != 1: return self.usage() if message and logfile: raise errors.ConaryError("options --message and --log-file are " "mutually exclusive") if logfile: # Read the checkin message from the file if logfile == '-': message = sys.stdin.read() else: try: message = open(logfile).read() except IOError, e: raise errors.ConaryError("While opening %s: %s" % (e.filename, e.strerror)) # Get rid of trailing white spaces, they're probably not # intended to be there anyway message = message.rstrip() checkin.commit(repos, cfg, message, callback=callback, test=test) log.setVerbosity(level)
def testLogWithPercentChar(self): log.resetErrorOccurred() log.setVerbosity(2) fooString = "Some message with a %s char in it" self.logCheck(log.error, (fooString,), "error: %s" % fooString) assert(log.errorOccurred()) self.logCheck(log.warning, (fooString,), "warning: %s" % fooString) self.logCheck(log.debug, (fooString,), "+ %s" % fooString) self.logCheck(log.info, (fooString,), "+ %s" % fooString) fooString = "This %s work" arg1 = "does" efooString = fooString % arg1 self.logCheck(log.error, (fooString, arg1), "error: %s" % efooString) assert(log.errorOccurred()) self.logCheck(log.warning, (fooString, arg1), "warning: %s" % efooString) self.logCheck(log.debug, (fooString, arg1), "+ %s" % efooString) self.logCheck(log.info, (fooString, arg1), "+ %s" % efooString)
def testLogWithPercentChar(self): log.resetErrorOccurred() log.setVerbosity(2) fooString = "Some message with a %s char in it" self.logCheck(log.error, (fooString, ), "error: %s" % fooString) assert (log.errorOccurred()) self.logCheck(log.warning, (fooString, ), "warning: %s" % fooString) self.logCheck(log.debug, (fooString, ), "+ %s" % fooString) self.logCheck(log.info, (fooString, ), "+ %s" % fooString) fooString = "This %s work" arg1 = "does" efooString = fooString % arg1 self.logCheck(log.error, (fooString, arg1), "error: %s" % efooString) assert (log.errorOccurred()) self.logCheck(log.warning, (fooString, arg1), "warning: %s" % efooString) self.logCheck(log.debug, (fooString, arg1), "+ %s" % efooString) self.logCheck(log.info, (fooString, arg1), "+ %s" % efooString)
def testResolutionKeepExistingLogging(self): self.addComponent('keepprov:runtime', '1.0', provides='trove:keepreq(1.0)', filePrimer=3) self.addComponent('keepprov:runtime', '2.0', provides='trove:keepreq(2.0)', filePrimer=4) self.addComponent('keepreq1:runtime', '1.0', requires='trove:keepreq(1.0)', filePrimer=5) self.addComponent('keepreq2:runtime', '1.0', requires='trove:keepreq(2.0)', filePrimer=6) self.updatePkg(['keepprov:runtime=1.0', 'keepreq1:runtime']) log.setVerbosity(log.DEBUG) self.logFilter.add() self.checkUpdate('keepreq2:runtime', ['keepreq2:runtime', 'keepprov:runtime=--2.0'], resolve=True, keepRequired=True) self.logFilter.compare([ '''+ Resolved: keepreq2:runtime=localhost@rpl:linux/1.0-1-1[] Required: trove: keepreq(2.0) Adding: keepprov:runtime=localhost@rpl:linux/2.0-1-1[]''', '''+ Update breaks dependencies!''', '''+ Broken dependency: (dep needed by the system but being removed): keepreq1:runtime=1.0-1-1 Requires: trove: keepreq(1.0) Provided by removed or updated packages: keepprov:runtime''', '''+ Attempting to update the following packages in order to remove their dependency on something being erased: keepreq1''', '''+ Resolved (installing side-by-side) keepreq1:runtime=1.0-1-1[] Required: trove: keepreq(1.0) Keeping: keepprov:runtime=1.0-1-1[]''', 'warning: keeping keepprov:runtime - required by at least keepreq1:runtime' ])
def testResolutionKeepExistingLogging(self): self.addComponent('keepprov:runtime', '1.0', provides='trove:keepreq(1.0)', filePrimer=3) self.addComponent('keepprov:runtime', '2.0', provides='trove:keepreq(2.0)', filePrimer=4) self.addComponent('keepreq1:runtime', '1.0', requires='trove:keepreq(1.0)', filePrimer=5) self.addComponent('keepreq2:runtime', '1.0', requires='trove:keepreq(2.0)', filePrimer=6) self.updatePkg(['keepprov:runtime=1.0', 'keepreq1:runtime']) log.setVerbosity(log.DEBUG) self.logFilter.add() self.checkUpdate('keepreq2:runtime', ['keepreq2:runtime', 'keepprov:runtime=--2.0'], resolve=True, keepRequired=True) self.logFilter.compare([ '''+ Resolved: keepreq2:runtime=localhost@rpl:linux/1.0-1-1[] Required: trove: keepreq(2.0) Adding: keepprov:runtime=localhost@rpl:linux/2.0-1-1[]''', '''+ Update breaks dependencies!''', '''+ Broken dependency: (dep needed by the system but being removed): keepreq1:runtime=1.0-1-1 Requires: trove: keepreq(1.0) Provided by removed or updated packages: keepprov:runtime''', '''+ Attempting to update the following packages in order to remove their dependency on something being erased: keepreq1''', '''+ Resolved (installing side-by-side) keepreq1:runtime=1.0-1-1[] Required: trove: keepreq(1.0) Keeping: keepprov:runtime=1.0-1-1[]''', 'warning: keeping keepprov:runtime - required by at least keepreq1:runtime'])
configFileList = [configFileList] for path in configFileList: try: cfg.read(path, exception=True) except IOError, msg: raise OptionError(msg, parser) for (arg, name) in cfgMap.items(): if argSet.has_key(arg): cfg.configLine("%s %s" % (name, argSet[arg])) del argSet[arg] if addVerboseOptions: if 'verbose' in argSet: if argSet['verbose'] is True: log.setVerbosity(log.INFO) del argSet['verbose'] if addDebugOptions: if argSet.has_key('debugger'): del argSet['debugger'] from conary.lib import debugger debugger.set_trace() sys.excepthook = util.genExcepthook(debug=cfg.debugExceptions, debugCtrlC=True) if 'debug' in argSet: if argSet['debug'] is True: log.setVerbosity(log.DEBUG) else: log.setVerbosity(log.LOWLEVEL)
def runCommand(self, client, cfg, argSet, args): command, jobId = self.requireParameters(args, 'jobId') log.setVerbosity(log.INFO) jobId = _getJobIdOrUUId(jobId) client.stopJob(jobId)
class RmakeMain(options.MainHandler): name = 'rmake' version = constants.version abstractCommand = command.rMakeCommand configClass = buildcfg.BuildConfiguration useConaryOptions = False commandList = command._commands def usage(self, rc=1, showAll=False): print 'rmake: front end to rMake build tool' if not showAll: print print 'Common Commands (use "rmake help" for the full list)' return options.MainHandler.usage(self, rc, showAll=showAll) def initializePlugins(self, argv): p = plugins.getPluginManager(argv, buildcfg.BuildConfiguration) p.callClientHook('client_preInit', self, argv) return p def getConfigFile(self, argv): pluginManager = self.initializePlugins(argv) if '--skip-default-config' in argv: argv.remove('--skip-default-config') read = False else: read = True buildConfig = buildcfg.BuildConfiguration(readConfigFiles=read) conaryConfig = conarycfg.ConaryConfiguration(readConfigFiles=read) return buildConfig, conaryConfig, pluginManager def runCommand(self, thisCommand, (buildConfig, conaryConfig, pluginManager), argSet, args): pluginManager.callClientHook('client_preCommand', self, thisCommand, (buildConfig, conaryConfig), argSet, args) compat.checkRequiredVersions() thisCommand.verbose = (log.getVerbosity() <= log.INFO) if args[1] != 'help': # NOTE: the help system assumes that the base level of output # you want is "warning", but rmake is more verbose than that. # Due to limitations in how configurable the help system is, # I can't easily fix that. Someday I should though. For now, # if we're running help, we make log.WARNING the default level, # and otherwise log.INFO is the default. log.setMinVerbosity(log.INFO) # don't let the buildFlavor be overridden yet client = helper.rMakeHelper(buildConfig=buildConfig, promptPassword=True) pluginManager.callClientHook('client_preCommand2', self, client, thisCommand) try: return options.MainHandler.runCommand(self, thisCommand, client, buildConfig, argSet, args) except errors.BadParameters: if not thisCommand.verbose: log.setVerbosity(log.WARNING) thisCommand.usage() raise
def testTroveDisplayInfo(self): trv = self.addComponent('test1:source=1.0-1') mi = self.createMetadataItem(categories=['cat1', 'cat2'], licenses=['GPL', 'GPLv3'], shortDesc='short', longDesc='long', crypto=['foo', 'foo2'], url=['url1']) self.addComponent('test1:run=1.0-1-1', metadata=mi, sourceName='test1:source') repos = self.openRepository() rs, s = self.captureOutput(queryrep.displayTroves, self.cfg, ['test1:run'], info=True) s = re.sub('Build time: .*', 'Build time: <TIME>', s) info = '''\ Name : test1:run Build time: <TIME> Version : 1.0-1-1 Label : localhost@rpl:linux Size : 14 Flavor : License : GPL License : GPLv3 Crypto : foo Crypto : foo2 Category : cat1 Category : cat2 Summary : short Url : url1 Description: long Source : test1:source Change log: () ''' assert(s == info) # --debug mode debugInfo = '''\ Name : test1:run Build time: <TIME> Version : 1.0-1-1 Label : localhost@rpl:linux Size : 14 Committed : <TIME> Flavor : License : GPL License : GPLv3 Crypto : foo Crypto : foo2 Category : cat1 Category : cat2 Summary : short Url : url1 Description: long Source : test1:source Change log: () Incomp. : False TroveVer : 10 Clone of : None Conary version : ''' verbosity = log.getVerbosity() log.setVerbosity(log.DEBUG) try: rs, s = self.captureOutput(queryrep.displayTroves, self.cfg, ['test1:run'], info=True) finally: log.setVerbosity(verbosity) s = re.sub('(Build time|Committed ): .*', r'\1: <TIME>', s) self.assertEqual(s, debugInfo) # test to make sure that we can display some info if :source # is not available to us noSourceInfo = '''\ Name : test1:run Build time: <TIME> Version : 1.0-1-1 Label : localhost@rpl:linux Size : 14 Flavor : License : GPL License : GPLv3 Crypto : foo Crypto : foo2 Category : cat1 Category : cat2 Summary : short Url : url1 Description: long ''' repos.deleteUserByName(self.cfg.buildLabel, 'anonymous') limitedRepos = self.setupUser(repos, self.cfg.buildLabel, 'limited', 'bar', '[^:]+(:(?!debuginfo$|source$).*|$)', self.cfg.buildLabel) self.cfg.user.addServerGlob('*', 'limited', 'bar') rs, s = self.captureOutput(queryrep.displayTroves, self.cfg, ['test1:run'], info=True) s = re.sub('Build time: .*', 'Build time: <TIME>', s) self.assertEqual(s, noSourceInfo)
class rMakeCommand(options.AbstractCommand): defaultGroup = 'Common Options' commandGroup = CG_MISC docs = { 'config': (VERBOSE_HELP, "Set config KEY to VALUE", "'KEY VALUE'"), 'config-file': (VERBOSE_HELP, "Read PATH config file", "PATH"), 'context': (VERBOSE_HELP, "Set the configuration context to use"), 'conary-config-file': (VERBOSE_HELP, "Read PATH conary config file", "PATH"), 'build-config-file': (VERBOSE_HELP, "Read PATH config file", "PATH"), 'rmake-config-file': (VERBOSE_HELP, "Read PATH config file", "PATH"), 'skip-default-config': (VERBOSE_HELP, "Don't read default configs"), 'verbose': (VERBOSE_HELP, "Display more detailed information where available") } def addParameters(self, argDef): d = {} d["context"] = ONE_PARAM d["config"] = MULT_PARAM d["build-config-file"] = MULT_PARAM d["conary-config-file"] = MULT_PARAM d["skip-default-config"] = NO_PARAM d["verbose"] = NO_PARAM argDef[self.defaultGroup] = d def _getContext(self, buildConfig, conaryConfig, argSet): context = conaryConfig.context if buildConfig.context: context = buildConfig.context if os.path.exists('CONARY'): conaryState = compat.ConaryVersion().ConaryStateFromFile( 'CONARY', parseSource=False) if conaryState.hasContext(): context = conaryState.getContext() context = os.environ.get('CONARY_CONTEXT', context) context = argSet.pop('context', context) return context def _setContext(self, buildConfig, conaryConfig, argSet): context = self._getContext(buildConfig, conaryConfig, argSet) usedContext = False if conaryConfig and context: if conaryConfig.hasSection(context): usedContext = True conaryConfig.setContext(context) buildConfig.useConaryConfig(conaryConfig) if context and buildConfig.hasSection(context): buildConfig.setContext(context) usedContext = True if not usedContext and context: raise errors.RmakeError('No such context "%s"' % context) def processConfigOptions(self, (buildConfig, conaryConfig, pluginManager), cfgMap, argSet): """ Manage any config maps we've set up, converting assigning them to the config object. """ configFileList = argSet.pop('build-config-file', []) if not isinstance(configFileList, list): configFileList = list(configFileList) configFileList.extend(argSet.pop('config-file', [])) for path in configFileList: buildConfig.read(path, exception=True) configFileList = argSet.pop('conary-config-file', []) if not isinstance(configFileList, list): configFileList = list(configFileList) if configFileList and not conaryConfig: conaryConfig = conarycfg.ConaryConfiguration(readConfigFiles=False) for path in configFileList: conaryConfig.read(path, exception=True) self._setContext(buildConfig, conaryConfig, argSet) for (arg, data) in cfgMap.items(): cfgName, paramType = data[0:2] value = argSet.pop(arg, None) if value is not None: if arg.startswith('no-'): value = not value buildConfig.configLine("%s %s" % (cfgName, value)) for line in argSet.pop('config', []): buildConfig.configLine(line) if argSet.pop('verbose', False): log.setVerbosity(log.DEBUG)
def testLookAsideTest1(self): raise testhelp.SkipTestException("tests shouldn't talk to the internet") self.resetCache() self.resetWork() self.resetRepository() repos = self.openRepository() cfg = self.cfg repCache = lookaside.RepositoryCache(repos, cfg=cfg) origDir = os.getcwd() os.chdir(self.workDir) self.newpkg('test') os.chdir('test') testdir = '/'.join((self.workDir, 'test')) log.setVerbosity(log.INFO) self.logFilter.add() # first, look for a file that does not exist assert(lookaside.findAll(self.cfg, repCache, 'http://example.conary.com/foo', 'test', (testdir,), allowNone=True) is None) # make sure that we got a negative cache entry assert(os.stat('/'.join((self.cacheDir, 'NEGATIVE/test/example.conary.com/foo')))) # now make sure that this works for ftp as well (urllib workaround) # XXX WORKAROUND until FTP works in eng lab #assert(lookaside.findAll(self.cfg, repCache, # 'ftp://download.rpath.com/blah', 'test', (testdir,), # allowNone=True) is None) # make sure that we got a negative cache entry #assert(os.stat('/'.join((self.cacheDir, # 'NEGATIVE/test/download.rpath.com/blah')))) # now we put a file in place f = file(os.sep.join((testdir, 'bar')), 'w') f.write('this is a test') f.close() c = lookaside.findAll(self.cfg, repCache, 'bar', 'test', (testdir,)) # it does not need to cache it; it is known to exist assert(c == os.sep.join((testdir, 'bar'))) # Test httpHeaders: c = util.normpath(lookaside.findAll(self.cfg, repCache, 'http://www.google.com/preferences', 'test', (testdir,), httpHeaders={'Accept-Language': 'es-es'})) assert(c == '/'.join((self.cacheDir, 'test/www.google.com/preferences'))) #open the page and check to see if it's in spanish f = open(c) contents = f.read() f.close() assert 'Preferencias globales' in contents # we need a web page to actually test the cache in operation # we do it a second time to make sure that the cache works for i in (0, 1): c = util.normpath(lookaside.findAll(self.cfg, repCache, 'http://wiki.rpath.com/wiki/Main_Page', 'test', (testdir,))) assert(c == '/'.join((self.cacheDir, 'test/wiki.rpath.com/wiki/Main_Page'))) self.logFilter.remove() self.logFilter.compare( ('+ Trying http://example.conary.com/foo...', # XXX WORKAROUND until FTP works in eng lab #'+ Trying ftp://download.rpath.com/blah...', #'+ Downloading ftp://download.rpath.com/blah...', '+ Trying http://www.google.com/preferences...', '+ Downloading http://www.google.com/preferences...', '+ Trying http://wiki.rpath.com/wiki/Main_Page...', '+ Downloading http://wiki.rpath.com/wiki/Main_Page...')) recipestr = """ class TestLookaside(PackageRecipe): name = 'test' version = '1' clearBuildReqs() def setup(r): r.addSource('bar', dest='/') """ self.writeFile('test.recipe', recipestr) self.addfile('test.recipe') self.addfile('bar', binary = True) self.commit() os.chdir(origDir) # ensure that a localOnly=True lookup in the repository works; # for this, we need a prepped recipeObj for its RepositoryCache # object recipeObj = self.getRecipeObjFromRepos('test', repos) self.logFilter.add() c = lookaside.findAll(self.cfg, recipeObj.laReposCache, 'bar', 'test', (), localOnly=True) self.logFilter.remove() self.logFilter.compare( '+ found bar in repository', ) assert(c == os.sep.join((self.cacheDir, 'test', 'bar')))
def testTroveDisplayInfo(self): trv = self.addComponent('test1:source=1.0-1') mi = self.createMetadataItem(categories=['cat1', 'cat2'], licenses=['GPL', 'GPLv3'], shortDesc='short', longDesc='long', crypto=['foo', 'foo2'], url=['url1']) self.addComponent('test1:run=1.0-1-1', metadata=mi, sourceName='test1:source') repos = self.openRepository() rs, s = self.captureOutput(queryrep.displayTroves, self.cfg, ['test1:run'], info=True) s = re.sub('Build time: .*', 'Build time: <TIME>', s) info = '''\ Name : test1:run Build time: <TIME> Version : 1.0-1-1 Label : localhost@rpl:linux Size : 14 Flavor : License : GPL License : GPLv3 Crypto : foo Crypto : foo2 Category : cat1 Category : cat2 Summary : short Url : url1 Description: long Source : test1:source Change log: () ''' assert (s == info) # --debug mode debugInfo = '''\ Name : test1:run Build time: <TIME> Version : 1.0-1-1 Label : localhost@rpl:linux Size : 14 Committed : <TIME> Flavor : License : GPL License : GPLv3 Crypto : foo Crypto : foo2 Category : cat1 Category : cat2 Summary : short Url : url1 Description: long Source : test1:source Change log: () Incomp. : False TroveVer : 10 Clone of : None Conary version : ''' verbosity = log.getVerbosity() log.setVerbosity(log.DEBUG) try: rs, s = self.captureOutput(queryrep.displayTroves, self.cfg, ['test1:run'], info=True) finally: log.setVerbosity(verbosity) s = re.sub('(Build time|Committed ): .*', r'\1: <TIME>', s) self.assertEqual(s, debugInfo) # test to make sure that we can display some info if :source # is not available to us noSourceInfo = '''\ Name : test1:run Build time: <TIME> Version : 1.0-1-1 Label : localhost@rpl:linux Size : 14 Flavor : License : GPL License : GPLv3 Crypto : foo Crypto : foo2 Category : cat1 Category : cat2 Summary : short Url : url1 Description: long ''' repos.deleteUserByName(self.cfg.buildLabel, 'anonymous') limitedRepos = self.setupUser(repos, self.cfg.buildLabel, 'limited', 'bar', '[^:]+(:(?!debuginfo$|source$).*|$)', self.cfg.buildLabel) self.cfg.user.addServerGlob('*', 'limited', 'bar') rs, s = self.captureOutput(queryrep.displayTroves, self.cfg, ['test1:run'], info=True) s = re.sub('Build time: .*', 'Build time: <TIME>', s) self.assertEqual(s, noSourceInfo)
def testDuplicateBasenames(self): self.resetCache() self.resetWork() self.resetRepository() repos = self.openRepository() cfg = self.cfg repCache = lookaside.RepositoryCache(repos, cfg=cfg) try: contentServer = rephelp.HTTPServerController(getRequester()) contentURL = contentServer.url() origDir = os.getcwd() os.chdir(self.workDir) self.newpkg('test') os.chdir('test') testdir = '/'.join((self.workDir, 'test')) log.setVerbosity(log.INFO) self.logFilter.add() # first, look for a file that does not exist assert lookaside.findAll(self.cfg, repCache, contentURL + '/404/foo', 'test', (testdir,), allowNone=True ) is None # make sure that we got a negative cache entry assert os.stat(self.cacheDir + '/NEGATIVE/test/404/foo') # now look for a file that does exist assert lookaside.findAll(self.cfg, repCache, contentURL + '/200/foo', 'test', (testdir,), allowNone=True ) is not None # make sure that we got a the cache entry assert os.stat(self.cacheDir + '/test/200/foo') # put two different files with the same name name in the cache fooDir = os.path.join(self.cacheDir,'test/foo.conary.com/foo/') os.makedirs(fooDir) self.writeFile(os.path.join(fooDir,'theFile'), 'Foo version of the file\n') barDir = os.path.join(self.cacheDir,'test/bar.conary.com/bar/') os.makedirs(barDir) self.writeFile(os.path.join(barDir,'theFile'), 'Bar version of the file\n') # this file shouldn't be found path = lookaside.findAll(self.cfg, repCache, 'http://baz.conary.com/foo/theFile', 'test', (testdir,), allowNone=True) self.assertEqual(path,None) # this file should be found and have the right contents path = lookaside.findAll(self.cfg, repCache, 'http://foo.conary.com/foo/theFile', 'test', (testdir,), allowNone=True) f = open(path) self.assertEqual(f.readline()[0:3],'Foo') f.close() # so should this one path = lookaside.findAll(self.cfg, repCache, 'http://bar.conary.com/bar/theFile', 'test', (testdir,), allowNone=True) f = open(path) self.assertEqual(f.readline()[0:3],'Bar') f.close() finally: contentServer.kill()
def testDuplicateBasenames(self): self.resetCache() self.resetWork() self.resetRepository() repos = self.openRepository() cfg = self.cfg repCache = lookaside.RepositoryCache(repos, cfg=cfg) try: contentServer = rephelp.HTTPServerController(getRequester()) contentURL = contentServer.url() origDir = os.getcwd() os.chdir(self.workDir) self.newpkg('test') os.chdir('test') testdir = '/'.join((self.workDir, 'test')) log.setVerbosity(log.INFO) self.logFilter.add() # first, look for a file that does not exist assert lookaside.findAll(self.cfg, repCache, contentURL + '/404/foo', 'test', (testdir, ), allowNone=True) is None # make sure that we got a negative cache entry assert os.stat(self.cacheDir + '/NEGATIVE/test/404/foo') # now look for a file that does exist assert lookaside.findAll(self.cfg, repCache, contentURL + '/200/foo', 'test', (testdir, ), allowNone=True) is not None # make sure that we got a the cache entry assert os.stat(self.cacheDir + '/test/200/foo') # put two different files with the same name name in the cache fooDir = os.path.join(self.cacheDir, 'test/foo.conary.com/foo/') os.makedirs(fooDir) self.writeFile(os.path.join(fooDir, 'theFile'), 'Foo version of the file\n') barDir = os.path.join(self.cacheDir, 'test/bar.conary.com/bar/') os.makedirs(barDir) self.writeFile(os.path.join(barDir, 'theFile'), 'Bar version of the file\n') # this file shouldn't be found path = lookaside.findAll(self.cfg, repCache, 'http://baz.conary.com/foo/theFile', 'test', (testdir, ), allowNone=True) self.assertEqual(path, None) # this file should be found and have the right contents path = lookaside.findAll(self.cfg, repCache, 'http://foo.conary.com/foo/theFile', 'test', (testdir, ), allowNone=True) f = open(path) self.assertEqual(f.readline()[0:3], 'Foo') f.close() # so should this one path = lookaside.findAll(self.cfg, repCache, 'http://bar.conary.com/bar/theFile', 'test', (testdir, ), allowNone=True) f = open(path) self.assertEqual(f.readline()[0:3], 'Bar') f.close() finally: contentServer.kill()
if argv is None: argv = argv = sys.argv[1:] try: options = parseArgs(argv) except OptionError, e: sys.stderr.write(e.errmsg) sys.stderr.write("\n") return e.errcode cfg = MirrorFileConfiguration() cfg.read(options.configFile, exception=True) callback = ChangesetCallback() if options.absolute: cfg.absoluteChangesets = True if options.verbose: log.setVerbosity(log.DEBUG) callback = VerboseChangesetCallback() if options.fastSync: # make --fast-sync imply --full-trove-sync options.sync = True callback.allowMissingFiles = cfg.excludeCapsuleContents try: mainWorkflow(cfg, callback, options.test, sync=options.sync, infoSync=options.infoSync, fastSync=options.fastSync, checkSync=options.checkSync) except KeyboardInterrupt: print >> sys.stderr print >> sys.stderr, 'Terminating due to user interrupt'
class CookCommand(CvcCommand): commands = ['cook'] paramHelp = '<file.recipe|troveName=<version>>[[flavor]]+' help = 'Build binary package and groups from a recipe' commandGroup = 'Recipe Building' docs = { 'cross': (VERBOSE_HELP, 'set macros for cross-compiling', '[(local|HOST)--]TARGET'), 'debug-exceptions': 'Enter debugger if a recipe fails in conary', 'flavor': 'build the trove with flavor FLAVOR', 'macro': ('set macro NAME to VALUE', "'NAME VALUE'"), 'macros': optparse.SUPPRESS_HELP, # can we get rid of this? 'no-clean': 'do not remove build directory even if build is' ' successful', 'no-deps': optparse.SUPPRESS_HELP, 'ignore-buildreqs': 'do not check build requirements', 'show-buildreqs': (VERBOSE_HELP, 'show build requirements for recipe'), 'prep': 'unpack, but do not build', 'download': 'download, but do not unpack or build', 'resume': ('resume building at given loc (default at failure)', '[LINENO|policy]'), 'to-file': (VERBOSE_HELP, 'Write changeset to file instead of' ' committing to the repository'), 'unknown-flags': (VERBOSE_HELP, 'Set all unknown flags that are used in the recipe to False') } def addParameters(self, argDef): CvcCommand.addParameters(self, argDef) argDef['debug-exceptions'] = NO_PARAM argDef['cross'] = ONE_PARAM argDef['flavor'] = ONE_PARAM argDef['macro'] = MULT_PARAM argDef['macros'] = ONE_PARAM argDef['no-clean'] = NO_PARAM argDef['no-deps'] = NO_PARAM argDef['ignore-buildreqs'] = NO_PARAM argDef['show-buildreqs'] = NO_PARAM argDef['prep'] = NO_PARAM argDef['download'] = NO_PARAM argDef['resume'] = STRICT_OPT_PARAM argDef['to-file'] = ONE_PARAM argDef['unknown-flags'] = NO_PARAM argDef['allow-flavor-change'] = NO_PARAM def runCommand(self, cfg, argSet, args, profile=False, callback=None, repos=None): args = args[1:] level = log.getVerbosity() macros = {} prep = 0 downloadOnly = False resume = None if argSet.has_key('flavor'): buildFlavor = deps.deps.parseFlavor(argSet['flavor'], raiseError=True) cfg.buildFlavor = deps.deps.overrideFlavor(cfg.buildFlavor, buildFlavor) del argSet['flavor'] if argSet.has_key('macros'): f = open(argSet['macros']) for row in f: row = row.strip() if not row or row[0] == '#': continue cfg.configLine('macros ' + row.strip()) f.close() del f del argSet['macros'] if argSet.has_key('macro'): for macro in argSet['macro']: cfg.configLine('macros ' + macro) del argSet['macro'] if argSet.has_key('prep'): del argSet['prep'] prep = 1 if argSet.has_key('ignore-buildreqs'): del argSet['ignore-buildreqs'] ignoreDeps = True elif argSet.has_key('no-deps'): del argSet['no-deps'] ignoreDeps = True else: ignoreDeps = False if argSet.has_key('download'): if argSet.has_key('prep') or prep == True: log.warn( 'download and prep should not be used together... prefering download only' ) del argSet['download'] ignoreDeps = True downloadOnly = True showBuildReqs = argSet.pop('show-buildreqs', False) if argSet.has_key('quiet'): cfg.quiet = True del argSet['quiet'] if 'no-clean' in argSet: cfg.cleanAfterCook = False del argSet['no-clean'] allowFlavorChange = argSet.pop('allow-flavor-change', False) if argSet.has_key('resume'): resume = argSet['resume'] del argSet['resume'] if argSet.has_key('unknown-flags'): unknownFlags = argSet['unknown-flags'] del argSet['unknown-flags'] else: unknownFlags = False if argSet.has_key('debug-exceptions'): del argSet['debug-exceptions'] cfg.debugRecipeExceptions = True crossCompile = argSet.pop('cross', None) if crossCompile: parts = crossCompile.split('--') isCrossTool = False if len(parts) == 1: crossTarget = crossCompile crossHost = None else: crossHost, crossTarget = parts if crossHost == 'local': crossHost = None isCrossTool = True crossCompile = (crossHost, crossTarget, isCrossTool) targetFile = argSet.pop("to-file", None) if argSet: return self.usage() groupOptions = cook.GroupCookOptions( alwaysBumpCount=True, errorOnFlavorChange=not allowFlavorChange, shortenFlavors=cfg.shortenGroupFlavors) # the remainder of the argument list are the things to build. # e.g., foo /path/to/bar.recipe, etc. items = args[1:] if not items: # if nothing was specified, try to build the package in the current # directory if os.path.isfile('CONARY'): conaryState = state.ConaryStateFromFile('CONARY', repos) items = [conaryState] if not items: # if we still don't have anything to build, throw a usage # message return self.usage() try: cook.cookCommand( cfg, items, prep, macros, resume=resume, allowUnknownFlags=unknownFlags, ignoreDeps=ignoreDeps, showBuildReqs=showBuildReqs, profile=profile, crossCompile=crossCompile, downloadOnly=downloadOnly, groupOptions=groupOptions, changeSetFile=targetFile, ) except builderrors.GroupFlavorChangedError, err: err.args = ( err.args[0] + '\n(Add the --allow-flavor-change flag to override this error)\n', ) raise log.setVerbosity(level)
def cookTrove(cfg, repos, logger, name, version, flavorList, targetLabel, loadSpecsList=None, logData=None, buildReqs=None, crossReqs=None): if not isinstance(flavorList, (tuple, list)): flavorList = [flavorList] util.mkdirChain(cfg.root + '/tmp') fd, csFile = tempfile.mkstemp(dir=cfg.root + '/tmp', prefix='rmake-%s-' % name, suffix='.ccs') os.chmod(csFile, 0644) os.close(fd) logPath = cfg.root + '/tmp/rmake/%s-%s.log' % (name, version.trailingRevision()) logFile = logfile.LogFile(logPath) os.chmod(logPath, 0664) os.chmod(cfg.root + '/tmp/rmake', 0775) results = CookResults(name, version, flavorList) # ignore child output problems signal.signal(signal.SIGTTOU, signal.SIG_IGN) inF, outF = pipereader.makePipes() pid = os.fork() if not pid: try: try: signal.signal(signal.SIGTERM, signal.SIG_DFL) os.close(inF) os.setpgid(0, 0) # don't accidentally make world writable files os.umask(0022) # don't allow us to create core dumps resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) log.setVerbosity(log.DEBUG) log.info("Cook process started (pid %s)" % os.getpid()) _cookTrove(cfg, repos, name, version, flavorList, targetLabel, loadSpecsList, csFile, buildReqs=buildReqs, crossReqs=crossReqs, failureFd=outF, logger=logger) except Exception, msg: if len(flavorList) > 1: errMsg = 'Error cooking %s=%s with flavors %s: %s' % \ (name, version, ', '.join([str(x) for x in flavorList]), str(msg)) else: errMsg = str(msg) _buildFailed(outF, errMsg, traceback.format_exc()) logFile.close() os._exit(1) else: logFile.close() os._exit(0)
def testLoadRecipe(self): self.repos = self.openRepository() self.overrideBuildFlavor('foo.flag,unknownflag') self.addTestPkg(1, version='1.0', header=''' if Use.readline: foo = '1.0-readline' else: foo = '1.0-noreadline' ''') self.cookTestPkg(1) self.updatePkg(self.cfg.root, 'test1', depCheck=False, justDatabase=True) # make a later version available self.addTestPkg(1, version='2.0', header=''' if Use.readline: foo = '2.0-readline' else: foo = '2.0-noreadline' ''') log.setVerbosity(log.INFO) self.logFilter.clear() self.logFilter.add() loader = self._testSubload(self.repos, 'loadInstalled("test1=:linux[readline]")') self.logFilter.compare(['+ Loaded TestRecipe1 from test1:source=/localhost@rpl:linux/1.0-1[]']) log.setVerbosity(log.WARNING) assert(loader.module.TestRecipe1.foo == '1.0-readline') FooLoaded = loader.getLoadedSpecs() assert(FooLoaded.keys() == ['test1=:linux[readline]']) assert(use.LocalFlags.flag) assert(len(loader.getLoadedTroves()) == 1) v1 = versions.VersionFromString('/localhost@rpl:linux/1.0-1') assert(loader.getLoadedTroves()[0] == ('test1:source', v1, deps.parseFlavor(''))) loader = self._testSubload(self.repos, 'loadInstalled("test1.recipe=:linux[readline]")') FooLoaded = loader.getLoadedSpecs() assert(FooLoaded.keys() == [ 'test1.recipe=:linux[readline]']) assert(loader.module.TestRecipe1.foo == '1.0-readline') # we don't have a version matching this installed, so just # default to the latest in the repos, but make sure the flavor # is respected loader = self._testSubload(self.repos, "loadInstalled('test1.recipe=:linux[!readline]')") assert(loader.module.TestRecipe1.foo == '2.0-noreadline') os.chdir(self.workDir) self.checkout('test1') os.chdir('test1') self.overrideBuildFlavor('!readline') # this is also testing that conary handles unknown flags in the build # flavor correctly if they're not referenced self.logFilter.add() self.cookItem(self.repos, self.cfg, 'test1.recipe') self.logFilter.remove() self.logFilter.compare( ['warning: ignoring unknown Use flag unknownflag']) self.updatePkg(self.cfg.root, 'test1-2.0.ccs', depCheck=False, keepExisting=True, justDatabase=True, replaceFiles=True) loader = self._testSubload(self.repos, "loadInstalled('test1.recipe=:linux[readline]')") assert(loader.module.TestRecipe1.foo == '1.0-readline') loader = self._testSubload(self.repos, "loadInstalled('test1.recipe=[!readline]')") assert(loader.module.TestRecipe1.foo == '2.0-noreadline') # now ensure we get the 2.0 recipe no matter what is installed # when we use loadSuperClass loader = self._testSubload(self.repos, "loadSuperClass('test1.recipe=:linux[readline]')") assert(loader.module.TestRecipe1.foo == '2.0-readline') loader = self._testSubload(self.repos, "loadSuperClass('test1.recipe=:linux[!readline]')") assert(loader.module.TestRecipe1.foo == '2.0-noreadline')
if argv is None: argv = argv=sys.argv[1:] try: options = parseArgs(argv) except OptionError, e: sys.stderr.write(e.errmsg) sys.stderr.write("\n") return e.errcode cfg = MirrorFileConfiguration() cfg.read(options.configFile, exception = True) callback = ChangesetCallback() if options.absolute: cfg.absoluteChangesets = True if options.verbose: log.setVerbosity(log.DEBUG) callback = VerboseChangesetCallback() if options.fastSync: # make --fast-sync imply --full-trove-sync options.sync = True callback.allowMissingFiles = cfg.excludeCapsuleContents try: mainWorkflow(cfg, callback, options.test, sync = options.sync, infoSync = options.infoSync, fastSync = options.fastSync, checkSync = options.checkSync) except KeyboardInterrupt: print >> sys.stderr print >> sys.stderr, 'Terminating due to user interrupt' sys.exit(1) def groupTroves(troveList):