def testCompatCheck(self): cv = compat.ConaryVersion('1.0.29') assert (not cv.supportsCloneCallback()) cv = compat.ConaryVersion('1.0.29_changeset') # _changeset must not make conary assume latest (RMK-1077) assert (not cv.supportsCloneCallback()) cv = compat.ConaryVersion('1.0.30') assert (cv.supportsCloneCallback()) self.logFilter.add() compat.ConaryVersion._warnedUser = False cv = compat.ConaryVersion('foo') self.logFilter.compare( ['warning: nonstandard conary version "foo". Assuming latest.']) assert (cv.supportsCloneCallback()) cv = compat.ConaryVersion('1.1.18') assert (cv.supportsCloneNoTracking()) cv = compat.ConaryVersion('1.1.16') assert (not cv.supportsCloneNoTracking()) cv = compat.ConaryVersion('1.0.30') assert (not cv.supportsCloneNoTracking()) cv = compat.ConaryVersion('2.0') assert (not cv.signAfterPromote()) cv = compat.ConaryVersion('1.2.7') assert (cv.signAfterPromote()) cv = compat.ConaryVersion('2.0.28') assert (cv.supportsDefaultBuildReqs())
def _doCommit(recipePath, repos, cfg, message): try: kw = {} if compat.ConaryVersion().supportsForceCommit(): kw.update(force=True) rv = checkin.commit(repos, cfg, message, **kw) except (conaryerrors.CvcError, conaryerrors.ConaryError), msg: raise errors.RmakeError("Could not commit changes to build" " recipe %s: %s" % (recipePath, msg))
def _getContext(self, buildConfig, conaryConfig, argSet): context = conaryConfig.context if buildConfig.context: context = buildConfig.context if os.path.exists('CONARY'): conaryState = compat.ConaryVersion().ConaryStateFromFile( 'CONARY', parseSource=False) if conaryState.hasContext(): context = conaryState.getContext() context = os.environ.get('CONARY_CONTEXT', context) context = argSet.pop('context', context) return context
def runCommand(self, client, cfg, argSet, args): if self.verbose: log.setVerbosity(log.DEBUG) else: log.setVerbosity(log.INFO) command, troveSpecs = self.requireParameters(args, 'troveSpec', appendExtra=True) if command == 'buildgroup': log.warning( '"buildgroup" is deprecated and will be removed in a future release - use "build --recurse" instead' ) rebuild = (command == 'rebuild') flavorSpec = argSet.pop('flavor', None) if flavorSpec: flavor = deps.parseFlavor(flavorSpec) if flavor is None: raise errors.ParseError("Invalid flavor: '%s'" % flavorSpec) newFlavor = deps.overrideFlavor(client.buildConfig.buildFlavor, flavor) client.buildConfig.buildFlavor = newFlavor newFlavors = [] for oldFlavor in client.buildConfig.flavor: newFlavors.append(deps.overrideFlavor(oldFlavor, flavor)) client.buildConfig.flavor = newFlavors matchSpecs = argSet.pop('match', []) hosts = argSet.pop('host', []) labels = argSet.pop('label', []) recurseGroups = argSet.pop('recurse', False) or command == 'buildgroup' if recurseGroups: if argSet.pop('binary-search', False): recurseGroups = client.BUILD_RECURSE_GROUPS_BINARY elif not compat.ConaryVersion().supportsFindGroupSources(): log.warning('Your conary does not support recursing a group' ' source component, defaulting to searching the' ' binary version') recurseGroups = client.BUILD_RECURSE_GROUPS_BINARY else: recurseGroups = client.BUILD_RECURSE_GROUPS_SOURCE self._prep(client, argSet) job = client.createBuildJob(troveSpecs, limitToHosts=hosts, limitToLabels=labels, recurseGroups=recurseGroups, matchSpecs=matchSpecs, rebuild=rebuild) return self._build(client, job, argSet)
class RmakeBuildContext(cfg.ConfigSection): _cfg_bases = [conarycfg.ConaryContext] bootstrapTroves = ( CfgList(CfgTroveSpec), [], "INTERNAL USE ONLY: Troves to be installed before the remaining " "chroot contents.") copyInConary = (CfgBool, False) copyInConfig = (CfgBool, False) # RMK-1052 rbuilderUrl = (cfgtypes.CfgString, 'https://localhost/') rmakeUser = (CfgUser, None) if not compat.ConaryVersion().supportsDefaultBuildReqs(): defaultBuildReqs = (CfgList(CfgString), [ 'bash:runtime', 'coreutils:runtime', 'filesystem', 'conary:runtime', 'conary-build:runtime', 'dev:runtime', 'grep:runtime', 'sed:runtime', 'findutils:runtime', 'gawk:runtime', ]) else: defaultBuildReqs = (CfgList(CfgString), []) resolveTroves = (CfgList(CfgQuotedLineList(CfgTroveSpec)), [[('group-dist', None, None)]]) matchTroveRule = (CfgList(CfgString), []) resolveTrovesOnly = (CfgBool, False) reuseRoots = (CfgBool, False) rpmRequirements = ( CfgList(CfgDependency), [], "INTERNAL USE ONLY: Dep provided by the RPM to be used for " "installation of the chroot.") strictMode = (CfgBool, False) targetLabel = (CfgLabel, versions.Label('NONE@local:NONE')) uuid = (CfgUUID, '') # DEPRECATED subscribe = CfgDict(CfgString) def __init__(self, parent, doc=None): cfg.ConfigSection.__init__(self, parent, doc=None) if hasattr(self, 'addConfigOption'): # Conary < 2.5 for info in conarycfg.ConaryContext._getConfigOptions(): if info[0] not in self: self.addConfigOption(*info)
def testGroupRecipe(self): if compat.ConaryVersion().conaryVersion[0:2] == [1,2]: raise testhelp.SkipTestException('test fails on 1.2') repos = self.openRmakeRepository() db = self.openRmakeDatabase() self.buildCfg.shortenGroupFlavors = True self.buildCfg.setSection('foo') # add context foo self.buildCfg.configLine('buildFlavor desktop is:x86') self.buildCfg.setSection('bar') self.buildCfg.configLine('buildFlavor !desktop is:x86') simple = self.addComponent('simple:runtime', '1.0-1-1', '') other1 = self.addComponent('other:runtime', '1.0-1-1', '!desktop', filePrimer=1) other2 = self.addComponent('other:runtime', '1.0-1-1', 'desktop', filePrimer=1) # Prevent the desktop flag from being pre-filtered out recipe = groupRecipe + '\n if Use.desktop: pass\n' trv = self.addComponent('group-foo:source', '/localhost@rpl:linux//rmakehost@local:linux/1:1.0-1', [('group-foo.recipe', recipe)]) troveList = [ trv.getNameVersionFlavor() + ('foo',), trv.getNameVersionFlavor() + ('bar',), ] job = self.newJob(*troveList) b = builder.Builder(self.rmakeCfg, job) logFile = logfile.LogFile(self.workDir + '/buildlog') logFile.redirectOutput() b.build() logFile.restoreOutput() assert b.job.isBuilt(), str(b.job.getFailureReason()) assert len(b.job.getBuiltTroveList()) == 2, b.job.getBuiltTroveList() for tup in b.job.getBuiltTroveList(): groupTrove = repos.getTrove(*tup) # this is just a very basic test of builder -> group build. # tests of the group cook code's ability to include the right # version in particular cases should be in cooktest.py if '!desktop' in str(groupTrove.getFlavor()): other = other1 else: other = other2 self.assertEqual(sorted(groupTrove.iterTroveList(strongRefs=True)), [other.getNameVersionFlavor(), simple.getNameVersionFlavor()])
def checkSanity(self): if self.copyInConary: # we're just overriding the version of conary used # as long as that't the only sanity check we can return # immediately return for job in self.jobList: if job[0] == 'conary:python': version = job[2][0].trailingRevision().getVersion() try: compat.ConaryVersion(version).checkRequiredVersion() except errors.RmakeError, error: errorMsg = str(error) + ( ' - tried to install version %s in chroot' % version) raise error.__class__(errorMsg)
def main(argv): log.setVerbosity(log.WARNING) try: argv = list(argv) debugAll = '--debug-all' in argv if debugAll: debuggerException = Exception argv.remove('--debug-all') else: debuggerException = errors.RmakeInternalError sys.excepthook = errors.genExcepthook(debug=debugAll, debugCtrlC=debugAll) compat.ConaryVersion().checkRequiredVersion() return RmakeMain().main(argv, debuggerException=debuggerException) except debuggerException, err: raise
def _old_commit(helper, job): ''' Commit a job to the target repository. @param job: rMake job ''' log.info('Starting commit') _start_time = time.time() okay, changeset, nbf_map = clone_job(helper, job) # Sanity check the resulting changeset and produce a mapping of # committed versions mapping = {job.jobId: {}} if okay: for trove in iter_new_troves(changeset, helper): # Make sure there are no references to the internal repos. for _, child_version, _ in trove.iterTroveList( strongRefs=True, weakRefs=True): assert child_version.getHost() \ != helper.cfg.reposName, \ "Trove %s references repository" % trove #assert not child_name.endswith(':testinfo'), \ # "Trove %s references :testinfo component" % trove trove_name, trove_version, trove_flavor = \ trove.getNameVersionFlavor() trove_branch = trove_version.branch() trove, _ = nbf_map[(trove_name, trove_branch, trove_flavor)] trove_nvfc = trove.getNameVersionFlavor(withContext=True) # map jobId -> trove -> binaries mapping[trove.jobId].setdefault(trove_nvfc, []).append( (trove_name, trove_version, trove_flavor)) else: raise RuntimeError('failed to clone finished build') if compat.ConaryVersion().signAfterPromote(): changeset = cook.signAbsoluteChangeset(changeset) helper.getRepos().commitChangeSet(changeset) _finish_time = time.time() log.info('Commit took %.03f seconds', _finish_time - _start_time) return mapping
def updateRecipes(repos, cfg, recipeList, committedSources): committedSourcesByNB = {} for name, version, flavor in committedSources: committedSourcesByNB[name, version.branch()] = version for recipe in recipeList: recipeDir = os.path.dirname(recipe) stateFilePath = recipeDir + '/CONARY' if not os.path.exists(stateFilePath): continue conaryStateFile = state.ConaryStateFromFile(stateFilePath) if not conaryStateFile.hasSourceState(): continue context = conaryStateFile.getContext() stateFile = conaryStateFile.getSourceState() troveName = stateFile.getName() branch = stateFile.getBranch() if (troveName, branch) not in committedSourcesByNB: continue stateVersion = stateFile.getVersion() newVersion = committedSourcesByNB[troveName, branch] if stateVersion != versions.NewVersion(): log.info('Updating %s after commit' % recipeDir) if compat.ConaryVersion().updateSrcTakesMultipleVersions(): try: # Added in CNY-3035 checkin.nologUpdateSrc(repos, [recipeDir]) except checkin.builderrors.UpToDate: pass # Don't mention if the source is already up to date except checkin.builderrors.CheckinError, e: e.logError() except AttributeError: checkin.updateSrc(repos, [recipeDir]) else: curDir = os.getcwd() try: os.chdir(recipeDir) checkin.updateSrc(repos) finally: os.chdir(curDir)
def _shadowAndCommit(conaryclient, cfg, recipeDir, stateFile, message): repos = conaryclient.getRepos() conaryCompat = compat.ConaryVersion() oldSourceVersion = stateFile.getVersion() targetLabel = cfg.getTargetLabel(oldSourceVersion) if not targetLabel: raise errors.RmakeError( 'Cannot cook local recipes unless a target label is set') skipped, cs = conaryclient.createShadowChangeSet(str(targetLabel), [stateFile.getNameVersionFlavor()]) recipePath = recipeDir + '/' + stateFile.getName().split(':')[0] + '.recipe' recipeClass, pathList = _getPathList(repos, cfg, recipePath, relative=True) troveName = stateFile.getName() troveVersion = stateFile.getVersion() if not skipped: signAbsoluteChangeset(cs, None) repos.commitChangeSet(cs) log.info("Shadowing %s to internal repository..." % troveName) shadowBranch = troveVersion.createShadow(targetLabel).branch() shadowVersion = repos.findTrove(None, (troveName, str(shadowBranch), None), None)[0][1] cwd = os.getcwd() prefix = 'rmake-shadow-%s-' % troveName.split(':')[0] shadowSourceDir = tempfile.mkdtemp(prefix=prefix) try: log.info("Committing local changes to %s to the" " internal repository..." % troveName) log.resetErrorOccurred() checkin.checkout(repos, cfg, shadowSourceDir, ['%s=%s' % (troveName, shadowVersion)]) if compat.ConaryVersion().stateFileVersion() > 0: kw = dict(repos=repos) else: kw = {} # grab new and old state and make any modifications due to adding # or deleting of files (we assume files that don't exist are # autosource and can be ignored) oldState = conaryCompat.ConaryStateFromFile(recipeDir + '/CONARY', repos=repos).getSourceState() newConaryState = conaryCompat.ConaryStateFromFile( shadowSourceDir + '/CONARY', repos=repos) newState = newConaryState.getSourceState() neededFiles = set(x[1] for x in oldState.iterFileList() if os.path.exists(os.path.join(recipeDir, x[1]))) neededFiles.update(pathList) autoSourceFiles = set(x[1] for x in oldState.iterFileList() if oldState.fileIsAutoSource(x[0])) existingFiles = set(x[1] for x in newState.iterFileList() if os.path.exists(os.path.join(shadowSourceDir, x[1]))) toCopy = neededFiles & existingFiles toDel = existingFiles - neededFiles toAdd = neededFiles - existingFiles for sourceFile in (toCopy | toAdd): newPath = os.path.join(shadowSourceDir, sourceFile) if os.path.dirname(sourceFile): util.mkdirChain(os.path.dirname(newPath)) if os.path.isdir(sourceFile): util.mkdirChain(newPath) else: shutil.copyfile(os.path.join(recipeDir, sourceFile), newPath) os.chdir(shadowSourceDir) if hasattr(cfg.sourceSearchDir, '_getUnexpanded'): cfg.configKey('sourceSearchDir', cfg.sourceSearchDir._getUnexpanded()) for f in toDel: checkin.removeFile(f) if toDel: # toDel modifies the CONARY file on disk, so reload with the # changes made there. newState = conaryCompat.ConaryStateFromFile( shadowSourceDir + '/CONARY', repos=repos).getSourceState() if conaryCompat.stateFileVersion() == 0: checkin.addFiles(toAdd) else: oldPathIds = dict((x[1], x[0]) for x in oldState.iterFileList()) for path in toAdd: if path in oldPathIds: isConfig = oldState.fileIsConfig(oldPathIds[path]) else: isConfig = _getConfigInfo(path) checkin.addFiles([path], binary=not isConfig, text=isConfig) if toAdd: # get the new pathIDs for all the added troves, # since we can't set the refresh setting without the # needed pathIds newState = conaryCompat.ConaryStateFromFile( shadowSourceDir + '/CONARY', repos=repos).getSourceState() newPathIds = dict((x[1], x[0]) for x in newState.iterFileList()) for path in (toCopy | toAdd): if path in oldPathIds: isConfig = oldState.fileIsConfig(oldPathIds[path]) else: isConfig = _getConfigInfo(path) newState.fileIsConfig(newPathIds[path], isConfig) for path in autoSourceFiles: if path in newPathIds: needsRefresh = oldState.fileNeedsRefresh(oldPathIds[path]) newState.fileNeedsRefresh(newPathIds[path], needsRefresh) # if the factory changed, update it if newState.getFactory() != oldState.getFactory(): newState.setFactory(oldState.getFactory()) # we may have modified the state file. Write it back out to # disk so it will be picked up by the commit. newConaryState.setSourceState(newState) newConaryState.write(shadowSourceDir + '/CONARY') if message is None and compat.ConaryVersion().supportsCloneCallback(): message = 'Automated rMake commit' _doCommit('%s/%s' % (recipeDir, troveName), repos, cfg, message) newState = state.ConaryStateFromFile(shadowSourceDir + '/CONARY', **kw) return newState.getSourceState().getNameVersionFlavor() finally: os.chdir(cwd) if hasattr(cfg.sourceSearchDir, '_getUnexpanded'): cfg.configKey('sourceSearchDir', cfg.sourceSearchDir._getUnexpanded()) shutil.rmtree(shadowSourceDir)
try: os.chdir('/tmp') # make sure we're in a directory # that we can write to. Although # this _shouldn't_ be an issue, # conary 1.0.{19,20} require it. # finally actually cook the recipe! groupOptions = cook.GroupCookOptions( alwaysBumpCount=False, shortenFlavors=cfg.shortenGroupFlavors, errorOnFlavorChange=False) m = macros.Macros() m._override('buildlabel', str(buildLabel)) m._override('buildbranch', str(buildBranch)) m._override('binarybranch', str(binaryBranch)) toCook = compat.ConaryVersion().getObjectsToCook( loaders, recipeClasses) built = cook.cookObject(repos, cfg, toCook, version, prep=False, macros=m, targetLabel=targetLabel, changeSetFile=csFile, alwaysBumpCount=False, ignoreDeps=False, logBuild=True, crossCompile=crossCompile, requireCleanSources=False, groupOptions=groupOptions) if not built:
def _commitRecipe(conaryclient, cfg, recipePath, message, branch=None): repos = conaryclient.getRepos() conaryCompat = compat.ConaryVersion() recipeClass, pathList = _getPathList(repos, cfg, recipePath) sourceName = recipeClass.name + ':source' log.info("Creating a copy of %s in the rMake internal repository..." % recipeClass.name) cwd = os.getcwd() recipeDir = tempfile.mkdtemp() log.resetErrorOccurred() try: fileNames = [] # Create a source trove that matches the recipe we're trying to cook if not branch: branch = versions.Branch([cfg.buildLabel]) targetLabel = cfg.getTargetLabel(branch) if compat.ConaryVersion().supportsNewPkgBranch(): buildBranch = branch.createShadow(targetLabel) kw = dict(buildBranch=buildBranch) else: buildBranch = versions.Branch([targetLabel]) kw={} cfg.buildLabel = targetLabel if not repos.getTroveLeavesByBranch( { sourceName : { buildBranch : None } }).get(sourceName, None): # we pass it None for repos to avoid the label-based check for # existing packages. checkin.newTrove(None, cfg, recipeClass.name, dir=recipeDir, **kw) else: # see if this package exists on our build branch checkin.checkout(repos, cfg, recipeDir, ['%s=%s' % (sourceName, buildBranch)]) os.chdir(recipeDir) sourceState = state.ConaryStateFromFile(recipeDir + '/CONARY').getSourceState() fileNames = dict((os.path.basename(x), x) for x in pathList) for (pathId, baseName, fileId, version) in list(sourceState.iterFileList()): # update or remove any currently existing files if baseName not in fileNames: sourceState.removeFilePath(baseName) else: shutil.copyfile(fileNames[baseName], os.path.join(recipeDir, baseName)) del fileNames[baseName] for baseName, path in fileNames.iteritems(): shutil.copyfile(path, os.path.join(recipeDir, baseName)) if conaryCompat.stateFileVersion() > 0: # mark all the files as binary - this this version can # never be checked in, it doesn't really matter, but # conary likes us to give a value. for fileName in fileNames: isConfig = _getConfigInfo(fileName) checkin.addFiles([fileName], binary=not isConfig, text=isConfig) else: checkin.addFiles(fileNames) _doCommit(recipePath, repos, cfg, 'Temporary recipe build for rmake') newState = conaryCompat.ConaryStateFromFile(recipeDir + '/CONARY', repos=repos) return newState.getSourceState().getNameVersionFlavor() finally: os.chdir(cwd) shutil.rmtree(recipeDir)
def startRepository(cfg, fork=True, logger=None): global conaryDir baseDir = cfg.serverDir if logger is None: logger = log reposDir = '%s/repos' % baseDir util.mkdirChain(reposDir) if not cfg.reposUser: passwordFile = reposDir + '/password' if os.path.exists(passwordFile): password = open(passwordFile).readline()[:-1] else: password = ''.join( [chr(random.randrange(ord('a'), ord('z'))) for x in range(10)]) open(passwordFile, 'w').write(password + '\n') os.chmod(reposDir + '/password', 0700) cfg.reposUser.addServerGlob(cfg.reposName, 'rmake', password) serverConfig = os.path.join(cfg.getReposDir(), 'serverrc') if os.path.exists(serverConfig): os.unlink(serverConfig) serverCfg = server.ServerConfig(os.path.join(cfg.getReposDir(), 'serverrc')) serverCfg.repositoryDB = ('sqlite', cfg.getReposDbPath()) serverCfg.contentsDir = cfg.getContentsPath() serverCfg.port = cfg.getReposInfo()[1] serverCfg.configKey('serverName', cfg.reposName) # this works with either # 1.0.16 or 1.0.17+ serverCfg.logFile = cfg.getReposDir() + '/repos.log' serverCfg.logFile = None # Transfer SSL settings from rMake config object if hasattr(server, 'SSL') and server.SSL: # The server supports starting in SSL mode serverCfg.useSSL = cfg.reposRequiresSsl() serverCfg.sslCert = cfg.sslCertPath serverCfg.sslKey = cfg.sslCertPath elif cfg.reposRequiresSsl(): raise errors.RmakeError( 'Tried to start repository at %s, but missing ssl server library: Please install m2crypto' % (cfg.getRepositoryUrl(), )) (driver, database) = serverCfg.repositoryDB db = dbstore.connect(database, driver) # Note - this will automatically migrate this repository! # Since this is a throwaway repos anyway, I think that's # acceptable. compat.ConaryVersion().loadServerSchema(db) db.commit() db.close() user, password = cfg.reposUser.find(cfg.reposName) addUser(serverCfg, user, password, write=True) if not serverCfg.useSSL: # allow anonymous access if we're not securing this repos # by using SSL - no use securing access if it's all going to be # viewable via tcpdump. addUser(serverCfg, 'anonymous', 'anonymous') if fork: pid = os.fork() if pid: try: pingServer(cfg) except: killServer(pid) raise logger.info('Started repository "%s" on port %s (pid %s)' % (cfg.reposName, serverCfg.port, pid)) return pid elif hasattr(logger, 'close'): logger.close() try: os.chdir(cfg.getReposDir()) serverrc = open(cfg.getReposConfigPath(), 'w') serverCfg.store(serverrc, includeDocs=False) util.mkdirChain(os.path.dirname(cfg.getReposLogPath())) logFile = logfile.LogFile(cfg.getReposLogPath()) logFile.redirectOutput(close=True) serverrc.close() os.execv('%s/server/server.py' % conaryDir, [ '%s/server/server.py' % conaryDir, '--config-file', cfg.getReposConfigPath() ]) except Exception, err: print >> sys.stderr, 'Could not start repository server: %s' % err os._exit(1)
def getRecipeObj(repos, name, version, flavor, trv, loadInstalledSource=None, installLabelPath=None, loadRecipeSpecs=None, buildLabel=None, groupRecipeSource=None, cfg=None): if cfg: cfg = copy.deepcopy(cfg) else: cfg = conarycfg.ConaryConfiguration(False) cfg.initializeFlavors() branch = version.branch() if not buildLabel: buildLabel = version.branch().label() if not installLabelPath: cfg.installLabelPath = [buildLabel] else: cfg.installLabelPath = installLabelPath cfg.buildFlavor = flavor cfg.defaultBasePackages = [] name = name.split(':')[0] use.LocalFlags._clear() assert (flavorutil.getArch(flavor)) use.setBuildFlagsFromFlavor(name, flavor, error=False) use.resetUsed() use.track(True) ignoreInstalled = not loadInstalledSource macros = { 'buildlabel': buildLabel.asString(), 'buildbranch': version.branch().asString() } cfg.lookaside = tempfile.mkdtemp() try: loader = RecipeLoaderFromSourceTrove(trv, repos, cfg, name + ':source', branch, ignoreInstalled=ignoreInstalled, db=loadInstalledSource, buildFlavor=flavor) recipeClass = loader.getRecipe() recipeClass._trove = trv if recipe.isGroupRecipe(recipeClass): recipeObj = recipeClass(repos, cfg, buildLabel, flavor, None, extraMacros=macros) recipeObj.sourceVersion = version recipeObj.loadPolicy() recipeObj.setup() elif (recipe.isPackageRecipe(recipeClass) or recipe.isFactoryRecipe(recipeClass) or recipe.isCapsuleRecipe(recipeClass)): if recipe.isFactoryRecipe(recipeClass): #This requires a specific capability in conary compat.ConaryVersion().requireFactoryRecipeGeneration() #Load the FactoryRecipe factoryClass = recipeClass loaded = cook.loadFactoryRecipe(factoryClass, cfg, repos, flavor) recipeClass = loaded.getRecipe() lcache = lookaside.RepositoryCache(repos) recipeObj = recipeClass(cfg, lcache, [], macros, lightInstance=True) recipeObj.sourceVersion = version recipeObj.populateLcache() if not recipeObj.needsCrossFlags(): recipeObj.crossRequires = [] recipeObj.loadPolicy() recipeObj.setup() elif recipe.isInfoRecipe(recipeClass): recipeObj = recipeClass(cfg, None, None, macros) recipeObj.sourceVersion = version recipeObj.setup() elif recipe.isRedirectRecipe(recipeClass): binaryBranch = version.getBinaryVersion().branch() recipeObj = recipeClass(repos, cfg, binaryBranch, flavor) recipeObj.sourceVersion = version recipeObj.setup() elif recipe.isFileSetRecipe(recipeClass): recipeObj = recipeClass(repos, cfg, buildLabel, flavor, extraMacros=macros) recipeObj.sourceVersion = version recipeObj.setup() else: raise RuntimeError, 'Unknown class type %s for recipe %s' % ( recipeClass, name) finally: util.rmtree(cfg.lookaside) return recipeObj, loader
def commitJobs(conaryclient, jobList, reposName, message=None, commitOutdatedSources=False, sourceOnly=False, excludeSpecs=None, writeToFile=None): jobsToCommit = {} alreadyCommitted = [] finalCs = changeset.ReadOnlyChangeSet() mapping = {} for job in jobList: if job.isCommitted(): alreadyCommitted.append(job) else: jobsToCommit[job.jobId] = job jobsToCommit = jobsToCommit.values() # dedup job list if not jobsToCommit: err = 'Job(s) already committed' return False, err allTroves = [] trovesByBranch = {} alreadyCommitted = False for job in jobsToCommit: mapping[job.jobId] = {} for trove in job.iterTroves(): allTroves.append(trove) troveVersion = trove.getVersion() if troveVersion.getHost() == reposName: if not troveVersion.branch().hasParentBranch(): message = ('Cannot commit filesystem cook %s - ' ' nowhere to commit to!' % trove.getName()) return False, message assert (allTroves) source = trovesource.SimpleTroveSource() if excludeSpecs: excludeSpecsWithContext = {} troveMap = {} for excludeSpec in excludeSpecs: if len(excludeSpec) == 4: context = excludeSpec[3] else: context = None excludeSpecsWithContext.setdefault(excludeSpec[:3], []).append(context) excludeSpecs = [x[:3] for x in excludeSpecs] for trove in allTroves: troveTup = (trove.getName().split(':')[0], trove.getVersion(), trove.getFlavor()) source.addTrove(*troveTup) troveMap.setdefault(troveTup, []).append(trove) source.searchAsDatabase() matches = source.findTroves(None, excludeSpecs, None, allowMissing=True) trvMatches = [] for excludeSpec, matchList in matches.iteritems(): contexts = excludeSpecsWithContext[excludeSpec] for match in matchList: for trv in troveMap[match]: if trv.context in contexts or None in contexts: trvMatches.append(trv) allTroves = [x for x in allTroves if x not in trvMatches] if not allTroves: message = ('All troves excluded - not committing') return False, message repos = conaryclient.getRepos() trovesByNBF = {} sourcesToCheck = [] branchMap = {} trovesToClone = [] for trove in allTroves: builtTroves = list(trove.iterBuiltTroves()) if not builtTroves: continue if builtTroves[0][1].getHost() != reposName: alreadyCommitted = True for n, v, f in builtTroves: trovesByNBF[n, v.branch(), f] = (trove, v) continue troveVersion = trove.getVersion() if troveVersion.getHost() == reposName: sourceTup = (trove.getName(), troveVersion, Flavor()) targetBranch = troveVersion.branch().parentBranch() branchMap[troveVersion.branch()] = targetBranch nbf = trove.getName(), targetBranch, Flavor() if nbf in trovesByNBF: if trovesByNBF[nbf][1] != troveVersion: badVersion = trovesByNBF[nbf][1] return False, ( "Cannot commit two different versions of source component %s:" " %s and %s" % (trove.getName(), troveVersion, badVersion)) trovesByNBF[nbf] = trove, troveVersion sourcesToCheck.append(sourceTup) if sourceOnly: continue for troveTup in builtTroves: branch = troveTup[1].branch() targetBranch = branch.parentBranch() # add mapping so that when the cloning is done # we can tell what commit resulted in what binaries. nbf = (troveTup[0], targetBranch, troveTup[2]) if nbf in trovesByNBF: otherBinary = trovesByNBF[nbf][0].getBinaryTroves()[0] if otherBinary[1].branch() == targetBranch: # this one's already committed. break # discard the later of the two commits. if trovesByNBF[nbf][0].getVersion() > trove.getVersion(): # we're the earlier one badTrove, badVersion = trovesByNBF[nbf] newTrove = trove newVersion = troveTup[1] else: badTrove = trove badVersion = troveTup[1] newTrove, newVersion = trovesByNBF[nbf] name = nbf[0] flavor = nbf[2] skipped = [] for badTroveTup in badTrove.iterBuiltTroves(): badNbf = (badTroveTup[0], targetBranch, badTroveTup[2]) if not ':' in badTroveTup[0]: skipped.append(badTroveTup[0]) if badNbf in trovesByNBF and badTrove is trovesByNBF[ badNbf][0]: del trovesByNBF[badNbf] skipped = '%s' % (', '.join(skipped)) log.warning("Not committing %s on %s[%s]%s - overridden by" " %s[%s]%s" % (skipped, badTroveTup[1], badTroveTup[2], badTrove.getContextStr(), newVersion, flavor, newTrove.getContextStr())) if trove is badTrove: break trovesByNBF[nbf] = trove, troveTup[1] branchMap[branch] = targetBranch for nbf, (trove, tupVersion) in trovesByNBF.items(): if tupVersion.branch() != nbf[1]: trovesToClone.append((nbf[0], tupVersion, nbf[2])) if not trovesToClone: if sourceOnly: err = 'Could not find sources to commit' elif alreadyCommitted: log.warning('All built troves have already been committed') return True, {} else: err = 'Can only commit built troves, none found' return False, err if sourcesToCheck and not commitOutdatedSources: outdated = _checkOutdatedSources(repos, sourcesToCheck) if outdated: outdated = ( '%s=%s (replaced by newer %s)' \ % (name, builtVer, newVer.trailingRevision()) for (name, builtVer, newVer) in outdated) err = ('The following source troves are out of date:\n%s\n\n' 'Use --commit-outdated-sources to commit anyway' % '\n'.join(outdated)) return False, err # only update build info if we'll be okay if some buildreqs are not # updated updateBuildInfo = compat.ConaryVersion().acceptsPartialBuildReqCloning() callback = callbacks.CloneCallback(conaryclient.cfg, message) passed, cs = conaryclient.createTargetedCloneChangeSet( branchMap, trovesToClone, updateBuildInfo=updateBuildInfo, cloneSources=False, trackClone=False, callback=callback, fullRecurse=False) if passed: oldTroves = [] for troveCs in cs.iterNewTroveList(): if troveCs.getOldVersion(): oldTroves.append(troveCs.getOldNameVersionFlavor()) if oldTroves: oldDict = {} for oldTrove in repos.getTroves(oldTroves): oldDict.setdefault(oldTrove.getNameVersionFlavor(), []).append(oldTrove) for troveCs in cs.iterNewTroveList(): if troveCs.getOldVersion(): trv = oldDict[troveCs.getOldNameVersionFlavor()].pop() trv.applyChangeSet(troveCs) else: trv = Trove(troveCs) for _, childVersion, _ in trv.iterTroveList(strongRefs=True, weakRefs=True): # make sure there are not any references to the internal # rmake repository - that would be a bad bug - easy to # do with the way we do cooking of groups. onRepos = childVersion.getHost() == reposName assert not onRepos, "Trove %s references repository" % trv n, v, f = troveCs.getNewNameVersionFlavor() trove, troveVersion = trovesByNBF[n, v.branch(), f] troveNVFC = trove.getNameVersionFlavor(withContext=True) # map jobId -> trove -> binaries mapping[trove.jobId].setdefault(troveNVFC, []).append((n, v, f)) else: return False, 'Creating clone failed' signatureKey = conaryclient.cfg.signatureKey if signatureKey and compat.ConaryVersion().signAfterPromote(): finalCs = signAbsoluteChangeset(cs, signatureKey) if writeToFile: cs.writeToFile(writeToFile) else: repos.commitChangeSet(cs, callback=callback) return True, mapping
def getTrovesToBuild(cfg, conaryclient, troveSpecList, message=None, recurseGroups=BUILD_RECURSE_GROUPS_NONE, matchSpecs=None, reposName=None, updateSpecs=None): toBuild = [] toFind = {} groupsToFind = [] if not matchSpecs: matchSpecs = [] if reposName is None: reposName = cfg.reposName repos = conaryclient.getRepos() cfg.resolveTroveTups = _getResolveTroveTups(cfg, repos) cfg.recurseGroups = int(recurseGroups) cfg.buildTroveSpecs = [] newTroveSpecs = [] recipesToCook = [] for troveSpec in list(troveSpecList): if not isinstance(troveSpec, tuple): troveSpec = cmdutil.parseTroveSpec(troveSpec) if len(troveSpec) == 3: context = '' else: context = troveSpec[3] troveSpec = troveSpec[:3] if (troveSpec[0].startswith('group-') and not recurseGroups and not compat.ConaryVersion().supportsCloneNonRecursive()): log.warning('You will not be able to commit this group build' ' without upgrading conary.') if troveSpec[2] is None: troveSpec = (troveSpec[0], troveSpec[1], deps.parseFlavor('')) if (not troveSpec[1] and not os.path.isdir(troveSpec[0]) and os.access(troveSpec[0], os.R_OK) and troveSpec[0].endswith('.recipe')): # don't rely on cwd, but do allow for symlinks to change # when restarting. Is that sane? Or should I just do realpath? troveSpec = (os.path.abspath(troveSpec[0]),) + troveSpec[1:] cfg.buildTroveSpecs.append(troveSpec) recipesToCook.append((os.path.realpath(troveSpec[0]), troveSpec[2])) continue cfg.buildTroveSpecs.append(troveSpec) if troveSpec[0].startswith('group-') and recurseGroups: groupsToFind.append(troveSpec) if recurseGroups == BUILD_RECURSE_GROUPS_SOURCE: newTroveSpecs.append(troveSpec) else: newTroveSpecs.append(troveSpec) localTroves = [(_getLocalCook(conaryclient, cfg, x[0], message), x[1]) for x in recipesToCook ] localTroves = [(x[0][0], x[0][1], x[1]) for x in localTroves] if recurseGroups == BUILD_RECURSE_GROUPS_SOURCE: compat.ConaryVersion().requireFindGroupSources() localGroupTroves = [ x for x in localTroves if x[0].startswith('group-') ] toBuild.extend(_findSourcesForSourceGroup(repos, reposName, cfg, groupsToFind, localGroupTroves, updateSpecs)) elif recurseGroups == BUILD_RECURSE_GROUPS_BINARY: newTroveSpecs.extend(_findSpecsForBinaryGroup(repos, reposName, cfg, groupsToFind, updateSpecs)) for troveSpec in newTroveSpecs: sourceName = troveSpec[0].split(':')[0] + ':source' s = toFind.setdefault((sourceName, troveSpec[1], None), []) if troveSpec[2] not in s: s.append(troveSpec[2]) results = repos.findTroves(cfg.buildLabel, toFind, None) for troveSpec, troveTups in results.iteritems(): flavorList = toFind[troveSpec] for troveTup in troveTups: for flavor in flavorList: toBuild.append((troveTup[0], troveTup[1], flavor)) toBuild.extend(localTroves) if matchSpecs: toBuild = _filterListByMatchSpecs(reposName, matchSpecs, toBuild) return toBuild
def clone_job(helper, job): ''' Create a changeset that will clone all built troves into the target label. ''' branch_map = {} # source_branch -> target_branch # nbf_map := name, target_branch, flavor -> trove, source_version # This maps a given name and built flavor back to the BuildTrove that # created it, allowing us to find duplicate builds and build a clone # job. nbf_map = {} for trove in job.iterTroves(): source_name, source_version, _ = trove.getNameVersionFlavor() #assert source_version.getHost() == helper.cfg.reposName # Determine which branch this will be committed to source_branch = origin_branch = source_version.branch() target_branch = origin_branch.createShadow( helper.plan.getTargetLabel()) # Mark said branch for final promote branch_map[source_branch] = target_branch # Check for different versions of the same trove nbf = source_name, target_branch, deps.Flavor() if nbf in nbf_map and nbf_map[nbf][1] != source_version: bad_version = nbf_map[nbf][1] raise RuntimeError("Cannot commit two different versions of " "source component %s: %s and %s" % (source_name, source_version, bad_version)) nbf_map[nbf] = trove, source_version # Add binary troves to mapping for bin_name, bin_version, bin_flavor in trove.iterBuiltTroves(): # Don't commit test info #if bin_name.endswith(':testinfo'): # continue nbf = bin_name, target_branch, bin_flavor if nbf in nbf_map: # Eliminate duplicate commits of the same NBF by keeping # only the newer package other_version = nbf_map[nbf][0].getBinaryTroves()[0][1] if other_version < bin_version: bad_trove, new_trove = nbf_map[nbf][0], trove new_version = bin_version else: new_trove, bad_trove = nbf_map[nbf][0], trove new_version = other_version new_name = new_trove.getName().split(':')[0] # Delete commit maps for the entire rejected package for bad_name, bad_version, bad_flavor \ in bad_trove.iterBuiltTroves(): bad_nbf = (bad_name, target_branch, bad_flavor) if not ':' in bad_name: log.warning('Not committing %s=%s[%s] - overridden ' 'by %s=%s', bad_name, bad_version, bad_flavor, new_name, new_version) if bad_nbf in nbf_map \ and bad_trove is nbf_map[bad_nbf][0]: log.debug('Purging %s=%s[%s]' % (bad_name, bad_version, bad_flavor)) del nbf_map[bad_nbf] # If this trove is the bad trove, stop processing it if trove is bad_trove: break nbf_map[nbf] = trove, bin_version # Determine what to clone troves_to_clone = [] for (trv_name, _, trv_flavor), (trove, trv_version) \ in nbf_map.iteritems(): troves_to_clone.append((trv_name, trv_version, trv_flavor)) # Do the clone update_build_info = compat.ConaryVersion()\ .acceptsPartialBuildReqCloning() callback = callbacks.CloneCallback(helper.cfg, helper.plan.commitMessage) okay, changeset = helper.getClient().createTargetedCloneChangeSet( branch_map, troves_to_clone, updateBuildInfo=update_build_info, cloneSources=False, trackClone=False, callback=callback, #cloneOnlyByDefaultTroves=True, fullRecurse=False) return okay, changeset, nbf_map