def reorderChangesets(f, csPath, initialSizes, maxisosize, isoblocksize, baseTrove): reservedTroves = [] sizedList = [] infoTroves = [] baseTroves = [] for line in f: csFile = line.split()[0] trvName = line.split()[1] if trvName in basicminimal: reservedTroves.append(line) spaceUsed = spaceused(join(csPath, csFile), isoblocksize) if trvName.startswith('info-'): infoTroves.append((spaceUsed, line)) elif trvName == baseTrove: baseTroves.append((spaceUsed, line)) else: sizedList.append((spaceUsed, line)) sizedList = [x for x in reversed(sorted(sizedList))] reservedList = [] if reservedTroves: trvNames = set() for trvLine in reservedTroves: trvNames.add(trvLine.split()[1]) cs = changeset.ChangeSetFromFile( \ join(csPath, trvLine.split()[0])) trv = Trove([x for x in cs.iterNewTroveList()][0]) for includedTrv in [x[0].split(':')[0] for x in \ trv.iterTroveList(strongRefs = True, weakRefs = True)]: trvNames.add(includedTrv) for size, line in sizedList[:]: if line.split()[1] in trvNames: reservedList.append((size, line)) sizedList.remove((size, line)) sizedList = infoTroves + reservedList + baseTroves + sizedList reOrdList = [[[], maxisosize - initialSizes[0]]] for size, line in sizedList: match = False for i, (changesets, limit) in enumerate(reOrdList): if maxisosize and (size <= limit): reOrdList[i][0].append(line) reOrdList[i][1] -= size match = True break if not match: reOrdList.append([[line], maxisosize - size - initialSizes[1]]) csList = [] for disc in [x[0] for x in reOrdList]: csList.extend(disc) return csList
def iter_new_troves(changeset, helper): ''' Take a changeset and yield trove objects corresponding to the new versions of all troves in that changeset. This involves fetching old troves and applying the changeset to them to produce the new troves. ''' # First get a list of all relative trove changesets old_troves = [] for trove_cs in changeset.iterNewTroveList(): if trove_cs.getOldVersion(): old_troves.append(trove_cs.getOldNameVersionFlavor()) # Now fetch trove objects corresponding to old versions old_dict = {} if old_troves: for old_trove in helper.getRepos().getTroves(old_troves): old_dict.setdefault(old_trove.getNameVersionFlavor(), []).append(old_trove) # Iterate over changeset again, yielding new trove objects for trove_cs in changeset.iterNewTroveList(): if trove_cs.getOldVersion(): trv = old_dict[trove_cs.getOldNameVersionFlavor()].pop() trv.applyChangeSet(trove_cs) yield trv else: yield Trove(trove_cs)
def _logDifferenceInPrebuiltReqs(self, trv, buildReqTups, preBuiltReqs): existsTrv = Trove('@update', versions.NewVersion(), deps.Flavor(), None) availableTrv = Trove('@update', versions.NewVersion(), deps.Flavor(), None) for troveNVF in preBuiltReqs: existsTrv.addTrove(*troveNVF) for troveNVF in buildReqTups: availableTrv.addTrove(*troveNVF) jobs = availableTrv.diff(existsTrv)[2] formatter = display.JobTupFormatter(affinityDb=None) formatter.dcfg.setTroveDisplay(fullVersions=True, fullFlavors=True, showComponents=True) formatter.dcfg.setJobDisplay(compressJobs=True) formatter.prepareJobLists([jobs]) self.logger.info('Could count %s=%s[%s]{%s} as prebuilt - the' ' following changes have been made in its' ' buildreqs:' % trv.getNameVersionFlavor( withContext=True)) for line in formatter.formatJobTups(jobs): self.logger.info(line) self.logger.info('...Rebuilding')
def _fetchOldChangeSets(self): """ Fetch old versions of each trove, where they can be found and are suitably sane. """ versionSpecs = [] latestSpecs = [] targetLabel = self.helper.plan.getTargetLabel() for package in self.packages: version = macro.expand(package.getBaseVersion(), package) versionSpecs.append((package.getName(), '%s/%s' % (targetLabel, version), None)) latestSpecs.append((package.getName(), str(targetLabel), None)) # Pick the new version for each package by querying all existing # versions (including markremoved ones) with the same version. results = self.helper.getRepos().findTroves(None, versionSpecs, allowMissing=True, getLeaves=False, troveTypes=trovesource.TROVE_QUERY_ALL) for package, (recipeText, recipeObj), query in zip( self.packages, self.recipes, versionSpecs): newVersion = _createVersion(package, self.helper, recipeObj.version) existingVersions = [x[1] for x in results.get(query, ())] while newVersion in existingVersions: newVersion.incrementSourceCount() package.nextVersion = newVersion # Grab the latest existing version so we can reuse autosources from it results = self.helper.getRepos().findTroves(None, latestSpecs, allowMissing=True) toGet = [] oldVersions = [] for package, query in zip(self.packages, latestSpecs): if not results.get(query): oldVersions.append(None) continue n, v, f = max(results[query]) toGet.append((n, (None, None), (v, f), True)) oldVersions.append((n, v, f)) self.oldChangeSet = self.helper.createChangeSet(toGet) for package, oldVersion in zip(self.packages, oldVersions): if oldVersion: trvCs = self.oldChangeSet.getNewTroveVersion(*oldVersion) self.oldTroves.append(Trove(trvCs)) else: self.oldTroves.append(None)
def _makeSourceTrove(package, helper): cs = ChangeSet() filesToAdd = {} ver = macro.expand(package.getBaseVersion(), package) version = _createVersion(package, helper, ver) latestSpec = (package.getName(), str(version.trailingLabel()), None) results = helper.getRepos().findTroves(None,[latestSpec],allowMissing=True, getLeaves=False,troveTypes=trovesource.TROVE_QUERY_ALL) if results: existingVersions = [x[1] for x in results.get(latestSpec, ())] while version in existingVersions: version.incrementSourceCount() new = Trove(package.name, version, deps.Flavor()) new.setFactory(package.targetConfig.factory) message = "Temporary Source for %s" % version message = message.rstrip() + "\n" new.changeChangeLog(ChangeLog(name=helper.cfg.name, contact=helper.cfg.contact, message=message)) for path, contents in package.recipeFiles.iteritems(): isText = path == package.getRecipeName() pathId = hashlib.md5(path).digest() fileHelper = filetypes.RegularFile(contents=contents, config=isText) fileStream = fileHelper.get(pathId) fileStream.flags.isSource(set=True) fileId = fileStream.fileId() fileVersion = new.getVersion() key = pathId + fileId filesToAdd[key] = (fileStream, fileHelper.contents, isText) new.addFile(pathId, path, fileVersion, fileId) new.invalidateDigests() new.computeDigests() for key, (fileObj, fileContents, cfgFile) in filesToAdd.items(): cs.addFileContents(fileObj.pathId(), fileObj.fileId(), ChangedFileTypes.file, fileContents, cfgFile) cs.addFile(None, fileObj.fileId(), fileObj.freeze()) cs.newTrove(new.diff(None, absolute=True)[0]) return new.getNameVersionFlavor(), cs
def _merge(self): changeSet = ChangeSet() deleteDirs = set() doCommit = False # If this is not None then all ephemeral sources will still be fetched # but will be placed in this directory instead. if self.helper.plan.ephemeralSourceDir: ephDir = self.helper.makeEphemeralDir() else: ephDir = None def _addFile(path, contents, isText): if path in oldFiles: # Always recycle pathId if available. pathId, _, oldFileId, oldFileVersion = oldFiles[path] else: pathId = hashlib.md5(path).digest() oldFileId = oldFileVersion = None fileHelper = filetypes.RegularFile(contents=contents, config=isText) fileStream = fileHelper.get(pathId) fileStream.flags.isSource(set=True) fileId = fileStream.fileId() # If the fileId matches, recycle the fileVersion too. if fileId == oldFileId: fileVersion = oldFileVersion else: fileVersion = newTrove.getVersion() filesToAdd[fileId] = (fileStream, fileHelper.contents, isText) newTrove.addFile(pathId, path, fileVersion, fileId) for package, (recipeText, recipeObj), oldTrove in zip( self.packages, self.recipes, self.oldTroves): filesToAdd = {} oldFiles = {} if oldTrove is not None: for pathId, path, fileId, fileVer in oldTrove.iterFileList(): oldFiles[path] = (pathId, path, fileId, fileVer) newTrove = Trove(package.name, package.nextVersion, deps.Flavor()) newTrove.setFactory(package.targetConfig.factory) # Add upstream files to new trove. Recycle pathids from the old # version. # LAZY: assume that everything other than the recipe is binary. # Conary has a magic module, but it only accepts filenames! for path, contents in package.recipeFiles.iteritems(): isText = path == package.getRecipeName() _addFile(path, contents, isText) # Collect requested auto sources from recipe. Unknown recipe types # will not be loaded so recipeObj will be the class, so assume # these have no sources. if not inspect.isclass(recipeObj): recipeFiles = dict((os.path.basename(x.getPath()), x) for x in recipeObj.getSourcePathList()) newFiles = set(x[1] for x in newTrove.iterFileList()) needFiles = set(recipeFiles) - newFiles for autoPath in needFiles: source = recipeFiles[autoPath] if getattr(source, 'contents', None ) and not source.sourcename: # Ignore trove scripts that have inline contents continue if not autoPath: log.error("bob does not support 'gussed' filenames; " "please use a full path for source '%s' in " "package %s", source.getPath(), package.name) raise RuntimeError("Unsupported source action") if (autoPath in oldFiles and not self.helper.plan.refreshSources and not source.ephemeral): # File exists in old version. pathId, path, fileId, fileVer = oldFiles[autoPath] newTrove.addFile(pathId, path, fileVer, fileId) continue if source.ephemeral and not ephDir: continue # File doesn't exist; need to create it. if source.ephemeral: laUrl = lookaside.laUrl(source.getPath()) tempDir = joinPaths(ephDir, os.path.dirname(laUrl.filePath())) mkdirChain(tempDir) else: tempDir = tempfile.mkdtemp() deleteDirs.add(tempDir) snapshot = _getSnapshot(self.helper, package, source, tempDir) if not source.ephemeral and snapshot: autoPathId = hashlib.md5(autoPath).digest() autoObj = FileFromFilesystem(snapshot, autoPathId) autoObj.flags.isAutoSource(set=True) autoObj.flags.isSource(set=True) autoFileId = autoObj.fileId() autoContents = filecontents.FromFilesystem(snapshot) filesToAdd[autoFileId] = (autoObj, autoContents, False) newTrove.addFile(autoPathId, autoPath, newTrove.getVersion(), autoFileId) # If the old and new troves are identical, just use the old one. if oldTrove and _sourcesIdentical( oldTrove, newTrove, [self.oldChangeSet, filesToAdd]): package.setDownstreamVersion(oldTrove.getVersion()) log.debug('Skipped %s=%s', oldTrove.getName(), oldTrove.getVersion()) continue # Add files and contents to changeset. for fileId, (fileObj, fileContents, cfgFile) in filesToAdd.items(): changeSet.addFileContents(fileObj.pathId(), fileObj.fileId(), ChangedFileTypes.file, fileContents, cfgFile) changeSet.addFile(None, fileObj.fileId(), fileObj.freeze()) # Create a changelog entry. changeLog = ChangeLog( name=self.helper.cfg.name, contact=self.helper.cfg.contact, message=self.helper.plan.commitMessage + '\n') newTrove.changeChangeLog(changeLog) # Calculate trove digests and add the trove to the changeset newTrove.invalidateDigests() newTrove.computeDigests() newTroveCs = newTrove.diff(None, absolute=True)[0] changeSet.newTrove(newTroveCs) doCommit = True package.setDownstreamVersion(newTrove.getVersion()) log.debug('Created %s=%s', newTrove.getName(), newTrove.getVersion()) if doCommit: cook.signAbsoluteChangesetByConfig(changeSet, self.helper.cfg) f = tempfile.NamedTemporaryFile(dir=os.getcwd(), suffix='.ccs', delete=False) f.close() changeSet.writeToFile(f.name) try: self.helper.getRepos().commitChangeSet(changeSet) except: log.error("Error committing changeset to repository, " "failed changeset is saved at %s", f.name) raise else: os.unlink(f.name) for path in deleteDirs: shutil.rmtree(path)
def commitJobs(conaryclient, jobList, reposName, message=None, commitOutdatedSources=False, sourceOnly = False, excludeSpecs=None, writeToFile=None): jobsToCommit = {} alreadyCommitted = [] finalCs = changeset.ReadOnlyChangeSet() mapping = {} for job in jobList: if job.isCommitted(): alreadyCommitted.append(job) else: jobsToCommit[job.jobId] = job jobsToCommit = jobsToCommit.values() # dedup job list if not jobsToCommit: err = 'Job(s) already committed' return False, err allTroves = [] trovesByBranch = {} alreadyCommitted = False for job in jobsToCommit: mapping[job.jobId] = {} for trove in job.iterTroves(): allTroves.append(trove) troveVersion = trove.getVersion() if troveVersion.getHost() == reposName: if not troveVersion.branch().hasParentBranch(): message = ('Cannot commit filesystem cook %s - ' ' nowhere to commit to!' % trove.getName()) return False, message assert(allTroves) source = trovesource.SimpleTroveSource() if excludeSpecs: excludeSpecsWithContext = {} troveMap = {} for excludeSpec in excludeSpecs: if len(excludeSpec) == 4: context = excludeSpec[3] else: context = None excludeSpecsWithContext.setdefault( excludeSpec[:3], []).append(context) excludeSpecs = [ x[:3] for x in excludeSpecs ] for trove in allTroves: troveTup = (trove.getName().split(':')[0], trove.getVersion(), trove.getFlavor()) source.addTrove(*troveTup) troveMap.setdefault(troveTup, []).append(trove) source.searchAsDatabase() matches = source.findTroves(None, excludeSpecs, None, allowMissing=True) trvMatches = [] for excludeSpec, matchList in matches.iteritems(): contexts = excludeSpecsWithContext[excludeSpec] for match in matchList: for trv in troveMap[match]: if trv.context in contexts or None in contexts: trvMatches.append(trv) allTroves = [ x for x in allTroves if x not in trvMatches ] if not allTroves: message = ('All troves excluded - not committing') return False, message repos = conaryclient.getRepos() trovesByNBF = {} sourcesToCheck = [] branchMap = {} trovesToClone = [] for trove in allTroves: builtTroves = list(trove.iterBuiltTroves()) if not builtTroves: continue if builtTroves[0][1].getHost() != reposName: alreadyCommitted = True for n,v,f in builtTroves: trovesByNBF[n, v.branch(), f] = (trove, v) continue troveVersion = trove.getVersion() if troveVersion.getHost() == reposName: sourceTup = (trove.getName(), troveVersion, Flavor()) targetBranch = troveVersion.branch().parentBranch() branchMap[troveVersion.branch()] = targetBranch nbf = trove.getName(), targetBranch, Flavor() if nbf in trovesByNBF: if trovesByNBF[nbf][1] != troveVersion: badVersion = trovesByNBF[nbf][1] return False, ("Cannot commit two different versions of source component %s:" " %s and %s" % (trove.getName(), troveVersion, badVersion)) trovesByNBF[nbf] = trove, troveVersion sourcesToCheck.append(sourceTup) if sourceOnly: continue for troveTup in builtTroves: branch = troveTup[1].branch() targetBranch = branch.parentBranch() # add mapping so that when the cloning is done # we can tell what commit resulted in what binaries. nbf = (troveTup[0], targetBranch, troveTup[2]) if nbf in trovesByNBF: otherBinary = trovesByNBF[nbf][0].getBinaryTroves()[0] if otherBinary[1].branch() == targetBranch: # this one's already committed. break # discard the later of the two commits. if trovesByNBF[nbf][0].getVersion() > trove.getVersion(): # we're the earlier one badTrove, badVersion = trovesByNBF[nbf] newTrove = trove newVersion = troveTup[1] else: badTrove = trove badVersion = troveTup[1] newTrove, newVersion = trovesByNBF[nbf] name = nbf[0] flavor = nbf[2] skipped = [] for badTroveTup in badTrove.iterBuiltTroves(): badNbf = (badTroveTup[0], targetBranch, badTroveTup[2]) if not ':' in badTroveTup[0]: skipped.append(badTroveTup[0]) if badNbf in trovesByNBF and badTrove is trovesByNBF[badNbf][0]: del trovesByNBF[badNbf] skipped = '%s' % (', '.join(skipped)) log.warning("Not committing %s on %s[%s]%s - overridden by" " %s[%s]%s" % (skipped, badTroveTup[1], badTroveTup[2], badTrove.getContextStr(), newVersion, flavor, newTrove.getContextStr())) if trove is badTrove: break trovesByNBF[nbf] = trove, troveTup[1] branchMap[branch] = targetBranch for nbf, (trove, tupVersion) in trovesByNBF.items(): if tupVersion.branch() != nbf[1]: trovesToClone.append((nbf[0], tupVersion, nbf[2])) if not trovesToClone: if sourceOnly: err = 'Could not find sources to commit' elif alreadyCommitted: log.warning('All built troves have already been committed') return True, {} else: err = 'Can only commit built troves, none found' return False, err if sourcesToCheck and not commitOutdatedSources: outdated = _checkOutdatedSources(repos, sourcesToCheck) if outdated: outdated = ( '%s=%s (replaced by newer %s)' \ % (name, builtVer, newVer.trailingRevision()) for (name, builtVer, newVer) in outdated) err = ('The following source troves are out of date:\n%s\n\n' 'Use --commit-outdated-sources to commit anyway' % '\n'.join(outdated)) return False, err # only update build info if we'll be okay if some buildreqs are not # updated callback = callbacks.CloneCallback(conaryclient.cfg, message) passed, cs = conaryclient.createTargetedCloneChangeSet( branchMap, trovesToClone, updateBuildInfo=True, cloneSources=False, trackClone=False, callback=callback, fullRecurse=False) if passed: oldTroves = [] for troveCs in cs.iterNewTroveList(): if troveCs.getOldVersion(): oldTroves.append(troveCs.getOldNameVersionFlavor()) if oldTroves: oldDict = {} for oldTrove in repos.getTroves(oldTroves): oldDict.setdefault(oldTrove.getNameVersionFlavor(), []).append(oldTrove) for troveCs in cs.iterNewTroveList(): if troveCs.getOldVersion(): trv = oldDict[troveCs.getOldNameVersionFlavor()].pop() trv.applyChangeSet(troveCs) else: trv = Trove(troveCs) for _, childVersion, _ in trv.iterTroveList(strongRefs=True, weakRefs=True): # make sure there are not any references to the internal # rmake repository - that would be a bad bug - easy to # do with the way we do cooking of groups. onRepos = childVersion.getHost() == reposName assert not onRepos, "Trove %s references repository" % trv n,v,f = troveCs.getNewNameVersionFlavor() trove, troveVersion = trovesByNBF[n, v.branch(), f] troveNVFC = trove.getNameVersionFlavor(withContext=True) # map jobId -> trove -> binaries mapping[trove.jobId].setdefault(troveNVFC, []).append((n,v,f)) else: return False, 'Creating clone failed' signatureKey = conaryclient.cfg.signatureKey if writeToFile: cs.writeToFile(writeToFile) else: repos.commitChangeSet(cs, callback=callback) return True, mapping
def commitJobs(conaryclient, jobList, reposName, message=None, commitOutdatedSources=False, sourceOnly=False, excludeSpecs=None, writeToFile=None): jobsToCommit = {} alreadyCommitted = [] finalCs = changeset.ReadOnlyChangeSet() mapping = {} for job in jobList: if job.isCommitted(): alreadyCommitted.append(job) else: jobsToCommit[job.jobId] = job jobsToCommit = jobsToCommit.values() # dedup job list if not jobsToCommit: err = 'Job(s) already committed' return False, err allTroves = [] trovesByBranch = {} alreadyCommitted = False for job in jobsToCommit: mapping[job.jobId] = {} for trove in job.iterTroves(): allTroves.append(trove) troveVersion = trove.getVersion() if troveVersion.getHost() == reposName: if not troveVersion.branch().hasParentBranch(): message = ('Cannot commit filesystem cook %s - ' ' nowhere to commit to!' % trove.getName()) return False, message assert (allTroves) source = trovesource.SimpleTroveSource() if excludeSpecs: excludeSpecsWithContext = {} troveMap = {} for excludeSpec in excludeSpecs: if len(excludeSpec) == 4: context = excludeSpec[3] else: context = None excludeSpecsWithContext.setdefault(excludeSpec[:3], []).append(context) excludeSpecs = [x[:3] for x in excludeSpecs] for trove in allTroves: troveTup = (trove.getName().split(':')[0], trove.getVersion(), trove.getFlavor()) source.addTrove(*troveTup) troveMap.setdefault(troveTup, []).append(trove) source.searchAsDatabase() matches = source.findTroves(None, excludeSpecs, None, allowMissing=True) trvMatches = [] for excludeSpec, matchList in matches.iteritems(): contexts = excludeSpecsWithContext[excludeSpec] for match in matchList: for trv in troveMap[match]: if trv.context in contexts or None in contexts: trvMatches.append(trv) allTroves = [x for x in allTroves if x not in trvMatches] if not allTroves: message = ('All troves excluded - not committing') return False, message repos = conaryclient.getRepos() trovesByNBF = {} sourcesToCheck = [] branchMap = {} trovesToClone = [] for trove in allTroves: builtTroves = list(trove.iterBuiltTroves()) if not builtTroves: continue if builtTroves[0][1].getHost() != reposName: alreadyCommitted = True for n, v, f in builtTroves: trovesByNBF[n, v.branch(), f] = (trove, v) continue troveVersion = trove.getVersion() if troveVersion.getHost() == reposName: sourceTup = (trove.getName(), troveVersion, Flavor()) targetBranch = troveVersion.branch().parentBranch() branchMap[troveVersion.branch()] = targetBranch nbf = trove.getName(), targetBranch, Flavor() if nbf in trovesByNBF: if trovesByNBF[nbf][1] != troveVersion: badVersion = trovesByNBF[nbf][1] return False, ( "Cannot commit two different versions of source component %s:" " %s and %s" % (trove.getName(), troveVersion, badVersion)) trovesByNBF[nbf] = trove, troveVersion sourcesToCheck.append(sourceTup) if sourceOnly: continue for troveTup in builtTroves: branch = troveTup[1].branch() targetBranch = branch.parentBranch() # add mapping so that when the cloning is done # we can tell what commit resulted in what binaries. nbf = (troveTup[0], targetBranch, troveTup[2]) if nbf in trovesByNBF: otherBinary = trovesByNBF[nbf][0].getBinaryTroves()[0] if otherBinary[1].branch() == targetBranch: # this one's already committed. break # discard the later of the two commits. if trovesByNBF[nbf][0].getVersion() > trove.getVersion(): # we're the earlier one badTrove, badVersion = trovesByNBF[nbf] newTrove = trove newVersion = troveTup[1] else: badTrove = trove badVersion = troveTup[1] newTrove, newVersion = trovesByNBF[nbf] name = nbf[0] flavor = nbf[2] skipped = [] for badTroveTup in badTrove.iterBuiltTroves(): badNbf = (badTroveTup[0], targetBranch, badTroveTup[2]) if not ':' in badTroveTup[0]: skipped.append(badTroveTup[0]) if badNbf in trovesByNBF and badTrove is trovesByNBF[ badNbf][0]: del trovesByNBF[badNbf] skipped = '%s' % (', '.join(skipped)) log.warning("Not committing %s on %s[%s]%s - overridden by" " %s[%s]%s" % (skipped, badTroveTup[1], badTroveTup[2], badTrove.getContextStr(), newVersion, flavor, newTrove.getContextStr())) if trove is badTrove: break trovesByNBF[nbf] = trove, troveTup[1] branchMap[branch] = targetBranch for nbf, (trove, tupVersion) in trovesByNBF.items(): if tupVersion.branch() != nbf[1]: trovesToClone.append((nbf[0], tupVersion, nbf[2])) if not trovesToClone: if sourceOnly: err = 'Could not find sources to commit' elif alreadyCommitted: log.warning('All built troves have already been committed') return True, {} else: err = 'Can only commit built troves, none found' return False, err if sourcesToCheck and not commitOutdatedSources: outdated = _checkOutdatedSources(repos, sourcesToCheck) if outdated: outdated = ( '%s=%s (replaced by newer %s)' \ % (name, builtVer, newVer.trailingRevision()) for (name, builtVer, newVer) in outdated) err = ('The following source troves are out of date:\n%s\n\n' 'Use --commit-outdated-sources to commit anyway' % '\n'.join(outdated)) return False, err # only update build info if we'll be okay if some buildreqs are not # updated updateBuildInfo = compat.ConaryVersion().acceptsPartialBuildReqCloning() callback = callbacks.CloneCallback(conaryclient.cfg, message) passed, cs = conaryclient.createTargetedCloneChangeSet( branchMap, trovesToClone, updateBuildInfo=updateBuildInfo, cloneSources=False, trackClone=False, callback=callback, fullRecurse=False) if passed: oldTroves = [] for troveCs in cs.iterNewTroveList(): if troveCs.getOldVersion(): oldTroves.append(troveCs.getOldNameVersionFlavor()) if oldTroves: oldDict = {} for oldTrove in repos.getTroves(oldTroves): oldDict.setdefault(oldTrove.getNameVersionFlavor(), []).append(oldTrove) for troveCs in cs.iterNewTroveList(): if troveCs.getOldVersion(): trv = oldDict[troveCs.getOldNameVersionFlavor()].pop() trv.applyChangeSet(troveCs) else: trv = Trove(troveCs) for _, childVersion, _ in trv.iterTroveList(strongRefs=True, weakRefs=True): # make sure there are not any references to the internal # rmake repository - that would be a bad bug - easy to # do with the way we do cooking of groups. onRepos = childVersion.getHost() == reposName assert not onRepos, "Trove %s references repository" % trv n, v, f = troveCs.getNewNameVersionFlavor() trove, troveVersion = trovesByNBF[n, v.branch(), f] troveNVFC = trove.getNameVersionFlavor(withContext=True) # map jobId -> trove -> binaries mapping[trove.jobId].setdefault(troveNVFC, []).append((n, v, f)) else: return False, 'Creating clone failed' signatureKey = conaryclient.cfg.signatureKey if signatureKey and compat.ConaryVersion().signAfterPromote(): finalCs = signAbsoluteChangeset(cs, signatureKey) if writeToFile: cs.writeToFile(writeToFile) else: repos.commitChangeSet(cs, callback=callback) return True, mapping
def _merge(self): changeSet = ChangeSet() deleteDirs = set() doCommit = False # If this is not None then all ephemeral sources will still be fetched # but will be placed in this directory instead. if self.helper.plan.ephemeralSourceDir: ephDir = self.helper.makeEphemeralDir() else: ephDir = None def _addFile(path, contents, isText): if path in oldFiles: # Always recycle pathId if available. pathId, _, oldFileId, oldFileVersion = oldFiles[path] else: pathId = hashlib.md5(path).digest() oldFileId = oldFileVersion = None fileHelper = filetypes.RegularFile(contents=contents, config=isText) fileStream = fileHelper.get(pathId) fileStream.flags.isSource(set=True) fileId = fileStream.fileId() # If the fileId matches, recycle the fileVersion too. if fileId == oldFileId: fileVersion = oldFileVersion else: fileVersion = newTrove.getVersion() filesToAdd[fileId] = (fileStream, fileHelper.contents, isText) newTrove.addFile(pathId, path, fileVersion, fileId) for package, (recipeText, recipeObj), oldTrove in zip( self.packages, self.recipes, self.oldTroves): filesToAdd = {} oldFiles = {} if oldTrove is not None: for pathId, path, fileId, fileVer in oldTrove.iterFileList(): oldFiles[path] = (pathId, path, fileId, fileVer) newTrove = Trove(package.name, package.nextVersion, deps.Flavor()) newTrove.setFactory(package.targetConfig.factory) # Add upstream files to new trove. Recycle pathids from the old # version. # LAZY: assume that everything other than the recipe is binary. # Conary has a magic module, but it only accepts filenames! for path, contents in package.recipeFiles.iteritems(): isText = path == package.getRecipeName() _addFile(path, contents, isText) # Collect requested auto sources from recipe. Unknown recipe types # will not be loaded so recipeObj will be the class, so assume # these have no sources. if not inspect.isclass(recipeObj): recipeFiles = dict((os.path.basename(x.getPath()), x) for x in recipeObj.getSourcePathList()) newFiles = set(x[1] for x in newTrove.iterFileList()) needFiles = set(recipeFiles) - newFiles for autoPath in needFiles: source = recipeFiles[autoPath] if (autoPath in oldFiles and not self.helper.plan.refreshSources and not source.ephemeral): # File exists in old version. pathId, path, fileId, fileVer = oldFiles[autoPath] newTrove.addFile(pathId, path, fileVer, fileId) continue if source.ephemeral and not ephDir: continue # File doesn't exist; need to create it. if source.ephemeral: laUrl = lookaside.laUrl(source.getPath()) tempDir = joinPaths(ephDir, os.path.dirname(laUrl.filePath())) mkdirChain(tempDir) else: tempDir = tempfile.mkdtemp() deleteDirs.add(tempDir) snapshot = _getSnapshot(self.helper, package, source, tempDir) if not source.ephemeral and snapshot: autoPathId = hashlib.md5(autoPath).digest() autoObj = FileFromFilesystem(snapshot, autoPathId) autoObj.flags.isAutoSource(set=True) autoObj.flags.isSource(set=True) autoFileId = autoObj.fileId() autoContents = filecontents.FromFilesystem(snapshot) filesToAdd[autoFileId] = (autoObj, autoContents, False) newTrove.addFile(autoPathId, autoPath, newTrove.getVersion(), autoFileId) # If the old and new troves are identical, just use the old one. if oldTrove and _sourcesIdentical( oldTrove, newTrove, [self.oldChangeSet, filesToAdd]): package.setDownstreamVersion(oldTrove.getVersion()) log.debug('Skipped %s=%s', oldTrove.getName(), oldTrove.getVersion()) continue # Add files and contents to changeset. for fileId, (fileObj, fileContents, cfgFile) in filesToAdd.items(): changeSet.addFileContents(fileObj.pathId(), fileObj.fileId(), ChangedFileTypes.file, fileContents, cfgFile) changeSet.addFile(None, fileObj.fileId(), fileObj.freeze()) # Create a changelog entry. changeLog = ChangeLog( name=self.helper.cfg.name, contact=self.helper.cfg.contact, message=self.helper.plan.commitMessage + '\n') newTrove.changeChangeLog(changeLog) # Calculate trove digests and add the trove to the changeset newTrove.invalidateDigests() newTrove.computeDigests() newTroveCs = newTrove.diff(None, absolute=True)[0] changeSet.newTrove(newTroveCs) doCommit = True package.setDownstreamVersion(newTrove.getVersion()) log.debug('Created %s=%s', newTrove.getName(), newTrove.getVersion()) if doCommit: cook.signAbsoluteChangesetByConfig(changeSet, self.helper.cfg) f = tempfile.NamedTemporaryFile(dir=os.getcwd(), suffix='.ccs', delete=False) f.close() changeSet.writeToFile(f.name) try: self.helper.getRepos().commitChangeSet(changeSet) except: log.error("Error committing changeset to repository, " "failed changeset is saved at %s", f.name) raise else: os.unlink(f.name) for path in deleteDirs: shutil.rmtree(path)