def testUnreachableSource(self): """createChangeSet must not mask network errors without a good reason @tests: CNY-3732 """ repos_alive = self.openRepository(0) repos_dead = self.openRepository(1) trv = self.addComponent('foo:runtime', 'localhost1@rpl:linux', repos=repos_dead) grp = self.addCollection('group-foo', [trv], repos=repos_alive) self.stopRepository(1) n, v, f = grp.getNameVersionFlavor() s = stack(repos_alive, repos_dead) self.assertRaises(errors.OpenError, s.createChangeSet, [(n, (None, None), (v, f), True)], recurse=True, withFiles=False)
def testTroveSourceStack(self): # Test the behaviour of TroveSourceStack instances s1 = SimplestFindTroveSource() s2 = SimplestFindTroveSource() s3 = SimplestFindTroveSource() nodeps = deps.parseFlavor('') count = 3 names = [] versions = [] for v in range(count): sv = str(v + 1) versions.append(ThawVersion('/localhost@rpl:devel/%s:1.0-1-1' % sv)) names.append("test%s" % sv) # Generate troves t = [] for i in range(count): l = [] t.append(l) name = names[i] for j in range(count): l.append((name, versions[j], nodeps)) s1.addTroves(t[0][1]) s2.addTroves(t[0][0], t[1][1]) s3.addTroves(t[0][2], t[1][0], t[2][2]) # Catch an early bug with a bad index try: ts1 = stack(s1) except Exception, e: # No exception expected self.fail("Caught exception: %s" % e)
def getSourceTrovesFromJob(job, troveList=None, repos=None, reposName=None): cfg = job.getMainConfig() if repos: cacheDir = None else: cacheDir = tempfile.mkdtemp(prefix='rmake-trovecache-') try: client = conaryclient.ConaryClient(cfg) repos = repocache.CachingTroveSource(client.getRepos(), cacheDir) except: util.rmtree(cacheDir) raise try: if troveList is None: troveList = list(job.iterLoadableTroves()) if not troveList: return {} if reposName is None: reposName = cfg.reposName resultSet = {} tupList = sorted(x.getNameVersionFlavor() for x in troveList) # create fake "packages" for all the troves we're building so that # they can be found for loadInstalled. if not cfg.isolateTroves: buildTrovePackages = [(x[0].split(':')[0], x[1], x[2]) for x in tupList] buildTroveSource = RemoveHostSource( trovesource.SimpleTroveSource(buildTrovePackages), reposName) else: buildTroveSource = None # don't search the internal repository explicitly for loadRecipe # sources - they may be a part of some bogus build. repos = RemoveHostRepos(repos, reposName) trovesByConfig = {} for trove in troveList: trovesByConfig.setdefault(trove.getContext(), []).append(trove) total = len(troveList) count = 0 for context, contextTroves in trovesByConfig.items(): buildCfg = contextTroves[0].cfg loadInstalledList = [ trovesource.TroveListTroveSource(repos, x) for x in buildCfg.resolveTroveTups ] loadInstalledList.append(repos) if buildTroveSource is not None: loadInstalledSource = trovesource.stack( buildTroveSource, *loadInstalledList) else: loadInstalledSource = trovesource.stack(*loadInstalledList) loadInstalledRepos = trovesource.stack(*loadInstalledList) if isinstance(loadInstalledRepos, trovesource.TroveSourceStack): for source in loadInstalledRepos.iterSources(): source._getLeavesOnly = True source.searchWithFlavor() # keep allowNoLabel set. else: loadInstalledRepos._getLeavesOnly = True loadInstalledRepos.searchWithFlavor() cachedRepos = CachingSource(loadInstalledRepos) resultSet.update( loadSourceTroves(job, cachedRepos, buildCfg.buildFlavor, contextTroves, total=total, count=count, loadInstalledSource=loadInstalledSource, installLabelPath=buildCfg.installLabelPath, internalHostName=reposName)) count = len(resultSet) finally: if cacheDir: util.rmtree(cacheDir) return resultSet
def loadSourceTrovesForJob(job, troveList=None, repos=None, reposName=None): cfg = job.getMainConfig() if repos: cacheDir = None else: cacheDir = tempfile.mkdtemp(prefix="rmake-trovecache-") try: client = conaryclient.ConaryClient(cfg) repos = repocache.CachingTroveSource(client.getRepos(), cacheDir) except: util.rmtree(cacheDir) raise try: if troveList is None: troveList = list(job.iterLoadableTroves()) if not troveList: return {} if reposName is None: reposName = cfg.reposName tupList = sorted(x.getNameVersionFlavor() for x in troveList) # create fake "packages" for all the troves we're building so that # they can be found for loadInstalled. if not cfg.isolateTroves: buildTrovePackages = [(x[0].split(":")[0], x[1], x[2]) for x in tupList] buildTroveSource = RemoveHostSource(trovesource.SimpleTroveSource(buildTrovePackages), reposName) else: buildTroveSource = None # don't search the internal repository explicitly for loadRecipe # sources - they may be a part of some bogus build. repos = RemoveHostRepos(repos, reposName) trovesByConfig = {} for trove in troveList: trovesByConfig.setdefault(trove.getContext(), []).append(trove) total = len(troveList) count = 0 for context, contextTroves in trovesByConfig.items(): buildCfg = contextTroves[0].cfg loadInstalledList = [trovesource.TroveListTroveSource(repos, x) for x in buildCfg.resolveTroveTups] loadInstalledList.append(repos) if buildTroveSource is not None: loadInstalledSource = trovesource.stack(buildTroveSource, *loadInstalledList) else: loadInstalledSource = trovesource.stack(*loadInstalledList) loadInstalledRepos = trovesource.stack(*loadInstalledList) if isinstance(loadInstalledRepos, trovesource.TroveSourceStack): for source in loadInstalledRepos.iterSources(): source._getLeavesOnly = True source.searchWithFlavor() # keep allowNoLabel set. else: loadInstalledRepos._getLeavesOnly = True loadInstalledRepos.searchWithFlavor() cachedRepos = CachingSource(loadInstalledRepos) loadSourceTroves( job, cachedRepos, buildCfg.buildFlavor, contextTroves, total=total, count=count, loadInstalledSource=loadInstalledSource, installLabelPath=buildCfg.installLabelPath, internalHostName=reposName, ) count += len(contextTroves) finally: if cacheDir: util.rmtree(cacheDir)
def getTroveSource(self): if len(self.sources) == 1: return self.sources[0].getTroveSource() return trovesource.stack(*[x.getTroveSource() for x in self.sources])
def displayChangeSet( db, cs, troveSpecs, cfg, asDiff=False, diffBinaries=False, # selection options exactFlavors=False, # trove options info=False, digSigs=False, deps=False, showBuildReqs=False, all=False, # file options ls=False, lsl=False, ids=False, sha1s=False, tags=False, fileDeps=False, fileVersions=False, fileFlavors=False, capsules=False, # collection options showTroves=False, recurse=None, showAllTroves=False, weakRefs=False, showTroveFlags=False, alwaysDisplayHeaders=False, recurseRepos=False, # job options showChanges=False, asJob=False, ): asDiff = asDiff or diffBinaries if all: deps = recurse = showTroveFlags = showAllTroves = True if ls: fileDeps = lsl = tags = True if showChanges: lsl = True if recurseRepos: recurse = True client = conaryclient.ConaryClient(cfg) repos = client.getRepos() if asDiff: troveSource = trovesource.SourceStack(client.getDatabase(), repos) for x in cs.gitDiff(troveSource, diffBinaries=diffBinaries): sys.stdout.write(x) elif not asJob and not showChanges and cs.isAbsolute(): changeSetSource = trovesource.ChangesetFilesTroveSource(None) changeSetSource.addChangeSet(cs) if not troveSpecs: troveTups = cs.getPrimaryTroveList() primary = True if not troveTups: log.warning("No primary troves in changeset, listing all troves") troveTups = [(x.getName(), x.getNewVersion(), x.getNewFlavor()) for x in cs.iterNewTroveList()] else: troveTups, primary = query.getTrovesToDisplay(changeSetSource, troveSpecs, exactFlavors=exactFlavors) if recurseRepos: querySource = trovesource.stack(changeSetSource, client.getRepos()) else: querySource = changeSetSource dcfg = display.DisplayConfig(querySource, client.db) dcfg.setTroveDisplay( deps=deps, info=info, showBuildReqs=showBuildReqs, digSigs=digSigs, fullFlavors=cfg.fullFlavors, showLabels=cfg.showLabels, baseFlavors=cfg.flavor, fullVersions=cfg.fullVersions, ) dcfg.setFileDisplay( ls=ls, lsl=lsl, ids=ids, sha1s=sha1s, tags=tags, fileDeps=fileDeps, fileVersions=fileVersions, fileFlavors=fileFlavors, capsules=capsules, ) recurseOne = showTroves or showAllTroves or weakRefs if recurse is None and not recurseOne: # if we didn't explicitly set recurse and we're not recursing one # level explicitly recurse = True in (ls, lsl, ids, sha1s, tags, deps, fileDeps, fileVersions, fileFlavors) dcfg.setChildDisplay( recurseAll=recurse, recurseOne=recurseOne, showNotByDefault=showAllTroves, showWeakRefs=weakRefs, checkExists=True, showNotExists=True, showTroveFlags=showTroveFlags, displayHeaders=alwaysDisplayHeaders or showTroveFlags, ) if primary: dcfg.setPrimaryTroves(set(troveTups)) formatter = display.TroveFormatter(dcfg) display.displayTroves(dcfg, formatter, troveTups) else: changeSetSource = trovesource.ChangeSetJobSource(repos, trovesource.stack(db, repos)) changeSetSource.addChangeSet(cs) jobs = getJobsToDisplay(changeSetSource, troveSpecs) dcfg = display.JobDisplayConfig(changeSetSource, client.db) dcfg.setJobDisplay(showChanges=showChanges, compressJobs=not cfg.showComponents) dcfg.setTroveDisplay( deps=deps, info=info, fullFlavors=cfg.fullFlavors, showLabels=cfg.showLabels, baseFlavors=cfg.flavor, fullVersions=cfg.fullVersions, ) dcfg.setFileDisplay( ls=ls, lsl=lsl, ids=ids, sha1s=sha1s, tags=tags, fileDeps=fileDeps, fileVersions=fileVersions, fileFlavors=fileFlavors, ) recurseOne = showTroves or showAllTroves or weakRefs if recurse is None and not recurseOne: # if we didn't explicitly set recurse and we're not recursing one # level explicitly and we specified troves (so everything won't # show up at the top level anyway), guess at whether to recurse recurse = True in (ls, lsl, ids, sha1s, tags, deps, fileDeps, fileVersions, fileFlavors) dcfg.setChildDisplay( recurseAll=recurse, recurseOne=recurseOne, showNotByDefault=showAllTroves, showWeakRefs=weakRefs, showTroveFlags=showTroveFlags, ) formatter = display.JobFormatter(dcfg) display.displayJobs(dcfg, formatter, jobs)
def displayChangeSet( db, cs, troveSpecs, cfg, asDiff=False, diffBinaries=False, # selection options exactFlavors=False, # trove options info=False, digSigs=False, deps=False, showBuildReqs=False, all=False, # file options ls=False, lsl=False, ids=False, sha1s=False, tags=False, fileDeps=False, fileVersions=False, fileFlavors=False, capsules=False, # collection options showTroves=False, recurse=None, showAllTroves=False, weakRefs=False, showTroveFlags=False, alwaysDisplayHeaders=False, recurseRepos=False, # job options showChanges=False, asJob=False): asDiff = asDiff or diffBinaries if all: deps = recurse = showTroveFlags = showAllTroves = True if ls: fileDeps = lsl = tags = True if showChanges: lsl = True if recurseRepos: recurse = True client = conaryclient.ConaryClient(cfg) repos = client.getRepos() if asDiff: troveSource = trovesource.SourceStack(client.getDatabase(), repos) for x in cs.gitDiff(troveSource, diffBinaries=diffBinaries): sys.stdout.write(x) elif not asJob and not showChanges and cs.isAbsolute(): changeSetSource = trovesource.ChangesetFilesTroveSource(None) changeSetSource.addChangeSet(cs) if not troveSpecs: troveTups = cs.getPrimaryTroveList() primary = True if not troveTups: log.warning( 'No primary troves in changeset, listing all troves') troveTups = [(x.getName(), x.getNewVersion(), x.getNewFlavor())\ for x in cs.iterNewTroveList()] else: troveTups, primary = query.getTrovesToDisplay( changeSetSource, troveSpecs, exactFlavors=exactFlavors) if recurseRepos: querySource = trovesource.stack(changeSetSource, client.getRepos()) else: querySource = changeSetSource dcfg = display.DisplayConfig(querySource, client.db) dcfg.setTroveDisplay( deps=deps, info=info, showBuildReqs=showBuildReqs, digSigs=digSigs, fullFlavors=cfg.fullFlavors, showLabels=cfg.showLabels, baseFlavors=cfg.flavor, fullVersions=cfg.fullVersions, ) dcfg.setFileDisplay(ls=ls, lsl=lsl, ids=ids, sha1s=sha1s, tags=tags, fileDeps=fileDeps, fileVersions=fileVersions, fileFlavors=fileFlavors, capsules=capsules) recurseOne = showTroves or showAllTroves or weakRefs if recurse is None and not recurseOne: # if we didn't explicitly set recurse and we're not recursing one # level explicitly recurse = True in (ls, lsl, ids, sha1s, tags, deps, fileDeps, fileVersions, fileFlavors) dcfg.setChildDisplay(recurseAll=recurse, recurseOne=recurseOne, showNotByDefault=showAllTroves, showWeakRefs=weakRefs, checkExists=True, showNotExists=True, showTroveFlags=showTroveFlags, displayHeaders=alwaysDisplayHeaders or showTroveFlags) if primary: dcfg.setPrimaryTroves(set(troveTups)) formatter = display.TroveFormatter(dcfg) display.displayTroves(dcfg, formatter, troveTups) else: changeSetSource = trovesource.ChangeSetJobSource( repos, trovesource.stack(db, repos)) changeSetSource.addChangeSet(cs) jobs = getJobsToDisplay(changeSetSource, troveSpecs) dcfg = display.JobDisplayConfig(changeSetSource, client.db) dcfg.setJobDisplay(showChanges=showChanges, compressJobs=not cfg.showComponents) dcfg.setTroveDisplay(deps=deps, info=info, fullFlavors=cfg.fullFlavors, showLabels=cfg.showLabels, baseFlavors=cfg.flavor, fullVersions=cfg.fullVersions) dcfg.setFileDisplay(ls=ls, lsl=lsl, ids=ids, sha1s=sha1s, tags=tags, fileDeps=fileDeps, fileVersions=fileVersions, fileFlavors=fileFlavors) recurseOne = showTroves or showAllTroves or weakRefs if recurse is None and not recurseOne: # if we didn't explicitly set recurse and we're not recursing one # level explicitly and we specified troves (so everything won't # show up at the top level anyway), guess at whether to recurse recurse = True in (ls, lsl, ids, sha1s, tags, deps, fileDeps, fileVersions, fileFlavors) dcfg.setChildDisplay(recurseAll=recurse, recurseOne=recurseOne, showNotByDefault=showAllTroves, showWeakRefs=weakRefs, showTroveFlags=showTroveFlags) formatter = display.JobFormatter(dcfg) display.displayJobs(dcfg, formatter, jobs)
class ChangesetFilesTest(rephelp.RepositoryHelper): def testSimple(self): r1 = self.addComponent('test:runtime', '1.0-1-1', fileContents=[], requires='trove: test:doc') r2 = self.addComponent( 'test:runtime', '2.0-1-1', fileContents=[], requires='soname: ELF32/foo.so.0(a b c) trove: test:doc') d1 = self.addComponent('test:doc', '1.0-1-1', filePrimer=1) l1 = self.addComponent('test:lib', '1.0-1-1', filePrimer=2, provides='soname: ELF32/foo.so.0(a b c d)') db = database.Database(':memory:', ':memory:') db.addTrove(r1) repos = self.openRepository() relCs = repos.createChangeSet([ ('test:runtime', (r1.getVersion(), r1.getFlavor()), (r2.getVersion(), r2.getFlavor()), False) ]) absCs = repos.createChangeSet([ ('test:doc', (None, None), (d1.getVersion(), d1.getFlavor()), False), ('test:lib', (None, None), (l1.getVersion(), l1.getFlavor()), False) ]) trvSrc = ChangesetFilesTroveSource(db, storeDeps=True) trvSrc.addChangeSet(relCs) trvSrc.addChangeSet(absCs) for trv in (r2, d1, l1): [otherTrv] = trvSrc.getTroves( [(trv.getName(), trv.getVersion(), trv.getFlavor())], withFiles=False) assert (trv == otherTrv) suggMap = trvSrc.resolveDependencies( None, [r2.getRequires(), r1.getRequires()]) assert (suggMap[r1.getRequires()] == [[(d1.getName(), d1.getVersion(), d1.getFlavor())]]) suggs = suggMap[r2.getRequires()] assert ([(d1.getName(), d1.getVersion(), d1.getFlavor())] in suggs) assert ([(l1.getName(), l1.getVersion(), l1.getFlavor())] in suggs) def testUselessRelChangeset(self): # don't consider a new version of a trove 'present' if it requires # that a relative changeset be applied to a trove that we don't # have installed. r1 = self.addCollection('foo', '1', [':run'], createComps=True) r2 = self.addCollection('foo', '2', [':run'], createComps=True) db = database.Database(':memory:', ':memory:') db.addTrove(r1) # this doesn't add foo:run to the db! chg = ('foo', (r1.getVersion(), r1.getFlavor()), (r2.getVersion(), r2.getFlavor()), False) repos = self.openRepository() relCs = repos.createChangeSet([chg], recurse=True) trvSrc = ChangesetFilesTroveSource(db, storeDeps=True) trvSrc.addChangeSet(relCs) fooRun = ('foo:run', r2.getVersion(), r2.getFlavor()) assert (trvSrc.hasTroves([fooRun]) == [False]) assert (trvSrc.hasTroves([r2.getNameVersionFlavor()]) == [True]) def testCreateNewChangesetNotInSource(self): # Test creating a relative changeset when all we have in the # source is absolute changesets, and vice versa. r1 = self.addCollection('foo', '1', [':run'], createComps=True) self.addComponent('foo:run', '2', filePrimer=2) r2 = self.addCollection('foo', '2', [':run']) # create two absolute changesets, we want one relative one out absChg = [ ('foo', (None, None), (r1.getVersion(), r1.getFlavor()), True), ('foo', (None, None), (r2.getVersion(), r2.getFlavor()), True) ] repos = self.openRepository() absCs = repos.createChangeSet(absChg, recurse=True) trvSrc = ChangesetFilesTroveSource(None) trvSrc.addChangeSet(absCs) relChg = ('foo', (r1.getVersion(), r1.getFlavor()), (r2.getVersion(), r2.getFlavor()), False) newCs, remainder = trvSrc.createChangeSet([relChg], recurse=False, withFiles=False, withFileContents=False) assert (not remainder) trvCs = newCs.iterNewTroveList().next() assert (trvCs.getNewVersion() == r2.getVersion()) assert (trvCs.getOldVersion() == r1.getVersion()) # now we'll try to get a relative changeset out when we don't # have the new trove that we'd need to make the relative changeset # work badChg = ('foo', (r1.getVersion(), r1.getFlavor()), (r2.getVersion(), deps.parseFlavor('foo')), False) newCs, remainder = trvSrc.createChangeSet([badChg], recurse=False, withFiles=False, withFileContents=False) assert (remainder) assert (newCs.isEmpty()) # now we'll try to get a relative changeset out when we don't # have the _old_ trove that we'd need to make the relative changeset # work badChg = ('foo', (r1.getVersion(), deps.parseFlavor('foo')), (r2.getVersion(), r2.getFlavor()), False) newCs, remainder = trvSrc.createChangeSet([badChg], recurse=False, withFiles=False, withFileContents=False) assert (remainder) assert (newCs.isEmpty()) # Now test getting absolute changesets when the trovesource only has # relative ones. This requires using the database to grab information # from the installed system about the old versions of troves. # First try creating those absolute changesets when there is # no information about the old trove on the system relCs = repos.createChangeSet([relChg], recurse=True) db = self.openDatabase() trvSrc = ChangesetFilesTroveSource(db) trvSrc.addChangeSet(relCs) newCs, remainder = trvSrc.createChangeSet(absChg, recurse=False, withFiles=False, withFileContents=False) assert (len(remainder) == 2) assert (newCs.isEmpty()) # now create them when the old trove has been installed. self.updatePkg(['foo=%s' % r1.getVersion()]) # FIXME - we need to recreate the changeset files source here - # apparently some information pertaining to what is installed is # cached trvSrc = ChangesetFilesTroveSource(db) trvSrc.addChangeSet(relCs) newCs, remainder = trvSrc.createChangeSet(absChg, recurse=False, withFiles=False, withFileContents=False) assert (not remainder) assert (len(list(newCs.iterNewTroveList())) == 2) def testTroveSourceStack(self): # Test the behaviour of TroveSourceStack instances s1 = SimplestFindTroveSource() s2 = SimplestFindTroveSource() s3 = SimplestFindTroveSource() nodeps = deps.parseFlavor('') count = 3 names = [] versions = [] for v in range(count): sv = str(v + 1) versions.append(ThawVersion('/localhost@rpl:devel/%s:1.0-1-1' % sv)) names.append("test%s" % sv) # Generate troves t = [] for i in range(count): l = [] t.append(l) name = names[i] for j in range(count): l.append((name, versions[j], nodeps)) s1.addTroves(t[0][1]) s2.addTroves(t[0][0], t[1][1]) s3.addTroves(t[0][2], t[1][0], t[2][2]) # Catch an early bug with a bad index try: ts1 = stack(s1) except Exception, e: # No exception expected self.fail("Caught exception: %s" % e) # The implementation of stack says so... self.assertEqual(ts1, s1) ret = ts1.hasTroves([t[0][0], t[1][1], t[2][2]]) self.assertEqual(ret, [True, False, False]) ts2 = stack(ts1) ret = ts1.hasTroves([t[0][2], t[1][0], t[2][1]]) self.assertEqual(ret, [True, False, False]) ts3 = stack(s1, s3) ts4 = stack(s2, ts3) # Same deal, but use the constructor for TroveSourceStack ts5 = TroveSourceStack(s2, ts3) for s in [s1, s2, s3]: self.assertTrue(ts4.hasSource(s)) self.assertTrue(ts5.hasSource(s)) for (rs, es) in zip(ts4.iterSources(), [s2, s1, s3]): self.assertEqual(rs, es) for (rs, es) in zip(ts5.iterSources(), [s2, s1, s3]): self.assertEqual(rs, es) # same source specified twice - obtain a stack ts6 = stack(s1, s1) self.assertNotEqual(ts6, s1) self.assertTrue(isinstance(ts6, TroveSourceStack)) ts7 = stack(ts6, ts6) self.assertEqual(ts7, ts6) ts8 = stack(ts3, ts4) # Old implementation was adding ts4 as a source for ts3, instead of # adding the subsources of ts4 to ts3 for s in ts8.iterSources(): self.assertFalse(isinstance(s, TroveSourceStack))
def getTroveSource(self): if len(self.sources) == 1: return self.sources[0].getTroveSource() return trovesource.stack(*[ x.getTroveSource() for x in self.sources])