def CloneTrove(cfg,
targetBranch,
troveSpecList,
updateBuildInfo=True,
info=False,
cloneSources=False,
message=None,
test=False,
fullRecurse=False,
ignoreConflicts=False,
exactFlavors=False):
client = ConaryClient(cfg)
repos = client.getRepos()
targetBranch = versions.VersionFromString(targetBranch)
if not isinstance(targetBranch, versions.Branch):
raise errors.ParseError(
'Cannot specify full version "%s" to clone to - must specify target branch'
% targetBranch)
troveSpecs = [cmdline.parseTroveSpec(x) for x in troveSpecList]
componentSpecs = [
x[0] for x in troveSpecs
if ':' in x[0] and x[0].split(':')[1] != 'source'
]
if componentSpecs:
raise errors.ParseError('Cannot clone components: %s' %
', '.join(componentSpecs))
trovesToClone = repos.findTroves(cfg.installLabelPath,
troveSpecs,
cfg.flavor,
exactFlavors=exactFlavors)
trovesToClone = list(set(itertools.chain(*trovesToClone.itervalues())))
if not client.cfg.quiet:
callback = client_callbacks.CloneCallback(client.cfg, message)
else:
callback = callbacks.CloneCallback()
okay, cs = client.createCloneChangeSet(targetBranch,
trovesToClone,
updateBuildInfo=updateBuildInfo,
infoOnly=info,
callback=callback,
fullRecurse=fullRecurse,
cloneSources=cloneSources)
if not okay:
return
return _finishClone(client,
cfg,
cs,
callback,
info=info,
test=test,
ignoreConflicts=ignoreConflicts)
def commitJobs(conaryclient,
jobList,
reposName,
message=None,
commitOutdatedSources=False,
sourceOnly=False,
excludeSpecs=None,
writeToFile=None):
jobsToCommit = {}
alreadyCommitted = []
finalCs = changeset.ReadOnlyChangeSet()
mapping = {}
for job in jobList:
if job.isCommitted():
alreadyCommitted.append(job)
else:
jobsToCommit[job.jobId] = job
jobsToCommit = jobsToCommit.values() # dedup job list
if not jobsToCommit:
err = 'Job(s) already committed'
return False, err
allTroves = []
trovesByBranch = {}
alreadyCommitted = False
for job in jobsToCommit:
mapping[job.jobId] = {}
for trove in job.iterTroves():
allTroves.append(trove)
troveVersion = trove.getVersion()
if troveVersion.getHost() == reposName:
if not troveVersion.branch().hasParentBranch():
message = ('Cannot commit filesystem cook %s - '
' nowhere to commit to!' % trove.getName())
return False, message
assert (allTroves)
source = trovesource.SimpleTroveSource()
if excludeSpecs:
excludeSpecsWithContext = {}
troveMap = {}
for excludeSpec in excludeSpecs:
if len(excludeSpec) == 4:
context = excludeSpec[3]
else:
context = None
excludeSpecsWithContext.setdefault(excludeSpec[:3],
[]).append(context)
excludeSpecs = [x[:3] for x in excludeSpecs]
for trove in allTroves:
troveTup = (trove.getName().split(':')[0], trove.getVersion(),
trove.getFlavor())
source.addTrove(*troveTup)
troveMap.setdefault(troveTup, []).append(trove)
source.searchAsDatabase()
matches = source.findTroves(None,
excludeSpecs,
None,
allowMissing=True)
trvMatches = []
for excludeSpec, matchList in matches.iteritems():
contexts = excludeSpecsWithContext[excludeSpec]
for match in matchList:
for trv in troveMap[match]:
if trv.context in contexts or None in contexts:
trvMatches.append(trv)
allTroves = [x for x in allTroves if x not in trvMatches]
if not allTroves:
message = ('All troves excluded - not committing')
return False, message
repos = conaryclient.getRepos()
trovesByNBF = {}
sourcesToCheck = []
branchMap = {}
trovesToClone = []
for trove in allTroves:
builtTroves = list(trove.iterBuiltTroves())
if not builtTroves:
continue
if builtTroves[0][1].getHost() != reposName:
alreadyCommitted = True
for n, v, f in builtTroves:
trovesByNBF[n, v.branch(), f] = (trove, v)
continue
troveVersion = trove.getVersion()
if troveVersion.getHost() == reposName:
sourceTup = (trove.getName(), troveVersion, Flavor())
targetBranch = troveVersion.branch().parentBranch()
branchMap[troveVersion.branch()] = targetBranch
nbf = trove.getName(), targetBranch, Flavor()
if nbf in trovesByNBF:
if trovesByNBF[nbf][1] != troveVersion:
badVersion = trovesByNBF[nbf][1]
return False, (
"Cannot commit two different versions of source component %s:"
" %s and %s" %
(trove.getName(), troveVersion, badVersion))
trovesByNBF[nbf] = trove, troveVersion
sourcesToCheck.append(sourceTup)
if sourceOnly:
continue
for troveTup in builtTroves:
branch = troveTup[1].branch()
targetBranch = branch.parentBranch()
# add mapping so that when the cloning is done
# we can tell what commit resulted in what binaries.
nbf = (troveTup[0], targetBranch, troveTup[2])
if nbf in trovesByNBF:
otherBinary = trovesByNBF[nbf][0].getBinaryTroves()[0]
if otherBinary[1].branch() == targetBranch:
# this one's already committed.
break
# discard the later of the two commits.
if trovesByNBF[nbf][0].getVersion() > trove.getVersion():
# we're the earlier one
badTrove, badVersion = trovesByNBF[nbf]
newTrove = trove
newVersion = troveTup[1]
else:
badTrove = trove
badVersion = troveTup[1]
newTrove, newVersion = trovesByNBF[nbf]
name = nbf[0]
flavor = nbf[2]
skipped = []
for badTroveTup in badTrove.iterBuiltTroves():
badNbf = (badTroveTup[0], targetBranch, badTroveTup[2])
if not ':' in badTroveTup[0]:
skipped.append(badTroveTup[0])
if badNbf in trovesByNBF and badTrove is trovesByNBF[
badNbf][0]:
del trovesByNBF[badNbf]
skipped = '%s' % (', '.join(skipped))
log.warning("Not committing %s on %s[%s]%s - overridden by"
" %s[%s]%s" %
(skipped, badTroveTup[1], badTroveTup[2],
badTrove.getContextStr(), newVersion, flavor,
newTrove.getContextStr()))
if trove is badTrove:
break
trovesByNBF[nbf] = trove, troveTup[1]
branchMap[branch] = targetBranch
for nbf, (trove, tupVersion) in trovesByNBF.items():
if tupVersion.branch() != nbf[1]:
trovesToClone.append((nbf[0], tupVersion, nbf[2]))
if not trovesToClone:
if sourceOnly:
err = 'Could not find sources to commit'
elif alreadyCommitted:
log.warning('All built troves have already been committed')
return True, {}
else:
err = 'Can only commit built troves, none found'
return False, err
if sourcesToCheck and not commitOutdatedSources:
outdated = _checkOutdatedSources(repos, sourcesToCheck)
if outdated:
outdated = ( '%s=%s (replaced by newer %s)' \
% (name, builtVer, newVer.trailingRevision())
for (name, builtVer, newVer) in outdated)
err = ('The following source troves are out of date:\n%s\n\n'
'Use --commit-outdated-sources to commit anyway' %
'\n'.join(outdated))
return False, err
# only update build info if we'll be okay if some buildreqs are not
# updated
updateBuildInfo = compat.ConaryVersion().acceptsPartialBuildReqCloning()
callback = callbacks.CloneCallback(conaryclient.cfg, message)
passed, cs = conaryclient.createTargetedCloneChangeSet(
branchMap,
trovesToClone,
updateBuildInfo=updateBuildInfo,
cloneSources=False,
trackClone=False,
callback=callback,
fullRecurse=False)
if passed:
oldTroves = []
for troveCs in cs.iterNewTroveList():
if troveCs.getOldVersion():
oldTroves.append(troveCs.getOldNameVersionFlavor())
if oldTroves:
oldDict = {}
for oldTrove in repos.getTroves(oldTroves):
oldDict.setdefault(oldTrove.getNameVersionFlavor(),
[]).append(oldTrove)
for troveCs in cs.iterNewTroveList():
if troveCs.getOldVersion():
trv = oldDict[troveCs.getOldNameVersionFlavor()].pop()
trv.applyChangeSet(troveCs)
else:
trv = Trove(troveCs)
for _, childVersion, _ in trv.iterTroveList(strongRefs=True,
weakRefs=True):
# make sure there are not any references to the internal
# rmake repository - that would be a bad bug - easy to
# do with the way we do cooking of groups.
onRepos = childVersion.getHost() == reposName
assert not onRepos, "Trove %s references repository" % trv
n, v, f = troveCs.getNewNameVersionFlavor()
trove, troveVersion = trovesByNBF[n, v.branch(), f]
troveNVFC = trove.getNameVersionFlavor(withContext=True)
# map jobId -> trove -> binaries
mapping[trove.jobId].setdefault(troveNVFC, []).append((n, v, f))
else:
return False, 'Creating clone failed'
signatureKey = conaryclient.cfg.signatureKey
if signatureKey and compat.ConaryVersion().signAfterPromote():
finalCs = signAbsoluteChangeset(cs, signatureKey)
if writeToFile:
cs.writeToFile(writeToFile)
else:
repos.commitChangeSet(cs, callback=callback)
return True, mapping
def promoteTroves(cfg,
troveSpecs,
targetList,
skipBuildInfo=False,
info=False,
message=None,
test=False,
ignoreConflicts=False,
cloneOnlyByDefaultTroves=False,
cloneSources=False,
allFlavors=False,
client=None,
targetFile=None,
exactFlavors=None,
excludeGroups=False):
targetMap = {}
searchPath = []
for fromLoc, toLoc in targetList:
context = cfg.buildLabel
fromLoc = _convertLabelOrBranch(fromLoc, context)
if fromLoc is not None:
if isinstance(fromLoc, versions.Branch):
context = fromLoc.label()
else:
context = fromLoc
searchPath.append(context)
toLoc = _convertLabelOrBranch(toLoc, context)
targetMap[fromLoc] = toLoc
troveSpecs = [cmdline.parseTroveSpec(x, False) for x in troveSpecs]
if exactFlavors:
allFlavors = False
elif allFlavors:
cfg.flavor = []
troveSpecFlavors = {}
for troveSpec in troveSpecs:
troveSpecFlavors.setdefault((troveSpec[0], troveSpec[1], None),
[]).append(troveSpec[2])
troveSpecs = list(troveSpecFlavors)
client = ConaryClient(cfg)
if not searchPath:
searchPath = cfg.buildLabel
searchSource = client.getSearchSource(installLabelPath=searchPath)
results = searchSource.findTroves(troveSpecs,
bestFlavor=not allFlavors,
exactFlavors=exactFlavors)
if allFlavors:
trovesToClone = []
for troveSpec, troveTups in results.items():
specFlavors = troveSpecFlavors[troveSpec]
for specFlavor in specFlavors:
if specFlavor is None:
matchingTups = troveTups
else:
matchingTups = [
x for x in troveTups
if x[2].stronglySatisfies(specFlavor)
]
# we only clone the latest version for all troves.
# bestFlavor=False resturns the leaves for all flavors, so
# we may need to cut some out.
latest = max([x[1] for x in matchingTups])
matchingTups = [x for x in matchingTups if x[1] == latest]
trovesToClone.extend(matchingTups)
else:
trovesToClone = itertools.chain(*results.itervalues())
trovesToClone = list(set(trovesToClone))
if not client.cfg.quiet:
callback = client_callbacks.CloneCallback(client.cfg, message)
else:
callback = callbacks.CloneCallback()
okay, cs = client.createSiblingCloneChangeSet(
targetMap,
trovesToClone,
updateBuildInfo=not skipBuildInfo,
infoOnly=info,
callback=callback,
cloneOnlyByDefaultTroves=cloneOnlyByDefaultTroves,
cloneSources=cloneSources,
excludeGroups=excludeGroups)
if not okay:
return False
return _finishClone(client,
cfg,
cs,
callback,
info=info,
test=test,
ignoreConflicts=ignoreConflicts,
targetFile=targetFile)
def clone_job(helper, job):
'''
Create a changeset that will clone all built troves into the target
label.
'''
branch_map = {} # source_branch -> target_branch
# nbf_map := name, target_branch, flavor -> trove, source_version
# This maps a given name and built flavor back to the BuildTrove that
# created it, allowing us to find duplicate builds and build a clone
# job.
nbf_map = {}
for trove in job.iterTroves():
source_name, source_version, _ = trove.getNameVersionFlavor()
#assert source_version.getHost() == helper.cfg.reposName
# Determine which branch this will be committed to
source_branch = origin_branch = source_version.branch()
target_branch = origin_branch.createShadow(
helper.plan.getTargetLabel())
# Mark said branch for final promote
branch_map[source_branch] = target_branch
# Check for different versions of the same trove
nbf = source_name, target_branch, deps.Flavor()
if nbf in nbf_map and nbf_map[nbf][1] != source_version:
bad_version = nbf_map[nbf][1]
raise RuntimeError("Cannot commit two different versions of "
"source component %s: %s and %s" % (source_name,
source_version, bad_version))
nbf_map[nbf] = trove, source_version
# Add binary troves to mapping
for bin_name, bin_version, bin_flavor in trove.iterBuiltTroves():
# Don't commit test info
#if bin_name.endswith(':testinfo'):
# continue
nbf = bin_name, target_branch, bin_flavor
if nbf in nbf_map:
# Eliminate duplicate commits of the same NBF by keeping
# only the newer package
other_version = nbf_map[nbf][0].getBinaryTroves()[0][1]
if other_version < bin_version:
bad_trove, new_trove = nbf_map[nbf][0], trove
new_version = bin_version
else:
new_trove, bad_trove = nbf_map[nbf][0], trove
new_version = other_version
new_name = new_trove.getName().split(':')[0]
# Delete commit maps for the entire rejected package
for bad_name, bad_version, bad_flavor \
in bad_trove.iterBuiltTroves():
bad_nbf = (bad_name, target_branch, bad_flavor)
if not ':' in bad_name:
log.warning('Not committing %s=%s[%s] - overridden '
'by %s=%s', bad_name, bad_version, bad_flavor,
new_name, new_version)
if bad_nbf in nbf_map \
and bad_trove is nbf_map[bad_nbf][0]:
log.debug('Purging %s=%s[%s]' % (bad_name, bad_version,
bad_flavor))
del nbf_map[bad_nbf]
# If this trove is the bad trove, stop processing it
if trove is bad_trove:
break
nbf_map[nbf] = trove, bin_version
# Determine what to clone
troves_to_clone = []
for (trv_name, _, trv_flavor), (trove, trv_version) \
in nbf_map.iteritems():
troves_to_clone.append((trv_name, trv_version, trv_flavor))
# Do the clone
update_build_info = compat.ConaryVersion()\
.acceptsPartialBuildReqCloning()
callback = callbacks.CloneCallback(helper.cfg,
helper.plan.commitMessage)
okay, changeset = helper.getClient().createTargetedCloneChangeSet(
branch_map, troves_to_clone,
updateBuildInfo=update_build_info,
cloneSources=False,
trackClone=False,
callback=callback,
#cloneOnlyByDefaultTroves=True,
fullRecurse=False)
return okay, changeset, nbf_map